diff --git a/.agents/skills/add-block/SKILL.md b/.agents/skills/add-block/SKILL.md index fb60d3c0649..24fd80608ce 100644 --- a/.agents/skills/add-block/SKILL.md +++ b/.agents/skills/add-block/SKILL.md @@ -1052,6 +1052,10 @@ After creating the block, you MUST validate it against every tool it references: 4. **Verify conditions** — each subBlock should only show for the operations that actually use it 5. **Verify `{Service}BlockMeta` is exported** with at least 7 templates, each having `icon`, `title`, `prompt`, `modules`, `category`, and `tags` 6. **If any tool outputs are still unknown**, explicitly tell the user instead of guessing block outputs +7. **Verify the tool execution boundary** — blocks never create or call API routes. Every referenced + tool must already be either a registered `InternalToolConfig.operation` or an absolute external + HTTP(S) `ToolConfig.request`. If transport needs to change, use the `add-tools` skill; do not add a + same-origin `/api/...` hop from the block. ## Option Lists: `selectorKey` or `options`, never a per-block fetcher diff --git a/.agents/skills/add-feature-flag/SKILL.md b/.agents/skills/add-feature-flag/SKILL.md index a741530f412..bb415d8585f 100644 --- a/.agents/skills/add-feature-flag/SKILL.md +++ b/.agents/skills/add-feature-flag/SKILL.md @@ -1,16 +1,16 @@ --- name: add-feature-flag -description: Add a runtime feature flag (AppConfig-backed on prod, secret fallback off-prod), global by default or optionally gated by org id, user id, or platform admin +description: Add a runtime feature flag (AppConfig-backed on prod, secret fallback off-prod), global by default or optionally gated by workspace id, org id, user id, or platform admin argument-hint: --- # Add Feature Flag Skill -You add a **runtime feature flag** to Sim that can change on prod with no redeploy (AWS AppConfig). Prefer a global on/off flag unless the rollout actually needs per-organization, per-user, or platform-admin targeting. When AppConfig isn't the source of truth, the flag falls back to a single **secret** (on/off only). +You add a **runtime feature flag** to Sim that can change on prod with no redeploy (AWS AppConfig). Prefer a global on/off flag unless the rollout actually needs per-workspace, per-organization, per-user, or platform-admin targeting. When AppConfig isn't the source of truth, the flag falls back to a single **secret** (on/off only). ## When to use this vs `env-flags.ts` -- **Feature flag** (`@/lib/core/config/feature-flags.ts`): runtime global on/off by default, optionally scoped by `userId`/`orgId`/admin. This skill. +- **Feature flag** (`@/lib/core/config/feature-flags.ts`): runtime global on/off by default, optionally scoped by `workspaceId`/`userId`/`orgId`/admin. This skill. - **Env flag** (`@/lib/core/config/env-flags.ts`): deploy-time capability/environment detection (`isProd`, `isHosted`, `isBillingEnabled`). A module-load boolean. **Do not add gated flags here.** If the user wants a fixed per-deployment toggle, send them to `env-flags.ts` instead. @@ -21,10 +21,11 @@ A flag's **gating rule lives only in the hosted AppConfig document**. It is ON f ```ts interface FeatureFlagRule { - enabled?: boolean // global default for everyone - orgIds?: string[] // allowlisted organization ids - userIds?: string[] // allowlisted user ids - adminEnabled?: boolean // platform admins (user.role === 'admin') + enabled?: boolean // global default for everyone + workspaceIds?: string[] // allowlisted workspace ids + orgIds?: string[] // allowlisted organization ids + userIds?: string[] // allowlisted user ids + adminEnabled?: boolean // platform admins (user.role === 'admin') } ``` @@ -34,10 +35,10 @@ Critically, **none of this is expressible in code** — gating (especially `admi 1. **Confirm the granularity before editing code.** If the user has not already specified it, stop and ask: - > Should `` be a global on/off flag (recommended), or does it need rollout targeting by organization, user, and/or platform admin? + > Should `` be a global on/off flag (recommended), or does it need rollout targeting by workspace, organization, user, and/or platform admin? - - Recommend **global**. Do not infer scoped gating merely because the call site already has a user or organization id. - - If the user chooses scoped gating but does not name the dimensions, ask which of organization, user, and platform admin it needs. Wire only the selected dimensions. + - Recommend **global**. Do not infer scoped gating merely because the call site already has a workspace, user, or organization id. + - If the user chooses scoped gating but does not name the dimensions, ask which of workspace, organization, user, and platform admin it needs. Wire only the selected dimensions. - If the user wants a fixed per-deployment toggle rather than a runtime AppConfig flag, use `env-flags.ts` instead. 2. **Define the flag.** Add one entry to the `FEATURE_FLAGS` registry in `apps/sim/lib/core/config/feature-flags.ts`. Each entry is the flag's whole definition — name (kebab-case key), `description`, and the `fallback` secret consulted when AppConfig isn't the source of truth (truthy ⇒ on globally): @@ -51,7 +52,7 @@ Critically, **none of this is expressible in code** — gating (especially `admi } ``` - `fallback` is the env/secret key (typed as `keyof typeof env`), so add `` to `apps/sim/lib/core/config/env.ts` first (and the deployment's secret store) — it won't typecheck otherwise. Do **not** add org/user/admin defaults here — that gating exists only in AppConfig. Adding the entry makes `` a valid `FeatureFlagName`. + `fallback` is the env/secret key (typed as `keyof typeof env`), so add `` to `apps/sim/lib/core/config/env.ts` first (and the deployment's secret store) — it won't typecheck otherwise. Do **not** add workspace/org/user/admin defaults here — that gating exists only in AppConfig. Adding the entry makes `` a valid `FeatureFlagName`. 3. **Gate the call site at the chosen granularity.** For the recommended global mode, pass no context: @@ -70,17 +71,17 @@ Critically, **none of this is expressible in code** — gating (especially `admi ```ts import { isFeatureEnabled } from '@/lib/core/config/feature-flags' - if (await isFeatureEnabled('', { userId, orgId })) { + if (await isFeatureEnabled('', { workspaceId, userId, orgId })) { // gated behavior } ``` - - Organization targeting uses `orgId`; user and platform-admin targeting require `userId`. + - Workspace targeting uses `workspaceId`; organization targeting uses `orgId`; user and platform-admin targeting require `userId`. - Missing ids are fine — a clause with no matching id is skipped; with no `userId`, the admin clause resolves to `false` without a DB read. - Admin routes that already know the caller is an admin may pass `{ userId, isAdmin: true }` to skip the role lookup. - **Client/UI flags:** resolve server-side (in a server component, route, or loader) and pass the boolean down as a prop. There is no client AppConfig. -4. **(Prod) configure in AppConfig.** The infra `feature-flags` profile schema is permissive, so a new flag needs **no infra change**. Operators add the flag to the hosted `feature-flags` document using `enabled` for global rollout or only the selected `orgIds`/`userIds`/`adminEnabled` clauses for scoped rollout, then start a `sim--fast` deployment (see the AppConfig runbook in the infra README — same flow as `access-control`). The fallback secret only applies when AppConfig is disabled. +4. **(Prod) configure in AppConfig.** The infra `feature-flags` profile schema is permissive, so a new flag needs **no infra change**. Operators add the flag to the hosted `feature-flags` document using `enabled` for global rollout or only the selected `workspaceIds`/`orgIds`/`userIds`/`adminEnabled` clauses for scoped rollout, then start a `sim--fast` deployment (see the AppConfig runbook in the infra README — same flow as `access-control`). The fallback secret only applies when AppConfig is disabled. 5. **Test.** Add a case to `apps/sim/lib/core/config/feature-flags.test.ts` that matches the chosen granularity. For a global flag, exercise `isFeatureEnabled('')` with an AppConfig `enabled` rule and toggle the fallback secret for the off-AppConfig path. For scoped rollout, cover only the selected clauses and mock `isPlatformAdmin` when testing `adminEnabled`. @@ -90,6 +91,6 @@ Critically, **none of this is expressible in code** — gating (especially `admi - Flag keys are `kebab-case`. - Never read flags via raw `fetch` or a new AppConfig client — always go through `isFeatureEnabled` / `getFeatureFlags`. -- Never bake gating into code. The fallback is a single boolean secret; org/user/admin scoping is AppConfig-only. +- Never bake gating into code. The fallback is a single boolean secret; workspace/org/user/admin scoping is AppConfig-only. - Never add or propagate request context unless the user chose scoped rollout. - The admin check reads the DB **replica** (`dbReplica`) and is resolved lazily, so an admin-gated flag adds at most one cheap replica read, and only when `adminEnabled` is the deciding clause. diff --git a/.agents/skills/add-integration/SKILL.md b/.agents/skills/add-integration/SKILL.md index 3f3bbaccf9a..b8e69d488bb 100644 --- a/.agents/skills/add-integration/SKILL.md +++ b/.agents/skills/add-integration/SKILL.md @@ -60,6 +60,18 @@ apps/sim/tools/{service}/ ### Key Patterns +Choose the tool boundary before writing the declaration: + +- Use `InternalToolConfig.operation` for same-process Sim/provider work. Put the handler under + `apps/sim/lib/internal/{service}/execute-tool.ts` and register every ID in + `apps/sim/lib/internal/tool-operations/registry.server.ts`. +- Use `ToolConfig.request` only for an absolute external HTTP(S) provider endpoint. + +Never point a tool at `/api/...`, construct an absolute URL back to Sim, declare +`request.internal`, or add an API route merely to reuse code, normalize files, or authorize +resources. A real external/browser route and an in-process tool may share the same operation, but +neither calls the other. Follow the full transport and handler rules in the `add-tools` skill. + **types.ts:** ```typescript import type { ToolResponse } from '@/tools/types' @@ -82,7 +94,7 @@ export interface {Service}Response extends ToolResponse { **Tool file pattern:** ```typescript -export const {service}{Action}Tool: ToolConfig = { +export const {service}{Action}Tool: InternalToolConfig = { id: '{service}_{action}', name: '{Service} {Action}', description: '...', @@ -95,16 +107,11 @@ export const {service}{Action}Tool: ToolConfig = { // ... other params }, - request: { url, method, headers, body }, - - transformResponse: async (response) => { - const data = await response.json() - return { - success: true, - output: { - field: data.field ?? null, // Always handle nullables - }, - } + operation: { + input: (params) => ({ + accessToken: params.accessToken, + // Map only the semantic operation input. + }), }, outputs: { /* ... */ }, @@ -135,7 +142,8 @@ and leave the field unannotated. sent with their normal request semantics. A URL, domain, resource ID, control field, or opaque payload is not model-visible merely because the provider is AI-backed or may process the referenced resource later. -- **Text or structured content consumed by an AI model:** declare `request.modelInput` with +- **Text or structured content consumed by an AI model:** declare `request.modelInput` for an + external provider request or `operation.modelInput` for an in-process operation, with `mode: 'project'` and select only the exact model-visible fields. The shared executor replaces activated Sim secrets with canonical `{{NAME}}` labels before request formatting. For nested or JSON-string fields, use a small shared selector plus `applyProjected`; verify that selecting the @@ -144,20 +152,19 @@ and leave the field unannotated. top-level param in `request.modelInput`. Project the private copy before the existing request formatter parses it; keep formatter behavior deterministic when a whole-value placeholder is not valid in the serialized grammar. Do not introduce a second hard-rejection path. -- **Opaque model input owned by an authenticated internal route** such as inline audio, image, - video, or document bytes: add `privateProvenance` to a projected request, or use +- **Opaque model input owned by an in-process operation** such as inline audio, image, video, or + document bytes: add `privateProvenance` to the operation model-input declaration, or use `mode: 'private-provenance'` when there is no textual projection. Do not select storage keys, - paths, signed URLs, or ordinary remote URLs as byte provenance; the owning route must authorize - stored bytes independently at model egress. The route must call + paths, signed URLs, or ordinary remote URLs as byte provenance; the owning operation must + authorize stored bytes independently at model egress. The operation must call `validateOpaqueModelInputProvenance` before downloading or sending content to the model and must apply the workspace-file provenance guard before reading a persisted workspace file. - **Sim-owned durable storage or internal execution handoff** that can later enter a workflow/model (table cells, Agent memory, knowledge documents/chunks, workspace-file contents, or child-workflow - input): transport encrypted field-scoped provenance with `request.secretProvenance`. The - authenticated receiver validates the exact selection and scope, strips the private envelope, and - persists, imports, or propagates it at the owning boundary. Preserve shared legacy behavior for - headerless internal calls and rows/files whose provenance marker is `NULL`; never invent a - tool-local migration rule. + input): transport encrypted field-scoped provenance with `operation.secretProvenance`. The + operation validates the exact selection and trusted scope, then persists, imports, or propagates + it at the owning boundary. Preserve shared legacy behavior for rows/files whose provenance marker + is `NULL`; never invent a tool-local migration rule. Hard rules: @@ -166,7 +173,8 @@ Hard rules: transport and strips private metadata from functional results. - Never attach private provenance to an external URL or to `directExecution`. Project proven model-visible external fields with `request.modelInput`; otherwise preserve ordinary request - semantics. Use an authenticated internal route when encrypted provenance must cross the boundary. + semantics. Use a registered in-process operation when encrypted provenance must cross the + boundary. - Never sanitize arbitrary third-party tool results. Projection applies only to secrets activated by Sim's resolved-secret provenance for that execution/tool call. - Do not add provenance merely because a value is persisted, returned by a tool, or appears in a @@ -596,6 +604,10 @@ If creating V2 versions (API-aligned outputs): - [ ] Created `tools/{service}/` directory - [ ] Created `types.ts` with all interfaces - [ ] Created tool file for each operation +- [ ] Chose exactly one boundary per tool: registered `InternalToolConfig.operation` or absolute + external HTTP(S) `ToolConfig.request` +- [ ] No tool points to `/api/...`, constructs a URL back to Sim, declares `request.internal`, or + has an HTTP fallback for an in-process operation - [ ] All params have correct visibility - [ ] All nullable fields use `?? null` - [ ] All optional outputs have `optional: true` @@ -607,6 +619,8 @@ If creating V2 versions (API-aligned outputs): external resource locators and control inputs retain their request semantics - [ ] Confirmed ordinary third-party tool results are not generically sanitized - [ ] Added provenance compatibility and fail-closed boundary tests where applicable +- [ ] `bun run check:tool-request-boundary` passes +- [ ] Internal-operation registry completeness test passes for every operation-backed tool ### Block - [ ] Created `blocks/blocks/{service}.ts` @@ -721,7 +735,8 @@ interface UserFile { ### File Input Pattern (Uploads) -For tools that accept file uploads, **always route through an internal API endpoint** rather than calling external APIs directly. This ensures proper file content retrieval. +File authorization, normalization, storage reads, provider upload, and response mapping belong in a +registered in-process operation. Do not create an internal API route for file tools. #### 1. Block SubBlocks for File Input @@ -757,137 +772,36 @@ Use the basic/advanced mode pattern: #### 2. Normalize File Input in Block Config -In `tools.config.tool`, use `normalizeFileInput` to handle all input variants: +`tools.config.tool` selects the tool before variable resolution and must not mutate or coerce input. +Use `tools.config.params`, which runs after variable resolution, to normalize all file variants: ```typescript import { normalizeFileInput } from '@/blocks/utils' tools: { config: { - tool: (params) => { - // Normalize file from basic (uploadFile), advanced (fileRef), or legacy (fileContent) - const normalizedFile = normalizeFileInput( - params.uploadFile || params.fileRef || params.fileContent, - { single: true } - ) - if (normalizedFile) { - params.file = normalizedFile - } - return `{service}_${params.operation}` + tool: (params) => `{service}_${params.operation}`, + params: (params) => { + // Serialization collapses the basic/advanced pair into the canonical `file` key. + const normalizedFile = normalizeFileInput(params.file, { single: true }) + return normalizedFile ? { file: normalizedFile } : {} }, }, } ``` -#### 3. Create Special Internal Tool Execution Route - -Create `apps/sim/app/api/tools/{service}/{action}/route.ts`. This raw route pattern is only for an integration's provider-execution boundary when it needs special file normalization, large-body handling, or protocol behavior. It is not the pattern for CRUD or other operations on protected Sim resources. For those, use the `migrate-application-operation` skill and an authorized application use case with the ordinary internal/v2 route builders. - -Internal tool routes are HTTP boundaries and follow the same contract policy as public routes — define the request/response shape in `apps/sim/lib/api/contracts/tools/{service}.ts` (or an existing aggregate) and validate with canonical helpers from `@/lib/api/server`. Never write a route-local Zod schema. Authenticate and perform cheap admission before parsing or downloading files. +#### 3. Define and register the in-process operation ```typescript -// apps/sim/lib/api/contracts/tools/{service}.ts -import { z } from 'zod' -import { defineRouteContract } from '@/lib/api/contracts' -import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' - -export const {service}UploadBodySchema = z.object({ - accessToken: z.string(), - file: FileInputSchema.optional().nullable(), - fileContent: z.string().optional().nullable(), - // ... other params -}) - -export const {service}UploadResponseSchema = z.object({ - success: z.boolean(), - output: z.object({ id: z.string(), url: z.string() }).optional(), - error: z.string().optional(), -}) - -export const {service}UploadContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/{service}/upload', - body: {service}UploadBodySchema, - response: { mode: 'json', schema: {service}UploadResponseSchema }, -}) - -export type {Service}UploadBody = z.input -export type {Service}UploadResponse = z.output -``` - -```typescript -// apps/sim/app/api/tools/{service}/upload/route.ts -import { createLogger } from '@sim/logger' -import { NextResponse, type NextRequest } from 'next/server' -import { {service}UploadContract } from '@/lib/api/contracts/tools/{service}' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { type RawFileInput } from '@/lib/uploads/utils/file-schemas' -import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' - -const logger = createLogger('{Service}UploadAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - // Auth always runs BEFORE parseRequest — never validate untrusted input before authenticating. - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest({service}UploadContract, request, {}) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - let fileBuffer: Buffer - let fileName: string - - // Prefer UserFile input, fall back to legacy base64 - if (data.file) { - const userFiles = processFilesToUserFiles([data.file as RawFileInput], requestId, logger) - if (userFiles.length === 0) { - return NextResponse.json({ success: false, error: 'Invalid file' }, { status: 400 }) - } - const userFile = userFiles[0] - fileBuffer = await downloadFileFromStorage(userFile, requestId, logger) - fileName = userFile.name - } else if (data.fileContent) { - // Legacy: base64 string (backwards compatibility) - fileBuffer = Buffer.from(data.fileContent, 'base64') - fileName = 'file' - } else { - return NextResponse.json({ success: false, error: 'File required' }, { status: 400 }) - } - - // Now call external API with fileBuffer - const response = await fetch('https://api.{service}.com/upload', { - method: 'POST', - headers: { Authorization: `Bearer ${data.accessToken}` }, - body: new Uint8Array(fileBuffer), // Convert Buffer for fetch - }) - - // ... handle response -}) -``` - -#### 4. Update Tool to Use Internal Route - -```typescript -export const {service}UploadTool: ToolConfig = { +export const {service}UploadTool: InternalToolConfig = { id: '{service}_upload', // ... params: { file: { type: 'file', required: false, visibility: 'user-or-llm' }, fileContent: { type: 'string', required: false, visibility: 'hidden' }, // Legacy }, - request: { - url: '/api/tools/{service}/upload', // Internal route - method: 'POST', - body: (params) => ({ + operation: { + input: (params) => ({ accessToken: params.accessToken, file: params.file, fileContent: params.fileContent, @@ -896,6 +810,13 @@ export const {service}UploadTool: ToolConfig = { } ``` +Implement `apps/sim/lib/internal/{service}/execute-tool.ts` and keep the file/provider work in typed +operations beside it. The handler validates `request.input`, derives storage authority only from +trusted `request.context`, authorizes every stored file before reading bytes, forwards +`request.signal`, enforces declared and actual byte caps, and returns the canonical tool response. +Register `{service}_upload` in `apps/sim/lib/internal/tool-operations/registry.server.ts` and add a +registry/direct-handler test. There is no HTTP fallback. + ### File Output Pattern (Downloads) For tools that return files, use `FileToolProcessor` to store files and return `UserFile` objects. @@ -923,11 +844,11 @@ transformResponse: async (response, context) => { } ``` -#### In API Route (for complex file handling) +#### In the operation handler (for complex file handling) ```typescript -// Return file data that FileToolProcessor can handle -return NextResponse.json({ +// Return file data that FileToolProcessor can handle. No API route is involved. +return Response.json({ success: true, output: { file: { diff --git a/.agents/skills/add-tools/SKILL.md b/.agents/skills/add-tools/SKILL.md index 6ae100ce86b..0b7a5cc1f6c 100644 --- a/.agents/skills/add-tools/SKILL.md +++ b/.agents/skills/add-tools/SKILL.md @@ -42,7 +42,29 @@ tools/{service}/ ## Tool Configuration Structure -Every tool MUST follow this exact structure: +### Choose the execution boundary first + +Every tool must use exactly one of these configurations: + +- **In-process operation (preferred):** use `InternalToolConfig` when the executor and the + implementation run in the same Sim process/trust/runtime plane. Materialize typed + `operation.input`, implement the handler under `apps/sim/lib/internal/{service}/execute-tool.ts`, + and register every tool ID in `apps/sim/lib/internal/tool-operations/registry.server.ts`. +- **External provider request:** use `ToolConfig.request` only when the URL is an absolute external + HTTP(S) provider endpoint. + +Never set a tool URL to `/api/...`, construct an absolute URL back to Sim, declare +`request.internal`, import a route module, or create an API route merely to normalize files, +authorize access, or reuse server code. A real browser/API route may remain as a thin adapter, but +the route and the tool must call the same operation directly. A true cross-process/capability +boundary uses an explicit server client and is not disguised as a tool self-hop. + +For protected Sim resources, the internal handler calls the domain's authorized application use +case with trusted execution context; use the `migrate-application-operation` skill. + +### External provider request + +Use this structure only for an absolute external provider API: ```typescript import type { {ServiceName}{Action}Params } from '@/tools/{service}/types' @@ -126,6 +148,38 @@ export const {serviceName}{Action}Tool: ToolConfig< } ``` +### In-process operation + +```typescript +import type { InternalToolConfig } from '@/tools/types' + +export const {serviceName}{Action}Tool: InternalToolConfig< + {ServiceName}{Action}Params, + {ServiceName}{Action}Response +> = { + id: '{service}_{action}', + name: '{Service} {Action}', + description: 'Brief description', + version: '1.0.0', + params: { + // Same canonical metadata as an external tool. + }, + operation: { + input: (params) => ({ + // Map resolved tool params into the typed semantic operation input. + }), + }, + outputs: { + // Define each output field. + }, +} +``` + +The registered handler accepts `InternalToolOperationCall`, validates `request.input`, uses only +trusted `request.context` for authority, forwards `request.signal`, and returns the same bounded +`Response` contract expected by the tool executor. It has no URL, method, request headers, fetch +fallback, or caller-controlled `_context` authority. + ## Critical Rules for Parameters ### Visibility Options @@ -149,17 +203,17 @@ export const {serviceName}{Action}Tool: ToolConfig< - Leave ordinary external API inputs and third-party results unchanged. Add provenance handling only when an exact field is proven to cross a Sim model, durable-storage, or internal-execution boundary. -- Project AI-consumed text/structured fields with the smallest exact `request.modelInput` selector. +- Project AI-consumed text/structured fields with the smallest exact model-input selector: + `request.modelInput` for an external request or `operation.modelInput` for an in-process operation. - Treat URLs, domains, resource IDs, and control fields as ordinary request values unless the exact field is proven model-visible. For serialized external model content, project the serialized top-level param through `request.modelInput` before the existing formatter parses it; do not add a separate hard-rejection mechanism. -- For authenticated internal routes, use `privateProvenance` for actual inline/raw model bytes or - `request.secretProvenance` for durable writes and execution handoffs. Do not treat a storage key, +- For in-process operations, use `operation.modelInput` for actual inline/raw model bytes or + `operation.secretProvenance` for durable writes and execution handoffs. Do not treat a storage key, path, signed URL, or remote URL as provenance for fetched bytes; authorize tracked stored bytes at - the owning model-egress boundary. Authenticate first, validate the exact selection and scope, - strip the private envelope, then import or propagate provenance at the receiving boundary. - Preserve documented headerless legacy behavior. + the owning model-egress boundary. Validate the exact selection and trusted scope, then import or + propagate provenance at the receiving operation boundary. - Never substitute secret plaintext into source, serialize plaintext provenance, hand-roll private headers, or blanket-sanitize tool results. - Add focused tests for named projection, identical unproven public text, malformed/incomplete @@ -466,6 +520,10 @@ All tool IDs MUST use `snake_case`: `{service}_{action}` (e.g., `x_create_tweet` ## Checklist Before Finishing - [ ] All tool IDs use snake_case +- [ ] Chose exactly one boundary: registered `InternalToolConfig.operation` or absolute external + HTTP(S) `ToolConfig.request` +- [ ] No tool request points to `/api/...`, constructs a URL back to Sim, or declares + `request.internal` - [ ] All params have explicit `required: true` or `required: false` - [ ] All params have appropriate `visibility` - [ ] All nullable response fields use `?? null` @@ -492,7 +550,9 @@ After creating all tools, you MUST validate every tool before finishing: - All required params are marked `required: true` - All optional params are marked `required: false` - Param types match the API (string, number, boolean, json) - - Request URL, method, headers, and body match the API spec + - For external tools, request URL, method, headers, and body match the provider API spec + - For internal tools, `operation.input` matches the handler schema and the handler is registered + with no HTTP fallback - `transformResponse` extracts the correct fields from the API response - All output fields match what the API actually returns - No fields are missing from outputs that the API provides diff --git a/.agents/skills/add-trigger/SKILL.md b/.agents/skills/add-trigger/SKILL.md index bfec917d60b..f3e776c848f 100644 --- a/.agents/skills/add-trigger/SKILL.md +++ b/.agents/skills/add-trigger/SKILL.md @@ -505,6 +505,11 @@ Two rules the checks enforce: ## Checklist +Webhook and polling routes are legitimate external ingress boundaries. They must not call this +Sim app's own API routes to reuse provider or business logic. Extract the shared provider operation +or authorized application use case and call it directly from the trigger handler and any other +server adapter. HTTP is reserved for an actual cross-process/capability boundary. + ### Trigger Definition - [ ] Created `utils.ts` with options, instructions, extra fields, and output builders - [ ] Primary trigger has `includeDropdown: true`; secondary triggers do NOT diff --git a/AGENTS.md b/AGENTS.md index ae4d76e0de0..b13fecd2b96 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,6 +40,14 @@ You are a professional software engineer. All code must follow best practices: a - Never substitute a billing owner, uploader, creator, or API-key owner for the acting principal. Fail fast when the identity model or operation policy cannot express the caller. - Use the `migrate-application-operation` skill whenever creating or migrating a protected endpoint, tool command, or resource method. +### Tool Execution Boundary + +- A tool has exactly one execution boundary. Use `InternalToolConfig.operation` when the executor can call the implementation in the same process and trust/runtime plane. Put the server handler under `apps/sim/lib/internal//execute-tool.ts` and register it in `apps/sim/lib/internal/tool-operations/registry.server.ts`. +- `ToolConfig.request` is only for absolute external HTTP(S) provider APIs. A tool definition must never point at `/api/...`, construct an absolute URL back to this Sim app, or declare an `internal` request policy. Do not add a same-origin route merely to reuse code, normalize files, or perform authorization. +- Real browser/API ingress and real cross-process capability boundaries may remain HTTP. Their route and any in-process tool adapter call the same application/provider operation; neither calls the other, and tool code never imports route modules. +- Protected Sim resources still enter through authorized application use cases. The internal tool handler is a trusted surface adapter, not an authorization or database bypass. +- `bun run check:tool-request-boundary` rejects detectable tool self-hops, the external request formatter rejects relative URLs at runtime, and the internal-operation registry test requires every operation-backed tool to have a loadable handler. + ### Root Structure ``` diff --git a/apps/docs/content/docs/de/blocks/agent.mdx b/apps/docs/content/docs/de/blocks/agent.mdx index f8f149da7e4..bbf5f47b43d 100644 --- a/apps/docs/content/docs/de/blocks/agent.mdx +++ b/apps/docs/content/docs/de/blocks/agent.mdx @@ -13,7 +13,7 @@ Der Agent-Block verbindet deinen Workflow mit Large Language Models (LLMs). Er v src="/static/blocks/agent.png" alt="Agent-Block-Konfiguration" width={500} - height={400} + height={450} className="my-6" /> diff --git a/apps/docs/content/docs/de/blocks/api.mdx b/apps/docs/content/docs/de/blocks/api.mdx index 43cacb97ce6..39dd3595e65 100644 --- a/apps/docs/content/docs/de/blocks/api.mdx +++ b/apps/docs/content/docs/de/blocks/api.mdx @@ -13,7 +13,7 @@ Der API-Block verbindet Ihren Workflow mit externen Diensten durch HTTP-Anfragen src="/static/blocks/api.png" alt="API-Block" width={500} - height={400} + height={422} className="my-6" /> diff --git a/apps/docs/content/docs/de/blocks/condition.mdx b/apps/docs/content/docs/de/blocks/condition.mdx index 5d8687e81d8..6a7a77b16ec 100644 --- a/apps/docs/content/docs/de/blocks/condition.mdx +++ b/apps/docs/content/docs/de/blocks/condition.mdx @@ -13,7 +13,7 @@ Der Bedingungsblock verzweigt die Workflow-Ausführung basierend auf booleschen src="/static/blocks/condition.png" alt="Bedingungsblock" width={500} - height={400} + height={317} className="my-6" /> diff --git a/apps/docs/content/docs/de/blocks/evaluator.mdx b/apps/docs/content/docs/de/blocks/evaluator.mdx index ed6e3766cd5..22d67b00f86 100644 --- a/apps/docs/content/docs/de/blocks/evaluator.mdx +++ b/apps/docs/content/docs/de/blocks/evaluator.mdx @@ -13,7 +13,7 @@ Der Evaluator-Block nutzt KI, um die Inhaltsqualität anhand benutzerdefinierter src="/static/blocks/evaluator.png" alt="Evaluator-Block-Konfiguration" width={500} - height={400} + height={337} className="my-6" /> diff --git a/apps/docs/content/docs/de/blocks/function.mdx b/apps/docs/content/docs/de/blocks/function.mdx index 3e974ef328a..5bb175956ad 100644 --- a/apps/docs/content/docs/de/blocks/function.mdx +++ b/apps/docs/content/docs/de/blocks/function.mdx @@ -11,7 +11,7 @@ Der Funktionsblock führt benutzerdefinierten JavaScript- oder TypeScript-Code i src="/static/blocks/function.png" alt="Funktionsblock mit Code-Editor" width={500} - height={400} + height={287} className="my-6" /> diff --git a/apps/docs/content/docs/de/blocks/guardrails.mdx b/apps/docs/content/docs/de/blocks/guardrails.mdx index 9d322fe219a..4e24df85d54 100644 --- a/apps/docs/content/docs/de/blocks/guardrails.mdx +++ b/apps/docs/content/docs/de/blocks/guardrails.mdx @@ -13,7 +13,7 @@ Der Guardrails-Block validiert und schützt Ihre KI-Workflows, indem er Inhalte src="/static/blocks/guardrails.png" alt="Guardrails-Block" width={500} - height={400} + height={398} className="my-6" /> @@ -90,7 +90,7 @@ Erkennt personenbezogene Daten mithilfe von Microsoft Presidio. Unterstützt üb src="/static/blocks/guardrails-2.png" alt="PII-Erkennungskonfiguration" width={700} - height={450} + height={335} className="my-6" /> diff --git a/apps/docs/content/docs/de/blocks/human-in-the-loop.mdx b/apps/docs/content/docs/de/blocks/human-in-the-loop.mdx index a6b14e0944a..b17167348a8 100644 --- a/apps/docs/content/docs/de/blocks/human-in-the-loop.mdx +++ b/apps/docs/content/docs/de/blocks/human-in-the-loop.mdx @@ -14,7 +14,7 @@ Der Human in the Loop Block pausiert die Workflow-Ausführung und wartet auf men src="/static/blocks/hitl-1.png" alt="Human in the Loop Block Konfiguration" width={500} - height={400} + height={355} className="my-6" /> @@ -26,7 +26,7 @@ Wenn die Ausführung diesen Block erreicht, pausiert der Workflow auf unbestimmt src="/static/blocks/hitl-2.png" alt="Human in the Loop Genehmigungsportal" width={700} - height={500} + height={467} className="my-6" /> diff --git a/apps/docs/content/docs/de/blocks/loop.mdx b/apps/docs/content/docs/de/blocks/loop.mdx index 288ebd8e54e..387054d7339 100644 --- a/apps/docs/content/docs/de/blocks/loop.mdx +++ b/apps/docs/content/docs/de/blocks/loop.mdx @@ -27,7 +27,7 @@ Wähle zwischen vier Arten von Schleifen: src="/static/blocks/loop-1.png" alt="For-Schleife mit Iterationen" width={500} - height={400} + height={267} className="my-6" /> @@ -53,7 +53,7 @@ Wähle zwischen vier Arten von Schleifen: src="/static/blocks/loop-2.png" alt="ForEach-Schleife mit Sammlung" width={500} - height={400} + height={273} className="my-6" /> @@ -77,7 +77,7 @@ Wähle zwischen vier Arten von Schleifen: src="/static/blocks/loop-3.png" alt="While-Schleife mit Bedingung" width={500} - height={400} + height={213} className="my-6" /> @@ -102,7 +102,7 @@ Wähle zwischen vier Arten von Schleifen: src="/static/blocks/loop-4.png" alt="Do-While-Schleife mit Bedingung" width={500} - height={400} + height={210} className="my-6" /> diff --git a/apps/docs/content/docs/de/blocks/parallel.mdx b/apps/docs/content/docs/de/blocks/parallel.mdx index 11ca47017dc..80fe4f7434e 100644 --- a/apps/docs/content/docs/de/blocks/parallel.mdx +++ b/apps/docs/content/docs/de/blocks/parallel.mdx @@ -27,7 +27,7 @@ Wählen Sie zwischen zwei Arten der parallelen Ausführung: src="/static/blocks/parallel-1.png" alt="Anzahlbasierte parallele Ausführung" width={500} - height={400} + height={231} className="my-6" /> @@ -56,7 +56,7 @@ const results = await blocks.parallel({ src="/static/blocks/parallel-2.png" alt="Sammlungsbasierte parallele Ausführung" width={500} - height={400} + height={220} className="my-6" /> diff --git a/apps/docs/content/docs/de/blocks/response.mdx b/apps/docs/content/docs/de/blocks/response.mdx index aba8a6380be..d9f7e98d283 100644 --- a/apps/docs/content/docs/de/blocks/response.mdx +++ b/apps/docs/content/docs/de/blocks/response.mdx @@ -13,7 +13,7 @@ Der Response-Block formatiert und sendet strukturierte HTTP-Antworten zurück an src="/static/blocks/response.png" alt="Konfiguration des Antwort-Blocks" width={500} - height={400} + height={382} className="my-6" /> diff --git a/apps/docs/content/docs/de/blocks/router.mdx b/apps/docs/content/docs/de/blocks/router.mdx index 42195595949..9445f5c9363 100644 --- a/apps/docs/content/docs/de/blocks/router.mdx +++ b/apps/docs/content/docs/de/blocks/router.mdx @@ -13,7 +13,7 @@ Der Router-Block verwendet KI, um Workflows basierend auf Inhaltsanalysen intell src="/static/blocks/router.png" alt="Router-Block mit mehreren Pfaden" width={500} - height={400} + height={342} className="my-6" /> diff --git a/apps/docs/content/docs/de/blocks/variables.mdx b/apps/docs/content/docs/de/blocks/variables.mdx index b370c88e533..9422180a61e 100644 --- a/apps/docs/content/docs/de/blocks/variables.mdx +++ b/apps/docs/content/docs/de/blocks/variables.mdx @@ -13,7 +13,7 @@ Der Variablen-Block aktualisiert Workflow-Variablen während der Ausführung. Va src="/static/blocks/variables.png" alt="Variablen-Block" width={500} - height={400} + height={246} className="my-6" /> diff --git a/apps/docs/content/docs/de/blocks/wait.mdx b/apps/docs/content/docs/de/blocks/wait.mdx index 2590c09628a..265d3e98992 100644 --- a/apps/docs/content/docs/de/blocks/wait.mdx +++ b/apps/docs/content/docs/de/blocks/wait.mdx @@ -13,7 +13,7 @@ Der Warten-Block pausiert deinen Workflow für eine bestimmte Zeit, bevor er mit src="/static/blocks/wait.png" alt="Warte-Block" width={500} - height={400} + height={295} className="my-6" /> diff --git a/apps/docs/content/docs/de/blocks/webhook.mdx b/apps/docs/content/docs/de/blocks/webhook.mdx index affaafcd6cb..2dbd0ab717d 100644 --- a/apps/docs/content/docs/de/blocks/webhook.mdx +++ b/apps/docs/content/docs/de/blocks/webhook.mdx @@ -12,7 +12,7 @@ Der Webhook-Block sendet HTTP-POST-Anfragen an externe Webhook-Endpunkte mit aut src="/static/blocks/webhook.png" alt="Webhook-Block" width={500} - height={400} + height={403} className="my-6" /> diff --git a/apps/docs/content/docs/de/blocks/workflow.mdx b/apps/docs/content/docs/de/blocks/workflow.mdx index af44f23cf7e..503be331815 100644 --- a/apps/docs/content/docs/de/blocks/workflow.mdx +++ b/apps/docs/content/docs/de/blocks/workflow.mdx @@ -13,7 +13,7 @@ import { Image } from '@/components/ui/image' src='/static/blocks/workflow.png' alt='Workflow-Block-Konfiguration' width={500} - height={400} + height={310} className='rounded-xl border border-border shadow-sm' /> @@ -30,7 +30,7 @@ Füge einen Workflow-Block hinzu, wenn du einen untergeordneten Workflow als Tei src='/static/blocks/workflow-2.png' alt='Workflow-Block mit Beispiel für Eingabezuordnung' width={700} - height={400} + height={287} className='rounded-xl border border-border shadow-sm' /> diff --git a/apps/docs/content/docs/de/copilot/index.mdx b/apps/docs/content/docs/de/copilot/index.mdx index 4ccdbc8b4c7..38542b9149f 100644 --- a/apps/docs/content/docs/de/copilot/index.mdx +++ b/apps/docs/content/docs/de/copilot/index.mdx @@ -27,7 +27,7 @@ Verwende das `@` Symbol, um auf verschiedene Ressourcen zu verweisen und Copilot src="/static/copilot/copilot-menu.png" alt="Copilot-Kontextmenü mit verfügbaren Referenzoptionen" width={600} - height={400} + height={521} /> Das `@` Menü bietet Zugriff auf: @@ -76,7 +76,7 @@ Diese kontextbezogenen Informationen helfen Copilot, genauere und relevantere Un src="/static/copilot/copilot-mode.png" alt="Copilot-Modusauswahl-Oberfläche" width={600} - height={400} + height={720} className="my-6" /> @@ -134,7 +134,7 @@ Du kannst einfach zwischen verschiedenen Denkmodi über die Modusauswahl in der src="/static/copilot/copilot-models.png" alt="Copilot-Modusauswahl zeigt den erweiterten Modus mit MAX-Umschalter" width={600} - height={300} + height={457} /> Die Oberfläche ermöglicht dir: diff --git a/apps/docs/content/docs/de/execution/basics.mdx b/apps/docs/content/docs/de/execution/basics.mdx index d7c9e2431b0..f5c8ba49f15 100644 --- a/apps/docs/content/docs/de/execution/basics.mdx +++ b/apps/docs/content/docs/de/execution/basics.mdx @@ -20,7 +20,7 @@ Mehrere Blöcke werden gleichzeitig ausgeführt, wenn sie nicht voneinander abh src="/static/execution/concurrency.png" alt="Mehrere Blöcke, die nach dem Start-Block parallel ausgeführt werden" width={800} - height={500} + height={685} /> In diesem Beispiel werden sowohl der Kundensupport- als auch der Deep-Researcher-Agentenblock gleichzeitig nach dem Start-Block ausgeführt, was die Effizienz maximiert. @@ -33,7 +33,7 @@ Wenn Blöcke mehrere Abhängigkeiten haben, wartet die Ausführungs-Engine autom src="/static/execution/combination.png" alt="Funktionsblock, der automatisch Ausgaben von mehreren vorherigen Blöcken empfängt" width={800} - height={500} + height={521} /> Der Funktionsblock erhält Ausgaben von beiden Agentenblöcken, sobald diese abgeschlossen sind, sodass Sie die kombinierten Ergebnisse verarbeiten können. @@ -46,7 +46,7 @@ Workflows können sich in mehrere Richtungen verzweigen, indem sie Routing-Blöc src="/static/execution/routing.png" alt="Workflow, der sowohl bedingte als auch router-basierte Verzweigungen zeigt" width={800} - height={500} + height={391} /> Dieser Workflow zeigt, wie die Ausführung unterschiedlichen Pfaden basierend auf Bedingungen oder KI-Entscheidungen folgen kann, wobei jeder Pfad unabhängig ausgeführt wird. diff --git a/apps/docs/content/docs/de/execution/costs.mdx b/apps/docs/content/docs/de/execution/costs.mdx index 5bdd5a23615..8d89e894478 100644 --- a/apps/docs/content/docs/de/execution/costs.mdx +++ b/apps/docs/content/docs/de/execution/costs.mdx @@ -34,7 +34,7 @@ Für Workflows mit KI-Blöcken können Sie detaillierte Kosteninformationen in d src="/static/logs/logs-cost.png" alt="Modellaufschlüsselung" width={600} - height={400} + height={153} className="my-6" /> diff --git a/apps/docs/content/docs/de/execution/index.mdx b/apps/docs/content/docs/de/execution/index.mdx index ce30568835b..a97ed481ac8 100644 --- a/apps/docs/content/docs/de/execution/index.mdx +++ b/apps/docs/content/docs/de/execution/index.mdx @@ -61,7 +61,7 @@ Alle öffentlichen Einstiegspunkte – API, Chat, Zeitplan, Webhook und manuelle src='/static/execution/deployment-versions.png' alt='Tabelle mit Deployment-Versionen' width={500} - height={280} + height={259} className='rounded-xl border border-border shadow-sm' /> diff --git a/apps/docs/content/docs/de/execution/logging.mdx b/apps/docs/content/docs/de/execution/logging.mdx index d8b38ff9b80..1a3d1a94fd8 100644 --- a/apps/docs/content/docs/de/execution/logging.mdx +++ b/apps/docs/content/docs/de/execution/logging.mdx @@ -21,7 +21,7 @@ Während der manuellen oder Chat-Workflow-Ausführung erscheinen Protokolle in E src="/static/logs/console.png" alt="Echtzeit-Konsolen-Panel" width={400} - height={300} + height={151} className="my-6" /> @@ -41,7 +41,7 @@ Alle Workflow-Ausführungen – ob manuell ausgelöst, über API, Chat, Zeitplan src="/static/logs/logs.png" alt="Protokollseite" width={600} - height={400} + height={451} className="my-6" /> @@ -61,7 +61,7 @@ Durch Klicken auf einen Protokolleintrag öffnet sich eine detaillierte Seitenle src="/static/logs/logs-sidebar.png" alt="Protokoll-Seitenleiste mit Details" width={600} - height={400} + height={880} className="my-6" /> @@ -104,7 +104,7 @@ Für jede protokollierte Ausführung klicken Sie auf "Snapshot anzeigen", um den src="/static/logs/logs-frozen-canvas.png" alt="Workflow-Snapshot" width={600} - height={400} + height={628} className="my-6" /> diff --git a/apps/docs/content/docs/de/getting-started/index.mdx b/apps/docs/content/docs/de/getting-started/index.mdx index 13cfe452170..f9ea00f9191 100644 --- a/apps/docs/content/docs/de/getting-started/index.mdx +++ b/apps/docs/content/docs/de/getting-started/index.mdx @@ -42,7 +42,7 @@ Einen Personenrecherche-Agenten, der: src="/static/getting-started/started-1.png" alt="Beispiel für erste Schritte" width={800} - height={500} + height={340} /> ## Schritt-für-Schritt-Anleitung diff --git a/apps/docs/content/docs/de/introduction/index.mdx b/apps/docs/content/docs/de/introduction/index.mdx index 277562dd39e..38ad402c634 100644 --- a/apps/docs/content/docs/de/introduction/index.mdx +++ b/apps/docs/content/docs/de/introduction/index.mdx @@ -14,7 +14,7 @@ Sim ist ein Open-Source-Tool zur visuellen Workflow-Erstellung für die Entwickl src="/static/introduction.png" alt="Sim visuelle Workflow-Leinwand" width={700} - height={450} + height={372} className="my-6" /> diff --git a/apps/docs/content/docs/de/knowledgebase/index.mdx b/apps/docs/content/docs/de/knowledgebase/index.mdx index d0b930c3d59..73f3b90527f 100644 --- a/apps/docs/content/docs/de/knowledgebase/index.mdx +++ b/apps/docs/content/docs/de/knowledgebase/index.mdx @@ -32,7 +32,7 @@ Sim unterstützt PDF, Word (DOC/DOCX), Klartext (TXT), Markdown (MD), HTML, Exce Sobald Ihre Dokumente verarbeitet sind, können Sie die einzelnen Chunks anzeigen und bearbeiten. Dies gibt Ihnen volle Kontrolle darüber, wie Ihre Inhalte organisiert und durchsucht werden. -Dokumentchunk-Ansicht mit verarbeiteten Inhalten +Dokumentchunk-Ansicht mit verarbeiteten Inhalten ### Chunk-Konfiguration @@ -66,7 +66,7 @@ Wenn mit Azure oder [Mistral OCR](https://docs.mistral.ai/ocr/) konfiguriert: Sobald Ihre Dokumente verarbeitet sind, können Sie sie in Ihren KI-Workflows über den Knowledge-Block verwenden. Dies ermöglicht Retrieval-Augmented Generation (RAG), wodurch Ihre KI-Agenten auf Ihre Dokumentinhalte zugreifen und darüber nachdenken können, um genauere, kontextbezogene Antworten zu liefern. -Verwendung des Knowledge-Blocks in Workflows +Verwendung des Knowledge-Blocks in Workflows ### Knowledge-Block-Funktionen - **Semantische Suche**: Relevante Inhalte mithilfe natürlichsprachlicher Abfragen finden diff --git a/apps/docs/content/docs/de/mcp/index.mdx b/apps/docs/content/docs/de/mcp/index.mdx index 12d5713bdd5..72e5e9fd70c 100644 --- a/apps/docs/content/docs/de/mcp/index.mdx +++ b/apps/docs/content/docs/de/mcp/index.mdx @@ -49,7 +49,7 @@ Sobald MCP-Server konfiguriert sind, werden ihre Tools in Ihren Agent-Blöcken v src="/static/blocks/mcp-2.png" alt="Using MCP Tool in Agent Block" width={700} - height={450} + height={353} className="my-6" /> @@ -68,7 +68,7 @@ Für eine präzisere Steuerung können Sie den dedizierten MCP-Tool-Block verwen src="/static/blocks/mcp-3.png" alt="Standalone MCP Tool Block" width={700} - height={450} + height={545} className="my-6" /> diff --git a/apps/docs/content/docs/de/triggers/index.mdx b/apps/docs/content/docs/de/triggers/index.mdx index 48e5a9c7412..6ef55b3037b 100644 --- a/apps/docs/content/docs/de/triggers/index.mdx +++ b/apps/docs/content/docs/de/triggers/index.mdx @@ -11,7 +11,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/triggers.png" alt="Triggers-Übersicht" width={500} - height={350} + height={348} className="my-6" /> diff --git a/apps/docs/content/docs/de/triggers/rss.mdx b/apps/docs/content/docs/de/triggers/rss.mdx index 10fc4d5b9e6..b917888dc25 100644 --- a/apps/docs/content/docs/de/triggers/rss.mdx +++ b/apps/docs/content/docs/de/triggers/rss.mdx @@ -12,7 +12,7 @@ Der RSS-Feed-Block überwacht RSS- und Atom-Feeds – wenn neue Einträge veröf src="/static/blocks/rss.png" alt="RSS-Feed-Block" width={500} - height={400} + height={228} className="my-6" /> diff --git a/apps/docs/content/docs/de/triggers/schedule.mdx b/apps/docs/content/docs/de/triggers/schedule.mdx index d85b1837b75..7ba129bcfa0 100644 --- a/apps/docs/content/docs/de/triggers/schedule.mdx +++ b/apps/docs/content/docs/de/triggers/schedule.mdx @@ -13,7 +13,7 @@ Der Zeitplan-Block löst Workflows automatisch nach einem wiederkehrenden Zeitpl src="/static/blocks/schedule.png" alt="Zeitplan-Block" width={500} - height={400} + height={336} className="my-6" /> @@ -67,7 +67,7 @@ Zeitpläne werden nach **100 aufeinanderfolgenden Fehlern** automatisch deaktivi src="/static/blocks/schedule-3.png" alt="Deaktivierter Zeitplan" width={500} - height={400} + height={310} className="my-6" /> diff --git a/apps/docs/content/docs/de/triggers/start.mdx b/apps/docs/content/docs/de/triggers/start.mdx index 7e7e7772df6..dce14681c49 100644 --- a/apps/docs/content/docs/de/triggers/start.mdx +++ b/apps/docs/content/docs/de/triggers/start.mdx @@ -13,7 +13,7 @@ Der Start-Block ist der Standard-Auslöser für Workflows, die in Sim erstellt w src="/static/start.png" alt="Start-Block mit Eingabeformat-Feldern" width={360} - height={380} + height={151} className="my-6" /> diff --git a/apps/docs/content/docs/de/triggers/webhook.mdx b/apps/docs/content/docs/de/triggers/webhook.mdx index 22892fca782..e7f79509506 100644 --- a/apps/docs/content/docs/de/triggers/webhook.mdx +++ b/apps/docs/content/docs/de/triggers/webhook.mdx @@ -18,7 +18,7 @@ Der generische Webhook-Block erstellt einen flexiblen Endpunkt, der beliebige Pa src="/static/blocks/webhook-trigger.png" alt="Generische Webhook-Konfiguration" width={500} - height={400} + height={405} className="my-6" /> diff --git a/apps/docs/content/docs/en/agents/mcp.mdx b/apps/docs/content/docs/en/agents/mcp.mdx index 9391b4d2171..6464ed8bb6c 100644 --- a/apps/docs/content/docs/en/agents/mcp.mdx +++ b/apps/docs/content/docs/en/agents/mcp.mdx @@ -26,7 +26,7 @@ To add one: src="/static/blocks/mcp-settings.png" alt="MCP Tools settings page" width={700} - height={450} + height={147} className="my-6" /> @@ -41,7 +41,7 @@ To add one: src="/static/blocks/mcp-add-modal.png" alt="Add New MCP Server modal" width={450} - height={290} + height={272} className="my-6" /> @@ -94,7 +94,7 @@ Once MCP servers are configured, their tools become available within your agent src="/static/blocks/mcp-agent-dropdown.png" alt="Using MCP Tool in Agent Block" width={700} - height={450} + height={304} className="my-6" /> @@ -108,7 +108,7 @@ Once MCP servers are configured, their tools become available within your agent src="/static/blocks/mcp-agent-tools.png" alt="MCP tools list for a selected server" width={400} - height={400} + height={356} className="my-6" /> @@ -129,7 +129,7 @@ For more granular control, you can use the dedicated MCP Tool block to execute s src="/static/blocks/mcp-tool-block.png" alt="Standalone MCP Tool Block" width={700} - height={450} + height={299} className="my-6" /> diff --git a/apps/docs/content/docs/en/chat/files.mdx b/apps/docs/content/docs/en/chat/files.mdx index 88071469c12..4efa3cec3fa 100644 --- a/apps/docs/content/docs/en/chat/files.mdx +++ b/apps/docs/content/docs/en/chat/files.mdx @@ -47,7 +47,7 @@ Open a file using `@filename` or the **+** menu, then describe the change: ## Presentations -Chat resource panel showing a generated Mothership-Use-Cases.pptx file open with the title slide and first use case slide visible +Chat resource panel showing a generated Mothership-Use-Cases.pptx file open with the title slide and first use case slide visible Sim can generate `.pptx` files: @@ -72,7 +72,7 @@ Sim can generate images using AI, and can use an existing image as a reference t - Attach an existing image to your message, then describe what you want: "Generate a new version of this banner with a blue color scheme instead of green" - "Create a variation of this diagram with the boxes rearranged horizontally [attach image]" -Chat resource panel showing a generated hero image of a Mothership-branded blimp flying over San Francisco at golden hour, alongside the chat response linking the file +Chat resource panel showing a generated hero image of a Mothership-branded blimp flying over San Francisco at golden hour, alongside the chat response linking the file Generated images are saved as workspace files. @@ -84,7 +84,7 @@ Sim can generate charts and data visualizations from data you describe or refere - "Create a line chart of token usage over the past 30 days from this data [paste data]" - "Generate a pie chart showing the distribution of lead sources from the leads table" -Chat resource panel showing a generated chart file with bar charts for backend 5xx errors and error rate over time +Chat resource panel showing a generated chart file with bar charts for backend 5xx errors and error rate over time Visualizations are saved as files and rendered in the resource panel. diff --git a/apps/docs/content/docs/en/chat/tables.mdx b/apps/docs/content/docs/en/chat/tables.mdx index 38be58527d8..b8e0c005f8a 100644 --- a/apps/docs/content/docs/en/chat/tables.mdx +++ b/apps/docs/content/docs/en/chat/tables.mdx @@ -6,7 +6,7 @@ description: Create, query, and manage workspace tables from Chat. import { Image } from '@/components/ui/image' import { FAQ } from '@/components/ui/faq' -Chat resource panel showing the pipeline_deals table with company, deal_owner, stage, and amount columns, alongside a chat summary of total pipeline value and breakdown by stage +Chat resource panel showing the pipeline_deals table with company, deal_owner, stage, and amount columns, alongside a chat summary of total pipeline value and breakdown by stage Create a table from a description or a CSV, query it in plain language, add or update rows, and export the results — all through conversation. Tables open in the resource panel as soon as they're created or referenced. diff --git a/apps/docs/content/docs/en/getting-started/index.mdx b/apps/docs/content/docs/en/getting-started/index.mdx index d4d1ed19461..734b28d4084 100644 --- a/apps/docs/content/docs/en/getting-started/index.mdx +++ b/apps/docs/content/docs/en/getting-started/index.mdx @@ -16,7 +16,7 @@ Build a people research agent in 10 minutes. It takes a name through a chat inte src="/static/getting-started/started-1.png" alt="The finished people research workflow" width={800} - height={500} + height={340} /> ## Tutorial diff --git a/apps/docs/content/docs/en/integrations/affinity.mdx b/apps/docs/content/docs/en/integrations/affinity.mdx index e21ec9e0784..0e9bc038725 100644 --- a/apps/docs/content/docs/en/integrations/affinity.mdx +++ b/apps/docs/content/docs/en/integrations/affinity.mdx @@ -2013,7 +2013,7 @@ Write one non-list field value on a company or person. The value type must match | `entityType` | string | Yes | Which entity to write the field on: companies or persons | | `entityId` | string | Yes | ID of that company or person | | `fieldId` | string | Yes | The field ID to write | -| `value` | json | Yes | The new value as \{type, data\}, where type matches the field\'s value type. Examples: \{"type":"text","data":"Series B"\}, \{"type":"number","data":42\}, \{"type":"dropdown","data":\{"dropdownOptionId":7\}\}, \{"type":"person","data":\{"id":123\}\}, \{"type":"person-multi","data":\[\{"id":123\}\]\}. Pass data as null to clear the field | +| `value` | json | Yes | The new value as \{type, data\}, where type matches the field's value type. Examples: \{"type":"text","data":"Series B"\}, \{"type":"number","data":42\}, \{"type":"dropdown","data":\{"dropdownOptionId":7\}\}, \{"type":"person","data":\{"id":123\}\}, \{"type":"person-multi","data":\[\{"id":123\}\]\}. Pass data as null to clear the field | #### Output @@ -2034,7 +2034,7 @@ Write one field value on a list row. Requires the "Export data from Lists" permi | `listId` | string | Yes | The list ID | | `listEntryId` | string | Yes | The list entry ID | | `fieldId` | string | Yes | The field ID to write | -| `value` | json | Yes | The new value as \{type, data\}, where type matches the field\'s value type. Examples: \{"type":"text","data":"Series B"\}, \{"type":"number","data":42\}, \{"type":"dropdown","data":\{"dropdownOptionId":7\}\}, \{"type":"person","data":\{"id":123\}\}, \{"type":"person-multi","data":\[\{"id":123\}\]\}. Pass data as null to clear the field | +| `value` | json | Yes | The new value as \{type, data\}, where type matches the field's value type. Examples: \{"type":"text","data":"Series B"\}, \{"type":"number","data":42\}, \{"type":"dropdown","data":\{"dropdownOptionId":7\}\}, \{"type":"person","data":\{"id":123\}\}, \{"type":"person-multi","data":\[\{"id":123\}\]\}. Pass data as null to clear the field | #### Output diff --git a/apps/docs/content/docs/en/integrations/agiloft.mdx b/apps/docs/content/docs/en/integrations/agiloft.mdx index 3c70095a9a3..2e2e8c76ec4 100644 --- a/apps/docs/content/docs/en/integrations/agiloft.mdx +++ b/apps/docs/content/docs/en/integrations/agiloft.mdx @@ -406,7 +406,7 @@ Search for records in an Agiloft table using a query. | `login` | string | Yes | Agiloft username | | `password` | string | Yes | Agiloft password | | `table` | string | Yes | Table name to search in \(e.g., "contracts", "contacts.employees"\) | -| `query` | string | No | Ad hoc EWSearch query. Combine conditions with && \(and\) or \|\| \(or\) and quote every value — e.g. \"summary~='test'&&priority='High'\". Required unless a saved search is given. | +| `query` | string | No | Ad hoc EWSearch query. Combine conditions with && \(and\) or \|\| \(or\) and quote every value — e.g. "summary~='test'&&priority='High'". Required unless a saved search is given. | | `search` | string | No | Label of a saved search defined on the table \(e.g., "C: Status is Closed"\). Can be combined with a query to narrow it further. | | `fields` | string | No | Comma-separated list of field names to include in the results | | `page` | string | No | Page number for paginated results \(starting from 0\) | @@ -435,7 +435,7 @@ Select record IDs matching a SQL WHERE clause from an Agiloft table. | `login` | string | Yes | Agiloft username | | `password` | string | Yes | Agiloft password | | `table` | string | Yes | Table name \(e.g., "contracts", "contacts.employees"\) | -| `where` | string | Yes | SQL WHERE clause using database column names \(e.g., "summary like \'%new%\'" or "assigned_person=\'John Doe\'"\). EWSelect has no page size and returns every matching ID, so append a database limit such as "limit 0,200" to bound the result. | +| `where` | string | Yes | SQL WHERE clause using database column names \(e.g., "summary like '%new%'" or "assigned_person='John Doe'"\). EWSelect has no page size and returns every matching ID, so append a database limit such as "limit 0,200" to bound the result. | #### Output diff --git a/apps/docs/content/docs/en/integrations/airtable.mdx b/apps/docs/content/docs/en/integrations/airtable.mdx index 1289dd63b5f..9957dfcc309 100644 --- a/apps/docs/content/docs/en/integrations/airtable.mdx +++ b/apps/docs/content/docs/en/integrations/airtable.mdx @@ -94,7 +94,7 @@ Read records from an Airtable table | `baseId` | string | Yes | Airtable base ID \(starts with "app", e.g., "appXXXXXXXXXXXXXX"\) | | `tableId` | string | Yes | Table ID \(starts with "tbl"\) or table name | | `maxRecords` | number | No | Maximum number of records to return \(default: all records\) | -| `filterFormula` | string | No | Formula to filter records \(e.g., "\(\{Field Name\} = \'Value\'\)"\) | +| `filterFormula` | string | No | Formula to filter records \(e.g., "\(\{Field Name\} = 'Value'\)"\) | #### Output diff --git a/apps/docs/content/docs/en/integrations/apify.mdx b/apps/docs/content/docs/en/integrations/apify.mdx index c866477c686..cca98ccdc02 100644 --- a/apps/docs/content/docs/en/integrations/apify.mdx +++ b/apps/docs/content/docs/en/integrations/apify.mdx @@ -97,7 +97,7 @@ Run a saved APIFY actor task synchronously and get dataset items (max 5 minutes) | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | APIFY API token from console.apify.com/account#/integrations | | `taskId` | string | Yes | Task ID or username/task-name. Examples: "janedoe/my-task", "moJRLRc85AitArpNN" | -| `input` | string | No | JSON string that overrides the task\'s saved input. Example: \{"startUrls": \[\{"url": "https://example.com"\}\]\} | +| `input` | string | No | JSON string that overrides the task's saved input. Example: \{"startUrls": \[\{"url": "https://example.com"\}\]\} | | `itemLimit` | number | No | Max dataset items to return \(1-250000\). Example: 500 | | `memory` | number | No | Memory in megabytes allocated for the run \(128-32768\). Example: 1024 for 1GB | | `timeout` | number | No | Timeout in seconds for the run. Example: 300 for 5 minutes | diff --git a/apps/docs/content/docs/en/integrations/apollo.mdx b/apps/docs/content/docs/en/integrations/apollo.mdx index 85c4f7f9808..81f27e521ce 100644 --- a/apps/docs/content/docs/en/integrations/apollo.mdx +++ b/apps/docs/content/docs/en/integrations/apollo.mdx @@ -56,7 +56,7 @@ Search Apollo's database for people using demographic filters | `organization_names` | array | No | Company names to search within \(legacy filter\) | | `organization_locations` | array | No | Headquarters locations of the people's current employer \(e.g., \['texas', 'tokyo', 'spain'\]\) | | `q_organization_domains_list` | array | No | Employer domain names \(e.g., \["apollo.io", "microsoft.com"\]\) — up to 1,000, no www. or @ | -| `organization_num_employees_ranges` | array | No | Employee count ranges for the person\'s current employer. Each entry is "min,max" \(e.g., \["1,10", "250,500", "10000,20000"\]\) | +| `organization_num_employees_ranges` | array | No | Employee count ranges for the person's current employer. Each entry is "min,max" \(e.g., \["1,10", "250,500", "10000,20000"\]\) | | `contact_email_status` | array | No | Email statuses to filter by: "verified", "unverified", "likely to engage", "unavailable" | | `q_keywords` | string | No | Keywords to search for | | `page` | number | No | Page number for pagination, default 1 \(e.g., 1, 2, 3\) | @@ -206,7 +206,7 @@ Create a new contact in your Apollo database | `title` | string | No | Job title \(e.g., "VP of Sales", "Software Engineer"\) | | `account_id` | string | No | Apollo account ID to associate with \(e.g., "acc_abc123"\) | | `owner_id` | string | No | User ID of the contact owner \(accepted by Apollo but not officially documented for POST /contacts\) | -| `organization_name` | string | No | Name of the contact\'s employer \(e.g., "Apollo"\) | +| `organization_name` | string | No | Name of the contact's employer \(e.g., "Apollo"\) | | `website_url` | string | No | Corporate website URL \(e.g., "https://www.apollo.io/"\) | | `label_names` | array | No | Lists/labels to add the contact to \(e.g., \["Prospects"\]\) | | `contact_stage_id` | string | No | Apollo ID for the contact stage | @@ -242,7 +242,7 @@ Update an existing contact in your Apollo database | `title` | string | No | Job title \(e.g., "VP of Sales", "Software Engineer"\) | | `account_id` | string | No | Apollo account ID \(e.g., "acc_abc123"\) | | `owner_id` | string | No | User ID of the contact owner \(accepted by Apollo but not officially documented for PATCH /contacts/\{id\}\) | -| `organization_name` | string | No | Name of the contact\'s employer \(e.g., "Apollo"\) | +| `organization_name` | string | No | Name of the contact's employer \(e.g., "Apollo"\) | | `website_url` | string | No | Corporate website URL \(e.g., "https://www.apollo.io/"\) | | `label_names` | array | No | Lists/labels to add the contact to \(e.g., \["Prospects"\]\) | | `contact_stage_id` | string | No | Apollo ID for the contact stage | diff --git a/apps/docs/content/docs/en/integrations/atlassian-service-account.mdx b/apps/docs/content/docs/en/integrations/atlassian-service-account.mdx index 5312576ca84..09fed2e3d68 100644 --- a/apps/docs/content/docs/en/integrations/atlassian-service-account.mdx +++ b/apps/docs/content/docs/en/integrations/atlassian-service-account.mdx @@ -58,7 +58,7 @@ The service account inherits permissions from the project/space roles you grant src="/static/credentials/atlassian/admin-auth-type-picker.png" alt="Atlassian admin — Choose authentication type with API token selected" width={700} - height={500} + height={551} className="my-4" /> @@ -100,7 +100,7 @@ The service account inherits permissions from the project/space roles you grant src="/static/credentials/atlassian/admin-scope-picker.png" alt="Atlassian token scope picker filtered to App: Jira and Scope type: Classic" width={1000} - height={600} + height={635} className="my-4" /> @@ -146,7 +146,7 @@ Your Atlassian site domain is the URL you use to access Jira or Confluence in yo src="/static/credentials/atlassian/sim-add-modal.png" alt="Add Atlassian Service Account dialog with API token and site domain filled in" width={420} - height={560} + height={419} className="my-6" /> @@ -169,7 +169,7 @@ Add a Jira, Jira Service Management, or Confluence block to your workflow. In th src="/static/credentials/atlassian/sim-jira-block-credential.png" alt="Jira block in a workflow with the Atlassian service account selected as the credential" width={1000} - height={500} + height={584} className="my-4" /> diff --git a/apps/docs/content/docs/en/integrations/azure_data_explorer.mdx b/apps/docs/content/docs/en/integrations/azure_data_explorer.mdx index ebe2f96ae3d..5210e62cf19 100644 --- a/apps/docs/content/docs/en/integrations/azure_data_explorer.mdx +++ b/apps/docs/content/docs/en/integrations/azure_data_explorer.mdx @@ -312,7 +312,7 @@ Push rows directly into an Azure Data Explorer table with .ingest inline. Data i | `resource` | string | No | Token audience override. Defaults to the cluster URI itself | | `database` | string | Yes | Database containing the target table | | `table` | string | Yes | Table to ingest into. Its schema is the assumed schema for the data | -| `data` | string | Yes | Rows to ingest, one record per line, parsed as CSV by default \(e.g., "Shoes,1000\\nWide Shoes,50"\) | +| `data` | string | Yes | Rows to ingest, one record per line, parsed as CSV by default \(e.g., "Shoes,1000\nWide Shoes,50"\) | | `ingestionProperties` | string | No | Ingestion properties clause contents, e.g. format="json", ingestionMappingReference="mymapping" | #### Output @@ -347,7 +347,7 @@ Materialize the result of a KQL query into a table with .set, .append, .set-or-a | `database` | string | Yes | Database containing the target table | | `table` | string | Yes | Table to ingest the query result into | | `mode` | string | No | set \(create, fail if it exists\), append \(add to an existing table\), set-or-append \(default\), or set-or-replace \(replace all data\) | -| `sourceQuery` | string | Yes | KQL query whose result becomes the ingested data \(e.g., LogsTable \| where Level == "Error" \| where Timestamp > ago\(1h\)\). Project the columns in the target table\'s order — matching is positional, not by name | +| `sourceQuery` | string | Yes | KQL query whose result becomes the ingested data \(e.g., LogsTable \| where Level == "Error" \| where Timestamp > ago\(1h\)\). Project the columns in the target table's order — matching is positional, not by name | | `async` | boolean | No | Return immediately with an OperationId and keep ingesting in the background. Check progress with Show Operations | | `ingestionProperties` | string | No | Optional ingestion properties clause contents, e.g. distributed=true, tags='\["daily"\]' | diff --git a/apps/docs/content/docs/en/integrations/azure_devops.mdx b/apps/docs/content/docs/en/integrations/azure_devops.mdx index 52723dc8611..25d3d38deb9 100644 --- a/apps/docs/content/docs/en/integrations/azure_devops.mdx +++ b/apps/docs/content/docs/en/integrations/azure_devops.mdx @@ -325,7 +325,7 @@ Execute a WIQL query to search for work items in Azure DevOps and return full fi | --------- | ---- | -------- | ----------- | | `organization` | string | Yes | Azure DevOps organization name | | `project` | string | Yes | Azure DevOps project name | -| `wiqlQuery` | string | Yes | WIQL query string \(e.g. "SELECT \[System.Id\] FROM workitems WHERE \[System.State\] = \'Doing\' ORDER BY \[System.Id\] ASC"\). Use TOP N to limit results. | +| `wiqlQuery` | string | Yes | WIQL query string \(e.g. "SELECT \[System.Id\] FROM workitems WHERE \[System.State\] = 'Doing' ORDER BY \[System.Id\] ASC"\). Use TOP N to limit results. | #### Output @@ -421,8 +421,8 @@ Create a new Basic-process work item (Issue, Task, or Epic) in Azure DevOps. Ret | `activity` | string | No | Activity \(Microsoft.VSTS.Common.Activity\). One of Deployment, Design, Development, Documentation, Requirements, Testing. Basic process: Task only. | | `remainingWork` | number | No | Remaining work hours \(Microsoft.VSTS.Scheduling.RemainingWork\). Basic process: Task only. | | `completedWork` | number | No | Completed work hours \(Microsoft.VSTS.Scheduling.CompletedWork\). Basic process: Task only. | -| `areaPath` | string | No | Area path for the work item, e.g. "MyProject\\\\Team" \(optional\) | -| `iterationPath` | string | No | Iteration path for the work item, e.g. "MyProject\\\\Sprint 1" \(optional\) | +| `areaPath` | string | No | Area path for the work item, e.g. "MyProject\\Team" \(optional\) | +| `iterationPath` | string | No | Iteration path for the work item, e.g. "MyProject\\Sprint 1" \(optional\) | | `tags` | string | No | Semicolon-separated tags, e.g. "issue; p1; auth" \(optional\) | #### Output diff --git a/apps/docs/content/docs/en/integrations/box.mdx b/apps/docs/content/docs/en/integrations/box.mdx index 630c50a1152..444761febcb 100644 --- a/apps/docs/content/docs/en/integrations/box.mdx +++ b/apps/docs/content/docs/en/integrations/box.mdx @@ -52,7 +52,6 @@ Upload a file to a Box folder | --------- | ---- | -------- | ----------- | | `parentFolderId` | string | Yes | The ID of the folder to upload the file to \(use "0" for root\) | | `file` | file | No | The file to upload \(UserFile object\) | -| `fileContent` | string | No | Legacy: base64 encoded file content | | `fileName` | string | No | Optional filename override | #### Output diff --git a/apps/docs/content/docs/en/integrations/cloudflare.mdx b/apps/docs/content/docs/en/integrations/cloudflare.mdx index a7d1257ee7e..411eecf5b0a 100644 --- a/apps/docs/content/docs/en/integrations/cloudflare.mdx +++ b/apps/docs/content/docs/en/integrations/cloudflare.mdx @@ -440,14 +440,14 @@ Lists SSL/TLS certificate packs for a zone. ### Cloudflare Get Zone Settings -Reads zone settings such as SSL mode, minimum TLS version, security level, and caching level. Cloudflare retired the endpoint that read every setting in one request, so each setting is read individually — name the ones you need to keep the read small. Defaults to $\{DEFAULT_ZONE_SETTING_IDS.join(', ')\}. +Reads zone settings such as SSL mode, minimum TLS version, security level, and caching level. Cloudflare retired the endpoint that read every setting in one request, so each setting is read individually — name the ones you need to keep the read small. Defaults to ssl, always_use_https, min_tls_version, tls_1_3, security_level, cache_level, browser_cache_ttl, development_mode, rocket_loader, email_obfuscation, hotlink_protection, ip_geolocation, http2, http3, websockets. #### Input | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `zoneId` | string | Yes | The zone ID to get settings for | -| `settingIds` | string | No | Comma-separated setting IDs to read, e.g. "ssl,min_tls_version,security_level". Leave blank to read the default set \($\{DEFAULT_ZONE_SETTING_IDS.join\(', '\)\}\). At most $\{MAX_ZONE_SETTING_IDS\} settings per call. | +| `settingIds` | string | No | Comma-separated setting IDs to read, e.g. "ssl,min_tls_version,security_level". Leave blank to read the default set \(ssl, always_use_https, min_tls_version, tls_1_3, security_level, cache_level, browser_cache_ttl, development_mode, rocket_loader, email_obfuscation, hotlink_protection, ip_geolocation, http2, http3, websockets\). At most 40 settings per call. | | `apiKey` | string | Yes | Cloudflare API Token | #### Output @@ -956,7 +956,7 @@ Updates a rate limiting rule in the http_ratelimit phase entry point ruleset of | `description` | string | No | Human-readable description of the rule | | `enabled` | boolean | No | Whether the rule is enabled | | `ref` | string | No | Reference tag that stays stable across rule updates. Because the update replaces the rule, omitting it resets the tag to the rule ID and breaks anything matching on the old value | -| `actionParameters` | string | No | JSON object of action-specific parameters for the mitigation action, e.g. \{"response":\{"status_code":429,"content":"\{\\"error\\":\\"rate limited\\"\}","content_type":"application/json"\}\} for a custom block response. Because the update replaces the rule, omitting it resets action_parameters to \{\} and the rule falls back to Cloudflare\'s default block page | +| `actionParameters` | string | No | JSON object of action-specific parameters for the mitigation action, e.g. \{"response":\{"status_code":429,"content":"\{\"error\":\"rate limited\"\}","content_type":"application/json"\}\} for a custom block response. Because the update replaces the rule, omitting it resets action_parameters to \{\} and the rule falls back to Cloudflare's default block page | | `logging` | string | No | JSON logging configuration to preserve, e.g. \{"enabled":true\}. Omitting it on a rule that had logging configured resets it to the default | | `apiKey` | string | Yes | Cloudflare API Token | diff --git a/apps/docs/content/docs/en/integrations/confluence.mdx b/apps/docs/content/docs/en/integrations/confluence.mdx index c4ef4f2d1bf..62398bc3491 100644 --- a/apps/docs/content/docs/en/integrations/confluence.mdx +++ b/apps/docs/content/docs/en/integrations/confluence.mdx @@ -43,7 +43,6 @@ Retrieve content from Confluence pages using the Confluence API. | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Confluence domain \(e.g., yourcompany.atlassian.net\) | | `pageId` | string | Yes | Confluence page ID to retrieve \(numeric ID from page URL or API\) | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -81,7 +80,6 @@ Update a Confluence page using the Confluence API. | `pageId` | string | Yes | Confluence page ID to update \(numeric ID from page URL or API\) | | `title` | string | No | New title for the page | | `content` | string | No | New content for the page in Confluence storage format | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -124,7 +122,6 @@ Create a new page in a Confluence space. | `title` | string | Yes | Title of the new page | | `content` | string | Yes | Page content in Confluence storage format \(HTML\) | | `parentId` | string | No | Parent page ID if creating a child page | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -165,7 +162,6 @@ Delete a Confluence page. By default moves to trash; use purge=true to permanent | `domain` | string | Yes | Your Confluence domain \(e.g., yourcompany.atlassian.net\) | | `pageId` | string | Yes | Confluence page ID to delete | | `purge` | boolean | No | If true, permanently deletes the page instead of moving to trash \(default: false\) | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -189,7 +185,6 @@ List all pages within a specific Confluence space. Supports pagination and filte | `status` | string | No | Filter pages by status: current, archived, trashed, or draft | | `bodyFormat` | string | No | Format for page body content: storage, atlas_doc_format, or view. If not specified, body is not included. | | `cursor` | string | No | Pagination cursor from previous response to get the next page of results | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -235,7 +230,6 @@ Get all child pages of a specific Confluence page. Useful for navigating page hi | `pageId` | string | Yes | The ID of the parent page to get children from | | `limit` | number | No | Maximum number of child pages to return \(default: 50, max: 250\) | | `cursor` | string | No | Pagination cursor from previous response to get the next page of results | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -263,7 +257,6 @@ Get the ancestor (parent) pages of a specific Confluence page. Returns the full | `domain` | string | Yes | Your Confluence domain \(e.g., yourcompany.atlassian.net\) | | `pageId` | string | Yes | The ID of the page to get ancestors for | | `limit` | number | No | Maximum number of ancestors to return \(default: 25, max: 250\) | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -290,7 +283,6 @@ List all versions (revision history) of a Confluence page. | `pageId` | string | Yes | The ID of the page to get versions for | | `limit` | number | No | Maximum number of versions to return \(default: 50, max: 250\) | | `cursor` | string | No | Pagination cursor from previous response | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -317,7 +309,6 @@ Get details about a specific version of a Confluence page. | `domain` | string | Yes | Your Confluence domain \(e.g., yourcompany.atlassian.net\) | | `pageId` | string | Yes | The ID of the page | | `versionNumber` | number | Yes | The version number to retrieve \(e.g., 1, 2, 3\) | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -353,7 +344,6 @@ List all custom properties (metadata) attached to a Confluence page. | `pageId` | string | Yes | The ID of the page to list properties from | | `limit` | number | No | Maximum number of properties to return \(default: 50, max: 250\) | | `cursor` | string | No | Pagination cursor from previous response | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -385,7 +375,6 @@ Create a new custom property (metadata) on a Confluence page. | `pageId` | string | Yes | The ID of the page to add the property to | | `key` | string | Yes | The key/name for the property | | `value` | json | Yes | The value for the property \(can be any JSON value\) | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -414,7 +403,6 @@ Delete a content property from a Confluence page by its property ID. | `domain` | string | Yes | Your Confluence domain \(e.g., yourcompany.atlassian.net\) | | `pageId` | string | Yes | The ID of the page containing the property | | `propertyId` | string | Yes | The ID of the property to delete | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -436,7 +424,6 @@ Search for content across Confluence pages, blog posts, and other content. | `domain` | string | Yes | Your Confluence domain \(e.g., yourcompany.atlassian.net\) | | `query` | string | Yes | Search query string | | `limit` | number | No | Maximum number of results to return \(default: 25\) | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -471,7 +458,6 @@ Search for content within a specific Confluence space. Optionally filter by text | `query` | string | No | Text search query. If not provided, returns all content in the space. | | `contentType` | string | No | Filter by content type: page, blogpost, attachment, or comment | | `limit` | number | No | Maximum number of results to return \(default: 25, max: 250\) | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -508,7 +494,6 @@ List all blog posts across all accessible Confluence spaces. | `status` | string | No | Filter by status: current, archived, trashed, or draft | | `sort` | string | No | Sort order: created-date, -created-date, modified-date, -modified-date, title, -title | | `cursor` | string | No | Pagination cursor from previous response | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -542,7 +527,6 @@ Get a specific Confluence blog post by ID, including its content. | `domain` | string | Yes | Your Confluence domain \(e.g., yourcompany.atlassian.net\) | | `blogPostId` | string | Yes | The ID of the blog post to retrieve | | `bodyFormat` | string | No | Format for blog post body: storage, atlas_doc_format, or view | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -586,7 +570,6 @@ Create a new blog post in a Confluence space. | `title` | string | Yes | Title of the blog post | | `content` | string | Yes | Blog post content in Confluence storage format \(HTML\) | | `status` | string | No | Blog post status: current \(default\) or draft | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -630,7 +613,6 @@ List all blog posts within a specific Confluence space. | `status` | string | No | Filter by status: current, archived, trashed, or draft | | `bodyFormat` | string | No | Format for blog post body: storage, atlas_doc_format, or view | | `cursor` | string | No | Pagination cursor from previous response | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -674,7 +656,6 @@ Add a comment to a Confluence page. | `domain` | string | Yes | Your Confluence domain \(e.g., yourcompany.atlassian.net\) | | `pageId` | string | Yes | Confluence page ID to comment on | | `comment` | string | Yes | Comment text in Confluence storage format | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -697,7 +678,6 @@ List all comments on a Confluence page. | `limit` | number | No | Maximum number of comments to return \(default: 25\) | | `bodyFormat` | string | No | Format for the comment body: storage, atlas_doc_format, view, or export_view \(default: storage\) | | `cursor` | string | No | Pagination cursor from previous response | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -735,7 +715,6 @@ Update an existing comment on a Confluence page. | `domain` | string | Yes | Your Confluence domain \(e.g., yourcompany.atlassian.net\) | | `commentId` | string | Yes | Confluence comment ID to update | | `comment` | string | Yes | Updated comment text in Confluence storage format | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -755,7 +734,6 @@ Delete a comment from a Confluence page. | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Confluence domain \(e.g., yourcompany.atlassian.net\) | | `commentId` | string | Yes | Confluence comment ID to delete | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -778,7 +756,6 @@ Upload a file as an attachment to a Confluence page. | `file` | file | Yes | The file to upload as an attachment | | `fileName` | string | No | Optional custom file name for the attachment | | `comment` | string | No | Optional comment to add to the attachment | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -804,7 +781,6 @@ List all attachments on a Confluence page. | `pageId` | string | Yes | Confluence page ID to list attachments from | | `limit` | number | No | Maximum number of attachments to return \(default: 50, max: 250\) | | `cursor` | string | No | Pagination cursor from previous response | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -840,7 +816,6 @@ Delete an attachment from a Confluence page (moves to trash). | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Confluence domain \(e.g., yourcompany.atlassian.net\) | | `attachmentId` | string | Yes | Confluence attachment ID to delete | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -862,7 +837,6 @@ List all labels on a Confluence page. | `pageId` | string | Yes | Confluence page ID to list labels from | | `limit` | number | No | Maximum number of labels to return \(default: 25, max: 250\) | | `cursor` | string | No | Pagination cursor from previous response | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -887,7 +861,6 @@ Add a label to a Confluence page for organization and categorization. | `pageId` | string | Yes | Confluence page ID to add the label to | | `labelName` | string | Yes | Name of the label to add | | `prefix` | string | No | Label prefix: global \(default\), my, team, or system | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -909,7 +882,6 @@ Remove a label from a Confluence page. | `domain` | string | Yes | Your Confluence domain \(e.g., yourcompany.atlassian.net\) | | `pageId` | string | Yes | Confluence page ID to remove the label from | | `labelName` | string | Yes | Name of the label to remove | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -932,7 +904,6 @@ Retrieve all pages that have a specific label applied. | `labelId` | string | Yes | The ID of the label to get pages for | | `limit` | number | No | Maximum number of pages to return \(default: 50, max: 250\) | | `cursor` | string | No | Pagination cursor from previous response | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -968,7 +939,6 @@ List all labels associated with a Confluence space. | `spaceId` | string | Yes | The ID of the Confluence space to list labels from | | `limit` | number | No | Maximum number of labels to return \(default: 25, max: 250\) | | `cursor` | string | No | Pagination cursor from previous response | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -992,7 +962,6 @@ Get details about a specific Confluence space. | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Confluence domain \(e.g., yourcompany.atlassian.net\) | | `spaceId` | string | Yes | Confluence space ID to retrieve | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -1024,7 +993,6 @@ Create a new Confluence space. | `name` | string | Yes | Name for the new space | | `key` | string | Yes | Unique key for the space \(uppercase, no spaces\) | | `description` | string | No | Description for the new space | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -1054,7 +1022,6 @@ Update a Confluence space name or description. | `spaceId` | string | Yes | ID of the space to update | | `name` | string | No | New name for the space | | `description` | string | No | New description for the space | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -1081,7 +1048,6 @@ Delete a Confluence space. | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Confluence domain \(e.g., yourcompany.atlassian.net\) | | `spaceId` | string | Yes | ID of the space to delete | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -1104,7 +1070,6 @@ List all Confluence spaces accessible to the user. | `domain` | string | Yes | Your Confluence domain \(e.g., yourcompany.atlassian.net\) | | `limit` | number | No | Maximum number of spaces to return \(default: 25, max: 250\) | | `cursor` | string | No | Pagination cursor from previous response | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -1137,7 +1102,6 @@ List properties on a Confluence space. | `spaceId` | string | Yes | Space ID to list properties for | | `limit` | number | No | Maximum number of properties to return \(default: 50, max: 250\) | | `cursor` | string | No | Pagination cursor from previous response | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -1163,7 +1127,6 @@ Create a property on a Confluence space. | `spaceId` | string | Yes | Space ID to create the property on | | `key` | string | Yes | Property key/name | | `value` | json | No | Property value \(JSON\) | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -1186,7 +1149,6 @@ Delete a property from a Confluence space. | `domain` | string | Yes | Your Confluence domain \(e.g., yourcompany.atlassian.net\) | | `spaceId` | string | Yes | Space ID the property belongs to | | `propertyId` | string | Yes | Property ID to delete | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -1209,7 +1171,6 @@ List permissions for a Confluence space. | `spaceId` | string | Yes | Space ID to list permissions for | | `limit` | number | No | Maximum number of permissions to return \(default: 50, max: 250\) | | `cursor` | string | No | Pagination cursor from previous response | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -1239,7 +1200,6 @@ Get all descendants of a Confluence page recursively. | `pageId` | string | Yes | Page ID to get descendants for | | `limit` | number | No | Maximum number of descendants to return \(default: 50, max: 250\) | | `cursor` | string | No | Pagination cursor from previous response | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -1273,7 +1233,6 @@ List inline tasks from Confluence. Optionally filter by page, space, assignee, o | `status` | string | No | Filter tasks by status \(complete or incomplete\) | | `limit` | number | No | Maximum number of tasks to return \(default: 50, max: 250\) | | `cursor` | string | No | Pagination cursor from previous response | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -1307,7 +1266,6 @@ Get a specific Confluence inline task by ID. | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Confluence domain \(e.g., yourcompany.atlassian.net\) | | `taskId` | string | Yes | The ID of the task to retrieve | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -1340,7 +1298,6 @@ Update the status of a Confluence inline task (complete or incomplete). | `domain` | string | Yes | Your Confluence domain \(e.g., yourcompany.atlassian.net\) | | `taskId` | string | Yes | The ID of the task to update | | `status` | string | Yes | New status for the task \(complete or incomplete\) | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -1374,7 +1331,6 @@ Update an existing Confluence blog post title and/or content. | `blogPostId` | string | Yes | The ID of the blog post to update | | `title` | string | No | New title for the blog post | | `content` | string | No | New content for the blog post in storage format | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -1398,7 +1354,6 @@ Delete a Confluence blog post. | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Confluence domain \(e.g., yourcompany.atlassian.net\) | | `blogPostId` | string | Yes | The ID of the blog post to delete | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -1418,7 +1373,6 @@ Get display name and profile info for a Confluence user by account ID. | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Confluence domain \(e.g., yourcompany.atlassian.net\) | | `accountId` | string | Yes | The Atlassian account ID of the user to look up | -| `cloudId` | string | No | Confluence Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output diff --git a/apps/docs/content/docs/en/integrations/crowdstrike.mdx b/apps/docs/content/docs/en/integrations/crowdstrike.mdx index 87637a0f06e..d43db472993 100644 --- a/apps/docs/content/docs/en/integrations/crowdstrike.mdx +++ b/apps/docs/content/docs/en/integrations/crowdstrike.mdx @@ -159,7 +159,7 @@ Run a read-only Real Time Response command in an open CrowdStrike Falcon session | `cloud` | string | Yes | CrowdStrike Falcon cloud region | | `sessionId` | string | Yes | RTR session ID returned by Init RTR Session | | `baseCommand` | string | Yes | Read-only RTR base command family, one of: cat, cd, clear, csrutil, env, eventlog, filehash, getsid, help, history, ifconfig, ipconfig, ls, mount, netstat, ps, reg, users. Subcommands belong in commandString, not here — and only reg query is read-tier, since reg set and reg delete are Active Responder commands. | -| `commandString` | string | Yes | Full command line to run, such as "ls C:\\Windows" or "reg query HKLM\\Software" | +| `commandString` | string | Yes | Full command line to run, such as "ls C:\Windows" or "reg query HKLM\Software" | #### Output diff --git a/apps/docs/content/docs/en/integrations/datadog.mdx b/apps/docs/content/docs/en/integrations/datadog.mdx index f8dabe5b7ea..c6ac7143f62 100644 --- a/apps/docs/content/docs/en/integrations/datadog.mdx +++ b/apps/docs/content/docs/en/integrations/datadog.mdx @@ -124,7 +124,7 @@ Create a new monitor/alert in Datadog. Monitors can track metrics, service check | --------- | ---- | -------- | ----------- | | `name` | string | Yes | Monitor name | | `type` | string | Yes | Monitor type: metric alert, service check, event alert, process alert, log alert, query alert, composite, synthetics alert, slo alert | -| `query` | string | Yes | Monitor query \(e.g., "avg\(last_5m\):avg:system.cpu.idle\{*\} < 20", "logs\(\"status:error\"\).index\(\"main\"\).rollup\(\"count\"\).last\(\"5m\"\) > 100"\) | +| `query` | string | Yes | Monitor query \(e.g., "avg\(last_5m\):avg:system.cpu.idle\{*\} < 20", "logs\("status:error"\).index\("main"\).rollup\("count"\).last\("5m"\) > 100"\) | | `message` | string | No | Message to include with notifications. Can include @-mentions and markdown. | | `tags` | string | No | Comma-separated list of tags | | `priority` | number | No | Monitor priority \(1-5, where 1 is highest\) | diff --git a/apps/docs/content/docs/en/integrations/daytona.mdx b/apps/docs/content/docs/en/integrations/daytona.mdx index 0bd13fc5ec2..6615b2296b4 100644 --- a/apps/docs/content/docs/en/integrations/daytona.mdx +++ b/apps/docs/content/docs/en/integrations/daytona.mdx @@ -216,7 +216,6 @@ Upload a file to a Daytona sandbox | `sandboxId` | string | Yes | ID of the sandbox to upload the file to | | `destinationPath` | string | Yes | Destination path in the sandbox \(a trailing slash uploads into that directory using the file name\) | | `file` | file | No | The file to upload | -| `fileContent` | string | No | Legacy: base64 encoded file content | | `fileName` | string | No | Optional file name override | #### Output diff --git a/apps/docs/content/docs/en/integrations/dropbox.mdx b/apps/docs/content/docs/en/integrations/dropbox.mdx index 83453f4cb9c..27b95131829 100644 --- a/apps/docs/content/docs/en/integrations/dropbox.mdx +++ b/apps/docs/content/docs/en/integrations/dropbox.mdx @@ -44,7 +44,6 @@ Upload a file to Dropbox | --------- | ---- | -------- | ----------- | | `path` | string | Yes | The path in Dropbox where the file should be saved \(e.g., /folder/document.pdf\) | | `file` | file | No | The file to upload \(UserFile object\) | -| `fileContent` | string | No | Legacy: base64 encoded file content | | `fileName` | string | No | Optional filename \(used if path is a folder\) | | `mode` | string | No | Write mode: add \(default\) or overwrite | | `autorename` | boolean | No | If true, rename the file if there is a conflict | diff --git a/apps/docs/content/docs/en/integrations/elasticsearch.mdx b/apps/docs/content/docs/en/integrations/elasticsearch.mdx index 17239c0896a..a0fb33108fb 100644 --- a/apps/docs/content/docs/en/integrations/elasticsearch.mdx +++ b/apps/docs/content/docs/en/integrations/elasticsearch.mdx @@ -199,7 +199,7 @@ Perform multiple index, create, delete, or update operations in a single request | `username` | string | No | Username for basic auth | | `password` | string | No | Password for basic auth | | `index` | string | No | Default index for operations \(e.g., "products", "logs-2024"\) | -| `operations` | string | Yes | Bulk operations as NDJSON string. Each operation is two lines: action metadata and optional document. Example: \{"index":\{"_index":"products","_id":"1"\}\}\\n\{"name":"Widget"\}\\n | +| `operations` | string | Yes | Bulk operations as NDJSON string. Each operation is two lines: action metadata and optional document. Example: \{"index":\{"_index":"products","_id":"1"\}\}\n\{"name":"Widget"\}\n | | `refresh` | string | No | Refresh policy: true, false, or wait_for | #### Output diff --git a/apps/docs/content/docs/en/integrations/extend.mdx b/apps/docs/content/docs/en/integrations/extend.mdx index e6ae6545a87..d8e7ed54605 100644 --- a/apps/docs/content/docs/en/integrations/extend.mdx +++ b/apps/docs/content/docs/en/integrations/extend.mdx @@ -27,11 +27,17 @@ Integrate Extend AI into the workflow. Parse and extract structured content from ### Extend Document Parser +Parse and extract content from documents using Extend AI + #### Input | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `file` | file | Yes | Document to be processed | +| `outputFormat` | string | No | Target output format \(markdown or spatial\). Defaults to markdown. | +| `chunking` | string | No | Chunking strategy \(page, document, or section\). Defaults to page. | +| `engine` | string | No | Parsing engine \(parse_performance or parse_light\). Defaults to parse_performance. | +| `apiKey` | string | Yes | Extend API key | #### Output diff --git a/apps/docs/content/docs/en/integrations/file.mdx b/apps/docs/content/docs/en/integrations/file.mdx index 554407ed50e..4222a602a7d 100644 --- a/apps/docs/content/docs/en/integrations/file.mdx +++ b/apps/docs/content/docs/en/integrations/file.mdx @@ -35,6 +35,8 @@ Read workspace file objects, extract the text content of files, fetch and parse ### File Read +Read workspace file objects from selected files or canonical workspace file IDs. + #### Input | Parameter | Type | Required | Description | @@ -73,8 +75,7 @@ Fetch and parse a file from a URL with optional custom headers. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `request` | string | No | No description | -| `outputs` | array | No | Array of parsed files with content and metadata | +| `fileUrl` | string | Yes | URL of the file to fetch and parse. | | `headers` | object | No | HTTP headers to include when fetching URL-based files. | #### Output diff --git a/apps/docs/content/docs/en/integrations/firecrawl.mdx b/apps/docs/content/docs/en/integrations/firecrawl.mdx index de218a6cfd7..12ebb7aed1f 100644 --- a/apps/docs/content/docs/en/integrations/firecrawl.mdx +++ b/apps/docs/content/docs/en/integrations/firecrawl.mdx @@ -52,7 +52,6 @@ Extract structured content from web pages with comprehensive metadata support. C | --------- | ---- | -------- | ----------- | | `url` | string | Yes | The URL to scrape content from \(e.g., "https://example.com/page"\) | | `formats` | json | No | Output formats supplied by existing Firecrawl block configurations | -| `scrapeOptions` | json | No | Options for content scraping | | `apiKey` | string | Yes | Firecrawl API key | #### Output @@ -90,7 +89,6 @@ Scrape multiple URLs in a single batch job and retrieve structured content from | `onlyMainContent` | boolean | No | Extract only main content from pages | | `maxConcurrency` | number | No | Maximum number of concurrent scrapes | | `ignoreInvalidURLs` | boolean | No | Skip invalid URLs instead of failing the batch \(default: true\) | -| `scrapeOptions` | json | No | Advanced scraping configuration options | | `zeroDataRetention` | boolean | No | Enable zero data retention | | `apiKey` | string | Yes | Firecrawl API key | @@ -159,7 +157,6 @@ Search for information on the web using Firecrawl | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `query` | string | Yes | The search query to use | -| `scrapeOptions` | json | No | Advanced scrape options supplied by existing configurations | | `apiKey` | string | Yes | Firecrawl API key | #### Output @@ -170,11 +167,11 @@ Search for information on the web using Firecrawl | ↳ `title` | string | Search result title from search engine | | ↳ `description` | string | Search result description/snippet from search engine | | ↳ `url` | string | URL of the search result | -| ↳ `markdown` | string | Page content in markdown \(when scrapeOptions.formats includes "markdown"\) | -| ↳ `html` | string | Processed HTML content \(when scrapeOptions.formats includes "html"\) | -| ↳ `rawHtml` | string | Unprocessed raw HTML \(when scrapeOptions.formats includes "rawHtml"\) | -| ↳ `links` | array | Links found on the page \(when scrapeOptions.formats includes "links"\) | -| ↳ `screenshot` | string | Screenshot URL \(expires after 24 hours, when scrapeOptions.formats includes "screenshot"\) | +| ↳ `markdown` | string | Page content in markdown; returned only when scraping was requested via the hidden scrapeOptions input | +| ↳ `html` | string | Processed HTML content; returned only when "html" is among the scrape formats requested via the hidden scrapeOptions input | +| ↳ `rawHtml` | string | Unprocessed raw HTML; returned only when "rawHtml" is among the scrape formats requested via the hidden scrapeOptions input | +| ↳ `links` | array | Links found on the page; returned only when "links" is among the scrape formats requested via the hidden scrapeOptions input | +| ↳ `screenshot` | string | Screenshot URL \(expires after 24 hours\); returned only when "screenshot" is among the scrape formats requested via the hidden scrapeOptions input | | ↳ `metadata` | object | Metadata about the search result page | | ↳ `title` | string | Page title | | ↳ `description` | string | Page meta description | @@ -195,7 +192,6 @@ Crawl entire websites and extract structured content from all accessible pages | `maxDepth` | number | No | Maximum depth to crawl from the starting URL \(e.g., 1, 2, 3\). Controls how many levels deep to follow links | | `formats` | json | No | Output formats for scraped content \(e.g., \["markdown"\], \["markdown", "html"\], \["markdown", "links"\]\) | | `prompt` | string | No | Natural-language crawl guidance supplied by existing configurations | -| `scrapeOptions` | json | No | Advanced scrape options supplied by existing configurations | | `excludePaths` | json | No | URL paths to exclude from crawling \(e.g., \["/blog/*", "/admin/*", "/*.pdf"\]\) | | `includePaths` | json | No | URL paths to include in crawling \(e.g., \["/docs/*", "/api/*"\]\). Only these paths will be crawled | | `onlyMainContent` | boolean | No | Extract only main content from pages | @@ -287,7 +283,6 @@ Get a complete list of URLs from any website quickly and reliably. Useful for di | `ignoreQueryParameters` | boolean | No | Exclude URLs containing query strings \(default: true\) | | `limit` | number | No | Maximum number of links to return \(e.g., 100, 1000, 5000\). Max: 100,000, default: 5,000 | | `timeout` | number | No | Request timeout in milliseconds | -| `location` | json | No | Geographic context for proxying \(country, languages\) | | `apiKey` | string | Yes | Firecrawl API key | #### Output @@ -313,7 +308,6 @@ Extract structured data from entire webpages using natural language prompts and | `includeSubdomains` | boolean | No | Extend scanning to subdomains \(default: true\) | | `showSources` | boolean | No | Return data sources in the response \(default: false\) | | `ignoreInvalidURLs` | boolean | No | Skip invalid URLs in the array \(default: true\) | -| `scrapeOptions` | json | No | Advanced scraping configuration options | | `apiKey` | string | Yes | Firecrawl API key | #### Output diff --git a/apps/docs/content/docs/en/integrations/github.mdx b/apps/docs/content/docs/en/integrations/github.mdx index 2997205fc26..120ad61de2f 100644 --- a/apps/docs/content/docs/en/integrations/github.mdx +++ b/apps/docs/content/docs/en/integrations/github.mdx @@ -120,11 +120,8 @@ Create comments on GitHub PRs | `body` | string | Yes | Comment content | | `pullNumber` | number | Yes | Pull request number | | `path` | string | No | File path for review comment | -| `position` | number | No | Line number for review comment | | `commentType` | string | No | Type of comment \(pr_comment or file_comment\) | | `line` | number | No | Line number for review comment | -| `side` | string | No | Side of the diff \(LEFT or RIGHT\) | -| `commitId` | string | No | The SHA of the commit to comment on | | `apiKey` | string | Yes | GitHub API token | #### Output diff --git a/apps/docs/content/docs/en/integrations/gitlab.mdx b/apps/docs/content/docs/en/integrations/gitlab.mdx index 45d4fe374f6..eaeff843f7a 100644 --- a/apps/docs/content/docs/en/integrations/gitlab.mdx +++ b/apps/docs/content/docs/en/integrations/gitlab.mdx @@ -1223,10 +1223,14 @@ Delete a GitLab user. Requires an administrator token with admin_mode on the ins ### GitLab Block User +Block a GitLab user, preventing them from signing in or accessing the instance + #### Input | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `userId` | number | Yes | The ID of the user to act on | #### Output @@ -1293,10 +1297,14 @@ Delete a GitLab user. Requires an administrator token with admin_mode on the ins ### GitLab Unblock User +Unblock a previously blocked GitLab user + #### Input | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `userId` | number | Yes | The ID of the user to act on | #### Output @@ -1363,10 +1371,14 @@ Delete a GitLab user. Requires an administrator token with admin_mode on the ins ### GitLab Deactivate User +Deactivate a dormant GitLab user + #### Input | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `userId` | number | Yes | The ID of the user to act on | #### Output @@ -1433,10 +1445,14 @@ Delete a GitLab user. Requires an administrator token with admin_mode on the ins ### GitLab Activate User +Reactivate a deactivated GitLab user + #### Input | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `userId` | number | Yes | The ID of the user to act on | #### Output @@ -1503,10 +1519,14 @@ Delete a GitLab user. Requires an administrator token with admin_mode on the ins ### GitLab Ban User +Ban a GitLab user + #### Input | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `userId` | number | Yes | The ID of the user to act on | #### Output @@ -1573,10 +1593,14 @@ Delete a GitLab user. Requires an administrator token with admin_mode on the ins ### GitLab Unban User +Unban a previously banned GitLab user + #### Input | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `userId` | number | Yes | The ID of the user to act on | #### Output @@ -1643,10 +1667,14 @@ Delete a GitLab user. Requires an administrator token with admin_mode on the ins ### GitLab Approve User +Approve a GitLab user whose signup is pending administrator approval + #### Input | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `userId` | number | Yes | The ID of the user to act on | #### Output @@ -1713,10 +1741,14 @@ Delete a GitLab user. Requires an administrator token with admin_mode on the ins ### GitLab Reject User +Reject a GitLab user whose signup is pending administrator approval + #### Input | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `host` | string | No | Self-managed GitLab host \(e.g. gitlab.example.com\). Defaults to gitlab.com. | +| `userId` | number | Yes | The ID of the user to act on | #### Output diff --git a/apps/docs/content/docs/en/integrations/google-service-account.mdx b/apps/docs/content/docs/en/integrations/google-service-account.mdx index 45af5739611..7bf63848d79 100644 --- a/apps/docs/content/docs/en/integrations/google-service-account.mdx +++ b/apps/docs/content/docs/en/integrations/google-service-account.mdx @@ -33,7 +33,7 @@ Before adding a service account to Sim, you need to configure it in the Google C src="/static/credentials/gcp-create-service-account.png" alt="Google Cloud Console — Create service account form" width={700} - height={500} + height={502} className="my-4" /> @@ -52,7 +52,7 @@ Before adding a service account to Sim, you need to configure it in the Google C src="/static/credentials/gcp-create-private-key.png" alt="Google Cloud Console — Create private key dialog with JSON selected" width={700} - height={400} + height={343} className="my-4" /> @@ -87,7 +87,7 @@ In the Google Cloud Console, go to **APIs & Services** → **Library** and enabl src="/static/credentials/gcp-add-client-id.png" alt="Google Workspace Admin Console — Add a new client ID with OAuth scopes" width={350} - height={300} + height={312} className="my-4" /> @@ -150,7 +150,7 @@ Once Google Cloud and Workspace are configured, add the service account as a cre src="/static/credentials/integrations-service-account.png" alt="Google Drive integration page with the service-account connect option" width={800} - height={150} + height={233} className="my-4" /> @@ -162,7 +162,7 @@ Once Google Cloud and Workspace are configured, add the service account as a cre src="/static/credentials/add-service-account.png" alt="Add Google Service Account dialog" width={350} - height={420} + height={407} className="my-6" /> @@ -188,7 +188,7 @@ Enter the email address of the Google Workspace user you want the service accoun src="/static/credentials/workflow-impersonated-account.png" alt="Gmail block in a workflow showing the Impersonated Account field with a service account credential" width={800} - height={350} + height={378} className="my-4" /> diff --git a/apps/docs/content/docs/en/integrations/google_calendar.mdx b/apps/docs/content/docs/en/integrations/google_calendar.mdx index 76216710dbd..43fc5aa5b90 100644 --- a/apps/docs/content/docs/en/integrations/google_calendar.mdx +++ b/apps/docs/content/docs/en/integrations/google_calendar.mdx @@ -89,7 +89,6 @@ List events from Google Calendar. Returns API-aligned fields only. | `maxResults` | number | No | Maximum number of events to return \(max 2500\) | | `pageToken` | string | No | Token for retrieving the next page of results | | `orderBy` | string | No | Order of events: startTime \(chronological, the default\) or updated \(last-modified\). startTime is always valid here because singleEvents is set. | -| `showDeleted` | boolean | No | Include deleted events | #### Output @@ -227,7 +226,6 @@ Get instances of a recurring event from Google Calendar. Returns API-aligned fie | `timeMax` | string | No | Upper bound for instances \(RFC3339 timestamp, e.g., 2025-06-04T00:00:00Z\) | | `maxResults` | number | No | Maximum number of instances to return \(default 250, max 2500\) | | `pageToken` | string | No | Token for retrieving subsequent pages of results | -| `showDeleted` | boolean | No | Include deleted instances | #### Output @@ -248,8 +246,6 @@ List all calendars in the user's calendar list. Returns API-aligned fields only. | `minAccessRole` | string | No | Minimum access role for returned calendars: freeBusyReader, reader, writer, or owner | | `maxResults` | number | No | Maximum number of calendars to return \(default 100, max 250\) | | `pageToken` | string | No | Token for retrieving subsequent pages of results | -| `showDeleted` | boolean | No | Include deleted calendars | -| `showHidden` | boolean | No | Include hidden calendars | #### Output diff --git a/apps/docs/content/docs/en/integrations/google_drive.mdx b/apps/docs/content/docs/en/integrations/google_drive.mdx index d6f781a2852..62db54a219a 100644 --- a/apps/docs/content/docs/en/integrations/google_drive.mdx +++ b/apps/docs/content/docs/en/integrations/google_drive.mdx @@ -105,7 +105,7 @@ List files and folders in Google Drive with complete metadata | ↳ `isAppAuthorized` | boolean | Whether created by requesting app | | ↳ `contentRestrictions` | json | Content restrictions | | ↳ `linkShareMetadata` | json | Link share metadata | -| `nextPageToken` | string | Token for fetching the next page of results | +| `nextPageToken` | string | Page token for the next page of files; absent from the response when the end of the files list has been reached | ### Get Google Drive File @@ -483,7 +483,7 @@ Search for files in Google Drive using advanced query syntax (e.g., fullText con | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `query` | string | Yes | Google Drive query string using advanced search syntax \(e.g., "fullText contains \'budget\'", "mimeType = \'application/pdf\'", "modifiedTime > \'2024-01-01\'"\) | +| `query` | string | Yes | Google Drive query string using advanced search syntax \(e.g., "fullText contains 'budget'", "mimeType = 'application/pdf'", "modifiedTime > '2024-01-01'"\) | | `pageSize` | number | No | Maximum number of files to return \(default: 100\) | | `pageToken` | string | No | Token for fetching the next page of results | @@ -518,7 +518,7 @@ Search for files in Google Drive using advanced query syntax (e.g., fullText con | ↳ `driveId` | string | Shared drive ID | | ↳ `capabilities` | json | User capabilities on file | | ↳ `version` | string | Version number | -| `nextPageToken` | string | Token for fetching the next page of results | +| `nextPageToken` | string | Page token for the next page of files; absent from the response when the end of the files list has been reached | ### Update Google Drive File @@ -692,7 +692,7 @@ List all permissions (who has access) for a file in Google Drive | ↳ `allowFileDiscovery` | boolean | Whether file is discoverable by grantee | | ↳ `pendingOwner` | boolean | Whether ownership transfer is pending | | ↳ `permissionDetails` | json | Details about inherited permissions | -| `nextPageToken` | string | Token for fetching the next page of permissions | +| `nextPageToken` | string | Page token for the next page of permissions; absent from the response when the end of the permissions list has been reached | ### Export Google Drive File @@ -741,7 +741,7 @@ List the revision history of a file in Google Drive | ↳ `md5Checksum` | string | MD5 checksum for binary revisions | | ↳ `size` | string | Size of the revision in bytes | | ↳ `exportLinks` | json | Export format links for the revision | -| `nextPageToken` | string | Token for fetching the next page of revisions | +| `nextPageToken` | string | Page token for the next page of revisions; absent from the response when the end of the revisions list has been reached | ### Get Google Drive Revision @@ -801,7 +801,7 @@ List comments on a file in Google Drive | ↳ `anchor` | string | Region of the document the comment refers to | | ↳ `quotedFileContent` | json | The file content the comment quotes | | ↳ `replies` | json | Threaded replies to the comment | -| `nextPageToken` | string | Token for fetching the next page of comments | +| `nextPageToken` | string | Page token for the next page of comments; absent from the response when the end of the comments list has been reached | ### Create Google Drive Comment diff --git a/apps/docs/content/docs/en/integrations/google_groups.mdx b/apps/docs/content/docs/en/integrations/google_groups.mdx index 9e6d263816e..db54d6c0961 100644 --- a/apps/docs/content/docs/en/integrations/google_groups.mdx +++ b/apps/docs/content/docs/en/integrations/google_groups.mdx @@ -43,7 +43,7 @@ List all groups in a Google Workspace domain | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `customer` | string | No | Customer ID or "my_customer" for the authenticated user\'s domain | +| `customer` | string | No | Customer ID or "my_customer" for the authenticated user's domain | | `domain` | string | No | Domain name to filter groups by | | `maxResults` | number | No | Maximum number of results to return \(1-200\). Example: 50 | | `pageToken` | string | No | Token for fetching the next page of results | diff --git a/apps/docs/content/docs/en/integrations/google_sheets.mdx b/apps/docs/content/docs/en/integrations/google_sheets.mdx index 70307178362..b6602c62a3e 100644 --- a/apps/docs/content/docs/en/integrations/google_sheets.mdx +++ b/apps/docs/content/docs/en/integrations/google_sheets.mdx @@ -75,7 +75,6 @@ Write data to a specific sheet in a Google Sheets spreadsheet | `cellRange` | string | No | The cell range to write to \(e.g. "A1:D10", "A1"\). Defaults to "A1" if not specified. | | `values` | array | Yes | The data to write as a 2D array \(e.g. \[\["Name", "Age"\], \["Alice", 30\], \["Bob", 25\]\]\) or array of objects. | | `valueInputOption` | string | No | The format of the data to write | -| `includeValuesInResponse` | boolean | No | Whether to include the written values in the response | #### Output @@ -102,7 +101,6 @@ Update data in a specific sheet in a Google Sheets spreadsheet | `cellRange` | string | No | The cell range to update \(e.g. "A1:D10", "A1"\). Defaults to "A1" if not specified. | | `values` | array | Yes | The data to update as a 2D array \(e.g. \[\["Name", "Age"\], \["Alice", 30\]\]\) or array of objects. | | `valueInputOption` | string | No | The format of the data to update | -| `includeValuesInResponse` | boolean | No | Whether to include the updated values in the response | #### Output @@ -129,7 +127,6 @@ Append data to the end of a specific sheet in a Google Sheets spreadsheet | `values` | array | Yes | The data to append as a 2D array \(e.g. \[\["Alice", 30\], \["Bob", 25\]\]\) or array of objects. | | `valueInputOption` | string | No | The format of the data to append | | `insertDataOption` | string | No | How to insert the data \(OVERWRITE or INSERT_ROWS\) | -| `includeValuesInResponse` | boolean | No | Whether to include the appended values in the response | #### Output diff --git a/apps/docs/content/docs/en/integrations/google_vault.mdx b/apps/docs/content/docs/en/integrations/google_vault.mdx index 13579318448..60ea81d70a9 100644 --- a/apps/docs/content/docs/en/integrations/google_vault.mdx +++ b/apps/docs/content/docs/en/integrations/google_vault.mdx @@ -173,9 +173,9 @@ Replace the name, query, and scope of an existing hold. This is a full-resource | `corpus` | string | Yes | Data corpus of the hold \(MAIL, DRIVE, GROUPS, HANGOUTS_CHAT, VOICE\) | | `accountEmails` | string | No | Comma-separated list of user emails covered by the hold \(e.g., "user1@example.com, user2@example.com"\) | | `orgUnitId` | string | No | Organization unit ID covered by the hold \(e.g., "id:03ph8a2z1enx5q0", alternative to accounts\) | -| `terms` | string | No | Search terms to filter held content \(e.g., "from:sender@example.com subject:invoice", for MAIL and GROUPS corpus\). Resupply the hold\'s current terms to keep them — this replaces the hold, so leaving it blank clears any existing filter. | -| `startTime` | string | No | Start time for date filtering \(ISO 8601 format, e.g., "2024-01-01T00:00:00Z", for MAIL and GROUPS corpus\). Resupply the hold\'s current value to keep it — leaving it blank clears any existing date filter. | -| `endTime` | string | No | End time for date filtering \(ISO 8601 format, e.g., "2024-12-31T23:59:59Z", for MAIL and GROUPS corpus\). Resupply the hold\'s current value to keep it — leaving it blank clears any existing date filter. | +| `terms` | string | No | Search terms to filter held content \(e.g., "from:sender@example.com subject:invoice", for MAIL and GROUPS corpus\). Resupply the hold's current terms to keep them — this replaces the hold, so leaving it blank clears any existing filter. | +| `startTime` | string | No | Start time for date filtering \(ISO 8601 format, e.g., "2024-01-01T00:00:00Z", for MAIL and GROUPS corpus\). Resupply the hold's current value to keep it — leaving it blank clears any existing date filter. | +| `endTime` | string | No | End time for date filtering \(ISO 8601 format, e.g., "2024-12-31T23:59:59Z", for MAIL and GROUPS corpus\). Resupply the hold's current value to keep it — leaving it blank clears any existing date filter. | | `includeSharedDrives` | boolean | No | Include files in shared drives \(for DRIVE corpus\). Resupply true if the hold currently includes shared drives — leaving it false/blank clears that setting. | #### Output diff --git a/apps/docs/content/docs/en/integrations/hubspot-setup.mdx b/apps/docs/content/docs/en/integrations/hubspot-setup.mdx index aaf56fd2064..fb9da7f16b6 100644 --- a/apps/docs/content/docs/en/integrations/hubspot-setup.mdx +++ b/apps/docs/content/docs/en/integrations/hubspot-setup.mdx @@ -28,9 +28,9 @@ You need: The Integrations page in Sim, with the sidebar entry, a search box, connected accounts, and featured integrations 3. The HubSpot page lists its skills and templates. Click **+ Add to Sim** in the top right. @@ -39,18 +39,11 @@ You need: src="/static/integrations/hubspot/hubspot-page-add-to-sim.png" alt="The HubSpot integration page in Sim with the Add to Sim button in the top right" width={800} - height={550} + height={717} /> 4. In the **Connect HubSpot** dialog, enter a **Display name** for the connection (for example "Sales HubSpot"), review the permissions requested, and click **Connect**. -The Connect HubSpot dialog showing the display name field and the list of permissions requested - 5. You are redirected to HubSpot. Sign in, then choose the HubSpot account you want to connect. The Integrations page showing the sidebar entry, search, connected accounts, and featured integrations The page's second tab, **Skills**, holds your workspace's [agent skills](/agents/skills). @@ -34,7 +34,7 @@ Open a service to see what it offers: src="/static/integrations/hubspot/hubspot-page-add-to-sim.png" alt="A service page (HubSpot) with its skills, templates, and the Add to Sim button" width={800} - height={550} + height={717} /> ## Connecting an account @@ -61,7 +61,7 @@ Blocks that require authentication (e.g. Gmail, Slack, HubSpot) display an accou src="/static/credentials/oauth-selector.png" alt="A block showing the account selector dropdown with connected accounts" width={500} - height={350} + height={295} /> You can also connect another account directly from the block by selecting **Connect another [service] account** at the bottom of the dropdown. @@ -82,7 +82,7 @@ In any block that requires an integration, click **Switch to manual ID** next to src="/static/integrations/switch-to-manual-id.png" alt="Block showing the Switch to manual ID button next to the account selector" width={500} - height={200} + height={193} /> Paste or reference the credential ID in that field. You can use a `{{SECRET}}` reference or a block output variable to make it dynamic. @@ -91,7 +91,7 @@ Paste or reference the credential ID in that field. You can use a `{{SECRET}}` r src="/static/integrations/manual-credential-id.png" alt="Block showing the Enter credential ID text field after switching to manual mode" width={500} - height={200} + height={197} /> ## Managing a connection diff --git a/apps/docs/content/docs/en/integrations/jira.mdx b/apps/docs/content/docs/en/integrations/jira.mdx index ffa2803f524..818ab031c6f 100644 --- a/apps/docs/content/docs/en/integrations/jira.mdx +++ b/apps/docs/content/docs/en/integrations/jira.mdx @@ -49,7 +49,6 @@ Retrieve detailed information about a specific Jira issue | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | | `issueKey` | string | Yes | Jira issue key to retrieve \(e.g., PROJ-123\) | | `includeAttachments` | boolean | No | Download attachment file contents and include them as files in the output | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -262,7 +261,6 @@ Update a Jira issue | `customFieldId` | string | No | Custom field ID to update \(e.g., customfield_10001\) | | `customFieldValue` | string | No | Value for the custom field | | `notifyUsers` | boolean | No | Whether to send email notifications about this update \(default: true\) | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -287,7 +285,6 @@ Create a new Jira issue | `description` | string | No | Description for the issue. Accepts plain text \(auto-wrapped in ADF\) or a raw ADF document object | | `priority` | string | No | Priority ID or name for the issue \(e.g., "10000" or "High"\) | | `assignee` | string | No | Assignee account ID for the issue | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | | `issueType` | string | Yes | Type of issue to create \(e.g., Task, Story, Bug, Epic, Sub-task\) | | `parent` | json | No | Parent issue key for creating subtasks \(e.g., \{ "key": "PROJ-123" \}\) | | `labels` | array | No | Labels for the issue \(array of label names\) | @@ -322,7 +319,6 @@ Retrieve multiple Jira issues from a project in bulk | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | | `projectId` | string | Yes | Jira project key \(e.g., PROJ\) | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -364,7 +360,6 @@ Delete a Jira issue | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | | `issueKey` | string | Yes | Jira issue key to delete \(e.g., PROJ-123\) | | `deleteSubtasks` | boolean | No | Whether to delete subtasks. If false, parent issues with subtasks cannot be deleted. | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -385,7 +380,6 @@ Assign a Jira issue to a user | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | | `issueKey` | string | Yes | Jira issue key to assign \(e.g., PROJ-123\) | | `accountId` | string | Yes | Account ID of the user to assign the issue to. Use "-1" for automatic assignment, or leave empty / pass "null" to unassign. | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -408,8 +402,7 @@ Move a Jira issue between workflow statuses (e.g., To Do -> In Progress) | `issueKey` | string | Yes | Jira issue key to transition \(e.g., PROJ-123\) | | `transitionId` | string | Yes | ID of the transition to execute \(e.g., "11" for "To Do", "21" for "In Progress"\) | | `comment` | string | No | Optional comment to add when transitioning the issue | -| `resolution` | string | No | Resolution name to set during transition \(e.g., "Fixed", "Won\'t Fix"\) | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | +| `resolution` | string | No | Resolution name to set during transition \(e.g., "Fixed", "Won't Fix"\) | #### Output @@ -437,7 +430,6 @@ Search for Jira issues using JQL (Jira Query Language) | `nextPageToken` | string | No | Cursor token for the next page of results. Omit for the first page. | | `maxResults` | number | No | Maximum number of results to return per page \(default: 50\) | | `fields` | array | No | Array of field names to return \(default: all fields\). | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -520,7 +512,6 @@ Add a comment to a Jira issue | `issueKey` | string | Yes | Jira issue key to add comment to \(e.g., PROJ-123\) | | `body` | string | Yes | Comment body text | | `visibility` | json | No | Restrict comment visibility. Object with "type" \("role" or "group"\) and "value" \(role/group name\). | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -555,7 +546,6 @@ Get all comments from a Jira issue | `startAt` | number | No | Index of the first comment to return \(default: 0\) | | `maxResults` | number | No | Maximum number of comments to return \(default: 50\) | | `orderBy` | string | No | Sort order for comments: "-created" for newest first, "created" for oldest first | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -605,7 +595,6 @@ Update an existing comment on a Jira issue | `commentId` | string | Yes | ID of the comment to update | | `body` | string | Yes | Updated comment text | | `visibility` | json | No | Restrict comment visibility. Object with "type" \("role" or "group"\) and "value" \(role/group name\). | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -638,7 +627,6 @@ Delete a comment from a Jira issue | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | | `issueKey` | string | Yes | Jira issue key containing the comment \(e.g., PROJ-123\) | | `commentId` | string | Yes | ID of the comment to delete | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -660,7 +648,6 @@ Get all attachments from a Jira issue | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | | `issueKey` | string | Yes | Jira issue key to get attachments from \(e.g., PROJ-123\) | | `includeAttachments` | boolean | No | Download attachment file contents and include them as files in the output | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -698,7 +685,6 @@ Add attachments to a Jira issue | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | | `issueKey` | string | Yes | Jira issue key to add attachments to \(e.g., PROJ-123\) | | `files` | file[] | Yes | Files to attach to the Jira issue | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -725,7 +711,6 @@ Delete an attachment from a Jira issue | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | | `attachmentId` | string | Yes | ID of the attachment to delete | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -749,7 +734,6 @@ Add a time tracking worklog entry to a Jira issue | `comment` | string | No | Optional comment for the worklog entry | | `started` | string | No | Optional start time in ISO format \(defaults to current time\) | | `visibility` | json | No | Restrict worklog visibility. Object with "type" \("role" or "group"\) and "value" \(role/group name\). | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -784,7 +768,6 @@ Get all worklog entries from a Jira issue | `issueKey` | string | Yes | Jira issue key to get worklogs from \(e.g., PROJ-123\) | | `startAt` | number | No | Index of the first worklog to return \(default: 0\) | | `maxResults` | number | No | Maximum number of worklogs to return \(default: 50\) | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -836,7 +819,6 @@ Update an existing worklog entry on a Jira issue | `comment` | string | No | Optional comment for the worklog entry | | `started` | string | No | Optional start time in ISO format | | `visibility` | json | No | Restrict worklog visibility. Object with "type" \("role" or "group"\) and "value" \(role/group name\). | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -880,7 +862,6 @@ Delete a worklog entry from a Jira issue | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | | `issueKey` | string | Yes | Jira issue key containing the worklog \(e.g., PROJ-123\) | | `worklogId` | string | Yes | ID of the worklog entry to delete | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -904,7 +885,6 @@ Create a link relationship between two Jira issues | `outwardIssueKey` | string | Yes | Jira issue key for the outward issue \(e.g., PROJ-456\) | | `linkType` | string | Yes | The type of link relationship \(e.g., "Blocks", "Relates to", "Duplicates"\) | | `comment` | string | No | Optional comment to add to the issue link | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -927,7 +907,6 @@ Delete a link between two Jira issues | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | | `linkId` | string | Yes | ID of the issue link to delete | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -948,7 +927,6 @@ Add a watcher to a Jira issue to receive notifications about updates | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | | `issueKey` | string | Yes | Jira issue key to add watcher to \(e.g., PROJ-123\) | | `accountId` | string | Yes | Account ID of the user to add as watcher | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -970,7 +948,6 @@ Remove a watcher from a Jira issue | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | | `issueKey` | string | Yes | Jira issue key to remove watcher from \(e.g., PROJ-123\) | | `accountId` | string | Yes | Account ID of the user to remove as watcher | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -993,7 +970,6 @@ Get Jira users. If an account ID is provided, returns a single user. Otherwise, | `accountId` | string | No | Optional account ID to get a specific user. If not provided, returns all users. | | `startAt` | number | No | The index of the first user to return \(for pagination, default: 0\) | | `maxResults` | number | No | Maximum number of users to return \(default: 50\) | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -1026,7 +1002,6 @@ Search for Jira users by email address or display name. Returns matching users w | `query` | string | Yes | A query string to search for users. Can be an email address, display name, or partial match. | | `maxResults` | number | No | Maximum number of users to return \(default: 50, max: 1000\) | | `startAt` | number | No | The index of the first user to return \(for pagination, default: 0\) | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -1058,7 +1033,6 @@ List Jira projects visible to the user, with optional name/key filtering and pag | `query` | string | No | Filter projects by partial name or key match | | `startAt` | number | No | The index of the first project to return \(for pagination, default: 0\) | | `maxResults` | number | No | Maximum number of projects to return \(default: 50, max: 100\) | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -1091,7 +1065,6 @@ Get the details of a single Jira project by its ID or key, including its type, l | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | | `projectId` | string | Yes | The project ID or key \(e.g., "PROJ" or "10000"\) | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -1124,7 +1097,6 @@ Get the workflow transitions available for an issue in its current status. Use t | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | | `issueKey` | string | Yes | The issue key or ID \(e.g., PROJ-123\) | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -1151,7 +1123,6 @@ List all issue types visible to the user across projects (e.g., Task, Bug, Story | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output @@ -1177,7 +1148,6 @@ Get all system and custom fields defined in the Jira instance. Useful for discov | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance. If not provided, it will be fetched using the domain. | #### Output diff --git a/apps/docs/content/docs/en/integrations/jira_service_management.mdx b/apps/docs/content/docs/en/integrations/jira_service_management.mdx index 3054fb35efe..87dcc4094db 100644 --- a/apps/docs/content/docs/en/integrations/jira_service_management.mdx +++ b/apps/docs/content/docs/en/integrations/jira_service_management.mdx @@ -45,7 +45,6 @@ Get all service desks from Jira Service Management | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `expand` | string | No | Comma-separated fields to expand in the response | | `start` | number | No | Start index for pagination \(e.g., 0, 50, 100\) | | `limit` | number | No | Maximum results to return \(e.g., 10, 25, 50\) | @@ -75,7 +74,6 @@ Get request types for a service desk in Jira Service Management | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `serviceDeskId` | string | Yes | Service Desk ID \(e.g., "1", "2"\) | | `searchQuery` | string | No | Filter request types by name | | `groupId` | string | No | Filter by request type group ID | @@ -110,7 +108,6 @@ Create a new service request in Jira Service Management | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `serviceDeskId` | string | Yes | Service Desk ID \(e.g., "1", "2"\) | | `requestTypeId` | string | Yes | Request Type ID \(e.g., "10", "15"\) | | `summary` | string | No | Summary/title for the service request \(required unless using Form Answers\) | @@ -145,7 +142,6 @@ Get a single service request from Jira Service Management | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `issueIdOrKey` | string | Yes | Issue ID or key \(e.g., SD-123\) | | `expand` | string | No | Comma-separated fields to expand: participant, status, sla, requestType, serviceDesk, attachment, comment, action | @@ -185,7 +181,6 @@ Get multiple service requests from Jira Service Management | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `serviceDeskId` | string | No | Filter by service desk ID \(e.g., "1", "2"\) | | `requestOwnership` | string | No | Filter by ownership: OWNED_REQUESTS, PARTICIPATED_REQUESTS, APPROVER, ALL_REQUESTS | | `requestStatus` | string | No | Filter by status: OPEN_REQUESTS, CLOSED_REQUESTS, ALL_REQUESTS | @@ -232,7 +227,6 @@ Add a comment (public or internal) to a service request in Jira Service Manageme | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `issueIdOrKey` | string | Yes | Issue ID or key \(e.g., SD-123\) | | `body` | string | Yes | Comment body text | | `isPublic` | boolean | Yes | Whether the comment is public \(visible to customer\) or internal \(true/false\) | @@ -263,7 +257,6 @@ Get comments for a service request in Jira Service Management | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `issueIdOrKey` | string | Yes | Issue ID or key \(e.g., SD-123\) | | `isPublic` | boolean | No | Filter to only public comments \(true/false\) | | `internal` | boolean | No | Filter to only internal comments \(true/false\) | @@ -300,7 +293,6 @@ Get customers for a service desk in Jira Service Management | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `serviceDeskId` | string | Yes | Service Desk ID \(e.g., "1", "2"\) | | `query` | string | No | Search query to filter customers \(e.g., "john", "acme"\) | | `start` | number | No | Start index for pagination \(e.g., 0, 50, 100\) | @@ -329,7 +321,6 @@ Add customers to a service desk in Jira Service Management | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `serviceDeskId` | string | Yes | Service Desk ID \(e.g., "1", "2"\) | | `accountIds` | string | Yes | Comma-separated Atlassian account IDs to add as customers | @@ -350,7 +341,6 @@ Get organizations for a service desk in Jira Service Management | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `serviceDeskId` | string | Yes | Service Desk ID \(e.g., "1", "2"\) | | `start` | number | No | Start index for pagination \(e.g., 0, 50, 100\) | | `limit` | number | No | Maximum results to return \(e.g., 10, 25, 50\) | @@ -375,7 +365,6 @@ Create a new organization in Jira Service Management | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `name` | string | Yes | Name of the organization to create | #### Output @@ -396,7 +385,6 @@ Add an organization to a service desk in Jira Service Management | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `serviceDeskId` | string | Yes | Service Desk ID \(e.g., "1", "2"\) | | `organizationId` | string | Yes | Organization ID to add to the service desk | @@ -418,7 +406,6 @@ Get queues for a service desk in Jira Service Management | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `serviceDeskId` | string | Yes | Service Desk ID \(e.g., "1", "2"\) | | `includeCount` | boolean | No | Include issue count for each queue \(true/false\) | | `start` | number | No | Start index for pagination \(e.g., 0, 50, 100\) | @@ -447,7 +434,6 @@ Get SLA information for a service request in Jira Service Management | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `issueIdOrKey` | string | Yes | Issue ID or key \(e.g., SD-123\) | | `start` | number | No | Start index for pagination \(e.g., 0, 50, 100\) | | `limit` | number | No | Maximum results to return \(e.g., 10, 25, 50\) | @@ -475,7 +461,6 @@ Get available transitions for a service request in Jira Service Management | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `issueIdOrKey` | string | Yes | Issue ID or key \(e.g., SD-123\) | | `start` | number | No | Start index for pagination \(e.g., 0, 50, 100\) | | `limit` | number | No | Maximum results to return \(e.g., 10, 25, 50\) | @@ -501,7 +486,6 @@ Transition a service request to a new status in Jira Service Management | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `issueIdOrKey` | string | Yes | Issue ID or key \(e.g., SD-123\) | | `transitionId` | string | Yes | Transition ID to apply | | `comment` | string | No | Optional comment to add during transition | @@ -524,7 +508,6 @@ Get participants for a request in Jira Service Management | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `issueIdOrKey` | string | Yes | Issue ID or key \(e.g., SD-123\) | | `start` | number | No | Start index for pagination \(e.g., 0, 50, 100\) | | `limit` | number | No | Maximum results to return \(e.g., 10, 25, 50\) | @@ -552,7 +535,6 @@ Add participants to a request in Jira Service Management | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `issueIdOrKey` | string | Yes | Issue ID or key \(e.g., SD-123\) | | `accountIds` | string | Yes | Comma-separated account IDs to add as participants | @@ -578,7 +560,6 @@ Get approvals for a request in Jira Service Management | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `issueIdOrKey` | string | Yes | Issue ID or key \(e.g., SD-123\) | | `start` | number | No | Start index for pagination \(e.g., 0, 50, 100\) | | `limit` | number | No | Maximum results to return \(e.g., 10, 25, 50\) | @@ -615,7 +596,6 @@ Approve or decline an approval request in Jira Service Management | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `issueIdOrKey` | string | Yes | Issue ID or key \(e.g., SD-123\) | | `approvalId` | string | Yes | Approval ID to answer | | `decision` | string | Yes | Decision: "approve" or "decline" | @@ -653,7 +633,6 @@ Get the fields required to create a request of a specific type in Jira Service M | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `serviceDeskId` | string | Yes | Service Desk ID \(e.g., "1", "2"\) | | `requestTypeId` | string | Yes | Request Type ID \(e.g., "10", "15"\) | @@ -686,7 +665,6 @@ List forms (ProForma/JSM Forms) in a Jira project to discover form IDs for reque | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `projectIdOrKey` | string | Yes | Jira project ID or key \(e.g., "10001" or "SD"\) | #### Output @@ -714,7 +692,6 @@ Get the full structure of a ProForma/JSM form including all questions, field typ | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `projectIdOrKey` | string | Yes | Jira project ID or key \(e.g., "10001" or "SD"\) | | `formId` | string | Yes | Form ID \(UUID from Get Form Templates\) | @@ -738,7 +715,6 @@ List forms (ProForma/JSM Forms) attached to a Jira issue with metadata (name, su | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `issueIdOrKey` | string | Yes | Issue ID or key \(e.g., "SD-123", "10001"\) | #### Output @@ -766,7 +742,6 @@ Attach a form template to an existing Jira issue or JSM request | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `issueIdOrKey` | string | Yes | Issue ID or key to attach the form to \(e.g., "SD-123"\) | | `formTemplateId` | string | Yes | Form template UUID \(from Get Form Templates\) | @@ -793,7 +768,6 @@ Save answers to a form attached to a Jira issue or JSM request | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `issueIdOrKey` | string | Yes | Issue ID or key \(e.g., "SD-123"\) | | `formId` | string | Yes | Form instance UUID \(from Attach Form or Get Issue Forms\) | | `answers` | json | Yes | Form answers using numeric question IDs as keys \(e.g., \{"1": \{"text": "Title"\}, "4": \{"choices": \["5"\]\}\}\) | @@ -817,7 +791,6 @@ Submit a form on a Jira issue or JSM request, locking it from further edits | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `issueIdOrKey` | string | Yes | Issue ID or key \(e.g., "SD-123"\) | | `formId` | string | Yes | Form instance UUID \(from Attach Form or Get Issue Forms\) | @@ -839,7 +812,6 @@ Get a single form with full design, state, and answers from a Jira issue | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `issueIdOrKey` | string | Yes | Issue ID or key \(e.g., "SD-123"\) | | `formId` | string | Yes | Form instance UUID \(from Attach Form or Get Issue Forms\) | @@ -863,7 +835,6 @@ Get simplified answers from a form attached to a Jira issue or JSM request | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `issueIdOrKey` | string | Yes | Issue ID or key \(e.g., "SD-123"\) | | `formId` | string | Yes | Form instance UUID \(from Attach Form or Get Issue Forms\) | @@ -885,7 +856,6 @@ Reopen a submitted form on a Jira issue or JSM request, allowing further edits | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `issueIdOrKey` | string | Yes | Issue ID or key \(e.g., "SD-123"\) | | `formId` | string | Yes | Form instance UUID \(from Get Issue Forms\) | @@ -907,7 +877,6 @@ Remove a form from a Jira issue or JSM request | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `issueIdOrKey` | string | Yes | Issue ID or key \(e.g., "SD-123"\) | | `formId` | string | Yes | Form instance UUID to delete | @@ -929,7 +898,6 @@ Make a form visible to customers on a Jira issue or JSM request | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `issueIdOrKey` | string | Yes | Issue ID or key \(e.g., "SD-123"\) | | `formId` | string | Yes | Form instance UUID | @@ -951,7 +919,6 @@ Make a form internal only (not visible to customers) on a Jira issue or JSM requ | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `issueIdOrKey` | string | Yes | Issue ID or key \(e.g., "SD-123"\) | | `formId` | string | Yes | Form instance UUID | @@ -973,7 +940,6 @@ Copy forms from one Jira issue to another | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `sourceIssueIdOrKey` | string | Yes | Source issue ID or key to copy forms from \(e.g., "SD-123"\) | | `targetIssueIdOrKey` | string | Yes | Target issue ID or key to copy forms to \(e.g., "SD-456"\) | | `formIds` | json | No | Optional JSON array of form UUIDs to copy \(e.g., \["uuid1", "uuid2"\]\). If omitted, copies all forms. | @@ -997,7 +963,6 @@ List Assets (Insight/CMDB) object schemas in Jira Service Management | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `workspaceId` | string | No | Assets workspace ID \(resolved automatically when omitted\) | | `startAt` | number | No | Pagination start index \(e.g., 0, 50\) | | `maxResults` | number | No | Maximum schemas to return \(e.g., 25, 50\) | @@ -1028,7 +993,6 @@ Get a single Assets (Insight/CMDB) object schema by ID | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `workspaceId` | string | No | Assets workspace ID \(resolved automatically when omitted\) | | `schemaId` | string | Yes | The Assets object schema ID | @@ -1055,7 +1019,6 @@ List object types within an Assets (Insight/CMDB) object schema | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `workspaceId` | string | No | Assets workspace ID \(resolved automatically when omitted\) | | `schemaId` | string | Yes | The Assets object schema ID to list object types for | | `excludeAbstract` | boolean | No | Exclude abstract object types from the result | @@ -1084,7 +1047,6 @@ Get the attribute definitions for an Assets (Insight/CMDB) object type. Use the | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `workspaceId` | string | No | Assets workspace ID \(resolved automatically when omitted\) | | `objectTypeId` | string | Yes | The Assets object type ID | | `onlyValueEditable` | boolean | No | Return only attributes whose values can be edited | @@ -1116,7 +1078,6 @@ Search Assets (Insight/CMDB) objects using AQL (Assets Query Language), e.g. obj | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `workspaceId` | string | No | Assets workspace ID \(resolved automatically when omitted\) | | `qlQuery` | string | Yes | AQL query string \(e.g., objectType = "Host" AND "Operating System" = "Ubuntu"\) | | `page` | number | No | Page number \(1-based, defaults to 1\) | @@ -1149,7 +1110,6 @@ Get a single Assets (Insight/CMDB) object by ID, including its attribute values | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `workspaceId` | string | No | Assets workspace ID \(resolved automatically when omitted\) | | `objectId` | string | Yes | The Assets object ID | @@ -1179,7 +1139,6 @@ Create an Assets (Insight/CMDB) object of a given object type. Attributes use ob | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `workspaceId` | string | No | Assets workspace ID \(resolved automatically when omitted\) | | `objectTypeId` | string | Yes | The object type ID to create the object under | | `attributes` | json | Yes | Array of attributes: \[\{ objectTypeAttributeId, objectAttributeValues: \[\{ value \}\] \}\] | @@ -1210,7 +1169,6 @@ Update an existing Assets (Insight/CMDB) object. Provide the attributes to chang | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `workspaceId` | string | No | Assets workspace ID \(resolved automatically when omitted\) | | `objectId` | string | Yes | The Assets object ID to update | | `attributes` | json | Yes | Array of attributes to set: \[\{ objectTypeAttributeId, objectAttributeValues: \[\{ value \}\] \}\] | @@ -1242,7 +1200,6 @@ Delete an Assets (Insight/CMDB) object by ID | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `domain` | string | Yes | Your Jira domain \(e.g., yourcompany.atlassian.net\) | -| `cloudId` | string | No | Jira Cloud ID for the instance | | `workspaceId` | string | No | Assets workspace ID \(resolved automatically when omitted\) | | `objectId` | string | Yes | The Assets object ID to delete | diff --git a/apps/docs/content/docs/en/integrations/jupyter.mdx b/apps/docs/content/docs/en/integrations/jupyter.mdx index ffa7b72c730..636d1af4922 100644 --- a/apps/docs/content/docs/en/integrations/jupyter.mdx +++ b/apps/docs/content/docs/en/integrations/jupyter.mdx @@ -117,7 +117,6 @@ Upload a file to a Jupyter server | `token` | string | Yes | Jupyter server authentication token | | `directory` | string | No | Destination directory, relative to the server root. Leave blank to upload to the root directory. | | `file` | file | No | The file to upload \(UserFile object\) | -| `fileContent` | string | No | Legacy: base64 encoded file content | | `fileName` | string | No | Optional filename override | #### Output diff --git a/apps/docs/content/docs/en/integrations/knowledge.mdx b/apps/docs/content/docs/en/integrations/knowledge.mdx index 30114c2e472..f1be279b661 100644 --- a/apps/docs/content/docs/en/integrations/knowledge.mdx +++ b/apps/docs/content/docs/en/integrations/knowledge.mdx @@ -48,7 +48,6 @@ Search for similar content in a knowledge base using vector similarity | `rerankerModel` | string | No | Cohere rerank model to use \(one of: rerank-v4.0-pro, rerank-v4.0-fast, rerank-v3.5\) | | `rerankerInputCount` | number | No | Number of vector results sent to the Cohere reranker \(1–100\). Defaults to topK × 4 capped at 100. | | `apiKey` | string | No | Cohere API key for reranker \(self-hosted deployments only\) | -| `tagFilters` | string | No | No description | #### Output @@ -108,7 +107,6 @@ Create a new document in a knowledge base | `name` | string | Yes | Name of the document | | `content` | string | Yes | Content of the document | | `documentTags` | object | No | Document tags | -| `documentTags` | string | No | No description | #### Output @@ -137,7 +135,6 @@ Create or update a document in a knowledge base. If a document with the given ID | `name` | string | Yes | Name of the document | | `content` | string | Yes | Content of the document | | `documentTags` | json | No | Document tags | -| `documentTags` | string | No | No description | #### Output diff --git a/apps/docs/content/docs/en/integrations/langsmith.mdx b/apps/docs/content/docs/en/integrations/langsmith.mdx index 8f28f774059..8ba298c5873 100644 --- a/apps/docs/content/docs/en/integrations/langsmith.mdx +++ b/apps/docs/content/docs/en/integrations/langsmith.mdx @@ -102,6 +102,13 @@ Patch an existing LangSmith run with outputs, status, or timing once it complete | `apiKey` | string | Yes | LangSmith API key | | `runId` | string | Yes | ID of the run to update | | `name` | string | No | Corrected run name | +| `end_time` | string | No | Run end time in ISO-8601 format | +| `outputs` | json | No | Outputs payload | +| `extra` | json | No | Additional metadata \(extra\) | +| `tags` | json | No | Array of tag strings | +| `status` | string | No | Run status | +| `error` | string | No | Error details | +| `events` | json | No | Structured events array | #### Output diff --git a/apps/docs/content/docs/en/integrations/latex.mdx b/apps/docs/content/docs/en/integrations/latex.mdx index ff72df88d4b..cb1088873cf 100644 --- a/apps/docs/content/docs/en/integrations/latex.mdx +++ b/apps/docs/content/docs/en/integrations/latex.mdx @@ -43,7 +43,7 @@ Compile a LaTeX document into a PDF via the public LaTeX-on-HTTP service (latex. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `content` | string | Yes | LaTeX source of the main document, from \\documentclass to \\end\{document\} | +| `content` | string | Yes | LaTeX source of the main document, from \documentclass to \end\{document\} | | `compiler` | string | No | LaTeX compiler: pdflatex \(default\), xelatex, lualatex, platex, uplatex, or context | | `fileName` | string | No | Name for the generated PDF file \(default: document.pdf\) | | `resources` | array | No | Supporting files for the compilation. Each entry has a "path" plus exactly one of "content" \(plain text\), "file" \(base64\), or "url" \(remote file\), e.g. \[\{"path": "refs.bib", "content": "..."\}, \{"path": "logo.png", "url": "https://..."\}\] | diff --git a/apps/docs/content/docs/en/integrations/linkedin.mdx b/apps/docs/content/docs/en/integrations/linkedin.mdx index 0ecee87159b..9e1c6164e1e 100644 --- a/apps/docs/content/docs/en/integrations/linkedin.mdx +++ b/apps/docs/content/docs/en/integrations/linkedin.mdx @@ -42,15 +42,6 @@ Share a post to your personal LinkedIn feed | --------- | ---- | -------- | ----------- | | `text` | string | Yes | The text content of your LinkedIn post | | `visibility` | string | No | Who can see this post: "PUBLIC" or "CONNECTIONS" \(default: "PUBLIC"\) | -| `request` | string | No | No description | -| `output` | string | No | No description | -| `output` | string | No | No description | -| `specificContent` | string | No | No description | -| `visibility` | string | No | No description | -| `headers` | string | No | No description | -| `output` | string | No | No description | -| `output` | string | No | No description | -| `output` | string | No | No description | #### Output diff --git a/apps/docs/content/docs/en/integrations/linq.mdx b/apps/docs/content/docs/en/integrations/linq.mdx index 5e21505f7cd..c90d57b6fed 100644 --- a/apps/docs/content/docs/en/integrations/linq.mdx +++ b/apps/docs/content/docs/en/integrations/linq.mdx @@ -108,7 +108,6 @@ Upload a file to Linq as a reusable attachment (max 100MB) and get an attachment | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Linq API key | | `file` | file | No | File to upload \(a UserFile from a file-upload field or a previous block\) | -| `fileContent` | string | No | Legacy base64-encoded file content fallback | | `filename` | string | No | Override the file name \(defaults to the uploaded file name\) | | `contentType` | string | No | Override the MIME type \(defaults to the uploaded file type\) | diff --git a/apps/docs/content/docs/en/integrations/managed_agent.mdx b/apps/docs/content/docs/en/integrations/managed_agent.mdx index 97572d8bcbd..014e011d748 100644 --- a/apps/docs/content/docs/en/integrations/managed_agent.mdx +++ b/apps/docs/content/docs/en/integrations/managed_agent.mdx @@ -54,7 +54,6 @@ Open a Claude Platform Managed Agent session and return the assistant response a | `memoryInstructions` | string | No | Per-attachment guidance for how the agent should use the memory store. | | `files` | array | No | File attachments \(cloud envs only\), as \[\{fileId, mountPath?\}\]. | | `sessionParameters` | object | No | Key/value session metadata forwarded to the session. | -| `modelInput` | string | No | No description | #### Output @@ -73,6 +72,7 @@ Create a Claude Platform Managed Agent session and return its id without waiting | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `credential` | string | Yes | Claude Platform credential \(Anthropic workspace API key\) to act with. | | `agent` | string | Yes | Managed-agent id inside the linked Claude workspace. | | `environment` | string | Yes | Environment id inside the linked Claude workspace. | | `environmentType` | string | No | Environment execution model hint \('cloud' \| 'self_hosted'\). | @@ -100,6 +100,8 @@ Send a user message to an existing Claude Platform Managed Agent session. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `credential` | string | Yes | Claude Platform credential \(Anthropic workspace API key\) to act with. | +| `sessionId` | string | Yes | Anthropic session id \(sesn_...\) to act on. | | `userMessage` | string | Yes | The user message to send to the session. | #### Output @@ -117,6 +119,8 @@ Read a Managed Agent session: status, stop reason, token usage, metadata, and an | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `credential` | string | Yes | Claude Platform credential \(Anthropic workspace API key\) to act with. | +| `sessionId` | string | Yes | Anthropic session id \(sesn_...\) to act on. | #### Output @@ -140,6 +144,8 @@ Read a Managed Agent session's event history and the agent's reply text. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `credential` | string | Yes | Claude Platform credential \(Anthropic workspace API key\) to act with. | +| `sessionId` | string | Yes | Anthropic session id \(sesn_...\) to act on. | | `eventTypes` | array | No | Optional event-type filter, e.g. \['agent.message'\]. Omit to return every event. | | `limit` | number | No | Maximum events to return, keeping the most recent \(default 500\). | @@ -161,6 +167,8 @@ Update a Managed Agent session's title or metadata. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `credential` | string | Yes | Claude Platform credential \(Anthropic workspace API key\) to act with. | +| `sessionId` | string | Yes | Anthropic session id \(sesn_...\) to act on. | | `title` | string | No | New session title. | | `sessionParameters` | object | No | Replacement metadata map \(replaces all stored metadata, not merged\). Leaving it empty leaves the stored metadata unchanged — use clearMetadata to remove it. | | `clearMetadata` | boolean | No | Removes all of the session's stored metadata. Overrides any map supplied above. | @@ -182,6 +190,8 @@ Stop a running Managed Agent session; it stays usable afterwards. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `credential` | string | Yes | Claude Platform credential \(Anthropic workspace API key\) to act with. | +| `sessionId` | string | Yes | Anthropic session id \(sesn_...\) to act on. | #### Output @@ -198,6 +208,8 @@ Allow or deny the tool calls a Managed Agent session is waiting on before it can | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `credential` | string | Yes | Claude Platform credential \(Anthropic workspace API key\) to act with. | +| `sessionId` | string | Yes | Anthropic session id \(sesn_...\) to act on. | | `toolUseIds` | array | Yes | Blocking tool-use EVENT ids, from Get Session pendingTools\[\].id where kind is 'confirmation' \(not toolu_ ids\). | | `decision` | string | Yes | 'allow' to let the tools run, or 'deny' to reject them. | | `denyMessage` | string | No | Reason surfaced to the agent. Only sent when the decision is deny. | @@ -218,6 +230,8 @@ Return the result of a custom tool a Managed Agent session is waiting on so it c | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `credential` | string | Yes | Claude Platform credential \(Anthropic workspace API key\) to act with. | +| `sessionId` | string | Yes | Anthropic session id \(sesn_...\) to act on. | | `customToolUseId` | string | Yes | The custom tool-use EVENT id being answered, from Get Session pendingTools\[\].id where kind is 'custom_tool_result'. | | `result` | string | Yes | The tool's output, returned to the agent as text. | | `isError` | boolean | No | Mark the result as a failure so the agent can adjust its approach. | @@ -237,6 +251,8 @@ Archive a Managed Agent session, preserving its history. Not reversible. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `credential` | string | Yes | Claude Platform credential \(Anthropic workspace API key\) to act with. | +| `sessionId` | string | Yes | Anthropic session id \(sesn_...\) to act on. | #### Output @@ -253,6 +269,8 @@ Permanently delete a Managed Agent session, its events, and its sandbox. Not rev | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `credential` | string | Yes | Claude Platform credential \(Anthropic workspace API key\) to act with. | +| `sessionId` | string | Yes | Anthropic session id \(sesn_...\) to act on. | #### Output diff --git a/apps/docs/content/docs/en/integrations/microsoft_dataverse.mdx b/apps/docs/content/docs/en/integrations/microsoft_dataverse.mdx index 8d7f2a1ec4e..03cbab8f6ea 100644 --- a/apps/docs/content/docs/en/integrations/microsoft_dataverse.mdx +++ b/apps/docs/content/docs/en/integrations/microsoft_dataverse.mdx @@ -205,7 +205,7 @@ Execute a bound or unbound Dataverse function. Functions are read-only operation | `functionName` | string | Yes | Function name \(e.g., RetrievePrincipalAccess, RetrieveTotalRecordCount\). Do not include the Microsoft.Dynamics.CRM. namespace prefix for unbound functions. | | `entitySetName` | string | No | Entity set name for bound functions \(e.g., systemusers\). Leave empty for unbound functions. | | `recordId` | string | No | Record GUID for bound functions. Leave empty for unbound functions. | -| `parameters` | string | No | Function parameters for the URL. Simple values can be inlined \(e.g., "LocalizedStandardName=\'Pacific Standard Time\ | +| `parameters` | string | No | Function parameters for the URL. Simple values can be inlined \(e.g., "LocalizedStandardName='Pacific Standard Time',LocaleId=1033"\), but values with reserved characters \(/ < > * % & : \ ? +\) must use parameter aliases: put the alias assignment in parentheses and the alias-to-value bindings after a "?", e.g. "LocalizedStandardName=@p1,LocaleId=@p2?@p1='Pacific Standard Time'&@p2=1033". Do not include the enclosing parentheses yourself. | #### Output @@ -387,7 +387,7 @@ Update an existing record in a Microsoft Dataverse table. Only send the columns ### Upload File to Microsoft Dataverse -Upload a file to a file or image column on a Dataverse record. Supports single-request upload for files up to 128 MB. The file content must be provided as a base64-encoded string. +Upload a file to a file or image column on a Dataverse record. Supports single-request upload for files up to 128 MB. Provide the file through the File input; its bytes are read from storage and sent as the raw request body. #### Input @@ -399,7 +399,6 @@ Upload a file to a file or image column on a Dataverse record. Supports single-r | `fileColumn` | string | Yes | File or image column logical name \(e.g., entityimage, cr_document\) | | `fileName` | string | Yes | Name of the file being uploaded \(e.g., document.pdf\) | | `file` | file | No | File to upload \(UserFile object\) | -| `fileContent` | string | No | Base64-encoded file content \(legacy\) | #### Output diff --git a/apps/docs/content/docs/en/integrations/microsoft_excel.mdx b/apps/docs/content/docs/en/integrations/microsoft_excel.mdx index fffb1d43366..ef12b24a054 100644 --- a/apps/docs/content/docs/en/integrations/microsoft_excel.mdx +++ b/apps/docs/content/docs/en/integrations/microsoft_excel.mdx @@ -74,7 +74,6 @@ Write data to a specific sheet in a Microsoft Excel spreadsheet | `cellRange` | string | No | The cell range to write to \(e.g., "A1:D10", "A1"\). Defaults to "A1" if not specified. | | `values` | array | Yes | The data to write as a 2D array \(e.g. \[\["Name", "Age"\], \["Alice", 30\], \["Bob", 25\]\]\) or array of objects. | | `valueInputOption` | string | No | The format of the data to write | -| `includeValuesInResponse` | boolean | No | Whether to include the written values in the response | #### Output diff --git a/apps/docs/content/docs/en/integrations/mistral_parse.mdx b/apps/docs/content/docs/en/integrations/mistral_parse.mdx index 3f9ec32583a..6af3e7ee15a 100644 --- a/apps/docs/content/docs/en/integrations/mistral_parse.mdx +++ b/apps/docs/content/docs/en/integrations/mistral_parse.mdx @@ -35,11 +35,16 @@ Integrate Mistral Parse into the workflow. Can extract text from uploaded PDF do ### Mistral PDF Parser +Parse PDF documents using Mistral OCR API + #### Input | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `file` | file | Yes | Normalized UserFile from file upload or file reference | +| `resultType` | string | No | Type of parsed result \(markdown, text, or json\). Defaults to markdown. | +| `pages` | array | No | Specific pages to process \(array of page numbers, starting from 0\) | +| `apiKey` | string | Yes | Mistral API key \(MISTRAL_API_KEY\) | #### Output @@ -54,15 +59,15 @@ Integrate Mistral Parse into the workflow. Can extract text from uploaded PDF do | ↳ `top_left_y` | number | Top-left Y coordinate in pixels | | ↳ `bottom_right_x` | number | Bottom-right X coordinate in pixels | | ↳ `bottom_right_y` | number | Bottom-right Y coordinate in pixels | -| ↳ `image_base64` | string | Base64-encoded image data \(when include_image_base64=true\) | +| ↳ `image_base64` | string | Base64-encoded image data; returned only when the hidden includeImageBase64 input is enabled | | ↳ `dimensions` | object | Page dimensions | | ↳ `dpi` | number | Dots per inch | | ↳ `height` | number | Page height in pixels | | ↳ `width` | number | Page width in pixels | -| ↳ `tables` | array | Extracted tables as HTML/markdown \(when table_format is set\). Referenced via placeholders like \[tbl-0.html\] | +| ↳ `tables` | array | Separate table objects, referenced from the markdown via placeholders like \[tbl-0.html\]. Mistral populates these only when table_format is "markdown" or "html"; Sim never sets it, so tables stay inline in the markdown and this list is empty | | ↳ `hyperlinks` | array | Array of URL strings detected in the page \(e.g., \["https://...", "mailto:..."\]\) | -| ↳ `header` | string | Page header content \(when extract_header=true\) | -| ↳ `footer` | string | Page footer content \(when extract_footer=true\) | +| ↳ `header` | string | Page header content. Mistral returns it only when extract_header is true \(it defaults to false\); Sim never sets it, so this is not returned | +| ↳ `footer` | string | Page footer content. Mistral returns it only when extract_footer is true \(it defaults to false\); Sim never sets it, so this is not returned | | `model` | string | Mistral OCR model identifier \(e.g., mistral-ocr-latest\) | | `usage_info` | object | Usage and processing statistics | | ↳ `pages_processed` | number | Total number of pages processed | diff --git a/apps/docs/content/docs/en/integrations/netsuite.mdx b/apps/docs/content/docs/en/integrations/netsuite.mdx index 5421a098097..18baa8e0ce4 100644 --- a/apps/docs/content/docs/en/integrations/netsuite.mdx +++ b/apps/docs/content/docs/en/integrations/netsuite.mdx @@ -56,7 +56,6 @@ List one page of a NetSuite record collection, optionally filtered with a q expr | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `recordType` | string | Yes | NetSuite REST record type script ID, such as customer or salesOrder | | `q` | string | No | NetSuite record collection filter expression | | `limit` | number | No | Results to return in this page \(1-1000; default 100\) | @@ -90,7 +89,6 @@ Retrieve one NetSuite record by internal or external ID. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `recordType` | string | Yes | NetSuite REST record type script ID, such as customer or salesOrder | | `recordId` | string | Yes | NetSuite internal ID or an external-ID reference beginning with eid: | | `fields` | string | No | Comma-separated record fields to return | @@ -113,7 +111,6 @@ Create a NetSuite record using the account-specific record metadata schema. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `recordType` | string | Yes | NetSuite REST record type script ID, such as customer or salesOrder | | `body` | json | Yes | Record fields matching the account-specific NetSuite metadata schema | | `replace` | string | No | Comma-separated sublists whose default lines should be replaced | @@ -135,7 +132,6 @@ Update fields on an existing NetSuite record with PATCH. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `recordType` | string | Yes | NetSuite REST record type script ID, such as customer or salesOrder | | `recordId` | string | Yes | NetSuite internal ID or an external-ID reference beginning with eid: | | `body` | json | Yes | Record fields matching the account-specific NetSuite metadata schema | @@ -158,7 +154,6 @@ Create or update a NetSuite record by external ID with PUT. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `recordType` | string | Yes | NetSuite REST record type script ID, such as customer or salesOrder | | `externalId` | string | Yes | External ID without the eid: prefix | | `body` | json | Yes | Record fields matching the account-specific NetSuite metadata schema | @@ -180,7 +175,6 @@ Delete one NetSuite record by internal or external ID. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `recordType` | string | Yes | NetSuite REST record type script ID, such as customer or salesOrder | | `recordId` | string | Yes | NetSuite internal ID or an external-ID reference beginning with eid: | @@ -200,7 +194,6 @@ Retrieve a record sublist, subrecord, referenced record, or nested subresource. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `recordType` | string | Yes | NetSuite REST record type script ID, such as customer or salesOrder | | `recordId` | string | Yes | NetSuite internal ID or an external-ID reference beginning with eid: | | `subresourcePath` | string | Yes | Slash-separated subresource path, such as item or item/1/inventoryDetail | @@ -221,7 +214,6 @@ Return a prepopulated create form, or an edit form when a record ID is supplied. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `recordType` | string | Yes | NetSuite REST record type script ID, such as customer or salesOrder | | `recordId` | string | No | Existing record ID; omit to request a create form | | `body` | json | No | Record fields matching the account-specific NetSuite metadata schema | @@ -245,7 +237,6 @@ Retrieve valid select values for one or more fields on a new or existing record. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `recordType` | string | Yes | NetSuite REST record type script ID, such as customer or salesOrder | | `recordId` | string | No | Existing record ID; omit to evaluate options for a new record | | `fields` | string | Yes | Comma-separated select field IDs | @@ -273,7 +264,6 @@ Attach a contact or file to another NetSuite record. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `recordType` | string | Yes | NetSuite REST record type script ID, such as customer or salesOrder | | `recordId` | string | Yes | NetSuite internal ID or an external-ID reference beginning with eid: | | `relatedType` | string | Yes | Related resource type: contact or file | @@ -297,7 +287,6 @@ Detach a contact or file from another NetSuite record. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `recordType` | string | Yes | NetSuite REST record type script ID, such as customer or salesOrder | | `recordId` | string | Yes | NetSuite internal ID or an external-ID reference beginning with eid: | | `relatedType` | string | Yes | Related resource type: contact or file | @@ -319,7 +308,6 @@ Execute a supported NetSuite record action such as approve, reject, or confirm. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `recordType` | string | Yes | NetSuite REST record type script ID, such as customer or salesOrder | | `recordId` | string | Yes | NetSuite internal ID or an external-ID reference beginning with eid: | | `action` | string | Yes | NetSuite record action ID without the @ prefix | @@ -342,7 +330,6 @@ Transform a supported source record into another NetSuite record type. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `recordType` | string | Yes | NetSuite REST record type script ID, such as customer or salesOrder | | `recordId` | string | Yes | NetSuite internal ID or an external-ID reference beginning with eid: | | `targetRecordType` | string | Yes | Target record type supported by the source record metadata | @@ -365,7 +352,6 @@ Submit an asynchronous request to retrieve up to 100 records of one type. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `recordType` | string | Yes | NetSuite REST record type script ID, such as customer or salesOrder | | `ids` | string | Yes | Up to 100 comma-separated internal IDs or eid: external-ID references | | `fields` | string | No | Comma-separated record fields to return | @@ -391,7 +377,6 @@ Submit an asynchronous batch that creates up to 100 records of one type. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `recordType` | string | Yes | NetSuite REST record type script ID, such as customer or salesOrder | | `items` | array | Yes | Array of 1-100 records matching the account-specific metadata schema | | `idempotencyKey` | string | No | Optional unique idempotency key for retrying the batch | @@ -414,7 +399,6 @@ Submit an asynchronous batch that updates up to 100 records of one type. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `recordType` | string | Yes | NetSuite REST record type script ID, such as customer or salesOrder | | `items` | array | Yes | Array of 1-100 records; every item must include an internal or external ID | | `idempotencyKey` | string | No | Optional unique idempotency key for retrying the batch | @@ -437,7 +421,6 @@ Submit an asynchronous batch that creates or updates up to 100 records by extern | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `recordType` | string | Yes | NetSuite REST record type script ID, such as customer or salesOrder | | `items` | array | Yes | Array of 1-100 records; every item must include externalId | | `idempotencyKey` | string | No | Optional unique idempotency key for retrying the batch | @@ -460,7 +443,6 @@ Submit an asynchronous request to delete up to 100 records of one type. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `recordType` | string | Yes | NetSuite REST record type script ID, such as customer or salesOrder | | `ids` | string | Yes | Up to 100 comma-separated internal IDs or eid: external-ID references | | `idempotencyKey` | string | No | Optional unique idempotency key for retrying the batch | @@ -483,7 +465,6 @@ Execute one page of a SuiteQL query through SuiteTalk REST web services. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `query` | string | Yes | SuiteQL SELECT query; use a complete unique ORDER BY when retrieving multiple pages | | `limit` | number | No | Results to return in this page \(1-1000; default 100\) | | `offset` | number | No | Zero-based result offset; must be divisible by limit and stay within the first 100,000 results and 1,000 pages | @@ -512,7 +493,6 @@ List one page of SuiteAnalytics Workbook datasets available to the authenticated | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `limit` | number | No | Results to return in this page \(1-1000; default 100\) | | `offset` | number | No | Zero-based result offset; must be divisible by limit and stay within the first 100,000 results and 1,000 pages | @@ -540,7 +520,6 @@ Execute one page of a standard or custom SuiteAnalytics Workbook dataset. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `datasetId` | string | Yes | SuiteAnalytics dataset script ID | | `limit` | number | No | Results to return in this page \(1-1000; default 100\) | | `offset` | number | No | Zero-based result offset; must be divisible by limit and stay within the first 100,000 results and 1,000 pages | @@ -569,7 +548,6 @@ List record types exposed to the authenticated role by the REST metadata catalog | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | #### Output @@ -596,7 +574,6 @@ Retrieve account-specific metadata for one NetSuite record type. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `recordType` | string | Yes | NetSuite REST record type script ID, such as customer or salesOrder | | `format` | string | No | Metadata representation: default, openapi, or json_schema | @@ -616,7 +593,6 @@ Retrieve job status, list job tasks, or retrieve one task status. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `jobId` | string | Yes | Asynchronous job ID | | `view` | string | No | Retrieve job status, list tasks for the job, or retrieve one task status | | `taskId` | string | No | Task ID; required when view is task | @@ -654,7 +630,6 @@ Retrieve the provider response for one task within a completed asynchronous job. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | | `jobId` | string | Yes | Asynchronous job ID | | `taskId` | string | Yes | Task ID within the asynchronous job | @@ -674,7 +649,6 @@ Retrieve the current UTC time from the NetSuite server. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | #### Output @@ -693,7 +667,6 @@ Retrieve REST web-services concurrency limits for the NetSuite account and integ | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | NetSuite OAuth 2.0 client-credentials service account | -| `instanceUrl` | string | No | SuiteTalk account origin injected by the executor from the selected credential | #### Output diff --git a/apps/docs/content/docs/en/integrations/notion.mdx b/apps/docs/content/docs/en/integrations/notion.mdx index ee43bba1373..c3a5c72bc75 100644 --- a/apps/docs/content/docs/en/integrations/notion.mdx +++ b/apps/docs/content/docs/en/integrations/notion.mdx @@ -258,6 +258,8 @@ Create a new database in Notion with custom properties ### Add Notion Database Row +Add a new row to a Notion database with specified properties + #### Input | Parameter | Type | Required | Description | diff --git a/apps/docs/content/docs/en/integrations/onepassword.mdx b/apps/docs/content/docs/en/integrations/onepassword.mdx index e6b5ee5fda3..3818631d5d1 100644 --- a/apps/docs/content/docs/en/integrations/onepassword.mdx +++ b/apps/docs/content/docs/en/integrations/onepassword.mdx @@ -188,7 +188,7 @@ Download the content of a file attached to an item | `serverUrl` | string | No | 1Password Connect server URL \(for Connect Server mode\) | | `vaultId` | string | Yes | The vault UUID | | `itemId` | string | Yes | The item UUID the file is attached to | -| `fileId` | string | Yes | The file ID \(from the item\'s "files" array, e.g. via Get Item\) | +| `fileId` | string | Yes | The file ID \(from the item's "files" array, e.g. via Get Item\) | #### Output diff --git a/apps/docs/content/docs/en/integrations/parallel_ai.mdx b/apps/docs/content/docs/en/integrations/parallel_ai.mdx index 9e9bb5c7fc6..c942beed9f4 100644 --- a/apps/docs/content/docs/en/integrations/parallel_ai.mdx +++ b/apps/docs/content/docs/en/integrations/parallel_ai.mdx @@ -98,7 +98,7 @@ Conduct comprehensive deep research across the web using Parallel AI. Synthesize | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `input` | string | Yes | Research query or question \(up to 15,000 characters\) | -| `processor` | string | No | Processing tier: pro, ultra, pro-fast, ultra-fast \(default: $\{DEFAULT_PROCESSOR\}\) | +| `processor` | string | No | Processing tier: pro, ultra, pro-fast, ultra-fast \(default: pro\) | | `include_domains` | string | No | Comma-separated list of domains to restrict research to \(source policy\) | | `exclude_domains` | string | No | Comma-separated list of domains to exclude from research \(source policy\) | | `apiKey` | string | Yes | Parallel AI API Key | diff --git a/apps/docs/content/docs/en/integrations/peopledatalabs.mdx b/apps/docs/content/docs/en/integrations/peopledatalabs.mdx index 1a22a24934b..db5bdafffde 100644 --- a/apps/docs/content/docs/en/integrations/peopledatalabs.mdx +++ b/apps/docs/content/docs/en/integrations/peopledatalabs.mdx @@ -110,7 +110,7 @@ Return up to 20 candidate person matches with confidence scores. Useful when you | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | People Data Labs API key | -| `email` | string | No | No description | +| `email` | string | No | Email | | `phone` | string | No | Phone number | | `profile` | string | No | LinkedIn profile URL | | `email_hash` | string | No | SHA-256 email hash | @@ -120,7 +120,7 @@ Return up to 20 candidate person matches with confidence scores. Useful when you | `middle_name` | string | No | Middle name | | `last_name` | string | No | Last name | | `company` | string | No | Company name or website | -| `school` | string | No | No description | +| `school` | string | No | School | | `location` | string | No | Location | | `street_address` | string | No | Street address | | `locality` | string | No | City | @@ -147,7 +147,7 @@ Search the People Data Labs person dataset using SQL or Elasticsearch DSL. Retur | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | People Data Labs API key | -| `sql` | string | No | PDL SQL query \(e.g., \"SELECT * FROM person WHERE job_title='engineer' AND location_country='united states'\"\) | +| `sql` | string | No | PDL SQL query \(e.g., "SELECT * FROM person WHERE job_title='engineer' AND location_country='united states'"\) | | `query` | string | No | Elasticsearch DSL query as JSON string. Use either sql or query, not both. | | `size` | number | No | Number of results to return \(1-100, default 1\) | | `scroll_token` | string | No | Pagination token returned from a prior response | @@ -277,7 +277,7 @@ Search the People Data Labs company dataset using SQL or Elasticsearch DSL. Retu | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | People Data Labs API key | -| `sql` | string | No | PDL SQL query \(e.g., \"SELECT * FROM company WHERE industry='computer software' AND size='51-200'\"\) | +| `sql` | string | No | PDL SQL query \(e.g., "SELECT * FROM company WHERE industry='computer software' AND size='51-200'"\) | | `query` | string | No | Elasticsearch DSL query as JSON string | | `size` | number | No | Number of results to return \(1-100, default 1\) | | `scroll_token` | string | No | Pagination token returned from a prior response | diff --git a/apps/docs/content/docs/en/integrations/pipedrive.mdx b/apps/docs/content/docs/en/integrations/pipedrive.mdx index 2dc380ceeef..1cda528615d 100644 --- a/apps/docs/content/docs/en/integrations/pipedrive.mdx +++ b/apps/docs/content/docs/en/integrations/pipedrive.mdx @@ -43,7 +43,6 @@ Retrieve all deals from Pipedrive with optional filters | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `authStyle` | string | No | Auth scheme for the token; set by the credential resolver for API-token service accounts | | `status` | string | No | Only fetch deals with a specific status. Values: open, won, lost. If omitted, all not deleted deals are returned | | `person_id` | string | No | If supplied, only deals linked to the specified person are returned \(e.g., "456"\) | | `org_id` | string | No | If supplied, only deals linked to the specified organization are returned \(e.g., "789"\) | @@ -88,7 +87,6 @@ Retrieve detailed information about a specific deal | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `authStyle` | string | No | Auth scheme for the token; set by the credential resolver for API-token service accounts | | `deal_id` | string | Yes | The ID of the deal to retrieve \(e.g., "123"\) | #### Output @@ -106,7 +104,6 @@ Create a new deal in Pipedrive | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `authStyle` | string | No | Auth scheme for the token; set by the credential resolver for API-token service accounts | | `title` | string | Yes | The title of the deal \(e.g., "Enterprise Software License"\) | | `value` | string | No | The monetary value of the deal \(e.g., "5000"\) | | `currency` | string | No | Currency code \(e.g., "USD", "EUR", "GBP"\) | @@ -132,7 +129,6 @@ Update an existing deal in Pipedrive | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `authStyle` | string | No | Auth scheme for the token; set by the credential resolver for API-token service accounts | | `deal_id` | string | Yes | The ID of the deal to update \(e.g., "123"\) | | `title` | string | No | New title for the deal \(e.g., "Updated Enterprise License"\) | | `value` | string | No | New monetary value for the deal \(e.g., "7500"\) | @@ -155,7 +151,6 @@ Retrieve files from Pipedrive with optional filters | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `authStyle` | string | No | Auth scheme for the token; set by the credential resolver for API-token service accounts | | `sort` | string | No | Sort files by field \(supported: "id", "update_time"\) | | `limit` | string | No | Number of results to return \(e.g., "50", default: 100, max: 100\) | | `start` | string | No | Pagination start offset \(0-based index of the first item to return\) | @@ -190,7 +185,6 @@ Retrieve mail threads from Pipedrive mailbox | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `authStyle` | string | No | Auth scheme for the token; set by the credential resolver for API-token service accounts | | `folder` | string | No | Filter by folder: inbox, drafts, sent, archive \(default: inbox\) | | `limit` | string | No | Number of results to return \(e.g., "25", default: 50\) | | `start` | string | No | Pagination start offset \(0-based index of the first item to return\) | @@ -213,7 +207,6 @@ Retrieve all messages from a specific mail thread | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `authStyle` | string | No | Auth scheme for the token; set by the credential resolver for API-token service accounts | | `thread_id` | string | Yes | The ID of the mail thread \(e.g., "12345"\) | #### Output @@ -232,7 +225,6 @@ Retrieve all pipelines from Pipedrive | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `authStyle` | string | No | Auth scheme for the token; set by the credential resolver for API-token service accounts | | `sort_by` | string | No | Field to sort by: id, update_time, add_time \(default: id\) | | `sort_direction` | string | No | Sorting direction: asc, desc \(default: asc\) | | `limit` | string | No | Number of results to return \(e.g., "50", default: 100, max: 500\) | @@ -264,7 +256,6 @@ Retrieve all deals in a specific pipeline | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `authStyle` | string | No | Auth scheme for the token; set by the credential resolver for API-token service accounts | | `pipeline_id` | string | Yes | The ID of the pipeline \(e.g., "1"\) | | `stage_id` | string | No | Filter by specific stage within the pipeline \(e.g., "2"\) | | `limit` | string | No | Number of results to return \(e.g., "50", default: 100, max: 500\) | @@ -286,7 +277,6 @@ Retrieve all projects or a specific project from Pipedrive | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `authStyle` | string | No | Auth scheme for the token; set by the credential resolver for API-token service accounts | | `project_id` | string | No | Optional: ID of a specific project to retrieve \(e.g., "123"\) | | `status` | string | No | Filter by project status: open, completed, deleted \(only for listing all\) | | `limit` | string | No | Number of results to return \(e.g., "50", default: 100, max: 500, only for listing all\) | @@ -311,7 +301,6 @@ Create a new project in Pipedrive | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `authStyle` | string | No | Auth scheme for the token; set by the credential resolver for API-token service accounts | | `title` | string | Yes | The title of the project \(e.g., "Q2 Marketing Campaign"\) | | `description` | string | No | Description of the project | | `start_date` | string | No | Project start date in YYYY-MM-DD format \(e.g., "2025-04-01"\) | @@ -332,7 +321,6 @@ Retrieve activities (tasks) from Pipedrive with optional filters | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `authStyle` | string | No | Auth scheme for the token; set by the credential resolver for API-token service accounts | | `user_id` | string | No | Filter activities by user ID \(e.g., "123"\) | | `type` | string | No | Filter by activity type \(call, meeting, task, deadline, email, lunch\) | | `done` | string | No | Filter by completion status: 0 for not done, 1 for done | @@ -370,7 +358,6 @@ Create a new activity (task) in Pipedrive | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `authStyle` | string | No | Auth scheme for the token; set by the credential resolver for API-token service accounts | | `subject` | string | Yes | The subject/title of the activity \(e.g., "Follow up call with John"\) | | `type` | string | Yes | Activity type: call, meeting, task, deadline, email, lunch | | `due_date` | string | Yes | Due date in YYYY-MM-DD format \(e.g., "2025-03-15"\) | @@ -396,7 +383,6 @@ Update an existing activity (task) in Pipedrive | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `authStyle` | string | No | Auth scheme for the token; set by the credential resolver for API-token service accounts | | `activity_id` | string | Yes | The ID of the activity to update \(e.g., "12345"\) | | `subject` | string | No | New subject/title for the activity \(e.g., "Updated meeting with client"\) | | `due_date` | string | No | New due date in YYYY-MM-DD format \(e.g., "2025-03-20"\) | @@ -420,7 +406,6 @@ Retrieve all leads or a specific lead from Pipedrive | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `authStyle` | string | No | Auth scheme for the token; set by the credential resolver for API-token service accounts | | `lead_id` | string | No | Optional: ID of a specific lead to retrieve \(e.g., "abc123-def456-ghi789"\) | | `archived` | string | No | Get archived leads instead of active ones \(e.g., "true" or "false"\) | | `owner_id` | string | No | Filter by owner user ID \(e.g., "123"\) | @@ -474,7 +459,6 @@ Create a new lead in Pipedrive | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `authStyle` | string | No | Auth scheme for the token; set by the credential resolver for API-token service accounts | | `title` | string | Yes | The name of the lead \(e.g., "Acme Corp - Website Redesign"\) | | `person_id` | string | No | ID of the person \(REQUIRED unless organization_id is provided\) \(e.g., "456"\) | | `organization_id` | string | No | ID of the organization \(REQUIRED unless person_id is provided\) \(e.g., "789"\) | @@ -499,7 +483,6 @@ Update an existing lead in Pipedrive | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `authStyle` | string | No | Auth scheme for the token; set by the credential resolver for API-token service accounts | | `lead_id` | string | Yes | The ID of the lead to update \(e.g., "abc123-def456-ghi789"\) | | `title` | string | No | New name for the lead \(e.g., "Updated Lead - Premium Package"\) | | `person_id` | string | No | New person ID \(e.g., "456"\) | @@ -525,7 +508,6 @@ Delete a specific lead from Pipedrive | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `authStyle` | string | No | Auth scheme for the token; set by the credential resolver for API-token service accounts | | `lead_id` | string | Yes | The ID of the lead to delete \(e.g., "abc123-def456-ghi789"\) | #### Output diff --git a/apps/docs/content/docs/en/integrations/pulse.mdx b/apps/docs/content/docs/en/integrations/pulse.mdx index 8206ce4cd2a..050b59519fb 100644 --- a/apps/docs/content/docs/en/integrations/pulse.mdx +++ b/apps/docs/content/docs/en/integrations/pulse.mdx @@ -39,11 +39,17 @@ Integrate Pulse into the workflow. Extract text from PDF documents, images, and ### Pulse Document Parser +Parse documents (PDF, images, Office docs) using Pulse OCR API + #### Input | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `file` | file | Yes | Document to be processed | +| `pages` | string | No | Page range to process \(1-indexed, e.g., "1-2,5"\) | +| `chunking` | string | No | Chunking strategies \(comma-separated: semantic, header, page, recursive\) | +| `chunkSize` | number | No | Maximum characters per chunk when chunking is enabled | +| `apiKey` | string | Yes | Pulse API key | #### Output @@ -54,9 +60,9 @@ Integrate Pulse into the workflow. Extract text from PDF documents, images, and | `job_id` | string | Unique job identifier | | `bounding_boxes` | json | Bounding box layout information | | `extraction_url` | string | URL for extraction results \(for large documents\) | -| `html` | string | HTML content if requested | -| `structured_output` | json | Structured output if schema was provided | +| `html` | string | HTML content; returned only when the hidden returnHtml input is enabled | +| `structured_output` | json | Structured output; Sim exposes no input for supplying a schema, so this is always null | | `chunks` | json | Chunked content if chunking was enabled | -| `figures` | json | Extracted figures if figure extraction was enabled | +| `figures` | json | Extracted figures; returned only when the hidden extractFigure input is enabled | diff --git a/apps/docs/content/docs/en/integrations/rabbitmq.mdx b/apps/docs/content/docs/en/integrations/rabbitmq.mdx index 9abc850a044..e76ea83c5ab 100644 --- a/apps/docs/content/docs/en/integrations/rabbitmq.mdx +++ b/apps/docs/content/docs/en/integrations/rabbitmq.mdx @@ -90,10 +90,10 @@ Retrieve messages from a RabbitMQ queue. Defaults to requeueing the messages so | `password` | string | Yes | RabbitMQ password | | `vhost` | string | No | Virtual host to operate on. Defaults to / | | `queue` | string | Yes | Queue to read messages from | -| `count` | number | No | Maximum number of messages to retrieve, from 1 to $\{MAX_MESSAGE_COUNT\}. Defaults to 1 | +| `count` | number | No | Maximum number of messages to retrieve, from 1 to 50. Defaults to 1 | | `ackmode` | string | No | How retrieved messages are handled: ack_requeue_true \(default, leaves messages in the queue\), ack_requeue_false \(removes them\), reject_requeue_true, or reject_requeue_false | | `encoding` | string | No | auto \(default\) returns readable text where possible, base64 always returns base64 | -| `truncate` | number | No | Truncate payloads longer than this many bytes. Defaults to $\{DEFAULT_TRUNCATE_BYTES\}, capped at $\{MAX_TRUNCATE_BYTES\}, and lowered further at high counts so the whole batch stays inside the response limit. Each message reports whether it was truncated | +| `truncate` | number | No | Truncate payloads longer than this many bytes. Defaults to 50000, capped at 1000000, and lowered further at high counts so the whole batch stays inside the response limit. Each message reports whether it was truncated | #### Output @@ -116,7 +116,7 @@ List queues in a RabbitMQ virtual host with their depth, consumer count, and con | `password` | string | Yes | RabbitMQ password | | `vhost` | string | No | Virtual host to operate on. Defaults to / | | `page` | number | No | Page of results to return, starting at 1 | -| `pageSize` | number | No | Queues per page, from 1 to $\{RABBITMQ_MAX_PAGE_SIZE\}. Defaults to $\{DEFAULT_PAGE_SIZE\} | +| `pageSize` | number | No | Queues per page, from 1 to 500. Defaults to 50 | | `name` | string | No | Filter queues whose name contains this value | | `useRegex` | boolean | No | Treat the name filter as a regular expression | @@ -234,7 +234,7 @@ List exchanges in a RabbitMQ virtual host with their type and declaration settin | `password` | string | Yes | RabbitMQ password | | `vhost` | string | No | Virtual host to operate on. Defaults to / | | `page` | number | No | Page of results to return, starting at 1 | -| `pageSize` | number | No | Exchanges per page, from 1 to $\{RABBITMQ_MAX_PAGE_SIZE\}. Defaults to $\{DEFAULT_PAGE_SIZE\} | +| `pageSize` | number | No | Exchanges per page, from 1 to 500. Defaults to 50 | | `name` | string | No | Filter exchanges whose name contains this value | | `useRegex` | boolean | No | Treat the name filter as a regular expression | @@ -532,7 +532,7 @@ List client connections to the broker with their user, state, and channel count. | `password` | string | Yes | RabbitMQ password | | `vhost` | string | No | Virtual host to operate on. Defaults to / | | `page` | number | No | Page of results to return, starting at 1 | -| `pageSize` | number | No | Connections per page, from 1 to $\{RABBITMQ_MAX_PAGE_SIZE\}. Defaults to $\{DEFAULT_PAGE_SIZE\} | +| `pageSize` | number | No | Connections per page, from 1 to 500. Defaults to 50 | | `name` | string | No | Filter connections whose name contains this value | | `useRegex` | boolean | No | Treat the name filter as a regular expression | @@ -559,7 +559,7 @@ List open channels with their prefetch limit and unacknowledged message count, w | `password` | string | Yes | RabbitMQ password | | `vhost` | string | No | Virtual host to operate on. Defaults to / | | `page` | number | No | Page of results to return, starting at 1 | -| `pageSize` | number | No | Channels per page, from 1 to $\{RABBITMQ_MAX_PAGE_SIZE\}. Defaults to $\{DEFAULT_PAGE_SIZE\} | +| `pageSize` | number | No | Channels per page, from 1 to 500. Defaults to 50 | | `name` | string | No | Filter channels whose name contains this value | | `useRegex` | boolean | No | Treat the name filter as a regular expression | @@ -626,7 +626,7 @@ Create or replace a RabbitMQ policy, applying settings such as dead-lettering, T | `password` | string | Yes | RabbitMQ password | | `vhost` | string | No | Virtual host to operate on. Defaults to / | | `policyName` | string | Yes | Name of the policy. Reusing an existing name replaces that policy | -| `pattern` | string | Yes | Regular expression matched against queue or exchange names, e.g. ^orders\\. to match every name starting with orders. | +| `pattern` | string | Yes | Regular expression matched against queue or exchange names, e.g. ^orders\. to match every name starting with orders. | | `definition` | string | Yes | Settings to apply, as a JSON object, e.g. \{"dead-letter-exchange":"dlx","message-ttl":86400000,"max-length":10000\} | | `priority` | number | No | Priority, defaulting to 0. When several policies match a resource only the highest-priority one applies — they do not merge | | `applyTo` | string | No | What the policy applies to: queues \(default\), classic_queues, quorum_queues, streams, exchanges, or all | diff --git a/apps/docs/content/docs/en/integrations/reducto.mdx b/apps/docs/content/docs/en/integrations/reducto.mdx index c3d5ae81ba5..ea70bbd72d6 100644 --- a/apps/docs/content/docs/en/integrations/reducto.mdx +++ b/apps/docs/content/docs/en/integrations/reducto.mdx @@ -37,11 +37,16 @@ Integrate Reducto Parse into the workflow. Can extract text from uploaded PDF do ### Reducto PDF Parser +Parse PDF documents using Reducto OCR API + #### Input | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `file` | file | Yes | PDF document to be processed | +| `pages` | array | No | Specific pages to process \(1-indexed page numbers\) | +| `tableOutputFormat` | string | No | Table output format \(`md` for Markdown or `html` for HTML\). Defaults to `md`. | +| `apiKey` | string | Yes | Reducto API key \(REDUCTO_API_KEY\) | #### Output diff --git a/apps/docs/content/docs/en/integrations/rippling.mdx b/apps/docs/content/docs/en/integrations/rippling.mdx index 10e3ab6871b..c1203881b0d 100644 --- a/apps/docs/content/docs/en/integrations/rippling.mdx +++ b/apps/docs/content/docs/en/integrations/rippling.mdx @@ -110,7 +110,7 @@ Get a specific worker by ID | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `id` | string | Yes | No description | +| `id` | string | Yes | Resource ID | | `expand` | string | No | Comma-separated fields to expand | #### Output @@ -196,7 +196,7 @@ Get a specific user by ID | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `id` | string | Yes | No description | +| `id` | string | Yes | Resource ID | #### Output @@ -336,7 +336,7 @@ Get a specific department by ID | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `id` | string | Yes | No description | +| `id` | string | Yes | Resource ID | | `expand` | string | No | Comma-separated fields to expand | #### Output @@ -390,7 +390,7 @@ Update an existing department | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `id` | string | Yes | No description | +| `id` | string | Yes | Department ID | | `name` | string | No | Department name | | `parentId` | string | No | Parent department ID | | `referenceCode` | string | No | Reference code | @@ -446,7 +446,7 @@ Get a specific team by ID | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `id` | string | Yes | No description | +| `id` | string | Yes | Resource ID | | `expand` | string | No | Comma-separated fields to expand | #### Output @@ -499,7 +499,7 @@ Get a specific employment type by ID | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `id` | string | Yes | No description | +| `id` | string | Yes | Resource ID | #### Output @@ -549,7 +549,7 @@ Get a specific title by ID | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `id` | string | Yes | No description | +| `id` | string | Yes | Resource ID | #### Output @@ -570,7 +570,7 @@ Create a new title | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `name` | string | Yes | No description | +| `name` | string | Yes | Title name | #### Output @@ -590,8 +590,8 @@ Update an existing title | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `id` | string | Yes | No description | -| `name` | string | No | No description | +| `id` | string | Yes | Title ID | +| `name` | string | No | Title name | #### Output @@ -681,7 +681,7 @@ Get a specific job function by ID | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `id` | string | Yes | No description | +| `id` | string | Yes | Resource ID | #### Output @@ -728,7 +728,7 @@ Get a specific work location by ID | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `id` | string | Yes | No description | +| `id` | string | Yes | Resource ID | #### Output @@ -752,7 +752,7 @@ Create a new work location | `apiKey` | string | Yes | Rippling API key | | `name` | string | Yes | Location name | | `streetAddress` | string | Yes | Street address | -| `locality` | string | No | No description | +| `locality` | string | No | City | | `region` | string | No | State/region | | `postalCode` | string | No | Postal code | | `country` | string | No | Country code | @@ -777,10 +777,10 @@ Update a work location | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `id` | string | Yes | No description | +| `id` | string | Yes | Location ID | | `name` | string | No | Location name | | `streetAddress` | string | No | Street address | -| `locality` | string | No | No description | +| `locality` | string | No | City | | `region` | string | No | State/region | | `postalCode` | string | No | Postal code | | `country` | string | No | Country code | @@ -852,7 +852,7 @@ Get a specific business partner by ID | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `id` | string | Yes | No description | +| `id` | string | Yes | Resource ID | | `expand` | string | No | Comma-separated fields to expand | #### Output @@ -946,7 +946,7 @@ Get a specific business partner group by ID | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `id` | string | Yes | No description | +| `id` | string | Yes | Resource ID | | `expand` | string | No | Comma-separated fields to expand | #### Output @@ -1051,7 +1051,7 @@ Get a specific supergroup by ID | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `id` | string | Yes | No description | +| `id` | string | Yes | Resource ID | #### Output @@ -1268,7 +1268,7 @@ Create a new custom object | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `name` | string | Yes | No description | +| `name` | string | Yes | Object name | | `description` | string | No | Description | | `category` | string | No | Category | @@ -1299,7 +1299,7 @@ Update a custom object | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | | `customObjectApiName` | string | Yes | Custom object API name | -| `name` | string | No | No description | +| `name` | string | No | Name | | `description` | string | No | Description | | `category` | string | No | Category | | `pluralLabel` | string | No | Plural label | @@ -1539,7 +1539,7 @@ Get a specific custom object record | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | | `customObjectApiName` | string | Yes | Custom object API name | -| `codrId` | string | Yes | No description | +| `codrId` | string | Yes | Record ID | #### Output @@ -1626,7 +1626,7 @@ Create a custom object record | `apiKey` | string | Yes | Rippling API key | | `customObjectApiName` | string | Yes | Custom object API name | | `externalId` | string | No | External ID for the record | -| `data` | json | Yes | No description | +| `data` | json | Yes | Record data | #### Output @@ -1785,7 +1785,7 @@ Get a specific custom app | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `id` | string | Yes | No description | +| `id` | string | Yes | Resource ID | #### Output @@ -1810,8 +1810,8 @@ Create a new custom app | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `name` | string | Yes | No description | -| `apiName` | string | Yes | No description | +| `name` | string | Yes | App name | +| `apiName` | string | Yes | API name | | `description` | string | No | Description | #### Output @@ -1836,8 +1836,8 @@ Update a custom app | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `id` | string | Yes | No description | -| `name` | string | No | No description | +| `id` | string | Yes | App ID | +| `name` | string | No | App name | | `apiName` | string | No | API name | | `description` | string | No | Description | @@ -1908,7 +1908,7 @@ Get a specific custom page | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `id` | string | Yes | No description | +| `id` | string | Yes | Resource ID | #### Output @@ -1934,7 +1934,7 @@ Create a new custom page | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `name` | string | Yes | No description | +| `name` | string | Yes | Page name | #### Output @@ -1959,8 +1959,8 @@ Update a custom page | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `id` | string | Yes | No description | -| `name` | string | No | No description | +| `id` | string | Yes | Page ID | +| `name` | string | No | Page name | #### Output @@ -2033,7 +2033,7 @@ Get a specific custom setting | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `id` | string | Yes | No description | +| `id` | string | Yes | Resource ID | #### Output @@ -2092,7 +2092,7 @@ Update a custom setting | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `id` | string | Yes | No description | +| `id` | string | Yes | Setting ID | | `displayName` | string | No | Display name | | `apiName` | string | No | Unique API name | | `dataType` | string | No | Data type of the setting | @@ -2165,7 +2165,7 @@ Get a specific object category | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `id` | string | Yes | No description | +| `id` | string | Yes | Resource ID | #### Output @@ -2208,7 +2208,7 @@ Update an object category | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `id` | string | Yes | No description | +| `id` | string | Yes | Category ID | | `name` | string | No | Category name | | `description` | string | No | Description | @@ -2248,7 +2248,7 @@ Get a report run by ID | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Rippling API key | -| `runId` | string | Yes | No description | +| `runId` | string | Yes | run id | #### Output diff --git a/apps/docs/content/docs/en/integrations/salesforce.mdx b/apps/docs/content/docs/en/integrations/salesforce.mdx index 4caf95090c2..3a51de3f3af 100644 --- a/apps/docs/content/docs/en/integrations/salesforce.mdx +++ b/apps/docs/content/docs/en/integrations/salesforce.mdx @@ -41,8 +41,6 @@ Retrieve accounts from Salesforce CRM | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | The ID token from Salesforce OAuth \(contains instance URL\) | -| `instanceUrl` | string | No | The Salesforce instance URL | | `limit` | string | No | Maximum number of results \(default: 100, max: 2000\) | | `fields` | string | No | Comma-separated field API names \(e.g., "Id,Name,Industry,Phone"\) | | `orderBy` | string | No | Field and direction for sorting \(e.g., "Name ASC" or "CreatedDate DESC"\) | @@ -71,8 +69,6 @@ Create a new account in Salesforce CRM | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `name` | string | Yes | Account name \(required\) | | `type` | string | No | Account type \(e.g., Customer, Partner, Prospect\) | | `industry` | string | No | Industry \(e.g., Technology, Healthcare, Finance\) | @@ -105,8 +101,6 @@ Update an existing account in Salesforce CRM | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `accountId` | string | Yes | Salesforce Account ID to update \(18-character string starting with 001\) | | `name` | string | No | Account name | | `type` | string | No | Account type \(e.g., Customer, Partner, Prospect\) | @@ -139,8 +133,6 @@ Delete an account from Salesforce CRM | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `accountId` | string | Yes | Salesforce Account ID to delete \(18-character string starting with 001\) | #### Output @@ -160,8 +152,6 @@ Get contact(s) from Salesforce - single contact if ID provided, or list if not | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `contactId` | string | No | Salesforce Contact ID \(18-character string starting with 003\) to get a single contact | | `limit` | string | No | Maximum number of results \(default: 100, max: 2000\). Only for list query. | | `fields` | string | No | Comma-separated field API names \(e.g., "Id,FirstName,LastName,Email,Phone"\) | @@ -193,14 +183,12 @@ Create a new contact in Salesforce CRM | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `lastName` | string | Yes | Last name \(required\) | | `firstName` | string | No | First name | | `email` | string | No | Email address | | `phone` | string | No | Phone number | | `accountId` | string | No | Salesforce Account ID \(18-character string starting with 001\) | -| `title` | string | No | No description | +| `title` | string | No | Job title | | `department` | string | No | Department | | `mailingStreet` | string | No | Mailing street | | `mailingCity` | string | No | Mailing city | @@ -227,15 +215,13 @@ Update an existing contact in Salesforce CRM | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `contactId` | string | Yes | Salesforce Contact ID to update \(18-character string starting with 003\) | | `lastName` | string | No | Last name | | `firstName` | string | No | First name | | `email` | string | No | Email address | | `phone` | string | No | Phone number | | `accountId` | string | No | Salesforce Account ID \(18-character string starting with 001\) | -| `title` | string | No | No description | +| `title` | string | No | Job title | | `department` | string | No | Department | | `mailingStreet` | string | No | Mailing street | | `mailingCity` | string | No | Mailing city | @@ -261,8 +247,6 @@ Delete a contact from Salesforce CRM | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `contactId` | string | Yes | Salesforce Contact ID to delete \(18-character string starting with 003\) | #### Output @@ -282,8 +266,6 @@ Retrieve lead(s) from Salesforce CRM | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `leadId` | string | No | Salesforce Lead ID \(18-character string starting with 00Q\) to get a single lead | | `limit` | string | No | Maximum number of results to return \(default: 100\) | | `fields` | string | No | Comma-separated list of field API names to return | @@ -315,8 +297,6 @@ Create a new lead | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `lastName` | string | Yes | Last name \(required\) | | `company` | string | Yes | Company name \(required\) | | `firstName` | string | No | First name | @@ -324,7 +304,7 @@ Create a new lead | `phone` | string | No | Phone number | | `status` | string | No | Lead status \(e.g., Open, Working, Closed\) | | `leadSource` | string | No | Lead source \(e.g., Web, Referral, Campaign\) | -| `title` | string | No | No description | +| `title` | string | No | Job title | | `description` | string | No | Lead description | #### Output @@ -345,8 +325,6 @@ Update an existing lead | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `leadId` | string | Yes | Salesforce Lead ID to update \(18-character string starting with 00Q\) | | `lastName` | string | No | Last name | | `company` | string | No | Company name | @@ -355,7 +333,7 @@ Update an existing lead | `phone` | string | No | Phone number | | `status` | string | No | Lead status \(e.g., Open, Working, Closed\) | | `leadSource` | string | No | Lead source \(e.g., Web, Referral, Campaign\) | -| `title` | string | No | No description | +| `title` | string | No | Job title | | `description` | string | No | Lead description | #### Output @@ -375,8 +353,6 @@ Delete a lead | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `leadId` | string | Yes | Salesforce Lead ID to delete \(18-character string starting with 00Q\) | #### Output @@ -396,8 +372,6 @@ Get opportunity(ies) from Salesforce | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `opportunityId` | string | No | Salesforce Opportunity ID \(18-character string starting with 006\) to get a single opportunity | | `limit` | string | No | Maximum number of results to return \(default: 100\) | | `fields` | string | No | Comma-separated list of field API names to return | @@ -428,8 +402,6 @@ Create a new opportunity | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `name` | string | Yes | Opportunity name \(required\) | | `stageName` | string | Yes | Stage name \(required, e.g., Prospecting, Qualification, Closed Won\) | | `closeDate` | string | Yes | Close date in YYYY-MM-DD format \(required\) | @@ -456,8 +428,6 @@ Update an existing opportunity | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `opportunityId` | string | Yes | Salesforce Opportunity ID to update \(18-character string starting with 006\) | | `name` | string | No | Opportunity name | | `stageName` | string | No | Stage name \(e.g., Prospecting, Qualification, Closed Won\) | @@ -484,8 +454,6 @@ Delete an opportunity | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `opportunityId` | string | Yes | Salesforce Opportunity ID to delete \(18-character string starting with 006\) | #### Output @@ -505,8 +473,6 @@ Get case(s) from Salesforce | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `caseId` | string | No | Salesforce Case ID \(18-character string starting with 500\) to get a single case | | `limit` | string | No | Maximum number of results to return \(default: 100\) | | `fields` | string | No | Comma-separated list of field API names to return | @@ -537,8 +503,6 @@ Create a new case | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `subject` | string | Yes | Case subject \(required\) | | `status` | string | No | Status \(e.g., New, Working, Escalated\) | | `priority` | string | No | Priority \(e.g., Low, Medium, High\) | @@ -565,8 +529,6 @@ Update an existing case | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `caseId` | string | Yes | Salesforce Case ID to update \(18-character string starting with 500\) | | `subject` | string | No | Case subject | | `status` | string | No | Status \(e.g., New, Working, Escalated, Closed\) | @@ -593,8 +555,6 @@ Delete a case | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `caseId` | string | Yes | Salesforce Case ID to delete \(18-character string starting with 500\) | #### Output @@ -614,8 +574,6 @@ Get task(s) from Salesforce | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `taskId` | string | No | Salesforce Task ID \(18-character string starting with 00T\) to get a single task | | `limit` | string | No | Maximum number of results to return \(default: 100\) | | `fields` | string | No | Comma-separated list of field API names to return | @@ -646,8 +604,6 @@ Create a new task | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `subject` | string | Yes | Task subject \(required\) | | `status` | string | No | Status \(e.g., Not Started, In Progress, Completed\) | | `priority` | string | No | Priority \(e.g., Low, Normal, High\) | @@ -674,8 +630,6 @@ Update an existing task | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `taskId` | string | Yes | Salesforce Task ID to update \(18-character string starting with 00T\) | | `subject` | string | No | Task subject | | `status` | string | No | Status \(e.g., Not Started, In Progress, Completed\) | @@ -702,8 +656,6 @@ Delete a task | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `taskId` | string | Yes | Salesforce Task ID to delete \(18-character string starting with 00T\) | #### Output @@ -723,8 +675,6 @@ Get a list of up to 200 recently viewed reports for the current user | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `searchTerm` | string | No | Filter reports by name \(case-insensitive partial match\) | #### Output @@ -745,8 +695,6 @@ Get the describe (definition and metadata) for a specific report | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `reportId` | string | Yes | Salesforce Report ID \(18-character string starting with 00O\) | #### Output @@ -767,8 +715,6 @@ Execute a report and retrieve the results | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `reportId` | string | Yes | Salesforce Report ID \(18-character string starting with 00O\) | | `includeDetails` | string | No | Include detail rows \(true/false, default: true\) | | `filters` | string | No | JSON array of report filter objects to apply | @@ -799,8 +745,6 @@ Get a list of available report types | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | #### Output @@ -820,8 +764,6 @@ Get a list of recently used dashboards for the current user | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | #### Output @@ -841,8 +783,6 @@ Get details and results for a specific dashboard | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `dashboardId` | string | Yes | Salesforce Dashboard ID \(18-character string starting with 01Z\) | #### Output @@ -867,8 +807,6 @@ Refresh a dashboard to get the latest data | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `dashboardId` | string | Yes | Salesforce Dashboard ID \(18-character string starting with 01Z\) | #### Output @@ -894,8 +832,6 @@ Execute a custom SOQL query to retrieve data from Salesforce | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `query` | string | Yes | SOQL query to execute \(e.g., SELECT Id, Name FROM Account LIMIT 10\) | #### Output @@ -919,8 +855,6 @@ Retrieve additional query results using the nextRecordsUrl from a previous query | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `nextRecordsUrl` | string | Yes | The nextRecordsUrl value from a previous query response \(e.g., /services/data/v59.0/query/01g...\) | #### Output @@ -943,8 +877,6 @@ Get metadata and field information for a Salesforce object | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `objectName` | string | Yes | Salesforce object API name \(e.g., Account, Contact, Lead, Custom_Object__c\) | #### Output @@ -998,8 +930,6 @@ Get a list of all available Salesforce objects | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | #### Output @@ -1037,8 +967,6 @@ Create a custom field on a Salesforce object (e.g., Account) using the Tooling A | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `objectName` | string | Yes | API name of the object to add the field to \(e.g., Account, Contact, Lead, MyObject__c\) | | `fieldName` | string | Yes | API name of the new field; the __c suffix is added automatically \(e.g., Region\) | | `label` | string | No | Display label shown in the UI \(defaults to the field name when omitted\) | @@ -1074,8 +1002,6 @@ Update an existing custom field on a Salesforce object using the Tooling API | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `fieldId` | string | Yes | Tooling API Id of the custom field to update \(find it via the Tooling Query tool\) | | `label` | string | No | Display label shown in the UI | | `length` | number | No | Maximum length for Text, LongTextArea, Html, or MultiselectPicklist fields | @@ -1107,8 +1033,6 @@ Delete a custom field from a Salesforce object using the Tooling API | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `fieldId` | string | Yes | Tooling API Id of the custom field to delete \(find it via the Tooling Query tool\) | #### Output @@ -1128,8 +1052,6 @@ Create a custom object in Salesforce using the Tooling API | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `objectName` | string | Yes | API name of the new object; the __c suffix is added automatically \(e.g., Project\) | | `label` | string | Yes | Singular display label for the object \(e.g., Project\) | | `pluralLabel` | string | Yes | Plural display label for the object \(e.g., Projects\) | @@ -1156,8 +1078,6 @@ Execute a SOQL query against the Tooling API to inspect metadata objects | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `idToken` | string | No | No description | -| `instanceUrl` | string | No | No description | | `query` | string | Yes | Tooling SOQL query \(e.g., SELECT Id, DeveloperName FROM CustomField WHERE TableEnumOrId = 'Account'\) | #### Output diff --git a/apps/docs/content/docs/en/integrations/sap_s4hana.mdx b/apps/docs/content/docs/en/integrations/sap_s4hana.mdx index 0758f7c8a81..7071d2e5aaa 100644 --- a/apps/docs/content/docs/en/integrations/sap_s4hana.mdx +++ b/apps/docs/content/docs/en/integrations/sap_s4hana.mdx @@ -72,7 +72,7 @@ List business partners from SAP S/4HANA Cloud (API_BUSINESS_PARTNER, A_BusinessP | `tokenUrl` | string | No | OAuth token URL \(Cloud Private / On-Premise + OAuth\) | | `username` | string | No | Username for HTTP Basic auth | | `password` | string | No | Password for HTTP Basic auth | -| `filter` | string | No | OData $filter expression \(e.g., "BusinessPartnerCategory eq \'1\'"\) | +| `filter` | string | No | OData $filter expression \(e.g., "BusinessPartnerCategory eq '1'"\) | | `top` | number | No | Maximum results to return \($top\) | | `skip` | number | No | Number of results to skip \($skip\) | | `orderBy` | string | No | OData $orderby expression | @@ -247,7 +247,7 @@ List customers from SAP S/4HANA Cloud (API_BUSINESS_PARTNER, A_Customer) with op | `tokenUrl` | string | No | OAuth token URL \(Cloud Private / On-Premise + OAuth\) | | `username` | string | No | Username for HTTP Basic auth | | `password` | string | No | Password for HTTP Basic auth | -| `filter` | string | No | OData $filter expression \(e.g., "CustomerAccountGroup eq \'Z001\'"\) | +| `filter` | string | No | OData $filter expression \(e.g., "CustomerAccountGroup eq 'Z001'"\) | | `top` | number | No | Maximum results to return \($top\) | | `skip` | number | No | Number of results to skip \($skip\) | | `orderBy` | string | No | OData $orderby expression | @@ -259,7 +259,7 @@ List customers from SAP S/4HANA Cloud (API_BUSINESS_PARTNER, A_Customer) with op | Parameter | Type | Description | | --------- | ---- | ----------- | | `status` | number | HTTP status code returned by SAP | -| `data` | json | Array of A_Customer entities, or `\{ results, __count?, __next? \}` when pagination metadata is present \(proxy unwraps the OData v2 `d` envelope\). Properties below describe each customer item. | +| `data` | json | Array of A_Customer entities, or `\{ results, __count?, __next? \}` when pagination metadata is present \(the integration unwraps the OData v2 `d` envelope\). Properties below describe each customer item. | | ↳ `Customer` | string | Customer key \(up to 10 characters\) | | ↳ `CustomerName` | string | Name of customer | | ↳ `CustomerFullName` | string | Full name of the customer | @@ -391,7 +391,7 @@ List suppliers from SAP S/4HANA Cloud (API_BUSINESS_PARTNER, A_Supplier) with op | `tokenUrl` | string | No | OAuth token URL \(Cloud Private / On-Premise + OAuth\) | | `username` | string | No | Username for HTTP Basic auth | | `password` | string | No | Password for HTTP Basic auth | -| `filter` | string | No | OData $filter expression \(e.g., "SupplierAccountGroup eq \'BP02\'"\) | +| `filter` | string | No | OData $filter expression \(e.g., "SupplierAccountGroup eq 'BP02'"\) | | `top` | number | No | Maximum results to return \($top\) | | `skip` | number | No | Number of results to skip \($skip\) | | `orderBy` | string | No | OData $orderby expression | @@ -568,7 +568,7 @@ List sales orders from SAP S/4HANA Cloud (API_SALES_ORDER_SRV, A_SalesOrder) wit | `tokenUrl` | string | No | OAuth token URL \(Cloud Private / On-Premise + OAuth\) | | `username` | string | No | Username for HTTP Basic auth | | `password` | string | No | Password for HTTP Basic auth | -| `filter` | string | No | OData $filter expression \(e.g., "SalesOrganization eq \'1010\'"\) | +| `filter` | string | No | OData $filter expression \(e.g., "SalesOrganization eq '1010'"\) | | `top` | number | No | Maximum results to return \($top\) | | `skip` | number | No | Number of results to skip \($skip\) | | `orderBy` | string | No | OData $orderby expression | @@ -777,7 +777,7 @@ List outbound deliveries from SAP S/4HANA Cloud (API_OUTBOUND_DELIVERY_SRV;v=000 | `tokenUrl` | string | No | OAuth token URL \(Cloud Private / On-Premise + OAuth\) | | `username` | string | No | Username for HTTP Basic auth | | `password` | string | No | Password for HTTP Basic auth | -| `filter` | string | No | OData $filter expression \(e.g., "OverallDeliveryStatus eq \'C\'"\) | +| `filter` | string | No | OData $filter expression \(e.g., "OverallDeliveryStatus eq 'C'"\) | | `top` | number | No | Maximum results to return \($top\) | | `skip` | number | No | Number of results to skip \($skip\) | | `orderBy` | string | No | OData $orderby expression | @@ -877,7 +877,7 @@ List inbound deliveries from SAP S/4HANA Cloud (API_INBOUND_DELIVERY_SRV;v=0002, | `tokenUrl` | string | No | OAuth token URL \(Cloud Private / On-Premise + OAuth\) | | `username` | string | No | Username for HTTP Basic auth | | `password` | string | No | Password for HTTP Basic auth | -| `filter` | string | No | OData $filter expression \(e.g., "ReceivingPlant eq \'1010\'"\) | +| `filter` | string | No | OData $filter expression \(e.g., "ReceivingPlant eq '1010'"\) | | `top` | number | No | Maximum results to return \($top\) | | `skip` | number | No | Number of results to skip \($skip\) | | `orderBy` | string | No | OData $orderby expression | @@ -973,7 +973,7 @@ List billing documents (customer invoices) from SAP S/4HANA Cloud (API_BILLING_D | `tokenUrl` | string | No | OAuth token URL \(Cloud Private / On-Premise + OAuth\) | | `username` | string | No | Username for HTTP Basic auth | | `password` | string | No | Password for HTTP Basic auth | -| `filter` | string | No | OData $filter expression \(e.g., "SoldToParty eq \'10100001\'"\) | +| `filter` | string | No | OData $filter expression \(e.g., "SoldToParty eq '10100001'"\) | | `top` | number | No | Maximum results to return \($top\) | | `skip` | number | No | Number of results to skip \($skip\) | | `orderBy` | string | No | OData $orderby expression | @@ -1100,7 +1100,7 @@ List products (materials) from SAP S/4HANA Cloud (API_PRODUCT_SRV, A_Product) wi | `tokenUrl` | string | No | OAuth token URL \(Cloud Private / On-Premise + OAuth\) | | `username` | string | No | Username for HTTP Basic auth | | `password` | string | No | Password for HTTP Basic auth | -| `filter` | string | No | OData $filter expression \(e.g., "ProductType eq \'FERT\'"\) | +| `filter` | string | No | OData $filter expression \(e.g., "ProductType eq 'FERT'"\) | | `top` | number | No | Maximum results to return \($top\) | | `skip` | number | No | Number of results to skip \($skip\) | | `orderBy` | string | No | OData $orderby expression | @@ -1243,7 +1243,7 @@ List material stock quantities from SAP S/4HANA Cloud (API_MATERIAL_STOCK_SRV, A | `tokenUrl` | string | No | OAuth token URL \(Cloud Private / On-Premise + OAuth\) | | `username` | string | No | Username for HTTP Basic auth | | `password` | string | No | Password for HTTP Basic auth | -| `filter` | string | No | OData $filter expression \(e.g., \"Material eq 'TG10' and Plant eq '1010' and InventoryStockType eq '01'\"\) | +| `filter` | string | No | OData $filter expression \(e.g., "Material eq 'TG10' and Plant eq '1010' and InventoryStockType eq '01'"\) | | `top` | number | No | Maximum results to return \($top\) | | `skip` | number | No | Number of results to skip \($skip\) | | `orderBy` | string | No | OData $orderby expression | @@ -1288,7 +1288,7 @@ List material document headers (goods movements) from SAP S/4HANA Cloud (API_MAT | `tokenUrl` | string | No | OAuth token URL \(Cloud Private / On-Premise + OAuth\) | | `username` | string | No | Username for HTTP Basic auth | | `password` | string | No | Password for HTTP Basic auth | -| `filter` | string | No | OData $filter expression \(e.g., \"MaterialDocumentYear eq '2024' and PostingDate ge datetime'2024-01-01T00:00:00'\"\) | +| `filter` | string | No | OData $filter expression \(e.g., "MaterialDocumentYear eq '2024' and PostingDate ge datetime'2024-01-01T00:00:00'"\) | | `top` | number | No | Maximum results to return \($top\) | | `skip` | number | No | Number of results to skip \($skip\) | | `orderBy` | string | No | OData $orderby expression | @@ -1380,7 +1380,7 @@ List purchase requisitions from SAP S/4HANA Cloud (API_PURCHASEREQ_PROCESS_SRV, | `tokenUrl` | string | No | OAuth token URL \(Cloud Private / On-Premise + OAuth\) | | `username` | string | No | Username for HTTP Basic auth | | `password` | string | No | Password for HTTP Basic auth | -| `filter` | string | No | OData $filter expression \(e.g., "PurchaseRequisitionType eq \'NB\'"\) | +| `filter` | string | No | OData $filter expression \(e.g., "PurchaseRequisitionType eq 'NB'"\) | | `top` | number | No | Maximum results to return \($top\) | | `skip` | number | No | Number of results to skip \($skip\) | | `orderBy` | string | No | OData $orderby expression | @@ -1523,7 +1523,7 @@ List purchase orders from SAP S/4HANA Cloud (API_PURCHASEORDER_PROCESS_SRV, A_Pu | `tokenUrl` | string | No | OAuth token URL \(Cloud Private / On-Premise + OAuth\) | | `username` | string | No | Username for HTTP Basic auth | | `password` | string | No | Password for HTTP Basic auth | -| `filter` | string | No | OData $filter expression \(e.g., "CompanyCode eq \'1010\'"\) | +| `filter` | string | No | OData $filter expression \(e.g., "CompanyCode eq '1010'"\) | | `top` | number | No | Maximum results to return \($top\) | | `skip` | number | No | Number of results to skip \($skip\) | | `orderBy` | string | No | OData $orderby expression | @@ -1698,7 +1698,7 @@ List supplier invoices from SAP S/4HANA Cloud (API_SUPPLIERINVOICE_PROCESS_SRV, | `tokenUrl` | string | No | OAuth token URL \(Cloud Private / On-Premise + OAuth\) | | `username` | string | No | Username for HTTP Basic auth | | `password` | string | No | Password for HTTP Basic auth | -| `filter` | string | No | OData $filter expression \(e.g., "InvoicingParty eq \'17300001\'"\) | +| `filter` | string | No | OData $filter expression \(e.g., "InvoicingParty eq '17300001'"\) | | `top` | number | No | Maximum results to return \($top\) | | `skip` | number | No | Number of results to skip \($skip\) | | `orderBy` | string | No | OData $orderby expression | @@ -1780,7 +1780,7 @@ Retrieve a single supplier invoice by composite key (SupplierInvoice + FiscalYea ### SAP S/4HANA OData Query -Make an arbitrary OData v2 call against any SAP S/4HANA Cloud whitelisted Communication Scenario. Use when no dedicated tool exists for the entity. The proxy handles auth, CSRF, and OData unwrapping. For write operations (POST/PUT/PATCH/MERGE/DELETE), pass an If-Match ETag obtained from a prior GET to avoid lost updates; misuse will mutate production data. +Make an arbitrary OData v2 call against any SAP S/4HANA Cloud whitelisted Communication Scenario. Use when no dedicated tool exists for the entity. The integration handles auth, CSRF, and OData unwrapping. For write operations (POST/PUT/PATCH/MERGE/DELETE), pass an If-Match ETag obtained from a prior GET to avoid lost updates; misuse will mutate production data. #### Input @@ -1797,9 +1797,9 @@ Make an arbitrary OData v2 call against any SAP S/4HANA Cloud whitelisted Commun | `username` | string | No | Username for HTTP Basic auth | | `password` | string | No | Password for HTTP Basic auth | | `service` | string | Yes | OData service name \(e.g., "API_BUSINESS_PARTNER", "API_SALES_ORDER_SRV"\) | -| `path` | string | Yes | Path inside the service \(e.g., "/A_BusinessPartner" or "/A_BusinessPartner\(\'1000123\'\)"\) | +| `path` | string | Yes | Path inside the service \(e.g., "/A_BusinessPartner" or "/A_BusinessPartner\('1000123'\)"\) | | `method` | string | No | HTTP method: GET \(default\), POST, PATCH, PUT, DELETE, MERGE | -| `query` | json | No | OData query parameters as JSON object or query string \(e.g., \{"$filter":"BusinessPartnerCategory eq \'1\'","$top":10\}\). $format=json is added automatically when omitted. | +| `query` | json | No | OData query parameters as JSON object or query string \(e.g., \{"$filter":"BusinessPartnerCategory eq '1'","$top":10\}\). $format=json is added automatically when omitted. | | `body` | json | No | JSON request body for write operations | | `ifMatch` | string | No | ETag value for the If-Match header \(required by SAP for PATCH/PUT/DELETE on existing entities\) | diff --git a/apps/docs/content/docs/en/integrations/sendgrid.mdx b/apps/docs/content/docs/en/integrations/sendgrid.mdx index b62bc65ea2d..72690aa178d 100644 --- a/apps/docs/content/docs/en/integrations/sendgrid.mdx +++ b/apps/docs/content/docs/en/integrations/sendgrid.mdx @@ -133,7 +133,7 @@ Search for contacts in SendGrid using a query | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | SendGrid API key | -| `query` | string | Yes | Search query \(e.g., \"email LIKE '%example.com%' AND CONTAINS\(list_ids, 'list-id'\)\"\) | +| `query` | string | Yes | Search query \(e.g., "email LIKE '%example.com%' AND CONTAINS\(list_ids, 'list-id'\)"\) | #### Output @@ -323,9 +323,7 @@ Get all email templates from SendGrid | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | SendGrid API key | | `generations` | string | No | Filter by generation \(legacy, dynamic, or both\) | -| `pageSize` | number | No | Number of templates to return per page \(default: 20, max: 200\). ' + - 'When paginating with pageToken, pass the same pageSize used on the first request ' + - 'to keep page boundaries consistent. | +| `pageSize` | number | No | Number of templates to return per page \(default: 20, max: 200\). When paginating with pageToken, pass the same pageSize used on the first request to keep page boundaries consistent. | | `pageToken` | string | No | Page token from a previous response \(nextPageToken\) to fetch the next page | #### Output diff --git a/apps/docs/content/docs/en/integrations/sentry.mdx b/apps/docs/content/docs/en/integrations/sentry.mdx index 26d0fd3b155..3a6ab8fb525 100644 --- a/apps/docs/content/docs/en/integrations/sentry.mdx +++ b/apps/docs/content/docs/en/integrations/sentry.mdx @@ -60,7 +60,7 @@ List issues from Sentry for a specific organization and optionally a specific pr | `statsPeriod` | string | No | Time window for the per-issue stats series \(e.g., "24h", "14d"\). Note: this controls the stats returned with each issue, not which issues are returned — use the query \(e.g., "age:-7d", "lastSeen:-24h"\) to filter results. | | `cursor` | string | No | Pagination cursor for retrieving next page of results | | `limit` | number | No | Number of issues to return per page \(max: 100\) | -| `status` | string | No | Filter by issue status: unresolved, resolved, or ignored. The legacy "ignored"/"muted" values map to Sentry\'s current "archived" search token. | +| `status` | string | No | Filter by issue status: unresolved, resolved, or ignored. The legacy "ignored"/"muted" values map to Sentry's current "archived" search token. | | `sort` | string | No | Sort order: date, new, trends, freq, or user \(default: date\) | #### Output diff --git a/apps/docs/content/docs/en/integrations/servicenow.mdx b/apps/docs/content/docs/en/integrations/servicenow.mdx index 2359530996b..a928106ff27 100644 --- a/apps/docs/content/docs/en/integrations/servicenow.mdx +++ b/apps/docs/content/docs/en/integrations/servicenow.mdx @@ -1230,7 +1230,7 @@ Search the ServiceNow CMDB for configuration items. Defaults to the base cmdb_ci | `instanceUrl` | string | Yes | ServiceNow instance URL \(e.g., https://instance.service-now.com\) | | `username` | string | Yes | ServiceNow username | | `password` | string | Yes | ServiceNow password | -| `ciClass` | string | No | CMDB class \(table\) to search, e.g., cmdb_ci_linux_server or cmdb_ci_app_server. Defaults to $\{SERVICENOW_TABLES.CI\}, which covers every CI class. | +| `ciClass` | string | No | CMDB class \(table\) to search, e.g., cmdb_ci_linux_server or cmdb_ci_app_server. Defaults to cmdb_ci, which covers every CI class. | | `name` | string | No | Text to match against the CI name using the ServiceNow LIKE operator, which matches anywhere in the field. | | `operationalStatus` | string | No | Operational status coded value \(operational_status\). The choice list is configured per instance. | | `query` | string | No | Additional ServiceNow encoded query, ANDed with the other filters \(e.g., "opened_at>=javascript:gs.beginningOfLastMonth\(\)"\). | diff --git a/apps/docs/content/docs/en/integrations/sixtyfour.mdx b/apps/docs/content/docs/en/integrations/sixtyfour.mdx index 844c3ac495b..772ec53a577 100644 --- a/apps/docs/content/docs/en/integrations/sixtyfour.mdx +++ b/apps/docs/content/docs/en/integrations/sixtyfour.mdx @@ -101,7 +101,7 @@ Enrich lead information with contact details, social profiles, and company data | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Sixtyfour API key | | `leadInfo` | string | Yes | Lead information as JSON object with key-value pairs \(e.g. name, company, title, linkedin\) | -| `struct` | string | Yes | Fields to collect as JSON object. Keys are field names, values are descriptions \(e.g. \{"email": "The individual\'s email address", "phone": "Phone number"\}\) | +| `struct` | string | Yes | Fields to collect as JSON object. Keys are field names, values are descriptions \(e.g. \{"email": "The individual's email address", "phone": "Phone number"\}\) | | `researchPlan` | string | No | Optional research plan to guide enrichment strategy | #### Output diff --git a/apps/docs/content/docs/en/integrations/slack.mdx b/apps/docs/content/docs/en/integrations/slack.mdx index c2ff9abf2af..56af83e283d 100644 --- a/apps/docs/content/docs/en/integrations/slack.mdx +++ b/apps/docs/content/docs/en/integrations/slack.mdx @@ -163,7 +163,6 @@ Create and share Slack canvases in channels. Canvases are collaborative document | `channel` | string | Yes | Slack channel ID \(e.g., C1234567890\) | | `title` | string | Yes | Title of the canvas | | `content` | string | Yes | Canvas content in markdown format | -| `document_content` | object | No | Structured canvas document content | #### Output diff --git a/apps/docs/content/docs/en/integrations/snowflake.mdx b/apps/docs/content/docs/en/integrations/snowflake.mdx index f10f6d15e9b..eb68cc57ebf 100644 --- a/apps/docs/content/docs/en/integrations/snowflake.mdx +++ b/apps/docs/content/docs/en/integrations/snowflake.mdx @@ -45,7 +45,6 @@ Execute one parameterized SQL statement through the Snowflake SQL API. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `warehouse` | string | No | Warehouse to use for this statement; defaults to the PAT user setting | @@ -93,7 +92,6 @@ Check a running or completed statement and retrieve exactly one result partition | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `statementHandle` | string | Yes | Statement handle returned by Snowflake | | `partition` | number | No | Zero-based result partition to retrieve; defaults to 0 | | `partitionCount` | number | No | Total number of result partitions, taken from the partitionCount of the first partition. Snowflake omits metadata from every later partition response, so supply this when fetching partition 1 or higher to keep truncated and nextPartition accurate | @@ -135,7 +133,6 @@ Cancel a running Snowflake SQL API statement. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `statementHandle` | string | Yes | Statement handle returned by Snowflake | #### Output @@ -175,7 +172,6 @@ Insert structured JSON rows using bound values. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `warehouse` | string | No | Warehouse to use for this statement; defaults to the PAT user setting | @@ -221,7 +217,6 @@ Update matching rows with a bound MERGE statement without inserting new rows. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `warehouse` | string | No | Warehouse to use for this statement; defaults to the PAT user setting | @@ -268,7 +263,6 @@ Update matching rows and insert unmatched rows with a bound MERGE statement. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `warehouse` | string | No | Warehouse to use for this statement; defaults to the PAT user setting | @@ -315,7 +309,6 @@ Delete rows matching a required set of bound column filters. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `warehouse` | string | No | Warehouse to use for this statement; defaults to the PAT user setting | @@ -361,7 +354,6 @@ Load files from an existing Snowflake stage with COPY INTO. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `warehouse` | string | No | Warehouse to use for this statement; defaults to the PAT user setting | @@ -414,7 +406,6 @@ Export a Snowflake table to files in a stage with COPY INTO. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `warehouse` | string | No | Warehouse to use for this statement; defaults to the PAT user setting | @@ -466,7 +457,6 @@ List the databases the credential can access. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `nameLike` | string | No | Optional SQL LIKE pattern for object names | @@ -509,7 +499,6 @@ List the schemas in a Snowflake database. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `database` | string | Yes | Database name | @@ -553,7 +542,6 @@ List the tables in a Snowflake schema. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `database` | string | Yes | Database name | @@ -598,7 +586,6 @@ List warehouses visible to the active Snowflake role. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `maxRows` | number | No | Maximum result rows; defaults to 1000 with a Sim safety limit of 10000 | @@ -641,7 +628,6 @@ Get the full details for a Snowflake virtual warehouse. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `warehouseName` | string | Yes | Warehouse name | @@ -683,7 +669,6 @@ Resume a Snowflake virtual warehouse if it is suspended. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `warehouseName` | string | Yes | Warehouse name | @@ -725,7 +710,6 @@ Suspend a Snowflake virtual warehouse. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `warehouseName` | string | Yes | Warehouse name | @@ -767,11 +751,10 @@ Resize a Snowflake warehouse or change its auto-suspend and auto-resume settings | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `warehouseName` | string | Yes | Warehouse name | -| `warehouseSize` | string | No | New warehouse size, one of $\{SNOWFLAKE_WAREHOUSE_SIZES.join\(', '\)\} | +| `warehouseSize` | string | No | New warehouse size, one of XSMALL, SMALL, MEDIUM, LARGE, XLARGE, XXLARGE, XXXLARGE, X4LARGE, X5LARGE, X6LARGE | | `autoSuspendSeconds` | number | No | Seconds of inactivity before the warehouse suspends. Snowflake polls every 30 seconds, so values under 30 or not a multiple of 30 may not behave as expected. 0 means the warehouse never suspends and keeps consuming credits | | `autoResume` | boolean | No | Whether the warehouse resumes automatically when a statement is submitted | @@ -812,7 +795,6 @@ List tasks in a Snowflake schema. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `database` | string | Yes | Database name | @@ -857,7 +839,6 @@ Describe a Snowflake task. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `database` | string | Yes | Database name | @@ -901,7 +882,6 @@ Run a Snowflake task immediately, optionally retrying its last failed graph. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `database` | string | Yes | Database name | @@ -946,7 +926,6 @@ Resume a suspended Snowflake task so its schedule runs again. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `database` | string | Yes | Database name | @@ -990,7 +969,6 @@ Suspend a Snowflake task so its schedule stops triggering runs. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `database` | string | Yes | Database name | @@ -1034,7 +1012,6 @@ Query up to seven days of Snowflake task history, capped at 10000 rows. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `warehouse` | string | No | Warehouse to use for this statement; defaults to the PAT user setting | @@ -1081,7 +1058,6 @@ Find one task history record by query ID within Snowflake’s seven-day window a | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `warehouse` | string | No | Warehouse to use for this statement; defaults to the PAT user setting | @@ -1127,7 +1103,6 @@ Cancel one running task query by query ID with SYSTEM$CANCEL_QUERY. Task runs al | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `warehouse` | string | No | Warehouse to use for this statement; defaults to the PAT user setting | @@ -1170,7 +1145,6 @@ Read a task query result with RESULT_SCAN during Snowflake’s 24-hour retention | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `warehouse` | string | No | Warehouse to use for this statement; defaults to the PAT user setting | @@ -1214,7 +1188,6 @@ List queries that completed in the last seven days, optionally filtered. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `warehouse` | string | No | Warehouse to use for this statement; defaults to the PAT user setting | @@ -1262,7 +1235,6 @@ List staged-file load results for a table over the last fourteen days. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `warehouse` | string | No | Warehouse to use for this statement; defaults to the PAT user setting | @@ -1310,7 +1282,6 @@ Inspect table and column metadata through Snowflake INFORMATION_SCHEMA views. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `warehouse` | string | No | Warehouse to use for this statement; defaults to the PAT user setting | @@ -1357,7 +1328,6 @@ Call a stored procedure with explicitly typed Snowflake bindings. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `oauthCredential` | string | Yes | Snowflake credential \(account host and programmatic access token\) | -| `domain` | string | No | Snowflake account host injected by the executor from the selected credential | | `role` | string | No | Snowflake role to use for this statement | | `statementTimeoutSeconds` | number | No | Statement timeout in seconds; 0 uses Snowflake maximum of 604800 seconds | | `warehouse` | string | No | Warehouse to use for this statement; defaults to the PAT user setting | diff --git a/apps/docs/content/docs/en/integrations/stripe.mdx b/apps/docs/content/docs/en/integrations/stripe.mdx index f9b6eebb366..a93493cc018 100644 --- a/apps/docs/content/docs/en/integrations/stripe.mdx +++ b/apps/docs/content/docs/en/integrations/stripe.mdx @@ -537,7 +537,7 @@ Search for Payment Intents using query syntax | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Stripe API key \(secret key\) | -| `query` | string | Yes | Search query \(e.g., \"status:'succeeded' AND currency:'usd'\"\) | +| `query` | string | Yes | Search query \(e.g., "status:'succeeded' AND currency:'usd'"\) | | `limit` | number | No | Number of results to return \(default 10, max 100\) | #### Output @@ -867,7 +867,7 @@ Search for customers using query syntax | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Stripe API key \(secret key\) | -| `query` | string | Yes | Search query \(e.g., "email:\'customer@example.com\'"\) | +| `query` | string | Yes | Search query \(e.g., "email:'customer@example.com'"\) | | `limit` | number | No | Number of results to return \(default 10, max 100\) | #### Output @@ -1399,7 +1399,7 @@ Search for subscriptions using query syntax | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Stripe API key \(secret key\) | -| `query` | string | Yes | Search query \(e.g., \"status:'active' AND customer:'cus_xxx'\"\) | +| `query` | string | Yes | Search query \(e.g., "status:'active' AND customer:'cus_xxx'"\) | | `limit` | number | No | Number of results to return \(default 10, max 100\) | #### Output @@ -2459,7 +2459,7 @@ Search for invoices using query syntax | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Stripe API key \(secret key\) | -| `query` | string | Yes | Search query \(e.g., "customer:\'cus_1234567890\'"\) | +| `query` | string | Yes | Search query \(e.g., "customer:'cus_1234567890'"\) | | `limit` | number | No | Number of results to return \(default 10, max 100\) | #### Output @@ -2603,7 +2603,7 @@ Search for charges using query syntax | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Stripe API key \(secret key\) | -| `query` | string | Yes | Search query \(e.g., \"status:'succeeded' AND currency:'usd'\"\) | +| `query` | string | Yes | Search query \(e.g., "status:'succeeded' AND currency:'usd'"\) | | `limit` | number | No | Number of results to return \(default 10, max 100\) | #### Output @@ -2735,7 +2735,7 @@ Search for products using query syntax | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Stripe API key \(secret key\) | -| `query` | string | Yes | Search query \(e.g., "name:\'shirt\'"\) | +| `query` | string | Yes | Search query \(e.g., "name:'shirt'"\) | | `limit` | number | No | Number of results to return \(default 10, max 100\) | #### Output @@ -2851,7 +2851,7 @@ Search for prices using query syntax | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Stripe API key \(secret key\) | -| `query` | string | Yes | Search query \(e.g., \"active:'true' AND currency:'usd'\"\) | +| `query` | string | Yes | Search query \(e.g., "active:'true' AND currency:'usd'"\) | | `limit` | number | No | Number of results to return \(default 10, max 100\) | #### Output diff --git a/apps/docs/content/docs/en/integrations/supabase.mdx b/apps/docs/content/docs/en/integrations/supabase.mdx index a9f884f7a65..f710213dc3e 100644 --- a/apps/docs/content/docs/en/integrations/supabase.mdx +++ b/apps/docs/content/docs/en/integrations/supabase.mdx @@ -255,6 +255,7 @@ Call a PostgreSQL function in Supabase | --------- | ---- | -------- | ----------- | | `projectId` | string | Yes | Your Supabase project ID \(e.g., jdrkgepadsdopsntdlom\) | | `functionName` | string | Yes | The name of the PostgreSQL function to call | +| `params` | object | No | Parameters to pass to the function as a JSON object | | `apiKey` | string | Yes | Your Supabase service role secret key | #### Output diff --git a/apps/docs/content/docs/en/integrations/table.mdx b/apps/docs/content/docs/en/integrations/table.mdx index a178a8778c6..744cbe6aa71 100644 --- a/apps/docs/content/docs/en/integrations/table.mdx +++ b/apps/docs/content/docs/en/integrations/table.mdx @@ -81,14 +81,14 @@ Insert a new row into a table. IMPORTANT: You must use the "data" parameter (not ### Batch Insert Rows -Insert multiple rows into a table at once (up to $\{TABLE_LIMITS.MAX_BATCH_INSERT_SIZE\} rows) +Insert multiple rows into a table at once (up to 1000 rows) #### Input | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `tableId` | string | Yes | Table ID | -| `rows` | array | Yes | Array of row data objects \(max $\{TABLE_LIMITS.MAX_BATCH_INSERT_SIZE\} rows\) | +| `rows` | array | Yes | Array of row data objects \(max 1000 rows\) | #### Output @@ -151,7 +151,7 @@ Update multiple rows that match filter criteria. Data is merged with existing ro | `tableId` | string | Yes | Table ID | | `filter` | object | Yes | Filter criteria using operators like $eq, $ne, $gt, $lt, $contains, $ncontains, $startsWith, $endsWith, $in, $nin, $empty, etc. | | `data` | object | Yes | Fields to update \(merged with existing data\) | -| `limit` | number | No | Maximum number of rows to update \(default: no limit, max: $\{TABLE_LIMITS.MAX_BULK_OPERATION_SIZE\}\) | +| `limit` | number | No | Maximum number of rows to update \(default: no limit, max: 1000\) | #### Output @@ -191,7 +191,7 @@ Delete multiple rows that match filter criteria. Use with caution - supports opt | --------- | ---- | -------- | ----------- | | `tableId` | string | Yes | Table ID | | `filter` | object | Yes | Filter criteria using operators like $eq, $ne, $gt, $lt, $contains, $ncontains, $startsWith, $endsWith, $in, $nin, $empty, etc. | -| `limit` | number | No | Maximum number of rows to delete \(default: no limit, max: $\{TABLE_LIMITS.MAX_BULK_OPERATION_SIZE\}\) | +| `limit` | number | No | Maximum number of rows to delete \(default: no limit, max: 1000\) | #### Output diff --git a/apps/docs/content/docs/en/integrations/textract.mdx b/apps/docs/content/docs/en/integrations/textract.mdx index 17b5bc046c4..6158f778c3c 100644 --- a/apps/docs/content/docs/en/integrations/textract.mdx +++ b/apps/docs/content/docs/en/integrations/textract.mdx @@ -35,11 +35,20 @@ Integrate AWS Textract into your workflow to extract text, tables, forms, and ke ### AWS Textract Parser +Parse documents using AWS Textract OCR and document analysis + #### Input | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | +| `accessKeyId` | string | Yes | AWS Access Key ID | +| `secretAccessKey` | string | Yes | AWS Secret Access Key | +| `region` | string | Yes | AWS region for Textract service \(e.g., us-east-1\) | +| `processingMode` | string | No | Document type: single-page or multi-page. Defaults to single-page. | | `file` | file | No | Document to be processed \(JPEG, PNG, or single-page PDF\). | +| `s3Uri` | string | No | S3 URI for multi-page processing \(s3://bucket/key\). | +| `featureTypes` | array | No | Feature types to detect: TABLES, FORMS, QUERIES, SIGNATURES, LAYOUT. If not specified, only text detection is performed. | +| `queries` | array | No | Custom queries to extract specific information. Only used when featureTypes includes QUERIES. | #### Output diff --git a/apps/docs/content/docs/en/integrations/tinybird.mdx b/apps/docs/content/docs/en/integrations/tinybird.mdx index 23e34238eae..c06f0e5e20b 100644 --- a/apps/docs/content/docs/en/integrations/tinybird.mdx +++ b/apps/docs/content/docs/en/integrations/tinybird.mdx @@ -46,7 +46,7 @@ Send events to a Tinybird Data Source using the Events API. Supports JSON and ND | --------- | ---- | -------- | ----------- | | `base_url` | string | Yes | Tinybird API base URL \(e.g., https://api.tinybird.co or https://api.us-east.tinybird.co\) | | `datasource` | string | Yes | Name of the Tinybird Data Source to send events to. Example: "events_raw", "user_analytics" | -| `data` | string | Yes | Data to send as NDJSON \(newline-delimited JSON\) or JSON string. Each event should be a valid JSON object. Example NDJSON: \{"user_id": 1, "event": "click"\}\\n\{"user_id": 2, "event": "view"\} | +| `data` | string | Yes | Data to send as NDJSON \(newline-delimited JSON\) or JSON string. Each event should be a valid JSON object. Example NDJSON: \{"user_id": 1, "event": "click"\}\n\{"user_id": 2, "event": "view"\} | | `wait` | boolean | No | Wait for database acknowledgment before responding. Enables safer retries but introduces latency. Defaults to false. | | `format` | string | No | Format of the events data: "ndjson" \(default\) or "json" | | `compression` | string | No | Compression format: "none" \(default\) or "gzip" | @@ -168,7 +168,7 @@ Delete rows from a Tinybird Data Source matching a SQL condition. | --------- | ---- | -------- | ----------- | | `base_url` | string | Yes | Tinybird API base URL \(e.g., https://api.tinybird.co\) | | `datasource` | string | Yes | Name of the Data Source to delete rows from. Example: "events_raw" | -| `delete_condition` | string | Yes | SQL WHERE-clause condition selecting the rows to delete. Example: "country = \'ES\'" or "event_date < \'2024-01-01\'" | +| `delete_condition` | string | Yes | SQL WHERE-clause condition selecting the rows to delete. Example: "country = 'ES'" or "event_date < '2024-01-01'" | | `dry_run` | boolean | No | When true, returns how many rows would be deleted without deleting them. Defaults to false. | | `token` | string | Yes | Tinybird API Token with DATASOURCES:CREATE scope | diff --git a/apps/docs/content/docs/en/integrations/tinyfish.mdx b/apps/docs/content/docs/en/integrations/tinyfish.mdx index e110597c167..8afda30e85c 100644 --- a/apps/docs/content/docs/en/integrations/tinyfish.mdx +++ b/apps/docs/content/docs/en/integrations/tinyfish.mdx @@ -7,7 +7,7 @@ import { BlockInfoCard } from "@/components/ui/block-info-card" {/* MANUAL-CONTENT-START:intro */} diff --git a/apps/docs/content/docs/en/integrations/trigger_dev.mdx b/apps/docs/content/docs/en/integrations/trigger_dev.mdx index 5a15f07ffc4..117550f390d 100644 --- a/apps/docs/content/docs/en/integrations/trigger_dev.mdx +++ b/apps/docs/content/docs/en/integrations/trigger_dev.mdx @@ -1712,7 +1712,7 @@ Execute a TRQL (SQL-like) query against Trigger.dev run data for reporting and a | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Trigger.dev secret API key \(starts with tr_\) | -| `query` | string | Yes | TRQL query to execute \(e.g., "SELECT run_id, status, triggered_at FROM runs WHERE status = \'Failed\' LIMIT 10"\) | +| `query` | string | Yes | TRQL query to execute \(e.g., "SELECT run_id, status, triggered_at FROM runs WHERE status = 'Failed' LIMIT 10"\) | | `scope` | string | No | Scope of data to query: environment \(default\), project, or organization | | `period` | string | No | Time period shorthand \(e.g., "1h", "7d", "30d"\). Cannot be combined with from/to | | `from` | string | No | Start of the time range as an ISO 8601 timestamp. Must be used with "to" | diff --git a/apps/docs/content/docs/en/integrations/vanta.mdx b/apps/docs/content/docs/en/integrations/vanta.mdx index 7ae2463a694..c25cb3dde74 100644 --- a/apps/docs/content/docs/en/integrations/vanta.mdx +++ b/apps/docs/content/docs/en/integrations/vanta.mdx @@ -330,9 +330,8 @@ Upload an evidence file to a Vanta document. Requires credentials with the vanta | `region` | string | No | Vanta API region: "us" \(api.vanta.com, default\) or "gov" \(api.vanta-gov.com\) | | `documentId` | string | Yes | Unique ID of the document to attach the file to | | `file` | file | No | The evidence file to upload | -| `fileContent` | string | No | Base64-encoded file content \(alternative to file\) | | `fileName` | string | No | Optional file name override | -| `mimeType` | string | No | MIME type of the file \(e.g., application/pdf\); used when uploading base64 content, since uploaded files already carry their own type | +| `mimeType` | string | No | MIME type of the file \(e.g., application/pdf\). Applies only to the base64 upload path; a file from the File input always sends the content type resolved from storage. | | `description` | string | No | Description of the uploaded evidence \(e.g., "Q3 access review evidence"\) | | `effectiveAtDate` | string | No | ISO 8601 date indicating when the document is effective from | diff --git a/apps/docs/content/docs/en/integrations/zoho_desk.mdx b/apps/docs/content/docs/en/integrations/zoho_desk.mdx index d7b58c75448..6786cba582c 100644 --- a/apps/docs/content/docs/en/integrations/zoho_desk.mdx +++ b/apps/docs/content/docs/en/integrations/zoho_desk.mdx @@ -51,7 +51,6 @@ List tickets from a Zoho Desk organization with optional filters. Returns a list | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `apiDomain` | string | No | Zoho Desk data-center REST base URL | | `orgId` | string | Yes | Zoho Desk organization ID | | `from` | number | No | Pagination start index \(0-based\) | | `limit` | number | No | Number of tickets to return \(1-100\) | @@ -111,7 +110,6 @@ Retrieve a single Zoho Desk ticket by ID. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `apiDomain` | string | No | Zoho Desk data-center REST base URL | | `orgId` | string | Yes | Zoho Desk organization ID | | `ticketId` | string | Yes | Ticket ID to retrieve | | `include` | string | No | Comma-separated related data to embed. Allowed: contacts, products, assignee, departments, contract, isRead, team, skills | @@ -162,7 +160,6 @@ Update fields on an existing Zoho Desk ticket. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `apiDomain` | string | No | Zoho Desk data-center REST base URL | | `orgId` | string | Yes | Zoho Desk organization ID | | `ticketId` | string | Yes | Ticket ID to update | | `subject` | string | No | Ticket subject | @@ -175,7 +172,7 @@ Update fields on an existing Zoho Desk ticket. | `dueDate` | string | No | Due date \(ISO 8601\) | | `description` | string | No | Ticket description | | `resolution` | string | No | Resolution notes recorded on the ticket | -| `classification` | string | No | Ticket classification. Zoho\'s system-defined values are Problem, Request, and Question; portals can define custom values. Pass "" to clear it. | +| `classification` | string | No | Ticket classification. Zoho's system-defined values are Problem, Request, and Question; portals can define custom values. Pass "" to clear it. | | `customFields` | json | No | Custom field values as a JSON object, keyed by custom field API name | #### Output @@ -224,7 +221,6 @@ List comments on a Zoho Desk ticket. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `apiDomain` | string | No | Zoho Desk data-center REST base URL | | `orgId` | string | Yes | Zoho Desk organization ID | | `ticketId` | string | Yes | Ticket ID | | `from` | number | No | Pagination start index \(0-based\) | @@ -267,7 +263,6 @@ Add a comment to a Zoho Desk ticket. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `apiDomain` | string | No | Zoho Desk data-center REST base URL | | `orgId` | string | Yes | Zoho Desk organization ID | | `ticketId` | string | Yes | Ticket ID | | `content` | string | Yes | Comment content | @@ -309,7 +304,6 @@ List conversation threads on a Zoho Desk ticket, newest first (Zoho sorts by sen | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `apiDomain` | string | No | Zoho Desk data-center REST base URL | | `orgId` | string | Yes | Zoho Desk organization ID | | `ticketId` | string | Yes | Ticket ID | | `from` | number | No | Pagination start index \(0-based\) | @@ -367,7 +361,6 @@ Retrieve the full content of a single Zoho Desk ticket thread. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `apiDomain` | string | No | Zoho Desk data-center REST base URL | | `orgId` | string | Yes | Zoho Desk organization ID | | `ticketId` | string | Yes | Ticket ID | | `threadId` | string | Yes | Thread ID | @@ -423,7 +416,6 @@ Retrieve a Zoho Desk contact by ID. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `apiDomain` | string | No | Zoho Desk data-center REST base URL | | `orgId` | string | Yes | Zoho Desk organization ID | | `contactId` | string | Yes | Contact ID to retrieve | | `include` | string | No | Comma-separated related data to embed. Allowed: accounts, owner | @@ -460,7 +452,6 @@ Download a Zoho Desk ticket attachment (from its href) as a file. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `apiDomain` | string | No | Zoho Desk data-center REST base URL | | `orgId` | string | Yes | Zoho Desk organization ID | | `href` | string | Yes | Attachment download href \(from a thread or comment attachment\) | | `fileName` | string | No | Optional file name for the downloaded file | @@ -479,7 +470,6 @@ List the Zoho Desk organizations (portals) the connected account can access. | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `apiDomain` | string | No | Zoho Desk data-center REST base URL | #### Output diff --git a/apps/docs/content/docs/en/introduction/index.mdx b/apps/docs/content/docs/en/introduction/index.mdx index 5df4f4dc723..898c4423ec9 100644 --- a/apps/docs/content/docs/en/introduction/index.mdx +++ b/apps/docs/content/docs/en/introduction/index.mdx @@ -14,9 +14,9 @@ Sim is the open-source AI workspace where teams build, deploy, and manage AI age
Sim workflow builder
diff --git a/apps/docs/content/docs/en/knowledgebase/connectors.mdx b/apps/docs/content/docs/en/knowledgebase/connectors.mdx index d95b9c2b88a..d659d835704 100644 --- a/apps/docs/content/docs/en/knowledgebase/connectors.mdx +++ b/apps/docs/content/docs/en/knowledgebase/connectors.mdx @@ -12,7 +12,7 @@ Connectors continuously sync documents from external services into your knowledg ## Available Connectors -Connect Source picker showing a searchable list of available connectors including Airtable, Asana, Confluence, Discord, Dropbox, Fireflies, GitHub, and Gmail +The current Connect Source picker showing searchable connectors including Airtable, Asana, Ashby, Azure DevOps, Bitbucket, Box, and ClickUp Sim ships with 64 built-in connectors: @@ -123,7 +123,7 @@ Click **Connect & Sync** to save the connector and trigger the first sync. Docum Open **Connected Sources** from the knowledge base to see all active connectors. Each card shows the connector's status, the last sync time and document count, and the next scheduled sync: -Connected Sources panel showing a Google Docs connector with Active status, last sync details, and a sync history log with dated entries +Connected Sources panel showing a Google Docs connector with Active status, last sync details, and a sync history log with dated entries The action buttons on each connector card: @@ -157,7 +157,7 @@ The log retains the most recent 10 sync runs. Sometimes a connector syncs documents you don't want in your knowledge base — drafts, templates, confidential pages, and so on. You can exclude them individually. -Edit Google Docs modal showing the Documents tab with Active (37) and Excluded (0) filter buttons and a 'No excluded documents' message +Edit Google Docs modal showing the Documents tab with Active (37) and Excluded (0) filter buttons and a 'No excluded documents' message To exclude a document, open the connector's settings modal, go to the **Documents** tab, and click **Exclude** next to any document. Excluded documents are skipped on every subsequent sync even if the source content changes. @@ -201,7 +201,7 @@ You can disable specific tag types during setup or at any time from the connecto ## Multiple Connectors -Knowledge base document list showing synced Google Docs documents with Name, Size, Tokens, Chunks, Uploaded date, Status, and Tags columns +Knowledge base document list showing synced Google Docs documents with Name, Size, Tokens, Chunks, Uploaded date, Status, and Tags columns You can add as many connectors as you need to a single knowledge base. Each manages its own documents independently, and all content is searchable together through the Knowledge block. Keep tag slot usage in mind when combining connectors that each populate metadata tags. diff --git a/apps/docs/content/docs/en/knowledgebase/tags.mdx b/apps/docs/content/docs/en/knowledgebase/tags.mdx index 4cccaa5387f..5ac083cb0d3 100644 --- a/apps/docs/content/docs/en/knowledgebase/tags.mdx +++ b/apps/docs/content/docs/en/knowledgebase/tags.mdx @@ -36,11 +36,11 @@ The type dropdown in the creation form shows current slot usage for each type (e Tag definitions live at the knowledge base level. To manage them, click the knowledge base name in the header to open the context menu and select **Tags**: -Knowledge base header showing the dropdown menu with Rename, Tags, and Delete options +Knowledge base header showing the dropdown menu with Rename, Tags, and Delete options This opens the Tags modal, which lists all defined tags and shows how many documents each one is assigned to. Click **Add Tag** to define a new one: -Tags modal showing 0 defined tags, a Tag Name input field, and a Type dropdown set to Text (0/7), with Cancel and Create Tag buttons +Tags modal showing 0 defined tags, a Tag Name input field, and a Type dropdown set to Text (0/7), with Cancel and Create Tag buttons Enter a **Tag Name** and pick a **Type**, then click **Create Tag**. The name must be unique within the knowledge base. The type dropdown only shows types that still have available slots. Press Enter to submit or Escape to cancel. @@ -58,7 +58,7 @@ This opens the tag panel for that document where you can set a value for each de The **Tags** column in the document list shows the current tag values for each document at a glance. Documents with no tags assigned show `– – –`: -Knowledge base document list showing Name, Size, Tokens, Chunks, Uploaded, Status, and Tags columns — Document1.txt shows no tags (– – –) while Document2.txt shows the value 'Waleed' +Knowledge base document list showing Name, Size, Tokens, Chunks, Uploaded, Status, and Tags columns — Document1.txt shows no tags (– – –) while Document2.txt shows the value 'Waleed' Use the **Filter** and **Sort** controls in the top right to narrow the list by tag values or sort by them. @@ -66,7 +66,7 @@ Use the **Filter** and **Sort** controls in the top right to narrow the list by In a workflow, open the Knowledge block and configure **Tag Filters** to restrict which documents are searched: -Knowledge block editor showing Operation: search, Knowledge Base: test, Search Query field (optional), Number of Results, and a Tag Filters section with Filter 1 containing Tag: Name, Operator: equals, and a Value field +Knowledge block editor showing Operation: search, Knowledge Base: test, Search Query field (optional), Number of Results, and a Tag Filters section with Filter 1 containing Tag: Name, Operator: equals, and a Value field Each filter has three parts: - **Tag** — select a tag definition from the knowledge base diff --git a/apps/docs/content/docs/en/logs-debugging/index.mdx b/apps/docs/content/docs/en/logs-debugging/index.mdx index 0b359bbb0e6..40fffa12457 100644 --- a/apps/docs/content/docs/en/logs-debugging/index.mdx +++ b/apps/docs/content/docs/en/logs-debugging/index.mdx @@ -15,8 +15,8 @@ The **Logs page** lists every run across your workspace, one row per run. The [L The Logs page: one row per run with workflow, date, status, cost in credits, trigger, and duration ## What a log records @@ -33,12 +33,12 @@ Open a run and **Log Details** shows the trace. The **Trace** tab lists each blo Log Details with the Trace tab open: a CRM sync run's spans, each block with its duration, 11.61s total -This is the level you debug at, because a run fails when one of its blocks fails, and a block usually fails because of the input it received. In this run, one glance at the spans shows where the time went: the HealingAgent took 7.83s of the 11.61s total. +This is the level you debug at, because a run fails when one of its blocks fails, and a block usually fails because of the input it received. In this run, one glance at the spans shows where the time went: Agent 1 took 7.84s of the 7.86s total. ### Input and output diff --git a/apps/docs/content/docs/en/logs-debugging/logging.mdx b/apps/docs/content/docs/en/logs-debugging/logging.mdx index 49d9f379413..43a83b0a883 100644 --- a/apps/docs/content/docs/en/logs-debugging/logging.mdx +++ b/apps/docs/content/docs/en/logs-debugging/logging.mdx @@ -16,9 +16,9 @@ During manual or chat runs, the Console panel in the editor shows each block as
Real-time Console Panel
@@ -30,9 +30,9 @@ Every run from every trigger — manual, API, chat, schedule, webhook — lands
Logs Page
@@ -44,9 +44,9 @@ Click any entry to open its sidebar: the run's timeline (start/end, total durati
Logs Sidebar Details
@@ -69,7 +69,7 @@ Click any entry to open its sidebar: the run's timeline (start/end, total durati src="/static/logs/logs-frozen-canvas.png" alt="Workflow Snapshot" width={600} - height={400} + height={628} className="my-6" /> diff --git a/apps/docs/content/docs/en/platform/costs.mdx b/apps/docs/content/docs/en/platform/costs.mdx index 4762b98c6e7..f057c37960b 100644 --- a/apps/docs/content/docs/en/platform/costs.mdx +++ b/apps/docs/content/docs/en/platform/costs.mdx @@ -39,7 +39,7 @@ For workflows using AI blocks, you can view detailed cost information in the log src="/static/logs/logs-cost.png" alt="Model Breakdown" width={600} - height={400} + height={153} className="my-6" /> diff --git a/apps/docs/content/docs/en/platform/credentials.mdx b/apps/docs/content/docs/en/platform/credentials.mdx index 387450643f8..4bdd757fd03 100644 --- a/apps/docs/content/docs/en/platform/credentials.mdx +++ b/apps/docs/content/docs/en/platform/credentials.mdx @@ -17,7 +17,7 @@ To manage secrets, open your workspace **Settings** and navigate to the **Secret src="/static/secrets/secrets-list.png" alt="Secrets tab showing Workspace and Personal sections with inline key-value rows" width={700} - height={500} + height={479} /> Secrets are organized into two sections: @@ -53,7 +53,7 @@ To reference a secret in any input field, type `{{` to open the variable dropdow src="/static/credentials/secret-dropdown.png" alt="Typing {{ in an input opens a dropdown showing available secrets" width={400} - height={250} + height={165} /> Select the secret you want to use. The reference appears highlighted in blue and is resolved to its actual value at runtime. @@ -62,7 +62,7 @@ Select the secret you want to use. The reference appears highlighted in blue and src="/static/credentials/secret-resolved.png" alt="A resolved secret reference shown as {{OPENAI_API_KEY}}" width={400} - height={200} + height={166} /> ### Execution log protection @@ -113,7 +113,7 @@ Click **Details** on any secret row to open its detail view. src="/static/secrets/secret-details.png" alt="Secret details view showing Key, Value, Description, and Members sections" width={700} - height={400} + height={351} /> From here you can: diff --git a/apps/docs/content/docs/en/platform/enterprise/access-control.mdx b/apps/docs/content/docs/en/platform/enterprise/access-control.mdx index 5f67e668124..3ff1b78f6f2 100644 --- a/apps/docs/content/docs/en/platform/enterprise/access-control.mdx +++ b/apps/docs/content/docs/en/platform/enterprise/access-control.mdx @@ -37,7 +37,7 @@ When a user runs a workflow or uses Chat, Sim reads the resolved group's configu Go to **Settings → Enterprise → Access Control** from any workspace in your organization. Permission groups are defined once at the organization level and apply to every workspace under it. Only organization owners and admins can manage them. -Access Control settings showing a list of permission groups: Contractors, Sales, Engineering, and Marketing, each with Details and Delete actions +Access Control settings showing a list of permission groups: Contractors, Sales, Engineering, and Marketing, each with Details and Delete actions ### 2. Create a permission group @@ -55,7 +55,7 @@ A workspace-scoped group with **no members** applies to everyone in its workspac Controls which AI model providers members of this group can use. -Model Providers tab showing a grid of AI providers including Ollama, vLLM, OpenAI, Anthropic, Google, Azure OpenAI, and others with checkboxes to allow or restrict access The list shows all providers available in Sim. +Model Providers tab showing a grid of AI providers including Ollama, vLLM, OpenAI, Anthropic, Google, Azure OpenAI, and others with checkboxes to allow or restrict access The list shows all providers available in Sim. - **All checked (default):** All providers are allowed. - **Subset checked:** Only the selected providers are allowed. Any workflow block or agent using a provider not on the list will fail at execution time. @@ -64,7 +64,7 @@ Controls which AI model providers members of this group can use. Controls which workflow blocks members can place and execute. -Blocks tab showing Core Blocks (Agent, API, Condition, Function, Knowledge, etc.) and Tools (integrations like 1Password, Ahrefs, Airtable, and more) with checkboxes to allow or restrict each Blocks are split into two sections: **Core Blocks** (Agent, API, Condition, Function, etc.) and **Tools** (all integration blocks). +Blocks tab showing Core Blocks (Agent, API, Condition, Function, Knowledge, etc.) and Tools (integrations like 1Password, Ahrefs, Airtable, and more) with checkboxes to allow or restrict each Blocks are split into two sections: **Core Blocks** (Agent, API, Condition, Function, etc.) and **Tools** (all integration blocks). - **All checked (default):** All blocks are allowed. - **Subset checked:** Only the selected blocks are allowed. Workflows that already contain a disallowed block will fail when run — they are not automatically modified. @@ -77,7 +77,7 @@ Controls which workflow blocks members can place and execute. Controls visibility of platform features and modules. -Platform tab showing feature toggles grouped by category: Sidebar (Knowledge Base, Tables), Workflow Panel (Copilot), Settings Tabs, Tools, Deploy Tabs, Features, Logs, and Collaboration Each checkbox maps to a specific feature; checking it hides or disables that feature for group members. +Platform tab showing feature toggles grouped by category: Sidebar (Knowledge Base, Tables), Workflow Panel (Copilot), Settings Tabs, Tools, Deploy Tabs, Features, Logs, and Collaboration Each checkbox maps to a specific feature; checking it hides or disables that feature for group members. **Sidebar** diff --git a/apps/docs/content/docs/en/platform/enterprise/audit-logs.mdx b/apps/docs/content/docs/en/platform/enterprise/audit-logs.mdx index b9d039c2a56..9be2eb67ad1 100644 --- a/apps/docs/content/docs/en/platform/enterprise/audit-logs.mdx +++ b/apps/docs/content/docs/en/platform/enterprise/audit-logs.mdx @@ -16,7 +16,7 @@ Audit logs give your organization a tamper-evident record of every significant a Go to **Settings → Enterprise → Audit Logs** in your workspace. Logs are displayed in a table with the following columns: -Audit Logs settings showing a table of events with columns for Timestamp, Event, Description, and Actor, along with search and filter controls +Audit Logs settings showing a table of events with columns for Timestamp, Event, Description, and Actor, along with search and filter controls | Column | Description | |--------|-------------| diff --git a/apps/docs/content/docs/en/platform/enterprise/forks.mdx b/apps/docs/content/docs/en/platform/enterprise/forks.mdx index ebaa22fb6ea..62c0c60574a 100644 --- a/apps/docs/content/docs/en/platform/enterprise/forks.mdx +++ b/apps/docs/content/docs/en/platform/enterprise/forks.mdx @@ -134,7 +134,7 @@ The setting belongs to **this workspace's copy** only. Excluding a workflow here **See activity** (or the Activity view from the Forks header) lists forks, pushes, pulls, and rollbacks that involve this workspace — including events recorded on the other side of the edge. -Activity view showing Fork and Push events with expandable detail rows +Activity view showing Fork and Push events with expandable detail rows Expand a row for names of workflows and resources that were created, updated, or archived, and any warnings (for example failed background copies or deploy failures). diff --git a/apps/docs/content/docs/en/platform/enterprise/index.mdx b/apps/docs/content/docs/en/platform/enterprise/index.mdx index fd3e46e4e1a..fa3f62af3b1 100644 --- a/apps/docs/content/docs/en/platform/enterprise/index.mdx +++ b/apps/docs/content/docs/en/platform/enterprise/index.mdx @@ -53,6 +53,12 @@ Track configuration and security-relevant actions across your organization for c --- +## Usage Tracking + +See where your organization's credits go — by member, workspace, model, and platform feature. See the [usage tracking guide](/platform/enterprise/usage-tracking). + +--- + ## Data Retention Configure how long execution logs, soft-deleted resources, and Chat data are kept before permanent deletion. See the [data retention guide](/platform/enterprise/data-retention). diff --git a/apps/docs/content/docs/en/platform/enterprise/meta.json b/apps/docs/content/docs/en/platform/enterprise/meta.json index cdf420f1ff3..48e14db8d21 100644 --- a/apps/docs/content/docs/en/platform/enterprise/meta.json +++ b/apps/docs/content/docs/en/platform/enterprise/meta.json @@ -10,6 +10,7 @@ "custom-blocks", "whitelabeling", "audit-logs", + "usage-tracking", "data-retention", "data-drains", "forks" diff --git a/apps/docs/content/docs/en/platform/enterprise/sso.mdx b/apps/docs/content/docs/en/platform/enterprise/sso.mdx index eef6b5d51f1..a79851039ac 100644 --- a/apps/docs/content/docs/en/platform/enterprise/sso.mdx +++ b/apps/docs/content/docs/en/platform/enterprise/sso.mdx @@ -39,7 +39,7 @@ Go to **Settings → Security → Single sign-on** in your organization settings ### 3. Fill in the form -Single Sign-On configuration form showing Provider Type (OIDC), Provider ID, Issuer URL, Domain, Client ID, Client Secret, Scopes, and Callback URL fields +Single Sign-On configuration form showing Provider Type (OIDC), Provider ID, Issuer URL, Domain, Client ID, Client Secret, Scopes, and Callback URL fields **Fields required for both protocols:** diff --git a/apps/docs/content/docs/en/platform/enterprise/usage-tracking.mdx b/apps/docs/content/docs/en/platform/enterprise/usage-tracking.mdx new file mode 100644 index 00000000000..da0e3d11952 --- /dev/null +++ b/apps/docs/content/docs/en/platform/enterprise/usage-tracking.mdx @@ -0,0 +1,158 @@ +--- +title: Usage Tracking +description: See where your organization's credits go, by member, workspace, and model +--- + +import { Callout } from 'fumadocs-ui/components/callout' +import { FAQ } from '@/components/ui/faq' +import { Image } from '@/components/ui/image' + +Usage tracking shows how your organization consumes credits across every part of the platform — which members, which workspaces, which models, and which product features. Use it to monitor spend against your commitment, find what is driving it, and export the underlying events for chargeback. + +All figures are in **credits** (1 credit = $0.005). See [cost calculation](/platform/costs) for how credits are derived. + +--- + +## Viewing usage + +Go to **Settings → Organization → Usage tracking** in your workspace. + +Usage tracking Overview tab showing the period selector, credits used against the organization limit, a daily usage chart, and a Sources section pairing a ranked list of sources with a radar chart of the same mix + +The period selector applies to every tab: + +| Period | What it covers | +|--------|----------------| +| **Current period** | Your organization's current billing period | +| **Previous period** | The period immediately before it | +| **Last 7 days** / **Last 30 days** | A rolling window ending now | +| **Custom range** | Any range up to 92 days | + + + Daily bars are drawn in your browser's timezone, while a billing period begins at a fixed UTC instant. The first and last bar of a period can therefore be partial, and two admins in different timezones will see the same total split across slightly different days. + + +### Tabs + +| Tab | Answers | +|-----|---------| +| **Overview** | How much have we used, against what limit, and what kind of work was it | +| **Members** | Which people are driving usage | +| **Workspaces** | Which workspaces are driving usage — select one to drill in | +| **Models** | Which models we are paying for | + +Selecting a workspace opens its detail view, which splits that workspace's usage into **Sources** (what kind of work) and **Workflows** (the individual workflow runs). **Open logs** jumps to [audit logs](/platform/enterprise/audit-logs) filtered to that workspace. + +A workspace's detail view with a Sources section listing Sim Chat and Workflow, and a Workflows section ranking individual workflows by credits + +The two sections answer different questions, and the difference is the point: **Sources** adds up to the workspace's total, while **Workflows** covers only the workflow-run part of it. In the example above, Sources totals 4,435 credits but the workflows list only accounts for the 161 credits under Workflow — the other 4,274 came from Chat, which no workflow produced. + +--- + +## What each source means + +A **source** is the part of the platform that consumed the credits. Every charge belongs to exactly one source, so the Sources breakdown always adds up to your total. + +| Source | Platform features that bill to it | +|--------|-----------------------------------| +| **Workflow** | Every workflow run — model calls made on Sim's hosted keys, hosted-key tool calls (web search, scraping, and similar), and the per-run base charge | +| **Sim Chat** | The Chat panel, agent calls made through the API, and the email Inbox | +| **Agent block** | The Agent block running inside a workflow | +| **Knowledge Base** | Embedding documents on upload or connector sync, and semantic search queries | +| **Wand** | Inline AI generation in editors | +| **Enrichment** | Table column enrichment | +| **Voice input** | Voice sessions | +| **Voice output** | Spoken responses in deployed chats | +| **Sim Chat (MCP)** | Retired. Appears only for historical periods | + +### Nuances worth knowing + +These follow from how charges are recorded, and they explain most questions about why a number looks the way it does. + +**Workflow covers the whole run, not just the base charge.** A workflow row includes three things: the models it called on Sim's hosted keys, any hosted-key tool calls, and the per-run base charge. Model usage typically dominates, but the per-run charge is not negligible — across a high-volume organization it commonly accounts for around a fifth of workflow spend, because it applies to every run whether or not the workflow calls a model. + +**Agent block is billed separately from the workflow that contains it.** An Agent block runs inside a workflow but is metered on its own, so its credits appear under **Agent block** rather than **Workflow**. Nothing is counted twice — but it also means Workflow alone understates what your workflows cost. Read the two lines together. + +**Sim Chat covers three surfaces.** The Chat panel, headless agent calls through the API, and the Inbox all record as **Sim Chat** and cannot be separated from one another. If you need per-surface attribution today, separate them by workspace instead. + +**Only workflow runs carry workflow attribution.** Chat, Agent block, Wand, Knowledge Base, and voice usage are not produced by a workflow and have no workflow attached. That is the gap between a workspace's **Sources** and **Workflows** sections shown above. + +**Some usage has no workspace.** Agent calls made through the API may not name a workspace, particularly from headless or self-hosted callers. That usage is grouped under **No workspace** on the Workspaces tab. It is still counted in your total. + +**Models excludes non-model charges.** The Models tab covers model usage only, so it will read lower than your period total — the difference is per-run base charges, hosted-key tool charges, and fixed charges such as voice sessions. + + + Each tab is a different slice of the same charges, not an additional set of them. Members, Workspaces, and Sources each add up to your period total on their own; adding two tabs together double-counts. + + +--- + +## Bring your own keys (BYOK) + +When a workspace or organization supplies its own provider key, Sim does not charge for that model usage. Those calls are still recorded, measured in **tokens** rather than credits — the credit cost is zero by definition — and they are not included in the credit totals anywhere in the panel. + + + This usage is not broken out in the panel today. Of what is recorded, only model usage in workflow runs is covered: tool calls and Chat usage made on your own keys are not recorded at all. + + +--- + +## Exporting + +**Export** downloads the events behind the current period and filters as a CSV with columns `Date, Source, Description, Workflow, Credits`. Credits are exported as plain numbers so the column can be summed, and carry decimals — an individual event often costs a fraction of a credit. + +**All events** opens the full ledger — every credit-consuming event, newest first, with its own filters and export. + +Very large exports are capped. When that happens the download still succeeds and Sim tells you it was truncated; narrow the date range to capture everything. + +--- + + + +--- + +## Self-hosted setup + +Self-hosted deployments use environment variables instead of the billing/plan check. + +### Environment variables + +```bash +USAGE_MONITORING_ENABLED=true +NEXT_PUBLIC_USAGE_MONITORING_ENABLED=true +``` + +Once enabled, usage tracking is viewable in **Settings → Organization → Usage tracking** for organization owners and admins. diff --git a/apps/docs/content/docs/en/platform/enterprise/whitelabeling.mdx b/apps/docs/content/docs/en/platform/enterprise/whitelabeling.mdx index fdaceff7bed..4f32a6a3c8a 100644 --- a/apps/docs/content/docs/en/platform/enterprise/whitelabeling.mdx +++ b/apps/docs/content/docs/en/platform/enterprise/whitelabeling.mdx @@ -16,7 +16,7 @@ Whitelabeling lets you replace Sim's default branding — logo, colors, and supp Go to **Settings → Enterprise → Whitelabeling** in your workspace. -Whitelabeling settings showing brand identity fields (Logo, Wordmark, Brand name), color pickers for primary and accent colors, and link fields for support email and documentation URL +Whitelabeling settings showing brand identity fields (Logo, Wordmark, Brand name), color pickers for primary and accent colors, and link fields for support email and documentation URL ### 2. Configure brand identity diff --git a/apps/docs/content/docs/en/tables/index.mdx b/apps/docs/content/docs/en/tables/index.mdx index be5f75947b5..e44ca483464 100644 --- a/apps/docs/content/docs/en/tables/index.mdx +++ b/apps/docs/content/docs/en/tables/index.mdx @@ -10,7 +10,7 @@ import { FAQ } from '@/components/ui/faq' A **table** is a grid of typed columns in your workspace, like a spreadsheet with a schema. Use one to hold reference data, collect what your workflows produce, or store the structured records your agents read and write.
- A table of typed columns: name, title, company, role + The current table editor showing typed position, driver, nationality, team, wins, and points columns
## Column types diff --git a/apps/docs/content/docs/en/workflows/blocks/credential.mdx b/apps/docs/content/docs/en/workflows/blocks/credential.mdx index 0970d40c3a9..ae28df764a2 100644 --- a/apps/docs/content/docs/en/workflows/blocks/credential.mdx +++ b/apps/docs/content/docs/en/workflows/blocks/credential.mdx @@ -98,7 +98,7 @@ Credential (List, Provider: Gmail) → ForEach Loop → Gmail (Send) using diff --git a/apps/docs/content/docs/en/workflows/blocks/guardrails.mdx b/apps/docs/content/docs/en/workflows/blocks/guardrails.mdx index ac614ae28bb..fedf8555758 100644 --- a/apps/docs/content/docs/en/workflows/blocks/guardrails.mdx +++ b/apps/docs/content/docs/en/workflows/blocks/guardrails.mdx @@ -55,7 +55,7 @@ Detects personally identifiable information with [Microsoft Presidio](https://mi src="/static/blocks/guardrails-2.png" alt="PII Detection Configuration" width={700} - height={450} + height={335} className="my-6" /> diff --git a/apps/docs/content/docs/en/workflows/blocks/human-in-the-loop.mdx b/apps/docs/content/docs/en/workflows/blocks/human-in-the-loop.mdx index 1a9d61fba34..6205e67e0ac 100644 --- a/apps/docs/content/docs/en/workflows/blocks/human-in-the-loop.mdx +++ b/apps/docs/content/docs/en/workflows/blocks/human-in-the-loop.mdx @@ -25,7 +25,7 @@ The **Human in the Loop block** pauses a run and waits for a person before it co src="/static/blocks/hitl-2.png" alt="Human in the Loop Approval Portal" width={700} - height={500} + height={467} className="my-6" /> diff --git a/apps/docs/content/docs/en/workflows/blocks/loop.mdx b/apps/docs/content/docs/en/workflows/blocks/loop.mdx index 685f40d4410..841df1ca885 100644 --- a/apps/docs/content/docs/en/workflows/blocks/loop.mdx +++ b/apps/docs/content/docs/en/workflows/blocks/loop.mdx @@ -18,28 +18,28 @@ The **Loop block** is a container that runs the blocks inside it repeatedly — A **For** loop runs a fixed number of times. Set the iteration count.
- For loop with iterations + For loop with iterations
A **ForEach** loop runs once per item in an array or object, exposing the current item as ``.
- ForEach loop with a collection + ForEach loop with a collection
A **While** loop runs as long as a condition is true. The condition is checked **before** each iteration, so the body may run zero times.
- While loop with a condition + While loop with a condition
A **Do-While** loop runs the body once, then repeats while a condition is true. The condition is checked **after** each iteration, so the body always runs at least once.
- Do-While loop with a condition + Do-While loop with a condition
diff --git a/apps/docs/content/docs/en/workflows/blocks/parallel.mdx b/apps/docs/content/docs/en/workflows/blocks/parallel.mdx index c15368b66f6..4cb5405fb67 100644 --- a/apps/docs/content/docs/en/workflows/blocks/parallel.mdx +++ b/apps/docs/content/docs/en/workflows/blocks/parallel.mdx @@ -18,14 +18,14 @@ The **Parallel block** is a container that runs the block inside it concurrently Runs a fixed number of identical instances at once. If the count is larger than the batch size, Sim runs serial batches and preserves the original result order.
- Count-based parallel execution + Count-based parallel execution
Distributes a collection across instances — each one processes a single item as ``. Large collections run in serial batches, preserving each item's index.
- Collection-based parallel execution + Collection-based parallel execution
diff --git a/apps/docs/content/docs/en/workflows/blocks/workflow.mdx b/apps/docs/content/docs/en/workflows/blocks/workflow.mdx index a6f1a02d9f3..5af492b86c4 100644 --- a/apps/docs/content/docs/en/workflows/blocks/workflow.mdx +++ b/apps/docs/content/docs/en/workflows/blocks/workflow.mdx @@ -23,7 +23,7 @@ Drop a Workflow block when you want to call a child workflow as part of a larger src='/static/blocks/workflow-2.png' alt='Workflow block with input mapping example' width={700} - height={400} + height={287} className='rounded-xl border border-border shadow-sm' /> diff --git a/apps/docs/content/docs/en/workflows/deployment/api.mdx b/apps/docs/content/docs/en/workflows/deployment/api.mdx index ab6cfe44a36..3e3c669372f 100644 --- a/apps/docs/content/docs/en/workflows/deployment/api.mdx +++ b/apps/docs/content/docs/en/workflows/deployment/api.mdx @@ -13,7 +13,7 @@ Deploy your workflow as a REST API endpoint that any application can call direct Open your workflow and click **Deploy**. The **General** tab opens first and shows you the current deployment state: -General tab of the Workflow Deployment modal showing a live workflow preview, a Versions table with v2 (live) and v1, and Undeploy / Update buttons +General tab of the Workflow Deployment modal showing a live workflow preview, a Versions table with v2 (live) and v1, and Undeploy / Update buttons The **General** tab contains: @@ -37,7 +37,7 @@ POST https://sim.ai/api/v2/workflows/{workflow-id}/execute When you modify the workflow canvas after deploying, an **Update deployment** badge appears at the bottom of the screen as a reminder that your live version is out of date: -Canvas toolbar showing the Update and Run buttons with an Update deployment tooltip +Canvas toolbar showing the Update and Run buttons with an Update deployment tooltip You can click the **Update** button directly from the canvas toolbar — you don't need to open the Deploy modal every time. @@ -45,7 +45,7 @@ You can click the **Update** button directly from the canvas toolbar — you don Every time you deploy or update, a new version is recorded in the Versions table. You can manage past versions using the context menu (⋮) next to any row: -Versions table showing v2 (live) and v1 with a context menu open offering Rename, Add description, Promote to live, and Load deployment options +Versions table showing v2 (live) and v1 with a context menu open offering Rename, Add description, Promote to live, and Load deployment options | Action | Description | |--------|-------------| @@ -82,7 +82,7 @@ Rollback re-activates an existing deployment version — the same operation as * Switch to the **API** tab in the Deploy modal to see ready-to-use code for all three execution modes: -API tab showing cURL, Python, JavaScript, and TypeScript language options, with Run workflow, Run workflow (stream response), and Run workflow (async) code sections +API tab showing cURL, Python, JavaScript, and TypeScript language options, with Run workflow, Run workflow (stream response), and Run workflow (async) code sections The language selector at the top lets you switch between **cURL**, **Python**, **JavaScript**, and **TypeScript**. Each mode — synchronous, streaming, and async — has its own code block that you can copy directly. The code is pre-filled with your workflow ID and a masked version of your API key. @@ -106,7 +106,7 @@ curl -X POST https://sim.ai/api/v2/workflows/{workflow-id}/execute \ Click **Edit API Info** to add a description and change the access mode: -Edit API Info modal with a Description textarea and an Access section toggling between API Key and Public modes +Edit API Info modal with a Description textarea and an Access section toggling between API Key and Public modes | Access Mode | Description | |-------------|-------------| @@ -170,7 +170,7 @@ Stream the response token-by-token as it is generated. Add `"stream": true` to y Use the **Select outputs** dropdown in the API tab to choose which fields to stream: -Select outputs dropdown open showing Agent 1 block with selectable output fields: content, model, tokens, toolCalls, providerTiming, cost +Select outputs dropdown open showing Agent 1 block with selectable output fields: content, model, tokens, toolCalls, providerTiming, cost The dropdown groups available outputs by block. The most common choice is `content` from an Agent block, which streams the generated text. You can select fields from multiple blocks simultaneously. diff --git a/apps/docs/content/docs/en/workflows/deployment/chat.mdx b/apps/docs/content/docs/en/workflows/deployment/chat.mdx index 030f1ef62b9..7f092b624d4 100644 --- a/apps/docs/content/docs/en/workflows/deployment/chat.mdx +++ b/apps/docs/content/docs/en/workflows/deployment/chat.mdx @@ -9,7 +9,7 @@ import { FAQ } from '@/components/ui/faq' Deploy your workflow as a conversational chat interface that users can interact with via a shareable link or embedded widget. Chat supports multi-turn conversations, file uploads, and voice input. -A deployed chat interface showing a conversation with Friendly Assistant +A deployed chat interface showing a conversation with Friendly Assistant Every chat message triggers a fresh workflow execution, with the full conversation history passed in as context. Responses stream back to the user in real time. @@ -21,7 +21,7 @@ Chat executions run against your workflow's active deployment snapshot. Publish Open your workflow, click **Deploy**, and select the **Chat** tab. You'll see the chat configuration panel: -Chat deployment configuration panel showing URL, Title, Output, Access control, and Welcome message fields +Chat deployment configuration panel showing URL, Title, Output, Access control, and Welcome message fields Configure the following fields, then click **Launch Chat**: @@ -37,13 +37,13 @@ Configure the following fields, then click **Launch Chat**: ### Output Selection -Output dropdown showing Agent 1 block with selectable fields: content, model, tokens, toolCalls, providerTiming, cost +Output dropdown showing Agent 1 block with selectable fields: content, model, tokens, toolCalls, providerTiming, cost The output dropdown groups available fields by block. For an Agent block, you can choose from `content`, `model`, `tokens`, `toolCalls`, `providerTiming`, and `cost`. In most cases, selecting `content` from the final Agent block is all you need — it streams the agent's text response directly to the user. ## Access Control -Access control section with Email tab selected, showing an Allowed emails field with @sim.ai domain added +Access control section with Email tab selected, showing an Allowed emails field with @sim.ai domain added | Mode | Description | |------|-------------| diff --git a/apps/docs/content/docs/en/workflows/deployment/mcp.mdx b/apps/docs/content/docs/en/workflows/deployment/mcp.mdx index 630dcbb6f51..17edf1302d7 100644 --- a/apps/docs/content/docs/en/workflows/deployment/mcp.mdx +++ b/apps/docs/content/docs/en/workflows/deployment/mcp.mdx @@ -25,7 +25,7 @@ MCP servers group your workflow tools together. Create and manage them in worksp src="/static/blocks/mcp-servers-settings.png" alt="MCP Servers settings page" width={700} - height={450} + height={303} className="my-6" /> @@ -40,7 +40,7 @@ MCP servers group your workflow tools together. Create and manage them in worksp src="/static/blocks/mcp-server-add-modal.png" alt="Add New MCP Server modal" width={550} - height={380} + height={371} className="my-6" /> @@ -53,7 +53,7 @@ MCP servers group your workflow tools together. Create and manage them in worksp src="/static/blocks/mcp-server-details.png" alt="MCP Server details view" width={700} - height={450} + height={491} className="my-6" /> @@ -77,7 +77,7 @@ Once your workflow is deployed, you can expose it as an MCP tool: src="/static/blocks/mcp-deploy-modal.png" alt="Workflow Deployment MCP tab" width={380} - height={470} + height={487} className="my-6" /> @@ -112,15 +112,15 @@ Sim generates a ready-to-paste configuration for every supported client. To get 1. Navigate to **Settings → MCP Servers** 2. Click **Details** on your server -3. Under **MCP Client**, select your client — **Cursor**, **Claude Code**, **Claude Desktop**, **VS Code**, or **Sim** -4. Copy the configuration, replacing `$SIM_API_KEY` with your Sim API key +3. Under **MCP Client**, select your client — **Cursor**, **Codex**, **Claude Code**, **Claude Desktop**, **VS Code**, or **Sim** +4. Copy the configuration and follow the authentication note below it
MCP client configuration panel
@@ -142,6 +142,18 @@ Cursor supports direct URL configuration. Add to your Cursor MCP settings (`.cur Cursor also provides a one-click install button in the server detail view. +### Codex + +Add this to your Codex configuration (`~/.codex/config.toml`): + +```toml +[mcp_servers."my-sim-workflows"] +url = "YOUR_SERVER_URL" +env_http_headers = { "X-API-Key" = "SIM_API_KEY" } +``` + +Set the `SIM_API_KEY` environment variable before starting Codex. The ChatGPT desktop app, Codex CLI, and Codex IDE extension share this configuration. + ### Claude Code Run this command in your terminal: @@ -185,7 +197,7 @@ For public servers, omit the `X-API-Key` header and `--header` arguments. Public -`$SIM_API_KEY` is a placeholder. For Claude Desktop and VS Code configs, replace it with your actual API key since these clients don't expand environment variables in JSON config files. Claude Code and Cursor handle variable expansion natively. +`$SIM_API_KEY` is a placeholder. For Claude Desktop and VS Code configs, replace it with your actual API key since these clients don't expand environment variables in JSON config files. Codex reads `SIM_API_KEY` from the environment, while Claude Code and Cursor handle variable expansion natively. ## Server Management @@ -226,5 +238,3 @@ Workflows execute using the same deployment version as API calls, ensuring consi { question: "What naming conventions should I follow for tool names?", answer: "Use lowercase letters, numbers, and underscores only. The name should be descriptive and follow MCP naming conventions, such as search_documents or send_email. This helps AI assistants understand and correctly invoke your tools." }, { question: "How are workflow inputs mapped to MCP tool parameters?", answer: "Your workflow's input format fields automatically become MCP tool parameters. Each parameter's description defaults to the description set on that input in the Start block, and you can override it per tool in the MCP configuration to help AI assistants understand what values to provide." }, ]} /> - - diff --git a/apps/docs/content/docs/en/workflows/triggers/schedule.mdx b/apps/docs/content/docs/en/workflows/triggers/schedule.mdx index 5c32cc1db41..911e79ede01 100644 --- a/apps/docs/content/docs/en/workflows/triggers/schedule.mdx +++ b/apps/docs/content/docs/en/workflows/triggers/schedule.mdx @@ -53,7 +53,7 @@ A schedule only runs once the workflow is deployed. Configure the trigger, then A schedule disables itself after **100 consecutive failures**, to stop runaway errors. A warning badge appears on the block and it stops running; click the badge to reactivate it. The counter resets to zero on any successful run.
- A disabled schedule + A disabled schedule
diff --git a/apps/docs/content/docs/en/workflows/triggers/start.mdx b/apps/docs/content/docs/en/workflows/triggers/start.mdx index fb1a65a714a..0520867ab1c 100644 --- a/apps/docs/content/docs/en/workflows/triggers/start.mdx +++ b/apps/docs/content/docs/en/workflows/triggers/start.mdx @@ -10,7 +10,7 @@ import { Image } from '@/components/ui/image' The **Start block** is the default trigger for a workflow. It defines the inputs the workflow takes and runs it from three surfaces with the same shape: a manual run in the editor, an [API deployment](/workflows/deployment/api), and a [chat deployment](/workflows/deployment/chat).
- Start block with Input Format fields + Start block with Input Format fields
diff --git a/apps/docs/content/docs/es/blocks/agent.mdx b/apps/docs/content/docs/es/blocks/agent.mdx index f8da7a466d6..c1af0ffcd16 100644 --- a/apps/docs/content/docs/es/blocks/agent.mdx +++ b/apps/docs/content/docs/es/blocks/agent.mdx @@ -13,7 +13,7 @@ El bloque Agente conecta tu flujo de trabajo con Modelos de Lenguaje Grandes (LL src="/static/blocks/agent.png" alt="Configuración del bloque Agente" width={500} - height={400} + height={450} className="my-6" /> diff --git a/apps/docs/content/docs/es/blocks/api.mdx b/apps/docs/content/docs/es/blocks/api.mdx index 1536aca695f..d30cc979c80 100644 --- a/apps/docs/content/docs/es/blocks/api.mdx +++ b/apps/docs/content/docs/es/blocks/api.mdx @@ -13,7 +13,7 @@ El bloque API conecta tu flujo de trabajo con servicios externos a través de pe src="/static/blocks/api.png" alt="Bloque API" width={500} - height={400} + height={422} className="my-6" /> diff --git a/apps/docs/content/docs/es/blocks/condition.mdx b/apps/docs/content/docs/es/blocks/condition.mdx index 48e36b4cd2e..165ed366d17 100644 --- a/apps/docs/content/docs/es/blocks/condition.mdx +++ b/apps/docs/content/docs/es/blocks/condition.mdx @@ -13,7 +13,7 @@ El bloque Condición ramifica la ejecución del flujo de trabajo basándose en e src="/static/blocks/condition.png" alt="Bloque de condición" width={500} - height={400} + height={317} className="my-6" /> diff --git a/apps/docs/content/docs/es/blocks/evaluator.mdx b/apps/docs/content/docs/es/blocks/evaluator.mdx index 9068b7e1a8c..5589be71963 100644 --- a/apps/docs/content/docs/es/blocks/evaluator.mdx +++ b/apps/docs/content/docs/es/blocks/evaluator.mdx @@ -13,7 +13,7 @@ El bloque Evaluador utiliza IA para puntuar y evaluar la calidad del contenido s src="/static/blocks/evaluator.png" alt="Configuración del bloque Evaluador" width={500} - height={400} + height={337} className="my-6" /> diff --git a/apps/docs/content/docs/es/blocks/function.mdx b/apps/docs/content/docs/es/blocks/function.mdx index 85bb63c5645..9b1e41fefa3 100644 --- a/apps/docs/content/docs/es/blocks/function.mdx +++ b/apps/docs/content/docs/es/blocks/function.mdx @@ -11,7 +11,7 @@ El bloque de Función ejecuta código JavaScript o TypeScript personalizado en t src="/static/blocks/function.png" alt="Bloque de función con editor de código" width={500} - height={400} + height={287} className="my-6" /> diff --git a/apps/docs/content/docs/es/blocks/guardrails.mdx b/apps/docs/content/docs/es/blocks/guardrails.mdx index 41b804ee89e..75182464c5f 100644 --- a/apps/docs/content/docs/es/blocks/guardrails.mdx +++ b/apps/docs/content/docs/es/blocks/guardrails.mdx @@ -13,7 +13,7 @@ El bloque Guardrails valida y protege tus flujos de trabajo de IA comprobando el src="/static/blocks/guardrails.png" alt="Bloque de barandillas de protección" width={500} - height={400} + height={398} className="my-6" /> @@ -90,7 +90,7 @@ Detecta información de identificación personal utilizando Microsoft Presidio. src="/static/blocks/guardrails-2.png" alt="Configuración de detección de PII" width={700} - height={450} + height={335} className="my-6" /> diff --git a/apps/docs/content/docs/es/blocks/human-in-the-loop.mdx b/apps/docs/content/docs/es/blocks/human-in-the-loop.mdx index 3f0ca976125..f06a63e43c0 100644 --- a/apps/docs/content/docs/es/blocks/human-in-the-loop.mdx +++ b/apps/docs/content/docs/es/blocks/human-in-the-loop.mdx @@ -14,7 +14,7 @@ El bloque Human in the Loop pausa la ejecución del flujo de trabajo y espera la src="/static/blocks/hitl-1.png" alt="Configuración del bloque Human in the Loop" width={500} - height={400} + height={355} className="my-6" /> @@ -26,7 +26,7 @@ Cuando la ejecución llega a este bloque, el flujo de trabajo se pausa indefinid src="/static/blocks/hitl-2.png" alt="Portal de aprobación Human in the Loop" width={700} - height={500} + height={467} className="my-6" /> diff --git a/apps/docs/content/docs/es/blocks/loop.mdx b/apps/docs/content/docs/es/blocks/loop.mdx index 8503eda6c58..ec82cc2f37d 100644 --- a/apps/docs/content/docs/es/blocks/loop.mdx +++ b/apps/docs/content/docs/es/blocks/loop.mdx @@ -27,7 +27,7 @@ Elige entre cuatro tipos de bucles: src="/static/blocks/loop-1.png" alt="Bucle For con iteraciones" width={500} - height={400} + height={267} className="my-6" /> @@ -53,7 +53,7 @@ Elige entre cuatro tipos de bucles: src="/static/blocks/loop-2.png" alt="Bucle ForEach con colección" width={500} - height={400} + height={273} className="my-6" /> @@ -77,7 +77,7 @@ Elige entre cuatro tipos de bucles: src="/static/blocks/loop-3.png" alt="Bucle While con condición" width={500} - height={400} + height={213} className="my-6" /> @@ -102,7 +102,7 @@ Elige entre cuatro tipos de bucles: src="/static/blocks/loop-4.png" alt="Bucle Do-While con condición" width={500} - height={400} + height={210} className="my-6" /> diff --git a/apps/docs/content/docs/es/blocks/parallel.mdx b/apps/docs/content/docs/es/blocks/parallel.mdx index fae05aaf6a9..31a85d28049 100644 --- a/apps/docs/content/docs/es/blocks/parallel.mdx +++ b/apps/docs/content/docs/es/blocks/parallel.mdx @@ -27,7 +27,7 @@ Elige entre dos tipos de ejecución paralela: src="/static/blocks/parallel-1.png" alt="Ejecución paralela basada en conteo" width={500} - height={400} + height={231} className="my-6" /> @@ -53,7 +53,7 @@ Elige entre dos tipos de ejecución paralela: src="/static/blocks/parallel-2.png" alt="Ejecución paralela basada en colección" width={500} - height={400} + height={220} className="my-6" /> diff --git a/apps/docs/content/docs/es/blocks/response.mdx b/apps/docs/content/docs/es/blocks/response.mdx index 698c3f51d30..1d7a407779d 100644 --- a/apps/docs/content/docs/es/blocks/response.mdx +++ b/apps/docs/content/docs/es/blocks/response.mdx @@ -13,7 +13,7 @@ El bloque de Respuesta formatea y envía respuestas HTTP estructuradas de vuelta src="/static/blocks/response.png" alt="Configuración del bloque de respuesta" width={500} - height={400} + height={382} className="my-6" /> diff --git a/apps/docs/content/docs/es/blocks/router.mdx b/apps/docs/content/docs/es/blocks/router.mdx index 97b984bb0af..d7fe8217b0b 100644 --- a/apps/docs/content/docs/es/blocks/router.mdx +++ b/apps/docs/content/docs/es/blocks/router.mdx @@ -13,7 +13,7 @@ El bloque Router utiliza IA para dirigir flujos de trabajo de manera inteligente src="/static/blocks/router.png" alt="Bloque Router con múltiples caminos" width={500} - height={400} + height={342} className="my-6" /> diff --git a/apps/docs/content/docs/es/blocks/variables.mdx b/apps/docs/content/docs/es/blocks/variables.mdx index cb9b18eae88..1d989bf7dc1 100644 --- a/apps/docs/content/docs/es/blocks/variables.mdx +++ b/apps/docs/content/docs/es/blocks/variables.mdx @@ -13,7 +13,7 @@ El bloque Variables actualiza las variables del flujo de trabajo durante la ejec src="/static/blocks/variables.png" alt="Bloque de variables" width={500} - height={400} + height={246} className="my-6" /> diff --git a/apps/docs/content/docs/es/blocks/wait.mdx b/apps/docs/content/docs/es/blocks/wait.mdx index d352994e7b3..b8ee0682762 100644 --- a/apps/docs/content/docs/es/blocks/wait.mdx +++ b/apps/docs/content/docs/es/blocks/wait.mdx @@ -13,7 +13,7 @@ El bloque Espera pausa tu flujo de trabajo durante un tiempo específico antes d src="/static/blocks/wait.png" alt="Bloque de espera" width={500} - height={400} + height={295} className="my-6" /> diff --git a/apps/docs/content/docs/es/blocks/webhook.mdx b/apps/docs/content/docs/es/blocks/webhook.mdx index f902df5079c..8bb57027988 100644 --- a/apps/docs/content/docs/es/blocks/webhook.mdx +++ b/apps/docs/content/docs/es/blocks/webhook.mdx @@ -12,7 +12,7 @@ El bloque Webhook envía solicitudes HTTP POST a endpoints de webhook externos c src="/static/blocks/webhook.png" alt="Bloque Webhook" width={500} - height={400} + height={403} className="my-6" /> diff --git a/apps/docs/content/docs/es/blocks/workflow.mdx b/apps/docs/content/docs/es/blocks/workflow.mdx index 40475884244..75a87a043e0 100644 --- a/apps/docs/content/docs/es/blocks/workflow.mdx +++ b/apps/docs/content/docs/es/blocks/workflow.mdx @@ -13,7 +13,7 @@ import { Image } from '@/components/ui/image' src='/static/blocks/workflow.png' alt='Configuración del bloque de flujo de trabajo' width={500} - height={400} + height={310} className='rounded-xl border border-border shadow-sm' /> @@ -30,7 +30,7 @@ Coloca un bloque de Flujo de trabajo cuando quieras llamar a un flujo de trabajo src='/static/blocks/workflow-2.png' alt='Bloque de flujo de trabajo con ejemplo de mapeo de entrada' width={700} - height={400} + height={287} className='rounded-xl border border-border shadow-sm' /> diff --git a/apps/docs/content/docs/es/copilot/index.mdx b/apps/docs/content/docs/es/copilot/index.mdx index 49d5cc2b717..a6aa0f31a9a 100644 --- a/apps/docs/content/docs/es/copilot/index.mdx +++ b/apps/docs/content/docs/es/copilot/index.mdx @@ -27,7 +27,7 @@ Usa el símbolo `@` para hacer referencia a varios recursos y proporcionar a Cop src="/static/copilot/copilot-menu.png" alt="Menú contextual de Copilot mostrando opciones de referencia disponibles" width={600} - height={400} + height={521} /> El menú `@` proporciona acceso a: @@ -76,7 +76,7 @@ Esta información contextual ayuda a Copilot a proporcionar asistencia más prec src="/static/copilot/copilot-mode.png" alt="Interfaz de selección de modo de Copilot" width={600} - height={400} + height={720} className="my-6" /> @@ -134,7 +134,7 @@ Puedes cambiar fácilmente entre diferentes modos de razonamiento utilizando el src="/static/copilot/copilot-models.png" alt="Selección de modo de Copilot mostrando el modo Avanzado con el interruptor MAX" width={600} - height={300} + height={457} /> La interfaz te permite: diff --git a/apps/docs/content/docs/es/execution/basics.mdx b/apps/docs/content/docs/es/execution/basics.mdx index 9ab9101c423..80dcdcddb3f 100644 --- a/apps/docs/content/docs/es/execution/basics.mdx +++ b/apps/docs/content/docs/es/execution/basics.mdx @@ -20,7 +20,7 @@ Múltiples bloques se ejecutan simultáneamente cuando no dependen entre sí. Es src="/static/execution/concurrency.png" alt="Múltiples bloques ejecutándose concurrentemente después del bloque de Inicio" width={800} - height={500} + height={685} /> En este ejemplo, tanto el bloque de agente de Atención al Cliente como el de Investigador Profundo se ejecutan simultáneamente después del bloque de Inicio, maximizando la eficiencia. @@ -33,7 +33,7 @@ Cuando los bloques tienen múltiples dependencias, el motor de ejecución espera src="/static/execution/combination.png" alt="Bloque de función recibiendo automáticamente salidas de múltiples bloques anteriores" width={800} - height={500} + height={521} /> El bloque de Función recibe las salidas de ambos bloques de agente tan pronto como se completan, permitiéndote procesar los resultados combinados. @@ -46,7 +46,7 @@ Los flujos de trabajo pueden ramificarse en múltiples direcciones utilizando bl src="/static/execution/routing.png" alt="Flujo de trabajo mostrando ramificación tanto condicional como basada en router" width={800} - height={500} + height={391} /> Este flujo de trabajo demuestra cómo la ejecución puede seguir diferentes caminos basados en condiciones o decisiones de IA, con cada camino ejecutándose independientemente. diff --git a/apps/docs/content/docs/es/execution/costs.mdx b/apps/docs/content/docs/es/execution/costs.mdx index efdc188fd93..d1005ac368a 100644 --- a/apps/docs/content/docs/es/execution/costs.mdx +++ b/apps/docs/content/docs/es/execution/costs.mdx @@ -34,7 +34,7 @@ Para flujos de trabajo que utilizan bloques de IA, puedes ver información detal src="/static/logs/logs-cost.png" alt="Desglose de modelos" width={600} - height={400} + height={153} className="my-6" /> diff --git a/apps/docs/content/docs/es/execution/index.mdx b/apps/docs/content/docs/es/execution/index.mdx index 9cff11ea754..8a803783c2e 100644 --- a/apps/docs/content/docs/es/execution/index.mdx +++ b/apps/docs/content/docs/es/execution/index.mdx @@ -61,7 +61,7 @@ Todos los puntos de entrada públicos—API, Chat, Programación, Webhook y ejec src='/static/execution/deployment-versions.png' alt='Tabla de versiones de despliegue' width={500} - height={280} + height={259} className='rounded-xl border border-border shadow-sm' /> diff --git a/apps/docs/content/docs/es/execution/logging.mdx b/apps/docs/content/docs/es/execution/logging.mdx index 98e4ff34cd0..3fa8e5870c1 100644 --- a/apps/docs/content/docs/es/execution/logging.mdx +++ b/apps/docs/content/docs/es/execution/logging.mdx @@ -21,7 +21,7 @@ Durante la ejecución manual o por chat del flujo de trabajo, los registros apar src="/static/logs/console.png" alt="Panel de consola en tiempo real" width={400} - height={300} + height={151} className="my-6" /> @@ -41,7 +41,7 @@ Todas las ejecuciones de flujos de trabajo, ya sean activadas manualmente, a tra src="/static/logs/logs.png" alt="Página de registros" width={600} - height={400} + height={451} className="my-6" /> @@ -61,7 +61,7 @@ Al hacer clic en cualquier entrada de registro se abre una vista detallada en la src="/static/logs/logs-sidebar.png" alt="Detalles de la barra lateral de registros" width={600} - height={400} + height={880} className="my-6" /> @@ -104,7 +104,7 @@ Para cualquier ejecución registrada, haz clic en "Ver instantánea" para ver el src="/static/logs/logs-frozen-canvas.png" alt="Instantánea del flujo de trabajo" width={600} - height={400} + height={628} className="my-6" /> diff --git a/apps/docs/content/docs/es/getting-started/index.mdx b/apps/docs/content/docs/es/getting-started/index.mdx index e0e1bb88309..7ebf8da2ae1 100644 --- a/apps/docs/content/docs/es/getting-started/index.mdx +++ b/apps/docs/content/docs/es/getting-started/index.mdx @@ -42,7 +42,7 @@ Un agente de investigación de personas que: src="/static/getting-started/started-1.png" alt="Ejemplo de primeros pasos" width={800} - height={500} + height={340} /> ## Tutorial paso a paso diff --git a/apps/docs/content/docs/es/introduction/index.mdx b/apps/docs/content/docs/es/introduction/index.mdx index e908e0b1a64..1f62e284dd8 100644 --- a/apps/docs/content/docs/es/introduction/index.mdx +++ b/apps/docs/content/docs/es/introduction/index.mdx @@ -14,7 +14,7 @@ Sim es un constructor de flujos de trabajo visuales de código abierto para crea src="/static/introduction.png" alt="Lienzo visual de flujos de trabajo de Sim" width={700} - height={450} + height={372} className="my-6" /> diff --git a/apps/docs/content/docs/es/knowledgebase/index.mdx b/apps/docs/content/docs/es/knowledgebase/index.mdx index 7d28228acc5..b367bbf0547 100644 --- a/apps/docs/content/docs/es/knowledgebase/index.mdx +++ b/apps/docs/content/docs/es/knowledgebase/index.mdx @@ -32,7 +32,7 @@ Sim admite archivos PDF, Word (DOC/DOCX), texto plano (TXT), Markdown (MD), HTML Una vez que tus documentos están procesados, puedes ver y editar los fragmentos individuales. Esto te da control total sobre cómo se organiza y busca tu contenido. -Vista de fragmentos de documentos mostrando contenido procesado +Vista de fragmentos de documentos mostrando contenido procesado ### Configuración de fragmentos @@ -66,7 +66,7 @@ Cuando se configura con Azure o [Mistral OCR](https://docs.mistral.ai/ocr/): Una vez que tus documentos estén procesados, puedes usarlos en tus flujos de trabajo de IA a través del bloque de conocimiento. Esto habilita la generación aumentada por recuperación (RAG), permitiendo que tus agentes de IA accedan y razonen sobre el contenido de tus documentos para proporcionar respuestas más precisas y contextuales. -Uso del bloque de conocimiento en flujos de trabajo +Uso del bloque de conocimiento en flujos de trabajo ### Características del bloque de conocimiento - **Búsqueda semántica**: encuentra contenido relevante usando consultas en lenguaje natural diff --git a/apps/docs/content/docs/es/mcp/index.mdx b/apps/docs/content/docs/es/mcp/index.mdx index 913a1746762..186ed3dd1dd 100644 --- a/apps/docs/content/docs/es/mcp/index.mdx +++ b/apps/docs/content/docs/es/mcp/index.mdx @@ -49,7 +49,7 @@ Una vez configurados los servidores MCP, sus herramientas están disponibles den src="/static/blocks/mcp-2.png" alt="Uso de herramienta MCP en bloque de agente" width={700} - height={450} + height={353} className="my-6" /> @@ -68,7 +68,7 @@ Para un control más granular, puedes usar el bloque de herramienta MCP dedicado src="/static/blocks/mcp-3.png" alt="Bloque de herramienta MCP independiente" width={700} - height={450} + height={545} className="my-6" /> diff --git a/apps/docs/content/docs/es/triggers/index.mdx b/apps/docs/content/docs/es/triggers/index.mdx index 730cdbcc803..b49cb2653fd 100644 --- a/apps/docs/content/docs/es/triggers/index.mdx +++ b/apps/docs/content/docs/es/triggers/index.mdx @@ -11,7 +11,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/triggers.png" alt="Resumen de disparadores" width={500} - height={350} + height={348} className="my-6" /> diff --git a/apps/docs/content/docs/es/triggers/rss.mdx b/apps/docs/content/docs/es/triggers/rss.mdx index 6978129d513..323911f0817 100644 --- a/apps/docs/content/docs/es/triggers/rss.mdx +++ b/apps/docs/content/docs/es/triggers/rss.mdx @@ -12,7 +12,7 @@ El bloque de Feed RSS monitorea feeds RSS y Atom – cuando se publican nuevos e src="/static/blocks/rss.png" alt="Bloque de Feed RSS" width={500} - height={400} + height={228} className="my-6" /> diff --git a/apps/docs/content/docs/es/triggers/schedule.mdx b/apps/docs/content/docs/es/triggers/schedule.mdx index 7ef0f750d51..626da9fd84e 100644 --- a/apps/docs/content/docs/es/triggers/schedule.mdx +++ b/apps/docs/content/docs/es/triggers/schedule.mdx @@ -13,7 +13,7 @@ El bloque de Programación activa automáticamente flujos de trabajo de forma re src="/static/blocks/schedule.png" alt="Bloque de programación" width={500} - height={400} + height={336} className="my-6" /> @@ -67,7 +67,7 @@ Las programaciones se desactivan automáticamente después de **100 fallos conse src="/static/blocks/schedule-3.png" alt="Programación desactivada" width={500} - height={400} + height={310} className="my-6" /> diff --git a/apps/docs/content/docs/es/triggers/start.mdx b/apps/docs/content/docs/es/triggers/start.mdx index 9f60a2b7380..1f516252051 100644 --- a/apps/docs/content/docs/es/triggers/start.mdx +++ b/apps/docs/content/docs/es/triggers/start.mdx @@ -13,7 +13,7 @@ El bloque Inicio es el disparador predeterminado para los flujos de trabajo crea src="/static/start.png" alt="Bloque de inicio con campos de formato de entrada" width={360} - height={380} + height={151} className="my-6" /> diff --git a/apps/docs/content/docs/es/triggers/webhook.mdx b/apps/docs/content/docs/es/triggers/webhook.mdx index 3eb70733181..08e2c5e56cf 100644 --- a/apps/docs/content/docs/es/triggers/webhook.mdx +++ b/apps/docs/content/docs/es/triggers/webhook.mdx @@ -18,7 +18,7 @@ El bloque de webhook genérico crea un punto de conexión flexible que puede rec src="/static/blocks/webhook-trigger.png" alt="Configuración de webhook genérico" width={500} - height={400} + height={405} className="my-6" /> diff --git a/apps/docs/content/docs/fr/blocks/agent.mdx b/apps/docs/content/docs/fr/blocks/agent.mdx index c2252b1de6d..381bfe18259 100644 --- a/apps/docs/content/docs/fr/blocks/agent.mdx +++ b/apps/docs/content/docs/fr/blocks/agent.mdx @@ -13,7 +13,7 @@ Le bloc Agent connecte votre flux de travail aux grands modèles de langage (LLM src="/static/blocks/agent.png" alt="Configuration du bloc Agent" width={500} - height={400} + height={450} className="my-6" /> diff --git a/apps/docs/content/docs/fr/blocks/api.mdx b/apps/docs/content/docs/fr/blocks/api.mdx index ca37ed96882..41fc2bf09ef 100644 --- a/apps/docs/content/docs/fr/blocks/api.mdx +++ b/apps/docs/content/docs/fr/blocks/api.mdx @@ -13,7 +13,7 @@ Le bloc API connecte votre flux de travail à des services externes via des requ src="/static/blocks/api.png" alt="Bloc API" width={500} - height={400} + height={422} className="my-6" /> diff --git a/apps/docs/content/docs/fr/blocks/condition.mdx b/apps/docs/content/docs/fr/blocks/condition.mdx index 14c00f53583..4a42e8a1b90 100644 --- a/apps/docs/content/docs/fr/blocks/condition.mdx +++ b/apps/docs/content/docs/fr/blocks/condition.mdx @@ -13,7 +13,7 @@ Le bloc Condition permet de ramifier l'exécution du flux de travail en fonction src="/static/blocks/condition.png" alt="Bloc de condition" width={500} - height={400} + height={317} className="my-6" /> diff --git a/apps/docs/content/docs/fr/blocks/evaluator.mdx b/apps/docs/content/docs/fr/blocks/evaluator.mdx index 419dc9010ec..245ab555d34 100644 --- a/apps/docs/content/docs/fr/blocks/evaluator.mdx +++ b/apps/docs/content/docs/fr/blocks/evaluator.mdx @@ -13,7 +13,7 @@ Le bloc Évaluateur utilise l'IA pour noter et évaluer la qualité du contenu s src="/static/blocks/evaluator.png" alt="Configuration du bloc Évaluateur" width={500} - height={400} + height={337} className="my-6" /> diff --git a/apps/docs/content/docs/fr/blocks/function.mdx b/apps/docs/content/docs/fr/blocks/function.mdx index 77d7c166e13..14cbb6a1a89 100644 --- a/apps/docs/content/docs/fr/blocks/function.mdx +++ b/apps/docs/content/docs/fr/blocks/function.mdx @@ -11,7 +11,7 @@ Le bloc Fonction exécute du code JavaScript ou TypeScript personnalisé dans vo src="/static/blocks/function.png" alt="Bloc de fonction avec éditeur de code" width={500} - height={400} + height={287} className="my-6" /> diff --git a/apps/docs/content/docs/fr/blocks/guardrails.mdx b/apps/docs/content/docs/fr/blocks/guardrails.mdx index 9b3e218d2bf..b1592599291 100644 --- a/apps/docs/content/docs/fr/blocks/guardrails.mdx +++ b/apps/docs/content/docs/fr/blocks/guardrails.mdx @@ -13,7 +13,7 @@ Le bloc Guardrails valide et protège vos flux de travail IA en vérifiant le co src="/static/blocks/guardrails.png" alt="Bloc de garde-fous" width={500} - height={400} + height={398} className="my-6" /> @@ -90,7 +90,7 @@ Détecte les informations personnelles identifiables à l'aide de Microsoft Pres src="/static/blocks/guardrails-2.png" alt="Configuration de détection PII" width={700} - height={450} + height={335} className="my-6" /> diff --git a/apps/docs/content/docs/fr/blocks/human-in-the-loop.mdx b/apps/docs/content/docs/fr/blocks/human-in-the-loop.mdx index 67f527d7ef9..44e2ac28c63 100644 --- a/apps/docs/content/docs/fr/blocks/human-in-the-loop.mdx +++ b/apps/docs/content/docs/fr/blocks/human-in-the-loop.mdx @@ -14,7 +14,7 @@ Le bloc Intervention humaine met en pause l'exécution du workflow et attend l'i src="/static/blocks/hitl-1.png" alt="Configuration du bloc Intervention humaine" width={500} - height={400} + height={355} className="my-6" /> @@ -26,7 +26,7 @@ Lorsque l'exécution atteint ce bloc, le workflow se met en pause indéfiniment src="/static/blocks/hitl-2.png" alt="Portail d'approbation de l'intervention humaine" width={700} - height={500} + height={467} className="my-6" /> diff --git a/apps/docs/content/docs/fr/blocks/loop.mdx b/apps/docs/content/docs/fr/blocks/loop.mdx index 116853d78bf..35e26f75d63 100644 --- a/apps/docs/content/docs/fr/blocks/loop.mdx +++ b/apps/docs/content/docs/fr/blocks/loop.mdx @@ -27,7 +27,7 @@ Choisissez entre quatre types de boucles : src="/static/blocks/loop-1.png" alt="Boucle For avec itérations" width={500} - height={400} + height={267} className="my-6" /> @@ -53,7 +53,7 @@ Choisissez entre quatre types de boucles : src="/static/blocks/loop-2.png" alt="Boucle ForEach avec collection" width={500} - height={400} + height={273} className="my-6" /> @@ -77,7 +77,7 @@ Choisissez entre quatre types de boucles : src="/static/blocks/loop-3.png" alt="Boucle While avec condition" width={500} - height={400} + height={213} className="my-6" /> @@ -102,7 +102,7 @@ Choisissez entre quatre types de boucles : src="/static/blocks/loop-4.png" alt="Boucle Do-While avec condition" width={500} - height={400} + height={210} className="my-6" /> diff --git a/apps/docs/content/docs/fr/blocks/parallel.mdx b/apps/docs/content/docs/fr/blocks/parallel.mdx index 9b2b84f70ae..a0804d5a53b 100644 --- a/apps/docs/content/docs/fr/blocks/parallel.mdx +++ b/apps/docs/content/docs/fr/blocks/parallel.mdx @@ -27,7 +27,7 @@ Choisissez entre deux types d'exécution parallèle : src="/static/blocks/parallel-1.png" alt="Exécution parallèle basée sur un nombre" width={500} - height={400} + height={231} className="my-6" /> @@ -53,7 +53,7 @@ Choisissez entre deux types d'exécution parallèle : src="/static/blocks/parallel-2.png" alt="Exécution parallèle basée sur une collection" width={500} - height={400} + height={220} className="my-6" /> diff --git a/apps/docs/content/docs/fr/blocks/response.mdx b/apps/docs/content/docs/fr/blocks/response.mdx index 3184ee1ae8a..031b269125a 100644 --- a/apps/docs/content/docs/fr/blocks/response.mdx +++ b/apps/docs/content/docs/fr/blocks/response.mdx @@ -13,7 +13,7 @@ Le bloc Réponse formate et envoie des réponses HTTP structurées aux appelants src="/static/blocks/response.png" alt="Configuration du bloc Réponse" width={500} - height={400} + height={382} className="my-6" /> diff --git a/apps/docs/content/docs/fr/blocks/router.mdx b/apps/docs/content/docs/fr/blocks/router.mdx index 0ae078f106a..37eea743360 100644 --- a/apps/docs/content/docs/fr/blocks/router.mdx +++ b/apps/docs/content/docs/fr/blocks/router.mdx @@ -13,7 +13,7 @@ Le bloc Routeur utilise l'IA pour diriger intelligemment les flux de travail en src="/static/blocks/router.png" alt="Bloc Routeur avec chemins multiples" width={500} - height={400} + height={342} className="my-6" /> diff --git a/apps/docs/content/docs/fr/blocks/variables.mdx b/apps/docs/content/docs/fr/blocks/variables.mdx index d29ad36eea7..4d8145cd2b4 100644 --- a/apps/docs/content/docs/fr/blocks/variables.mdx +++ b/apps/docs/content/docs/fr/blocks/variables.mdx @@ -13,7 +13,7 @@ Le bloc Variables met à jour les variables du workflow pendant l'exécution. Le src="/static/blocks/variables.png" alt="Bloc de variables" width={500} - height={400} + height={246} className="my-6" /> diff --git a/apps/docs/content/docs/fr/blocks/wait.mdx b/apps/docs/content/docs/fr/blocks/wait.mdx index 163e7849620..fcd93ee5cbc 100644 --- a/apps/docs/content/docs/fr/blocks/wait.mdx +++ b/apps/docs/content/docs/fr/blocks/wait.mdx @@ -13,7 +13,7 @@ Le bloc Attente met en pause votre flux de travail pendant une durée spécifié src="/static/blocks/wait.png" alt="Bloc d'attente" width={500} - height={400} + height={295} className="my-6" /> diff --git a/apps/docs/content/docs/fr/blocks/webhook.mdx b/apps/docs/content/docs/fr/blocks/webhook.mdx index c9d0ca93528..12ee50c4afa 100644 --- a/apps/docs/content/docs/fr/blocks/webhook.mdx +++ b/apps/docs/content/docs/fr/blocks/webhook.mdx @@ -12,7 +12,7 @@ Le bloc Webhook envoie des requêtes HTTP POST vers des points de terminaison we src="/static/blocks/webhook.png" alt="Bloc Webhook" width={500} - height={400} + height={403} className="my-6" /> diff --git a/apps/docs/content/docs/fr/blocks/workflow.mdx b/apps/docs/content/docs/fr/blocks/workflow.mdx index 90ac83bb211..277b9f4fffe 100644 --- a/apps/docs/content/docs/fr/blocks/workflow.mdx +++ b/apps/docs/content/docs/fr/blocks/workflow.mdx @@ -13,7 +13,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/workflow.png" alt="Configuration du bloc de flux de travail" width={500} - height={400} + height={310} className="rounded-xl border border-border shadow-sm" /> @@ -30,7 +30,7 @@ Déposez un bloc de flux de travail lorsque vous souhaitez appeler un flux de tr src="/static/blocks/workflow-2.png" alt="Exemple de mappage d'entrée du bloc de flux de travail" width={700} - height={400} + height={287} className="rounded-xl border border-border shadow-sm" /> diff --git a/apps/docs/content/docs/fr/copilot/index.mdx b/apps/docs/content/docs/fr/copilot/index.mdx index 4a04abb26ed..fd3b74afd04 100644 --- a/apps/docs/content/docs/fr/copilot/index.mdx +++ b/apps/docs/content/docs/fr/copilot/index.mdx @@ -27,7 +27,7 @@ Utilisez le symbole `@` pour référencer diverses ressources et donner à Copil src="/static/copilot/copilot-menu.png" alt="Menu contextuel de Copilot montrant les options de référence disponibles" width={600} - height={400} + height={521} /> Le menu `@` donne accès à : @@ -76,7 +76,7 @@ Ces informations contextuelles aident Copilot à fournir une assistance plus pr src="/static/copilot/copilot-mode.png" alt="Interface de sélection du mode Copilot" width={600} - height={400} + height={720} className="my-6" /> @@ -134,7 +134,7 @@ Vous pouvez facilement basculer entre différents modes de raisonnement à l'aid src="/static/copilot/copilot-models.png" alt="Sélection du mode Copilot montrant le mode Avancé avec l'option MAX" width={600} - height={300} + height={457} /> L'interface vous permet de : diff --git a/apps/docs/content/docs/fr/execution/basics.mdx b/apps/docs/content/docs/fr/execution/basics.mdx index 5153741f004..dd22ea6a4f4 100644 --- a/apps/docs/content/docs/fr/execution/basics.mdx +++ b/apps/docs/content/docs/fr/execution/basics.mdx @@ -20,7 +20,7 @@ Plusieurs blocs s'exécutent simultanément lorsqu'ils ne dépendent pas les uns src="/static/execution/concurrency.png" alt="Plusieurs blocs s'exécutant simultanément après le bloc de démarrage" width={800} - height={500} + height={685} /> Dans cet exemple, les blocs d'agent de support client et de chercheur approfondi s'exécutent simultanément après le bloc de démarrage, maximisant ainsi l'efficacité. @@ -33,7 +33,7 @@ Lorsque des blocs ont plusieurs dépendances, le moteur d'exécution attend auto src="/static/execution/combination.png" alt="Bloc de fonction recevant automatiquement les sorties de plusieurs blocs précédents" width={800} - height={500} + height={521} /> Le bloc de fonction reçoit les sorties des deux blocs d'agent dès qu'ils sont terminés, vous permettant de traiter les résultats combinés. @@ -46,7 +46,7 @@ Les workflows peuvent se ramifier dans plusieurs directions en utilisant des blo src="/static/execution/routing.png" alt="Workflow montrant à la fois des ramifications conditionnelles et basées sur un routeur" width={800} - height={500} + height={391} /> Ce flux de travail démontre comment l'exécution peut suivre différents chemins basés sur des conditions ou des décisions d'IA, chaque chemin s'exécutant indépendamment. diff --git a/apps/docs/content/docs/fr/execution/costs.mdx b/apps/docs/content/docs/fr/execution/costs.mdx index dd43d873f76..cb98f3868bb 100644 --- a/apps/docs/content/docs/fr/execution/costs.mdx +++ b/apps/docs/content/docs/fr/execution/costs.mdx @@ -34,7 +34,7 @@ Pour les flux de travail utilisant des blocs d'IA, vous pouvez consulter des inf src="/static/logs/logs-cost.png" alt="Répartition des modèles" width={600} - height={400} + height={153} className="my-6" /> diff --git a/apps/docs/content/docs/fr/execution/index.mdx b/apps/docs/content/docs/fr/execution/index.mdx index 4cc09865aa5..cf9f1749d39 100644 --- a/apps/docs/content/docs/fr/execution/index.mdx +++ b/apps/docs/content/docs/fr/execution/index.mdx @@ -61,7 +61,7 @@ Tous les points d'entrée publics — API, Chat, Planification, Webhook et exéc src='/static/execution/deployment-versions.png' alt='Tableau des versions de déploiement' width={500} - height={280} + height={259} className='rounded-xl border border-border shadow-sm' /> diff --git a/apps/docs/content/docs/fr/execution/logging.mdx b/apps/docs/content/docs/fr/execution/logging.mdx index 1f6927e5779..6af100ac07e 100644 --- a/apps/docs/content/docs/fr/execution/logging.mdx +++ b/apps/docs/content/docs/fr/execution/logging.mdx @@ -21,7 +21,7 @@ Pendant l'exécution manuelle ou par chat d'un flux de travail, les journaux app src="/static/logs/console.png" alt="Panneau de console en temps réel" width={400} - height={300} + height={151} className="my-6" /> @@ -41,7 +41,7 @@ Toutes les exécutions de flux de travail — qu'elles soient déclenchées manu src="/static/logs/logs.png" alt="Page de journaux" width={600} - height={400} + height={451} className="my-6" /> @@ -61,7 +61,7 @@ Cliquer sur n'importe quelle entrée de journal ouvre une vue détaillée dans l src="/static/logs/logs-sidebar.png" alt="Détails de la barre latérale des journaux" width={600} - height={400} + height={880} className="my-6" /> @@ -104,7 +104,7 @@ Pour toute exécution enregistrée, cliquez sur « Voir l'instantané » pour vi src="/static/logs/logs-frozen-canvas.png" alt="Instantané de workflow" width={600} - height={400} + height={628} className="my-6" /> diff --git a/apps/docs/content/docs/fr/getting-started/index.mdx b/apps/docs/content/docs/fr/getting-started/index.mdx index df0912ff2f4..ae4e7885ca0 100644 --- a/apps/docs/content/docs/fr/getting-started/index.mdx +++ b/apps/docs/content/docs/fr/getting-started/index.mdx @@ -42,7 +42,7 @@ Un agent de recherche de personnes qui : src="/static/getting-started/started-1.png" alt="Exemple de premiers pas" width={800} - height={500} + height={340} /> ## Tutoriel étape par étape diff --git a/apps/docs/content/docs/fr/introduction/index.mdx b/apps/docs/content/docs/fr/introduction/index.mdx index e0259ed35bb..d49b0dd4468 100644 --- a/apps/docs/content/docs/fr/introduction/index.mdx +++ b/apps/docs/content/docs/fr/introduction/index.mdx @@ -14,7 +14,7 @@ Sim est un constructeur de flux de travail visuel open-source pour créer et dé src="/static/introduction.png" alt="Canevas de flux de travail visuel Sim" width={700} - height={450} + height={372} className="my-6" /> diff --git a/apps/docs/content/docs/fr/knowledgebase/index.mdx b/apps/docs/content/docs/fr/knowledgebase/index.mdx index 956ea8f1f18..e6fd70debde 100644 --- a/apps/docs/content/docs/fr/knowledgebase/index.mdx +++ b/apps/docs/content/docs/fr/knowledgebase/index.mdx @@ -32,7 +32,7 @@ Sim prend en charge les fichiers PDF, Word (DOC/DOCX), texte brut (TXT), Markdow Une fois vos documents traités, vous pouvez visualiser et modifier les segments individuels. Cela vous donne un contrôle total sur l'organisation et la recherche de votre contenu. -Vue des segments de document montrant le contenu traité +Vue des segments de document montrant le contenu traité ### Configuration des fragments @@ -66,7 +66,7 @@ Lorsqu'il est configuré avec Azure ou [Mistral OCR](https://docs.mistral.ai/ocr Une fois vos documents traités, vous pouvez les utiliser dans vos workflows d'IA via le bloc de connaissances. Cela active la génération augmentée par récupération (RAG), permettant à vos agents d'IA d'accéder à votre contenu documentaire et de raisonner dessus pour fournir des réponses plus précises et contextuelles. -Utilisation du bloc de connaissances dans les workflows +Utilisation du bloc de connaissances dans les workflows ### Fonctionnalités du bloc de connaissances - **Recherche sémantique** : trouvez du contenu pertinent à l'aide de requêtes en langage naturel diff --git a/apps/docs/content/docs/fr/mcp/index.mdx b/apps/docs/content/docs/fr/mcp/index.mdx index e0525996072..cbe19912027 100644 --- a/apps/docs/content/docs/fr/mcp/index.mdx +++ b/apps/docs/content/docs/fr/mcp/index.mdx @@ -50,7 +50,7 @@ Une fois les serveurs MCP configurés, leurs outils deviennent disponibles dans src="/static/blocks/mcp-2.png" alt="Utilisation de l'outil MCP dans un bloc d'agent" width={700} - height={450} + height={353} className="my-6" /> @@ -69,7 +69,7 @@ Pour un contrôle plus précis, vous pouvez utiliser le bloc d'outil MCP dédié src="/static/blocks/mcp-3.png" alt="Bloc d'outil MCP autonome" width={700} - height={450} + height={545} className="my-6" /> diff --git a/apps/docs/content/docs/fr/triggers/index.mdx b/apps/docs/content/docs/fr/triggers/index.mdx index ef9b408df06..17152b4b26d 100644 --- a/apps/docs/content/docs/fr/triggers/index.mdx +++ b/apps/docs/content/docs/fr/triggers/index.mdx @@ -11,7 +11,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/triggers.png" alt="Aperçu des déclencheurs" width={500} - height={350} + height={348} className="my-6" /> diff --git a/apps/docs/content/docs/fr/triggers/rss.mdx b/apps/docs/content/docs/fr/triggers/rss.mdx index d14507b9a7a..5c5dc16757d 100644 --- a/apps/docs/content/docs/fr/triggers/rss.mdx +++ b/apps/docs/content/docs/fr/triggers/rss.mdx @@ -12,7 +12,7 @@ Le bloc Flux RSS surveille les flux RSS et Atom – lorsque de nouveaux élémen src="/static/blocks/rss.png" alt="Bloc Flux RSS" width={500} - height={400} + height={228} className="my-6" /> diff --git a/apps/docs/content/docs/fr/triggers/schedule.mdx b/apps/docs/content/docs/fr/triggers/schedule.mdx index 746b92a88f0..fc471ef3967 100644 --- a/apps/docs/content/docs/fr/triggers/schedule.mdx +++ b/apps/docs/content/docs/fr/triggers/schedule.mdx @@ -13,7 +13,7 @@ Le bloc Planification déclenche automatiquement des workflows de manière récu src="/static/blocks/schedule.png" alt="Bloc de planification" width={500} - height={400} + height={336} className="my-6" /> @@ -67,7 +67,7 @@ Les planifications se désactivent automatiquement après **100 échecs consécu src="/static/blocks/schedule-3.png" alt="Planification désactivée" width={500} - height={400} + height={310} className="my-6" /> diff --git a/apps/docs/content/docs/fr/triggers/start.mdx b/apps/docs/content/docs/fr/triggers/start.mdx index a62f3a7f61b..23ca584ed31 100644 --- a/apps/docs/content/docs/fr/triggers/start.mdx +++ b/apps/docs/content/docs/fr/triggers/start.mdx @@ -13,7 +13,7 @@ Le bloc Démarrer est le déclencheur par défaut pour les flux de travail cré src="/static/start.png" alt="Bloc de démarrage avec champs de format d'entrée" width={360} - height={380} + height={151} className="my-6" /> diff --git a/apps/docs/content/docs/fr/triggers/webhook.mdx b/apps/docs/content/docs/fr/triggers/webhook.mdx index c1d7fa009c2..87687b23959 100644 --- a/apps/docs/content/docs/fr/triggers/webhook.mdx +++ b/apps/docs/content/docs/fr/triggers/webhook.mdx @@ -18,7 +18,7 @@ Le bloc Webhook générique crée un point de terminaison flexible qui peut rece src="/static/blocks/webhook-trigger.png" alt="Configuration du webhook générique" width={500} - height={400} + height={405} className="my-6" /> diff --git a/apps/docs/content/docs/ja/blocks/agent.mdx b/apps/docs/content/docs/ja/blocks/agent.mdx index 4eb5c8e8eb8..6213a36aae5 100644 --- a/apps/docs/content/docs/ja/blocks/agent.mdx +++ b/apps/docs/content/docs/ja/blocks/agent.mdx @@ -13,7 +13,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/agent.png" alt="エージェントブロックの設定" width={500} - height={400} + height={450} className="my-6" /> diff --git a/apps/docs/content/docs/ja/blocks/api.mdx b/apps/docs/content/docs/ja/blocks/api.mdx index 1efff82c411..6dae7523d25 100644 --- a/apps/docs/content/docs/ja/blocks/api.mdx +++ b/apps/docs/content/docs/ja/blocks/api.mdx @@ -13,7 +13,7 @@ APIブロックは、HTTPリクエストを通じてワークフローを外部 src="/static/blocks/api.png" alt="APIブロック" width={500} - height={400} + height={422} className="my-6" /> diff --git a/apps/docs/content/docs/ja/blocks/condition.mdx b/apps/docs/content/docs/ja/blocks/condition.mdx index a1f5b6cfbe3..c25df185534 100644 --- a/apps/docs/content/docs/ja/blocks/condition.mdx +++ b/apps/docs/content/docs/ja/blocks/condition.mdx @@ -13,7 +13,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/condition.png" alt="条件ブロック" width={500} - height={400} + height={317} className="my-6" /> diff --git a/apps/docs/content/docs/ja/blocks/evaluator.mdx b/apps/docs/content/docs/ja/blocks/evaluator.mdx index 8fcc6d5ec7f..a753f7ca7be 100644 --- a/apps/docs/content/docs/ja/blocks/evaluator.mdx +++ b/apps/docs/content/docs/ja/blocks/evaluator.mdx @@ -13,7 +13,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/evaluator.png" alt="評価者ブロックの設定画面" width={500} - height={400} + height={337} className="my-6" /> diff --git a/apps/docs/content/docs/ja/blocks/function.mdx b/apps/docs/content/docs/ja/blocks/function.mdx index 566e7a43fb3..e62c9ef64fb 100644 --- a/apps/docs/content/docs/ja/blocks/function.mdx +++ b/apps/docs/content/docs/ja/blocks/function.mdx @@ -11,7 +11,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/function.png" alt="コードエディタ付き関数ブロック" width={500} - height={400} + height={287} className="my-6" /> diff --git a/apps/docs/content/docs/ja/blocks/guardrails.mdx b/apps/docs/content/docs/ja/blocks/guardrails.mdx index 85083fa8e8c..0481be21e24 100644 --- a/apps/docs/content/docs/ja/blocks/guardrails.mdx +++ b/apps/docs/content/docs/ja/blocks/guardrails.mdx @@ -13,7 +13,7 @@ import { Video } from '@/components/ui/video' src="/static/blocks/guardrails.png" alt="ガードレールブロック" width={500} - height={400} + height={398} className="my-6" /> @@ -90,7 +90,7 @@ Microsoft Presidioを使用して個人を特定できる情報を検出しま src="/static/blocks/guardrails-2.png" alt="PII検出の設定" width={700} - height={450} + height={335} className="my-6" /> diff --git a/apps/docs/content/docs/ja/blocks/human-in-the-loop.mdx b/apps/docs/content/docs/ja/blocks/human-in-the-loop.mdx index bcd54b2a488..d6c2ebb5411 100644 --- a/apps/docs/content/docs/ja/blocks/human-in-the-loop.mdx +++ b/apps/docs/content/docs/ja/blocks/human-in-the-loop.mdx @@ -14,7 +14,7 @@ import { Video } from '@/components/ui/video' src="/static/blocks/hitl-1.png" alt="ヒューマン・イン・ザ・ループブロックの設定" width={500} - height={400} + height={355} className="my-6" /> @@ -26,7 +26,7 @@ import { Video } from '@/components/ui/video' src="/static/blocks/hitl-2.png" alt="ヒューマン・イン・ザ・ループ承認ポータル" width={700} - height={500} + height={467} className="my-6" /> diff --git a/apps/docs/content/docs/ja/blocks/loop.mdx b/apps/docs/content/docs/ja/blocks/loop.mdx index 362cdb6d031..53831187d97 100644 --- a/apps/docs/content/docs/ja/blocks/loop.mdx +++ b/apps/docs/content/docs/ja/blocks/loop.mdx @@ -27,7 +27,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/loop-1.png" alt="反復回数を使用したFor ループ" width={500} - height={400} + height={267} className="my-6" /> @@ -53,7 +53,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/loop-2.png" alt="コレクションを使用したForEach ループ" width={500} - height={400} + height={273} className="my-6" /> @@ -77,7 +77,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/loop-3.png" alt="条件付きWhile ループ" width={500} - height={400} + height={213} className="my-6" /> @@ -102,7 +102,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/loop-4.png" alt="条件付きDo-While ループ" width={500} - height={400} + height={210} className="my-6" /> diff --git a/apps/docs/content/docs/ja/blocks/parallel.mdx b/apps/docs/content/docs/ja/blocks/parallel.mdx index eea83a188c1..20ac59ea66b 100644 --- a/apps/docs/content/docs/ja/blocks/parallel.mdx +++ b/apps/docs/content/docs/ja/blocks/parallel.mdx @@ -27,7 +27,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/parallel-1.png" alt="カウントベースの並列実行" width={500} - height={400} + height={231} className="my-6" /> @@ -53,7 +53,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/parallel-2.png" alt="コレクションベースの並列実行" width={500} - height={400} + height={220} className="my-6" /> diff --git a/apps/docs/content/docs/ja/blocks/response.mdx b/apps/docs/content/docs/ja/blocks/response.mdx index 8d2f629ddce..0700c5c4279 100644 --- a/apps/docs/content/docs/ja/blocks/response.mdx +++ b/apps/docs/content/docs/ja/blocks/response.mdx @@ -13,7 +13,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/response.png" alt="レスポンスブロックの設定" width={500} - height={400} + height={382} className="my-6" /> diff --git a/apps/docs/content/docs/ja/blocks/router.mdx b/apps/docs/content/docs/ja/blocks/router.mdx index 8aa16bfdf9e..1e0664ad7fa 100644 --- a/apps/docs/content/docs/ja/blocks/router.mdx +++ b/apps/docs/content/docs/ja/blocks/router.mdx @@ -13,7 +13,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/router.png" alt="複数のパスを持つルーターブロック" width={500} - height={400} + height={342} className="my-6" /> diff --git a/apps/docs/content/docs/ja/blocks/variables.mdx b/apps/docs/content/docs/ja/blocks/variables.mdx index a0c80af60e1..d8b44abf52f 100644 --- a/apps/docs/content/docs/ja/blocks/variables.mdx +++ b/apps/docs/content/docs/ja/blocks/variables.mdx @@ -13,7 +13,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/variables.png" alt="変数ブロック" width={500} - height={400} + height={246} className="my-6" /> diff --git a/apps/docs/content/docs/ja/blocks/wait.mdx b/apps/docs/content/docs/ja/blocks/wait.mdx index 1961892276c..1c48158e0db 100644 --- a/apps/docs/content/docs/ja/blocks/wait.mdx +++ b/apps/docs/content/docs/ja/blocks/wait.mdx @@ -13,7 +13,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/wait.png" alt="待機ブロック" width={500} - height={400} + height={295} className="my-6" /> diff --git a/apps/docs/content/docs/ja/blocks/webhook.mdx b/apps/docs/content/docs/ja/blocks/webhook.mdx index 13f1cd3b178..a1502e2bffa 100644 --- a/apps/docs/content/docs/ja/blocks/webhook.mdx +++ b/apps/docs/content/docs/ja/blocks/webhook.mdx @@ -12,7 +12,7 @@ Webhookブロックは、自動的なWebhookヘッダーとオプションのHMA src="/static/blocks/webhook.png" alt="Webhookブロック" width={500} - height={400} + height={403} className="my-6" /> diff --git a/apps/docs/content/docs/ja/blocks/workflow.mdx b/apps/docs/content/docs/ja/blocks/workflow.mdx index 5a95d693113..f7f7daf3f26 100644 --- a/apps/docs/content/docs/ja/blocks/workflow.mdx +++ b/apps/docs/content/docs/ja/blocks/workflow.mdx @@ -13,7 +13,7 @@ import { Image } from '@/components/ui/image' src='/static/blocks/workflow.png' alt='ワークフローブロックの設定' width={500} - height={400} + height={310} className='rounded-xl border border-border shadow-sm' /> @@ -30,7 +30,7 @@ import { Image } from '@/components/ui/image' src='/static/blocks/workflow-2.png' alt='入力マッピング例を示すワークフローブロック' width={700} - height={400} + height={287} className='rounded-xl border border-border shadow-sm' /> diff --git a/apps/docs/content/docs/ja/copilot/index.mdx b/apps/docs/content/docs/ja/copilot/index.mdx index e586d36c25f..ceae47375b9 100644 --- a/apps/docs/content/docs/ja/copilot/index.mdx +++ b/apps/docs/content/docs/ja/copilot/index.mdx @@ -27,7 +27,7 @@ Copilotはエディター内のアシスタントで、Sim Copilotを使用し src="/static/copilot/copilot-menu.png" alt="利用可能な参照オプションを表示するCopilotコンテキストメニュー" width={600} - height={400} + height={521} /> `@` メニューから以下にアクセスできます: @@ -76,7 +76,7 @@ Copilotはエディター内のアシスタントで、Sim Copilotを使用し src="/static/copilot/copilot-mode.png" alt="Copilotモード選択インターフェース" width={600} - height={400} + height={720} className="my-6" /> @@ -134,7 +134,7 @@ Copilotインターフェースのモードセレクターを使用して、異 src="/static/copilot/copilot-models.png" alt="MAXトグル付きの高度モードを表示するCopilotモード選択" width={600} - height={300} + height={457} /> このインターフェースでは以下のことが可能です: diff --git a/apps/docs/content/docs/ja/execution/basics.mdx b/apps/docs/content/docs/ja/execution/basics.mdx index b5f7825fe7f..96beb830d96 100644 --- a/apps/docs/content/docs/ja/execution/basics.mdx +++ b/apps/docs/content/docs/ja/execution/basics.mdx @@ -20,7 +20,7 @@ Simの実行エンジンは依存関係を分析し、最も効率的な順序 src="/static/execution/concurrency.png" alt="スタートブロックの後に複数のブロックが同時に実行されている" width={800} - height={500} + height={685} /> この例では、カスタマーサポートとディープリサーチャーの両方のエージェントブロックがスタートブロックの後に同時に実行され、効率を最大化しています。 @@ -33,7 +33,7 @@ Simの実行エンジンは依存関係を分析し、最も効率的な順序 src="/static/execution/combination.png" alt="関数ブロックが複数の前のブロックからの出力を自動的に受け取る" width={800} - height={500} + height={521} /> 関数ブロックは両方のエージェントブロックが完了するとすぐにそれらの出力を受け取り、結合された結果を処理することができます。 @@ -46,7 +46,7 @@ Simの実行エンジンは依存関係を分析し、最も効率的な順序 src="/static/execution/routing.png" alt="条件分岐とルーターベースの分岐の両方を示すワークフロー" width={800} - height={500} + height={391} /> このワークフローは、条件やAIの判断に基づいて実行が異なる経路をたどる方法を示しており、各経路は独立して実行されます。 diff --git a/apps/docs/content/docs/ja/execution/costs.mdx b/apps/docs/content/docs/ja/execution/costs.mdx index c5cf8b1c801..1a9c06c3f30 100644 --- a/apps/docs/content/docs/ja/execution/costs.mdx +++ b/apps/docs/content/docs/ja/execution/costs.mdx @@ -34,7 +34,7 @@ AIブロックを使用するワークフローでは、ログで詳細なコス src="/static/logs/logs-cost.png" alt="モデル内訳" width={600} - height={400} + height={153} className="my-6" /> diff --git a/apps/docs/content/docs/ja/execution/index.mdx b/apps/docs/content/docs/ja/execution/index.mdx index 00c54fb744a..d2d645e3a17 100644 --- a/apps/docs/content/docs/ja/execution/index.mdx +++ b/apps/docs/content/docs/ja/execution/index.mdx @@ -60,7 +60,7 @@ Simの実行エンジンは、ブロックを正しい順序で処理し、デ src='/static/execution/deployment-versions.png' alt='デプロイメントバージョン一覧表' width={500} - height={280} + height={259} className='rounded-xl border border-border shadow-sm' /> diff --git a/apps/docs/content/docs/ja/execution/logging.mdx b/apps/docs/content/docs/ja/execution/logging.mdx index 23e6539fe3d..e4310ea6260 100644 --- a/apps/docs/content/docs/ja/execution/logging.mdx +++ b/apps/docs/content/docs/ja/execution/logging.mdx @@ -21,7 +21,7 @@ Simは異なるワークフローとユースケースに対応する2つの補 src="/static/logs/console.png" alt="リアルタイムコンソールパネル" width={400} - height={300} + height={151} className="my-6" /> @@ -41,7 +41,7 @@ Simは異なるワークフローとユースケースに対応する2つの補 src="/static/logs/logs.png" alt="ログページ" width={600} - height={400} + height={451} className="my-6" /> @@ -61,7 +61,7 @@ Simは異なるワークフローとユースケースに対応する2つの補 src="/static/logs/logs-sidebar.png" alt="ログサイドバーの詳細" width={600} - height={400} + height={880} className="my-6" /> @@ -104,7 +104,7 @@ Simは異なるワークフローとユースケースに対応する2つの補 src="/static/logs/logs-frozen-canvas.png" alt="ワークフロースナップショット" width={600} - height={400} + height={628} className="my-6" /> diff --git a/apps/docs/content/docs/ja/getting-started/index.mdx b/apps/docs/content/docs/ja/getting-started/index.mdx index 35642ac457f..8bb4cfe7603 100644 --- a/apps/docs/content/docs/ja/getting-started/index.mdx +++ b/apps/docs/content/docs/ja/getting-started/index.mdx @@ -42,7 +42,7 @@ import { Image } from '@/components/ui/image' src="/static/getting-started/started-1.png" alt="はじめての例" width={800} - height={500} + height={340} /> ## ステップバイステップのチュートリアル diff --git a/apps/docs/content/docs/ja/introduction/index.mdx b/apps/docs/content/docs/ja/introduction/index.mdx index dd37ac43fe7..6b21f1e8a11 100644 --- a/apps/docs/content/docs/ja/introduction/index.mdx +++ b/apps/docs/content/docs/ja/introduction/index.mdx @@ -14,7 +14,7 @@ Simはオープンソースのビジュアルワークフロービルダーで src="/static/introduction.png" alt="Simビジュアルワークフローキャンバス" width={700} - height={450} + height={372} className="my-6" /> diff --git a/apps/docs/content/docs/ja/knowledgebase/index.mdx b/apps/docs/content/docs/ja/knowledgebase/index.mdx index 23ecd63868c..301913dd168 100644 --- a/apps/docs/content/docs/ja/knowledgebase/index.mdx +++ b/apps/docs/content/docs/ja/knowledgebase/index.mdx @@ -31,7 +31,7 @@ SimはPDF、Word(DOC/DOCX)、プレーンテキスト(TXT)、Markdown( ドキュメントが処理されると、個々のチャンクを閲覧および編集できます。これにより、コンテンツの整理方法と検索方法を完全に制御できます。 -処理されたコンテンツを表示するドキュメントチャンクビュー +処理されたコンテンツを表示するドキュメントチャンクビュー ### チャンク設定 @@ -65,7 +65,7 @@ Azureまたは[Mistral OCR](https://docs.mistral.ai/ocr/)で設定されてい ドキュメントが処理されると、ナレッジブロックを通じてAIワークフローで使用できます。これにより検索拡張生成(RAG)が可能になり、AIエージェントがドキュメントコンテンツにアクセスして推論し、より正確でコンテキストに沿った応答を提供できます。 -ワークフローでのナレッジブロックの使用 +ワークフローでのナレッジブロックの使用 ### ナレッジブロックの機能 - **セマンティック検索**: 自然言語クエリを使用して関連コンテンツを検索 diff --git a/apps/docs/content/docs/ja/mcp/index.mdx b/apps/docs/content/docs/ja/mcp/index.mdx index be0fc699ca8..a09fcf50d43 100644 --- a/apps/docs/content/docs/ja/mcp/index.mdx +++ b/apps/docs/content/docs/ja/mcp/index.mdx @@ -49,7 +49,7 @@ MCPサーバーが設定されると、そのツールがエージェントブ src="/static/blocks/mcp-2.png" alt="エージェントブロックでのMCPツールの使用" width={700} - height={450} + height={353} className="my-6" /> @@ -68,7 +68,7 @@ MCPサーバーが設定されると、そのツールがエージェントブ src="/static/blocks/mcp-3.png" alt="スタンドアロンMCPツールブロック" width={700} - height={450} + height={545} className="my-6" /> diff --git a/apps/docs/content/docs/ja/triggers/index.mdx b/apps/docs/content/docs/ja/triggers/index.mdx index 7bf66bd07c9..4a9863cd460 100644 --- a/apps/docs/content/docs/ja/triggers/index.mdx +++ b/apps/docs/content/docs/ja/triggers/index.mdx @@ -11,7 +11,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/triggers.png" alt="トリガーの概要" width={500} - height={350} + height={348} className="my-6" /> diff --git a/apps/docs/content/docs/ja/triggers/rss.mdx b/apps/docs/content/docs/ja/triggers/rss.mdx index 72737a2a80a..1baa8e36f7d 100644 --- a/apps/docs/content/docs/ja/triggers/rss.mdx +++ b/apps/docs/content/docs/ja/triggers/rss.mdx @@ -12,7 +12,7 @@ RSSフィードブロックはRSSとAtomフィードを監視します - 新し src="/static/blocks/rss.png" alt="RSSフィードブロック" width={500} - height={400} + height={228} className="my-6" /> diff --git a/apps/docs/content/docs/ja/triggers/schedule.mdx b/apps/docs/content/docs/ja/triggers/schedule.mdx index c7ffaf42c79..b91ed317a38 100644 --- a/apps/docs/content/docs/ja/triggers/schedule.mdx +++ b/apps/docs/content/docs/ja/triggers/schedule.mdx @@ -13,7 +13,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/schedule.png" alt="スケジュールブロック" width={500} - height={400} + height={336} className="my-6" /> @@ -67,7 +67,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/schedule-3.png" alt="無効化されたスケジュール" width={500} - height={400} + height={310} className="my-6" /> diff --git a/apps/docs/content/docs/ja/triggers/start.mdx b/apps/docs/content/docs/ja/triggers/start.mdx index e7932d3c54d..9d544300621 100644 --- a/apps/docs/content/docs/ja/triggers/start.mdx +++ b/apps/docs/content/docs/ja/triggers/start.mdx @@ -13,7 +13,7 @@ import { Image } from '@/components/ui/image' src="/static/start.png" alt="入力フォーマットフィールドを持つスタートブロック" width={360} - height={380} + height={151} className="my-6" /> diff --git a/apps/docs/content/docs/ja/triggers/webhook.mdx b/apps/docs/content/docs/ja/triggers/webhook.mdx index 1eb9b0bf13f..7250ab9d3bc 100644 --- a/apps/docs/content/docs/ja/triggers/webhook.mdx +++ b/apps/docs/content/docs/ja/triggers/webhook.mdx @@ -18,7 +18,7 @@ Webhookを使用すると、外部サービスがHTTPリクエストを送信し src="/static/blocks/webhook-trigger.png" alt="汎用Webhook設定" width={500} - height={400} + height={405} className="my-6" /> diff --git a/apps/docs/content/docs/zh/blocks/agent.mdx b/apps/docs/content/docs/zh/blocks/agent.mdx index 0c31e8d5f65..6fd5932a8ba 100644 --- a/apps/docs/content/docs/zh/blocks/agent.mdx +++ b/apps/docs/content/docs/zh/blocks/agent.mdx @@ -13,7 +13,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/agent.png" alt="代理模块配置" width={500} - height={400} + height={450} className="my-6" /> diff --git a/apps/docs/content/docs/zh/blocks/api.mdx b/apps/docs/content/docs/zh/blocks/api.mdx index 0c3812c8714..c9f629d902b 100644 --- a/apps/docs/content/docs/zh/blocks/api.mdx +++ b/apps/docs/content/docs/zh/blocks/api.mdx @@ -13,7 +13,7 @@ API 模块通过 HTTP 请求将您的工作流连接到外部服务。支持 GET src="/static/blocks/api.png" alt="API 模块" width={500} - height={400} + height={422} className="my-6" /> diff --git a/apps/docs/content/docs/zh/blocks/condition.mdx b/apps/docs/content/docs/zh/blocks/condition.mdx index a66e1ac2bdf..0939c45193c 100644 --- a/apps/docs/content/docs/zh/blocks/condition.mdx +++ b/apps/docs/content/docs/zh/blocks/condition.mdx @@ -13,7 +13,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/condition.png" alt="条件块" width={500} - height={400} + height={317} className="my-6" /> diff --git a/apps/docs/content/docs/zh/blocks/evaluator.mdx b/apps/docs/content/docs/zh/blocks/evaluator.mdx index 3fcb8e4bd15..8bbe4f893b5 100644 --- a/apps/docs/content/docs/zh/blocks/evaluator.mdx +++ b/apps/docs/content/docs/zh/blocks/evaluator.mdx @@ -13,7 +13,7 @@ Evaluator 模块使用 AI 根据自定义指标对内容质量进行评分和评 src="/static/blocks/evaluator.png" alt="Evaluator 模块配置" width={500} - height={400} + height={337} className="my-6" /> diff --git a/apps/docs/content/docs/zh/blocks/function.mdx b/apps/docs/content/docs/zh/blocks/function.mdx index 1174d79abd5..1d3f2615b9a 100644 --- a/apps/docs/content/docs/zh/blocks/function.mdx +++ b/apps/docs/content/docs/zh/blocks/function.mdx @@ -11,7 +11,7 @@ Function 模块在您的工作流中执行自定义 JavaScript 或 TypeScript src="/static/blocks/function.png" alt="函数块与代码编辑器" width={500} - height={400} + height={287} className="my-6" /> diff --git a/apps/docs/content/docs/zh/blocks/guardrails.mdx b/apps/docs/content/docs/zh/blocks/guardrails.mdx index cd2854e25ca..28305e57487 100644 --- a/apps/docs/content/docs/zh/blocks/guardrails.mdx +++ b/apps/docs/content/docs/zh/blocks/guardrails.mdx @@ -13,7 +13,7 @@ Guardrails 模块通过针对多种验证类型检查内容,验证并保护您 src="/static/blocks/guardrails.png" alt="防护栏块" width={500} - height={400} + height={398} className="my-6" /> @@ -90,7 +90,7 @@ Guardrails 模块通过针对多种验证类型检查内容,验证并保护您 src="/static/blocks/guardrails-2.png" alt="PII 检测配置" width={700} - height={450} + height={335} className="my-6" /> diff --git a/apps/docs/content/docs/zh/blocks/human-in-the-loop.mdx b/apps/docs/content/docs/zh/blocks/human-in-the-loop.mdx index 023ffd98760..6dc1a3bdb72 100644 --- a/apps/docs/content/docs/zh/blocks/human-in-the-loop.mdx +++ b/apps/docs/content/docs/zh/blocks/human-in-the-loop.mdx @@ -14,7 +14,7 @@ import { Video } from '@/components/ui/video' src="/static/blocks/hitl-1.png" alt="人工干预模块配置" width={500} - height={400} + height={355} className="my-6" /> @@ -26,7 +26,7 @@ import { Video } from '@/components/ui/video' src="/static/blocks/hitl-2.png" alt="人工干预审批门户" width={700} - height={500} + height={467} className="my-6" /> diff --git a/apps/docs/content/docs/zh/blocks/loop.mdx b/apps/docs/content/docs/zh/blocks/loop.mdx index 8eb966d0467..100140fb5d1 100644 --- a/apps/docs/content/docs/zh/blocks/loop.mdx +++ b/apps/docs/content/docs/zh/blocks/loop.mdx @@ -27,7 +27,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/loop-1.png" alt="带有迭代的 For 循环" width={500} - height={400} + height={267} className="my-6" /> @@ -53,7 +53,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/loop-2.png" alt="带有集合的 ForEach 循环" width={500} - height={400} + height={273} className="my-6" /> @@ -77,7 +77,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/loop-3.png" alt="带有条件的 While 循环" width={500} - height={400} + height={213} className="my-6" /> @@ -102,7 +102,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/loop-4.png" alt="带有条件的 Do-While 循环" width={500} - height={400} + height={210} className="my-6" /> diff --git a/apps/docs/content/docs/zh/blocks/parallel.mdx b/apps/docs/content/docs/zh/blocks/parallel.mdx index 1714b68eac8..14d5962fe6b 100644 --- a/apps/docs/content/docs/zh/blocks/parallel.mdx +++ b/apps/docs/content/docs/zh/blocks/parallel.mdx @@ -27,7 +27,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/parallel-1.png" alt="基于计数的并行执行" width={500} - height={400} + height={231} className="my-6" /> @@ -53,7 +53,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/parallel-2.png" alt="基于集合的并行执行" width={500} - height={400} + height={220} className="my-6" /> diff --git a/apps/docs/content/docs/zh/blocks/response.mdx b/apps/docs/content/docs/zh/blocks/response.mdx index c7cde6068cc..45e235bf340 100644 --- a/apps/docs/content/docs/zh/blocks/response.mdx +++ b/apps/docs/content/docs/zh/blocks/response.mdx @@ -13,7 +13,7 @@ Response 块用于格式化并将结构化的 HTTP 响应发送回 API 调用方 src="/static/blocks/response.png" alt="响应模块配置" width={500} - height={400} + height={382} className="my-6" /> diff --git a/apps/docs/content/docs/zh/blocks/router.mdx b/apps/docs/content/docs/zh/blocks/router.mdx index b8efa1c8f06..63ed4eff64c 100644 --- a/apps/docs/content/docs/zh/blocks/router.mdx +++ b/apps/docs/content/docs/zh/blocks/router.mdx @@ -13,7 +13,7 @@ Router 块使用 AI 基于内容分析智能地路由工作流。与使用简单 src="/static/blocks/router.png" alt="具有多路径的路由器模块" width={500} - height={400} + height={342} className="my-6" /> diff --git a/apps/docs/content/docs/zh/blocks/variables.mdx b/apps/docs/content/docs/zh/blocks/variables.mdx index 485d0ecba6a..d264e27840e 100644 --- a/apps/docs/content/docs/zh/blocks/variables.mdx +++ b/apps/docs/content/docs/zh/blocks/variables.mdx @@ -13,7 +13,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/variables.png" alt="变量块" width={500} - height={400} + height={246} className="my-6" /> diff --git a/apps/docs/content/docs/zh/blocks/wait.mdx b/apps/docs/content/docs/zh/blocks/wait.mdx index e1555701c1e..03b89c4e781 100644 --- a/apps/docs/content/docs/zh/blocks/wait.mdx +++ b/apps/docs/content/docs/zh/blocks/wait.mdx @@ -13,7 +13,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/wait.png" alt="等待模块" width={500} - height={400} + height={295} className="my-6" /> diff --git a/apps/docs/content/docs/zh/blocks/webhook.mdx b/apps/docs/content/docs/zh/blocks/webhook.mdx index d7324c16672..bd46f8813e6 100644 --- a/apps/docs/content/docs/zh/blocks/webhook.mdx +++ b/apps/docs/content/docs/zh/blocks/webhook.mdx @@ -12,7 +12,7 @@ Webhook 模块会向外部 webhook 端点发送 HTTP POST 请求,自动附加 src="/static/blocks/webhook.png" alt="Webhook 模块" width={500} - height={400} + height={403} className="my-6" /> diff --git a/apps/docs/content/docs/zh/blocks/workflow.mdx b/apps/docs/content/docs/zh/blocks/workflow.mdx index 0853596abdb..cd2689a51c5 100644 --- a/apps/docs/content/docs/zh/blocks/workflow.mdx +++ b/apps/docs/content/docs/zh/blocks/workflow.mdx @@ -13,7 +13,7 @@ import { Image } from '@/components/ui/image' src='/static/blocks/workflow.png' alt='工作流模块配置' width={500} - height={400} + height={310} className='rounded-xl border border-border shadow-sm' /> @@ -30,7 +30,7 @@ import { Image } from '@/components/ui/image' src='/static/blocks/workflow-2.png' alt='带有输入映射示例的工作流模块' width={700} - height={400} + height={287} className='rounded-xl border border-border shadow-sm' /> diff --git a/apps/docs/content/docs/zh/copilot/index.mdx b/apps/docs/content/docs/zh/copilot/index.mdx index 38fdf56c9f7..de570ba115b 100644 --- a/apps/docs/content/docs/zh/copilot/index.mdx +++ b/apps/docs/content/docs/zh/copilot/index.mdx @@ -27,7 +27,7 @@ Copilot 是您编辑器中的助手,可帮助您使用 Sim Copilot 构建和 src="/static/copilot/copilot-menu.png" alt="Copilot 上下文菜单显示可用的引用选项" width={600} - height={400} + height={521} /> `@` 菜单提供以下访问: @@ -76,7 +76,7 @@ Copilot 是您编辑器中的助手,可帮助您使用 Sim Copilot 构建和 src="/static/copilot/copilot-mode.png" alt="Copilot 模式选择界面" width={600} - height={400} + height={720} className="my-6" /> @@ -134,7 +134,7 @@ Copilot 是您编辑器中的助手,可帮助您使用 Sim Copilot 构建和 src="/static/copilot/copilot-models.png" alt="Copilot 模式选择界面显示高级模式和 MAX 切换选项" width={600} - height={300} + height={457} /> 该界面允许您: diff --git a/apps/docs/content/docs/zh/execution/basics.mdx b/apps/docs/content/docs/zh/execution/basics.mdx index 2e84b8fda28..9d17798cbd1 100644 --- a/apps/docs/content/docs/zh/execution/basics.mdx +++ b/apps/docs/content/docs/zh/execution/basics.mdx @@ -20,7 +20,7 @@ Sim 的执行引擎通过分析依赖关系,以最有效的顺序智能地处 src="/static/execution/concurrency.png" alt="多个块在起始块之后并发运行" width={800} - height={500} + height={685} /> 在此示例中,客户支持和深度研究代理块在起始块之后同时执行,从而最大化效率。 @@ -33,7 +33,7 @@ Sim 的执行引擎通过分析依赖关系,以最有效的顺序智能地处 src="/static/execution/combination.png" alt="函数块自动接收来自多个前置块的输出" width={800} - height={500} + height={521} /> 函数块在两个代理块完成后接收它们的输出,从而可以处理合并的结果。 @@ -46,7 +46,7 @@ Sim 的执行引擎通过分析依赖关系,以最有效的顺序智能地处 src="/static/execution/routing.png" alt="显示条件分支和基于路由器的分支的工作流" width={800} - height={500} + height={391} /> 此工作流演示了如何根据条件或 AI 决策沿不同路径执行,每条路径独立运行。 diff --git a/apps/docs/content/docs/zh/execution/costs.mdx b/apps/docs/content/docs/zh/execution/costs.mdx index 6a0a220a936..4d24310a765 100644 --- a/apps/docs/content/docs/zh/execution/costs.mdx +++ b/apps/docs/content/docs/zh/execution/costs.mdx @@ -34,7 +34,7 @@ totalCost = baseExecutionCharge + modelCost src="/static/logs/logs-cost.png" alt="模型成本明细" width={600} - height={400} + height={153} className="my-6" /> diff --git a/apps/docs/content/docs/zh/execution/index.mdx b/apps/docs/content/docs/zh/execution/index.mdx index faa621fee75..3c6f60ec61e 100644 --- a/apps/docs/content/docs/zh/execution/index.mdx +++ b/apps/docs/content/docs/zh/execution/index.mdx @@ -60,7 +60,7 @@ Sim 的执行引擎通过按正确的顺序处理模块、管理数据流并优 src='/static/execution/deployment-versions.png' alt='部署版本表' width={500} - height={280} + height={259} className='rounded-xl border border-border shadow-sm' /> diff --git a/apps/docs/content/docs/zh/execution/logging.mdx b/apps/docs/content/docs/zh/execution/logging.mdx index 62285c57af3..2f9d66288c8 100644 --- a/apps/docs/content/docs/zh/execution/logging.mdx +++ b/apps/docs/content/docs/zh/execution/logging.mdx @@ -21,7 +21,7 @@ Sim 提供了两种互补的日志界面,以适应不同的工作流和使用 src="/static/logs/console.png" alt="实时控制台面板" width={400} - height={300} + height={151} className="my-6" /> @@ -41,7 +41,7 @@ Sim 提供了两种互补的日志界面,以适应不同的工作流和使用 src="/static/logs/logs.png" alt="日志页面" width={600} - height={400} + height={451} className="my-6" /> @@ -61,7 +61,7 @@ Sim 提供了两种互补的日志界面,以适应不同的工作流和使用 src="/static/logs/logs-sidebar.png" alt="日志侧边栏详情" width={600} - height={400} + height={880} className="my-6" /> @@ -104,7 +104,7 @@ Sim 提供了两种互补的日志界面,以适应不同的工作流和使用 src="/static/logs/logs-frozen-canvas.png" alt="工作流快照" width={600} - height={400} + height={628} className="my-6" /> diff --git a/apps/docs/content/docs/zh/getting-started/index.mdx b/apps/docs/content/docs/zh/getting-started/index.mdx index 01c2df4fe8b..9c233d57ca3 100644 --- a/apps/docs/content/docs/zh/getting-started/index.mdx +++ b/apps/docs/content/docs/zh/getting-started/index.mdx @@ -42,7 +42,7 @@ import { Image } from '@/components/ui/image' src="/static/getting-started/started-1.png" alt="入门示例" width={800} - height={500} + height={340} /> ## 分步教程 diff --git a/apps/docs/content/docs/zh/introduction/index.mdx b/apps/docs/content/docs/zh/introduction/index.mdx index 2b396f51bf1..c9842a54953 100644 --- a/apps/docs/content/docs/zh/introduction/index.mdx +++ b/apps/docs/content/docs/zh/introduction/index.mdx @@ -14,7 +14,7 @@ Sim 是一个开源的可视化工作流构建器,用于构建和部署 AI 代 src="/static/introduction.png" alt="Sim 可视化工作流画布" width={700} - height={450} + height={372} className="my-6" /> diff --git a/apps/docs/content/docs/zh/knowledgebase/index.mdx b/apps/docs/content/docs/zh/knowledgebase/index.mdx index 73b750fcf46..40602865c05 100644 --- a/apps/docs/content/docs/zh/knowledgebase/index.mdx +++ b/apps/docs/content/docs/zh/knowledgebase/index.mdx @@ -31,7 +31,7 @@ Sim 支持 PDF、Word (DOC/DOCX)、纯文本 (TXT)、Markdown (MD)、HTML、Exce 文档处理完成后,您可以查看和编辑各个分块。这使您可以完全控制内容的组织和搜索方式。 -显示已处理内容的文档分块视图 +显示已处理内容的文档分块视图 ### 分块配置 @@ -65,7 +65,7 @@ Sim 支持 PDF、Word (DOC/DOCX)、纯文本 (TXT)、Markdown (MD)、HTML、Exce 文档处理完成后,您可以通过知识块在 AI 工作流中使用它们。这实现了 RAG(检索增强生成),让您的 AI 智能体能够访问并理解文档内容,从而提供更准确、有上下文的回复。 -在工作流中使用知识块 +在工作流中使用知识块 ### 知识块功能 - **语义搜索**:通过自然语言查询查找相关内容 diff --git a/apps/docs/content/docs/zh/mcp/index.mdx b/apps/docs/content/docs/zh/mcp/index.mdx index b81792cebeb..08b5a61eda0 100644 --- a/apps/docs/content/docs/zh/mcp/index.mdx +++ b/apps/docs/content/docs/zh/mcp/index.mdx @@ -49,7 +49,7 @@ MCP 服务器提供工具集合,供您的代理使用。您可以在工作区 src="/static/blocks/mcp-2.png" alt="在 Agent 模块中使用 MCP 工具" width={700} - height={450} + height={353} className="my-6" /> @@ -68,7 +68,7 @@ MCP 服务器提供工具集合,供您的代理使用。您可以在工作区 src="/static/blocks/mcp-3.png" alt="独立 MCP 工具模块" width={700} - height={450} + height={545} className="my-6" /> diff --git a/apps/docs/content/docs/zh/triggers/index.mdx b/apps/docs/content/docs/zh/triggers/index.mdx index bdbf48cc1b4..e43eed581ec 100644 --- a/apps/docs/content/docs/zh/triggers/index.mdx +++ b/apps/docs/content/docs/zh/triggers/index.mdx @@ -11,7 +11,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/triggers.png" alt="触发器概览" width={500} - height={350} + height={348} className="my-6" /> diff --git a/apps/docs/content/docs/zh/triggers/rss.mdx b/apps/docs/content/docs/zh/triggers/rss.mdx index fd4080ccbd0..7b72e752079 100644 --- a/apps/docs/content/docs/zh/triggers/rss.mdx +++ b/apps/docs/content/docs/zh/triggers/rss.mdx @@ -12,7 +12,7 @@ RSS 订阅源模块监控 RSS 和 Atom 订阅源——当有新内容发布时 src="/static/blocks/rss.png" alt="RSS 订阅源模块" width={500} - height={400} + height={228} className="my-6" /> diff --git a/apps/docs/content/docs/zh/triggers/schedule.mdx b/apps/docs/content/docs/zh/triggers/schedule.mdx index fca74cb4e78..91a1b900e7f 100644 --- a/apps/docs/content/docs/zh/triggers/schedule.mdx +++ b/apps/docs/content/docs/zh/triggers/schedule.mdx @@ -13,7 +13,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/schedule.png" alt="计划块" width={500} - height={400} + height={336} className="my-6" /> @@ -67,7 +67,7 @@ import { Image } from '@/components/ui/image' src="/static/blocks/schedule-3.png" alt="已禁用的计划" width={500} - height={400} + height={310} className="my-6" /> diff --git a/apps/docs/content/docs/zh/triggers/start.mdx b/apps/docs/content/docs/zh/triggers/start.mdx index 237ce39bd64..cc063a483b7 100644 --- a/apps/docs/content/docs/zh/triggers/start.mdx +++ b/apps/docs/content/docs/zh/triggers/start.mdx @@ -13,7 +13,7 @@ import { Image } from '@/components/ui/image' src="/static/start.png" alt="带有输入格式字段的开始模块" width={360} - height={380} + height={151} className="my-6" /> diff --git a/apps/docs/content/docs/zh/triggers/webhook.mdx b/apps/docs/content/docs/zh/triggers/webhook.mdx index c801ff4f0a6..bae1a6f00e2 100644 --- a/apps/docs/content/docs/zh/triggers/webhook.mdx +++ b/apps/docs/content/docs/zh/triggers/webhook.mdx @@ -18,7 +18,7 @@ Webhook 允许外部服务通过向您的工作流发送 HTTP 请求来触发工 src="/static/blocks/webhook-trigger.png" alt="通用 Webhook 配置" width={500} - height={400} + height={405} className="my-6" /> diff --git a/apps/docs/public/static/blocks/mcp-agent-dropdown.png b/apps/docs/public/static/blocks/mcp-agent-dropdown.png index 580286a18b6..06c78b1fa24 100644 Binary files a/apps/docs/public/static/blocks/mcp-agent-dropdown.png and b/apps/docs/public/static/blocks/mcp-agent-dropdown.png differ diff --git a/apps/docs/public/static/blocks/mcp-settings.png b/apps/docs/public/static/blocks/mcp-settings.png index cb4c420814c..a3b2bc06d02 100644 Binary files a/apps/docs/public/static/blocks/mcp-settings.png and b/apps/docs/public/static/blocks/mcp-settings.png differ diff --git a/apps/docs/public/static/blocks/mcp-tool-block.png b/apps/docs/public/static/blocks/mcp-tool-block.png index d4c28cacb0d..ead01d5912e 100644 Binary files a/apps/docs/public/static/blocks/mcp-tool-block.png and b/apps/docs/public/static/blocks/mcp-tool-block.png differ diff --git a/apps/docs/public/static/connectors/connectors-sources.png b/apps/docs/public/static/connectors/connectors-sources.png index 0ac2323ba54..3aa4181a846 100644 Binary files a/apps/docs/public/static/connectors/connectors-sources.png and b/apps/docs/public/static/connectors/connectors-sources.png differ diff --git a/apps/docs/public/static/credentials/integrations-service-account.png b/apps/docs/public/static/credentials/integrations-service-account.png index 83f232afea3..0e53ab4ff80 100644 Binary files a/apps/docs/public/static/credentials/integrations-service-account.png and b/apps/docs/public/static/credentials/integrations-service-account.png differ diff --git a/apps/docs/public/static/credentials/workflow-impersonated-account.png b/apps/docs/public/static/credentials/workflow-impersonated-account.png index 12a23fc56e8..9b203ed2cce 100644 Binary files a/apps/docs/public/static/credentials/workflow-impersonated-account.png and b/apps/docs/public/static/credentials/workflow-impersonated-account.png differ diff --git a/apps/docs/public/static/enterprise/usage-tracking-overview.png b/apps/docs/public/static/enterprise/usage-tracking-overview.png new file mode 100644 index 00000000000..64c6d5eb9fc Binary files /dev/null and b/apps/docs/public/static/enterprise/usage-tracking-overview.png differ diff --git a/apps/docs/public/static/enterprise/usage-tracking-workspace-detail.png b/apps/docs/public/static/enterprise/usage-tracking-workspace-detail.png new file mode 100644 index 00000000000..70003a33efb Binary files /dev/null and b/apps/docs/public/static/enterprise/usage-tracking-workspace-detail.png differ diff --git a/apps/docs/public/static/integrations/hubspot/hubspot-page-add-to-sim.png b/apps/docs/public/static/integrations/hubspot/hubspot-page-add-to-sim.png index 4e3c0cabd36..c09d76e929c 100644 Binary files a/apps/docs/public/static/integrations/hubspot/hubspot-page-add-to-sim.png and b/apps/docs/public/static/integrations/hubspot/hubspot-page-add-to-sim.png differ diff --git a/apps/docs/public/static/integrations/hubspot/integrations-page.png b/apps/docs/public/static/integrations/hubspot/integrations-page.png index ed9d8e828fc..e153084e312 100644 Binary files a/apps/docs/public/static/integrations/hubspot/integrations-page.png and b/apps/docs/public/static/integrations/hubspot/integrations-page.png differ diff --git a/apps/docs/public/static/introduction.png b/apps/docs/public/static/introduction.png index 989d72d6688..99f3c97e1bb 100644 Binary files a/apps/docs/public/static/introduction.png and b/apps/docs/public/static/introduction.png differ diff --git a/apps/docs/public/static/logs/console.png b/apps/docs/public/static/logs/console.png index 46a8415c3c3..bea125e5724 100644 Binary files a/apps/docs/public/static/logs/console.png and b/apps/docs/public/static/logs/console.png differ diff --git a/apps/docs/public/static/logs/log-trace.png b/apps/docs/public/static/logs/log-trace.png index d2d286fac17..af54cd6e613 100644 Binary files a/apps/docs/public/static/logs/log-trace.png and b/apps/docs/public/static/logs/log-trace.png differ diff --git a/apps/docs/public/static/logs/logs-sidebar.png b/apps/docs/public/static/logs/logs-sidebar.png index 3316625277c..0f98a5d4069 100644 Binary files a/apps/docs/public/static/logs/logs-sidebar.png and b/apps/docs/public/static/logs/logs-sidebar.png differ diff --git a/apps/docs/public/static/logs/logs.png b/apps/docs/public/static/logs/logs.png index 9d0629a30eb..25c96cae066 100644 Binary files a/apps/docs/public/static/logs/logs.png and b/apps/docs/public/static/logs/logs.png differ diff --git a/apps/docs/public/static/logs/workspace-logs.png b/apps/docs/public/static/logs/workspace-logs.png index 4605b80e8d6..25c96cae066 100644 Binary files a/apps/docs/public/static/logs/workspace-logs.png and b/apps/docs/public/static/logs/workspace-logs.png differ diff --git a/apps/docs/public/static/quick-reference/run-workflow.png b/apps/docs/public/static/quick-reference/run-workflow.png index 4175cde99dc..cb62983b3ce 100644 Binary files a/apps/docs/public/static/quick-reference/run-workflow.png and b/apps/docs/public/static/quick-reference/run-workflow.png differ diff --git a/apps/docs/public/static/quick-reference/terminal.png b/apps/docs/public/static/quick-reference/terminal.png index 5395824fbea..bea125e5724 100644 Binary files a/apps/docs/public/static/quick-reference/terminal.png and b/apps/docs/public/static/quick-reference/terminal.png differ diff --git a/apps/docs/public/static/quick-reference/test-chat.png b/apps/docs/public/static/quick-reference/test-chat.png index e64064676c4..46c28011a9e 100644 Binary files a/apps/docs/public/static/quick-reference/test-chat.png and b/apps/docs/public/static/quick-reference/test-chat.png differ diff --git a/apps/docs/public/static/quick-reference/update-deployment.png b/apps/docs/public/static/quick-reference/update-deployment.png index 4f9639e7147..0b91c44a645 100644 Binary files a/apps/docs/public/static/quick-reference/update-deployment.png and b/apps/docs/public/static/quick-reference/update-deployment.png differ diff --git a/apps/docs/public/static/skills/add-skill-create.png b/apps/docs/public/static/skills/add-skill-create.png index db7323bcf9e..5c5027b56c8 100644 Binary files a/apps/docs/public/static/skills/add-skill-create.png and b/apps/docs/public/static/skills/add-skill-create.png differ diff --git a/apps/docs/public/static/skills/manage-skills.png b/apps/docs/public/static/skills/manage-skills.png index c54762ce80a..5ff4d8fb7de 100644 Binary files a/apps/docs/public/static/skills/manage-skills.png and b/apps/docs/public/static/skills/manage-skills.png differ diff --git a/apps/docs/public/static/skills/skills-tab.png b/apps/docs/public/static/skills/skills-tab.png index 3ff2b8c1ebb..5ff4d8fb7de 100644 Binary files a/apps/docs/public/static/skills/skills-tab.png and b/apps/docs/public/static/skills/skills-tab.png differ diff --git a/apps/docs/public/static/tables/tables-overview.png b/apps/docs/public/static/tables/tables-overview.png index 1c622ae253e..c2167c169b3 100644 Binary files a/apps/docs/public/static/tables/tables-overview.png and b/apps/docs/public/static/tables/tables-overview.png differ diff --git a/apps/sim/app/api/audit-logs/export/route.test.ts b/apps/sim/app/api/audit-logs/export/route.test.ts index 6f177ab7b9f..7c0667a9de7 100644 --- a/apps/sim/app/api/audit-logs/export/route.test.ts +++ b/apps/sim/app/api/audit-logs/export/route.test.ts @@ -164,4 +164,29 @@ describe('GET /api/audit-logs/export', () => { expect(response.status).toBe(400) expect(mockQueryAuditLogs).not.toHaveBeenCalled() }) + + /** + * The export has to filter by everything the on-screen feed does. It did not + * forward `workspaceId`, so an admin exporting from a workspace-scoped feed + * downloaded the whole organization — silently, because every field of + * `AuditLogFilterParams` is optional and dropping one still type-checks. + */ + it('forwards the workspace filter the on-screen feed applies', async () => { + mockGetOrgWorkspaceIds.mockResolvedValue(['workspace-1']) + + await GET(makeRequest('?workspaceId=workspace-1')) + + expect(mockBuildFilterConditions).toHaveBeenCalledWith( + expect.objectContaining({ workspaceId: 'workspace-1' }) + ) + }) + + it('rejects a workspaceId outside the organization, as the list route does', async () => { + mockGetOrgWorkspaceIds.mockResolvedValue(['workspace-1']) + + const response = await GET(makeRequest('?workspaceId=workspace-elsewhere')) + + expect(response.status).toBe(400) + expect(mockQueryAuditLogs).not.toHaveBeenCalled() + }) }) diff --git a/apps/sim/app/api/audit-logs/export/route.ts b/apps/sim/app/api/audit-logs/export/route.ts index c4860a85092..bcf02a37d72 100644 --- a/apps/sim/app/api/audit-logs/export/route.ts +++ b/apps/sim/app/api/audit-logs/export/route.ts @@ -66,8 +66,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { } const { organizationId, orgMemberIds } = authResult.context - const { search, action, resourceType, actorId, startDate, endDate, includeDeparted } = - parsed.data.query + const { actorId, workspaceId, includeDeparted } = parsed.data.query if (actorId && !orgMemberIds.includes(actorId)) { return NextResponse.json( @@ -77,20 +76,34 @@ export const GET = withRouteHandler(async (request: NextRequest) => { } const orgWorkspaceIds = await getOrgWorkspaceIds(organizationId) + /** + * The same refusal `listAuditLogs` gives. The scope predicate already makes an + * out-of-organization id return nothing, but an empty CSV and a 400 that names the + * problem are very different answers to the same bad request, and the two paths + * disagreeing about which one you get is what an audit trail cannot afford. + */ + if (workspaceId && !orgWorkspaceIds.includes(workspaceId)) { + return NextResponse.json( + { error: 'workspaceId does not belong to your organization' }, + { status: 400 } + ) + } const scopeCondition = buildOrgScopeCondition({ organizationId, orgWorkspaceIds, orgMemberIds, includeDeparted, }) - const filterConditions = buildFilterConditions({ - action, - resourceType, - actorId, - search, - startDate, - endDate, - }) + /** + * The whole parsed query, not a hand-listed subset. + * + * Every field of `AuditLogFilterParams` is optional, so dropping one type-checks + * silently — which is how `workspaceId` came to be accepted by the contract, + * honoured by the list route, and ignored here: an admin looking at one + * workspace's feed downloaded the entire organization's, under a truncation + * warning that blamed the date range. + */ + const filterConditions = buildFilterConditions(parsed.data.query) const conditions = [scopeCondition, ...filterConditions] const rows: ReturnType[] = [] diff --git a/apps/sim/app/api/audit-logs/route.ts b/apps/sim/app/api/audit-logs/route.ts index f2cc96a2b3f..f63e99575c2 100644 --- a/apps/sim/app/api/audit-logs/route.ts +++ b/apps/sim/app/api/audit-logs/route.ts @@ -27,6 +27,7 @@ export const GET = defineInternalJsonRoute({ action: query.action, resourceType: query.resourceType, actorId: query.actorId, + workspaceId: query.workspaceId, startDate: query.startDate, endDate: query.endDate, }, diff --git a/apps/sim/app/api/copilot/tools/execute/route.test.ts b/apps/sim/app/api/copilot/tools/execute/route.test.ts index 87e7d1e88f2..716fbeb9ffd 100644 --- a/apps/sim/app/api/copilot/tools/execute/route.test.ts +++ b/apps/sim/app/api/copilot/tools/execute/route.test.ts @@ -95,12 +95,26 @@ describe('POST /api/copilot/tools/execute (in-band)', () => { expect(body.error).toBe('File not found: files/a.md') }) - it('withholds results when no egress registry can be built', async () => { - mockPrepareEnvironmentContext.mockRejectedValue(new Error('env unavailable')) + /** + * Running the tool without a catalog used to produce the worst pair of outcomes available: + * the side effect happened and the caller got a bare `{success: true}` naming neither the + * cause nor whether anything had changed. + */ + it('refuses the call, without running the tool, when no egress registry can be built', async () => { + mockPrepareEnvironmentContext.mockRejectedValue(new Error('Workspace ws-gone does not exist')) mockHandler.mockResolvedValue({ success: true, output: { content: 'sensitive' } }) + const res = await POST(makeRequest({ ...BASE_BODY, messageId: 'msg-no-registry' }) as never) const body = await res.json() - expect(body).toEqual({ success: true }) + + expect(mockHandler).not.toHaveBeenCalled() + expect(body.success).toBe(false) + expect(body.output).toEqual({ resultWithheld: true, effect: 'not_attempted' }) + // The thrown reason is an unprojectable environment failure — the catalog that would + // vouch for it is the very thing missing — so it stays in the log. + expect(body.error).not.toContain('does not exist') + expect(body.error).toContain(BASE_BODY.workspaceId) + expect(body.error).toContain('could not be resolved') }) it('reuses one turn registry across calls that share a messageId', async () => { diff --git a/apps/sim/app/api/copilot/tools/execute/route.ts b/apps/sim/app/api/copilot/tools/execute/route.ts index 9da0848f6ca..f645a5cb522 100644 --- a/apps/sim/app/api/copilot/tools/execute/route.ts +++ b/apps/sim/app/api/copilot/tools/execute/route.ts @@ -4,11 +4,13 @@ import { type NextRequest, NextResponse } from 'next/server' import { copilotToolExecuteInternalBodySchema } from '@/lib/api/contracts/copilot' import { validationErrorResponse } from '@/lib/api/server' import { prepareCopilotEnvironmentContext } from '@/lib/copilot/environment-context' +import { MothershipStreamV1ToolOutcome } from '@/lib/copilot/generated/mothership-stream-v1' import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1' import { TraceSpan } from '@/lib/copilot/generated/trace-spans-v1' import { checkInternalApiKey } from '@/lib/copilot/request/http' import { withIncomingGoSpan } from '@/lib/copilot/request/otel' import { + describeWithholdingCause, inspectToolResultForCopilot, projectToolErrorMessageForCopilot, } from '@/lib/copilot/request/tools/resolved-secret-result' @@ -16,6 +18,7 @@ import { handleResourceSideEffects } from '@/lib/copilot/request/tools/resources import type { ToolCallResult } from '@/lib/copilot/request/types' import { ensureHandlersRegistered } from '@/lib/copilot/tool-executor' import { executeTool } from '@/lib/copilot/tool-executor/executor' +import { TOOL_EFFECT_PHASE } from '@/lib/copilot/tool-executor/types' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' @@ -115,17 +118,43 @@ export const POST = withRouteHandler((request: NextRequest) => [TraceAttr.UserId]: userId, }) - let toolRegistry: ResolvedSecretTraceRegistry | undefined - let turnRegistry: ResolvedSecretTraceRegistry | undefined + let toolRegistry: ResolvedSecretTraceRegistry + let turnRegistry: ResolvedSecretTraceRegistry try { turnRegistry = await getTurnEgressRegistry(userId, workspaceId, messageId) toolRegistry = turnRegistry.forkForInputPaths([]) } catch (err) { - logger.error('In-band egress registry unavailable; results will be withheld', { + /** + * Without a catalog the projection can vouch for nothing, so every result this call + * could produce would be withheld. Running the tool anyway was the worst of both + * outcomes: the side effect happened and the caller got an opaque sentinel that named + * neither the cause nor whether anything had changed. Refusing before dispatch is + * both truthful and the only answer that leaves nothing behind. + * + * The cause is almost always the workspace itself — a deleted or inaccessible id + * reaching this lane — which is actionable, so it is reported rather than swallowed. + */ + logger.error('In-band egress registry unavailable; refusing the call', { toolName, toolCallId, + userId, + workspaceId, error: getErrorMessage(err), }) + rootSpan.setAttributes({ [TraceAttr.ToolOutcome]: MothershipStreamV1ToolOutcome.error }) + /** + * The thrown reason stays in the log. It is an environment or database failure that + * nothing here can project — the catalog it needed is the very thing that is missing — + * so this is the one message on this route that must be fixed text. The workspace id + * is echoed because the caller supplied it, and it is what makes this actionable. + */ + return NextResponse.json({ + success: false, + error: workspaceId + ? `${toolName} was not run: its workspace (${workspaceId}) could not be resolved. Check that the workspace exists and is accessible before retrying.` + : `${toolName} was not run: its execution environment could not be resolved.`, + output: { resultWithheld: true, effect: TOOL_EFFECT_PHASE.notAttempted }, + }) } try { @@ -148,9 +177,23 @@ export const POST = withRouteHandler((request: NextRequest) => }) const projection = inspectToolResultForCopilot(result, toolRegistry, toolName) const projected = projection.result - if (projection.safe && toolRegistry?.isComplete() && turnRegistry) { + if (projection.safe && toolRegistry.isComplete()) { turnRegistry.mergeToolCallRegistry(toolRegistry) } + if (!projection.safe) { + /** + * Reported on its own rather than folded into the failure branch below: a withheld + * SUCCESS keeps `projected.success` true, so gating on failure meant the one case + * that leaves no other trace — the model reads a bare success — was also the one + * case whose cause was never written down. + */ + logger.warn('In-band tool result withheld by egress projection', { + toolName, + toolCallId, + runtimeSucceeded: result.success, + ...describeWithholdingCause(projection.cause), + }) + } if (!projected.success) { logger.warn('In-band tool execution failed', { toolName, diff --git a/apps/sim/app/api/emails/preview/route.ts b/apps/sim/app/api/emails/preview/route.ts index 63cfb4994dd..d880aaf1dae 100644 --- a/apps/sim/app/api/emails/preview/route.ts +++ b/apps/sim/app/api/emails/preview/route.ts @@ -160,7 +160,7 @@ const emailTemplates = { scope: 'organization', currentUsage: 500, limit: 500, - ctaLink: 'https://sim.ai/organization/org_123/settings/billing', + ctaLink: 'https://sim.ai/workspace/ws_123/settings/billing', }), // Operational notification emails diff --git a/apps/sim/app/api/files/parse/route.test.ts b/apps/sim/app/api/files/parse/route.test.ts deleted file mode 100644 index 26c01caa2f5..00000000000 --- a/apps/sim/app/api/files/parse/route.test.ts +++ /dev/null @@ -1,962 +0,0 @@ -/** - * Tests for file parse API route - * - * @vitest-environment node - */ -import { - authMockFns, - createMockRequest, - hybridAuthMockFns, - inputValidationMock, - inputValidationMockFns, - permissionsMock, - permissionsMockFns, - storageServiceMock, - storageServiceMockFns, -} from '@sim/testing' -import { NextRequest } from 'next/server' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { FileParserError } from '@/lib/file-parsers/errors' - -const { - mockVerifyFileAccess, - mockVerifyWorkspaceFileAccess, - mockGetStorageProvider, - mockIsUsingCloudStorage, - mockIsSupportedFileType, - mockParseFile, - mockParseBuffer, - mockFsAccess, - mockFsStat, - mockFsReadFile, - mockFsWriteFile, - mockJoin, - actualPath, - mockUploadWorkspaceFile, -} = vi.hoisted(() => { - // eslint-disable-next-line @typescript-eslint/no-require-imports - const actualPath = require('path') as typeof import('path') - return { - mockVerifyFileAccess: vi.fn().mockResolvedValue(true), - mockVerifyWorkspaceFileAccess: vi.fn().mockResolvedValue(true), - mockGetStorageProvider: vi.fn().mockReturnValue('s3'), - mockIsUsingCloudStorage: vi.fn().mockReturnValue(true), - mockIsSupportedFileType: vi.fn().mockReturnValue(true), - mockParseFile: vi.fn().mockResolvedValue({ - content: 'parsed content', - metadata: { pageCount: 1 }, - }), - mockParseBuffer: vi.fn().mockResolvedValue({ - content: 'parsed buffer content', - metadata: { pageCount: 1 }, - }), - mockFsAccess: vi.fn().mockResolvedValue(undefined), - mockFsStat: vi.fn().mockImplementation(() => ({ isFile: () => true, size: 17 })), - mockFsReadFile: vi.fn().mockResolvedValue(Buffer.from('test file content')), - mockFsWriteFile: vi.fn().mockResolvedValue(undefined), - mockJoin: vi.fn((...args: string[]): string => { - if (args[0] === '/test/uploads') { - return `/test/uploads/${args[args.length - 1]}` - } - return actualPath.join(...args) - }), - actualPath, - mockUploadWorkspaceFile: vi - .fn() - .mockImplementation( - async (workspaceId: string, _userId: string, _buffer: Buffer, fileName: string) => ({ - id: 'wf_test', - name: fileName, - size: 0, - type: 'application/octet-stream', - url: `/api/files/serve/${workspaceId}/${fileName}`, - key: `${workspaceId}/${fileName}`, - context: 'workspace', - }) - ), - } -}) - -vi.mock('@/app/api/files/authorization', () => ({ - verifyFileAccess: mockVerifyFileAccess, - verifyWorkspaceFileAccess: mockVerifyWorkspaceFileAccess, -})) - -vi.mock('@/lib/uploads', () => ({ - getStorageProvider: mockGetStorageProvider, - isUsingCloudStorage: mockIsUsingCloudStorage, - StorageService: storageServiceMock, -})) - -vi.mock('@/lib/file-parsers', () => ({ - isSupportedFileType: mockIsSupportedFileType, - parseFile: mockParseFile, - parseBuffer: mockParseBuffer, -})) - -vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) - -vi.mock('path', () => ({ - default: actualPath, - ...actualPath, - join: mockJoin, - basename: actualPath.basename, - extname: actualPath.extname, -})) - -vi.mock('@/lib/uploads/core/setup.server', () => ({ - UPLOAD_DIR_SERVER: '/test/uploads', -})) - -vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) - -vi.mock('@/lib/core/utils/logging', () => ({ - sanitizeUrlForLog: vi.fn((url: string) => url), -})) - -vi.mock('@/lib/uploads/contexts/execution', () => ({ - uploadExecutionFile: vi.fn(), -})) - -vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ - uploadWorkspaceFile: mockUploadWorkspaceFile, -})) - -vi.mock('@/lib/uploads/server/metadata', () => ({ - getFileMetadataByKey: vi.fn(), -})) - -vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock) - -vi.mock('fs/promises', () => ({ - default: { - access: mockFsAccess, - stat: mockFsStat, - readFile: mockFsReadFile, - writeFile: mockFsWriteFile, - }, - access: mockFsAccess, - stat: mockFsStat, - readFile: mockFsReadFile, - writeFile: mockFsWriteFile, -})) - -import { POST } from '@/app/api/files/parse/route' - -function setupFileApiMocks( - options: { - authenticated?: boolean - storageProvider?: 's3' | 'blob' | 'local' - cloudEnabled?: boolean - } = {} -) { - const { authenticated = true, storageProvider = 's3', cloudEnabled = true } = options - - if (authenticated) { - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'test-user-id', email: 'test@example.com' }, - }) - } else { - authMockFns.mockGetSession.mockResolvedValue(null) - } - - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: authenticated, - userId: authenticated ? 'test-user-id' : undefined, - error: authenticated ? undefined : 'Unauthorized', - }) - - hybridAuthMockFns.mockCheckHybridAuth.mockResolvedValue({ - success: authenticated, - userId: authenticated ? 'test-user-id' : undefined, - error: authenticated ? undefined : 'Unauthorized', - }) - - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: authenticated, - userId: authenticated ? 'test-user-id' : undefined, - error: authenticated ? undefined : 'Unauthorized', - }) - - mockGetStorageProvider.mockReturnValue(storageProvider) - mockIsUsingCloudStorage.mockReturnValue(cloudEnabled) -} - -describe('File Parse API Route', () => { - beforeEach(() => { - vi.clearAllMocks() - - setupFileApiMocks({ - authenticated: true, - }) - - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue({ canView: true }) - storageServiceMockFns.mockHasCloudStorage.mockReturnValue(true) - storageServiceMockFns.mockDownloadFile.mockResolvedValue(Buffer.from('test file content')) - mockFsStat.mockResolvedValue({ isFile: () => true, size: 17 }) - mockFsReadFile.mockResolvedValue(Buffer.from('test file content')) - mockIsSupportedFileType.mockReturnValue(true) - mockUploadWorkspaceFile.mockClear() - mockParseFile.mockResolvedValue({ - content: 'parsed content', - metadata: { pageCount: 1 }, - }) - mockParseBuffer.mockResolvedValue({ - content: 'parsed buffer content', - metadata: { pageCount: 1 }, - }) - }) - - afterEach(() => { - vi.clearAllMocks() - }) - - it('should handle missing file path', async () => { - const req = createMockRequest('POST', {}) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data).toHaveProperty('error', 'No file path provided') - }) - - it('should accept and process a local file', async () => { - setupFileApiMocks({ - cloudEnabled: false, - storageProvider: 'local', - authenticated: true, - }) - - const req = createMockRequest('POST', { - filePath: '/api/files/serve/test-file.txt', - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data).not.toBeNull() - - if (data.success === true) { - expect(data).toHaveProperty('output') - } else { - expect(data).toHaveProperty('error') - expect(typeof data.error).toBe('string') - } - }) - - it('should process S3 files', async () => { - setupFileApiMocks({ - cloudEnabled: true, - storageProvider: 's3', - authenticated: true, - }) - - const req = createMockRequest('POST', { - filePath: '/api/files/serve/s3/test-file.pdf', - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - - if (data.success === true) { - expect(data).toHaveProperty('output') - } else { - expect(data).toHaveProperty('error') - } - }) - - it('should keep known binary extensions as binary even when the bytes are valid UTF-8', async () => { - setupFileApiMocks({ - cloudEnabled: true, - storageProvider: 's3', - authenticated: true, - }) - mockIsSupportedFileType.mockReturnValue(false) - storageServiceMockFns.mockDownloadFile.mockResolvedValue(Buffer.from('valid utf8 bytes')) - - const req = createMockRequest('POST', { - filePath: '/api/files/serve/execution/workspace-1/workflow-1/execution-1/image.png', - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.output.content).toBe('[Binary PNG file - 16 bytes]') - }) - - it('should parse unknown extensions as text when the bytes look like UTF-8 text', async () => { - setupFileApiMocks({ - cloudEnabled: true, - storageProvider: 's3', - authenticated: true, - }) - mockIsSupportedFileType.mockReturnValue(false) - storageServiceMockFns.mockDownloadFile.mockResolvedValue(Buffer.from('plain text content')) - - const req = createMockRequest('POST', { - filePath: '/api/files/serve/execution/workspace-1/workflow-1/execution-1/readme.customtext', - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.output.content).toBe('plain text content') - }) - - it('should reject parser complexity limits instead of returning raw text', async () => { - setupFileApiMocks({ - cloudEnabled: true, - storageProvider: 's3', - authenticated: true, - }) - storageServiceMockFns.mockDownloadFile.mockResolvedValue(Buffer.from('{"value":true}')) - mockParseBuffer.mockRejectedValueOnce( - new FileParserError('complexity_limit', 'JSON document exceeds the complexity limit') - ) - - const req = createMockRequest('POST', { - filePath: '/api/files/serve/execution/workspace-1/workflow-1/execution-1/data.json', - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(false) - expect(data.error).toContain('complexity limit') - expect(data).not.toHaveProperty('output') - }) - - it('should handle multiple files', async () => { - setupFileApiMocks({ - cloudEnabled: false, - storageProvider: 'local', - authenticated: true, - }) - - const req = createMockRequest('POST', { - filePath: ['/api/files/serve/file1.txt', '/api/files/serve/file2.txt'], - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data).toHaveProperty('success') - expect(data).toHaveProperty('results') - expect(Array.isArray(data.results)).toBe(true) - expect(data.results).toHaveLength(2) - }) - - it('should keep the multi-file download cap independent from the remaining parsed-output cap', async () => { - inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ - isValid: true, - resolvedIP: '203.0.113.10', - }) - inputValidationMockFns.mockSecureFetchWithPinnedIP - .mockResolvedValueOnce( - new Response('file content', { - status: 200, - headers: { 'content-type': 'text/plain' }, - }) - ) - .mockResolvedValueOnce( - new Response('second file content', { - status: 200, - headers: { - 'content-length': String(20 * 1024 * 1024), - 'content-type': 'text/plain', - }, - }) - ) - - const fourMbContent = 'a'.repeat(4 * 1024 * 1024) - mockParseBuffer - .mockResolvedValueOnce({ - content: fourMbContent, - metadata: { pageCount: 1 }, - }) - .mockResolvedValueOnce({ - content: 'second file', - metadata: { pageCount: 1 }, - }) - - const req = createMockRequest('POST', { - filePath: ['https://example.com/file1.txt', 'https://example.com/file2.txt'], - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.results).toHaveLength(2) - expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenNthCalledWith( - 1, - 'https://example.com/file1.txt', - '203.0.113.10', - expect.objectContaining({ maxResponseBytes: 100 * 1024 * 1024 }) - ) - expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenNthCalledWith( - 2, - 'https://example.com/file2.txt', - '203.0.113.10', - expect.objectContaining({ maxResponseBytes: 100 * 1024 * 1024 }) - ) - }) - - it('should never dedup external URL fetches by path filename — two URLs sharing image.png both download', async () => { - inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ - isValid: true, - resolvedIP: '203.0.113.10', - }) - inputValidationMockFns.mockSecureFetchWithPinnedIP - .mockResolvedValueOnce( - new Response('first image bytes', { - status: 200, - headers: { 'content-type': 'image/png' }, - }) - ) - .mockResolvedValueOnce( - new Response('second image bytes — different content', { - status: 200, - headers: { 'content-type': 'image/png' }, - }) - ) - mockIsSupportedFileType.mockReturnValue(false) - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('write') - - const req = createMockRequest('POST', { - filePath: [ - 'https://files.slack.com/files-pri/T07-FAAA/download/image.png', - 'https://files.slack.com/files-pri/T07-FBBB/download/image.png', - ], - workspaceId: 'workspace-id', - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.results).toHaveLength(2) - expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenCalledTimes(2) - expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenNthCalledWith( - 1, - 'https://files.slack.com/files-pri/T07-FAAA/download/image.png', - '203.0.113.10', - expect.any(Object) - ) - expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenNthCalledWith( - 2, - 'https://files.slack.com/files-pri/T07-FBBB/download/image.png', - '203.0.113.10', - expect.any(Object) - ) - expect(mockUploadWorkspaceFile).toHaveBeenCalledTimes(2) - expect(storageServiceMockFns.mockDownloadFile).not.toHaveBeenCalled() - }) - - it('should stop multi-file parsing once the combined parsed output is too large', async () => { - inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ - isValid: true, - resolvedIP: '203.0.113.10', - }) - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValue( - new Response('file content', { - status: 200, - headers: { 'content-type': 'text/plain' }, - }) - ) - - mockParseBuffer.mockResolvedValueOnce({ - content: 'a'.repeat(5 * 1024 * 1024 + 1), - metadata: { pageCount: 1 }, - }) - - const req = createMockRequest('POST', { - filePath: ['https://example.com/file1.txt', 'https://example.com/file2.txt'], - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(413) - expect(data.success).toBe(false) - expect(data.error).toContain('too large') - expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenCalledTimes(1) - }) - - it('should include successful multi-file parse results when a later file exceeds the cap', async () => { - inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ - isValid: true, - resolvedIP: '203.0.113.10', - }) - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValue( - new Response('file content', { - status: 200, - headers: { 'content-type': 'text/plain' }, - }) - ) - - mockParseBuffer - .mockResolvedValueOnce({ - content: 'first file', - metadata: { pageCount: 1 }, - }) - .mockResolvedValueOnce({ - content: 'a'.repeat(5 * 1024 * 1024), - metadata: { pageCount: 1 }, - }) - - const req = createMockRequest('POST', { - filePath: ['https://example.com/file1.txt', 'https://example.com/file2.txt'], - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.error).toContain('too large') - expect(data.results).toHaveLength(1) - expect(data.results[0].output.content).toBe('first file') - expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenCalledTimes(2) - }) - - it('should pass custom headers when fetching external URLs', async () => { - inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ - isValid: true, - resolvedIP: '203.0.113.10', - }) - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValue( - new Response('private file content', { - status: 200, - headers: { 'content-type': 'text/plain' }, - }) - ) - - const headers = { Authorization: 'Bearer xoxb-test-token' } - const req = createMockRequest('POST', { - filePath: 'https://files.slack.com/files-pri/T000-F000/download/report.txt', - headers, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenCalledWith( - 'https://files.slack.com/files-pri/T000-F000/download/report.txt', - '203.0.113.10', - expect.objectContaining({ - timeout: 30000, - headers, - }) - ) - }) - - it('should reject oversized external downloads before reading the body', async () => { - inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ - isValid: true, - resolvedIP: '203.0.113.10', - }) - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValue( - new Response('oversized', { - status: 200, - headers: { 'content-length': '104857601', 'content-type': 'text/plain' }, - }) - ) - - const req = createMockRequest('POST', { - filePath: 'https://example.com/large.txt', - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(false) - expect(data.error).toContain('too large') - expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenCalledWith( - 'https://example.com/large.txt', - '203.0.113.10', - expect.objectContaining({ - maxResponseBytes: 104857600, - }) - ) - }) - - it('should reject oversized local files before materializing them', async () => { - setupFileApiMocks({ - cloudEnabled: false, - storageProvider: 'local', - authenticated: true, - }) - mockFsStat.mockResolvedValue({ isFile: () => true, size: 104857601 }) - - const req = createMockRequest('POST', { - filePath: 'workspace/large.txt', - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(false) - expect(data.error).toContain('too large') - expect(mockFsReadFile).not.toHaveBeenCalled() - }) - - it('should process execution file URLs with context query param', async () => { - setupFileApiMocks({ - cloudEnabled: true, - storageProvider: 's3', - authenticated: true, - }) - - const req = createMockRequest('POST', { - filePath: - '/api/files/serve/s3/6vzIweweXAS1pJ1mMSrr9Flh6paJpHAx/79dac297-5ebb-410b-b135-cc594dfcb361/c36afbb0-af50-42b0-9b23-5dae2d9384e8/Confirmation.pdf?context=execution', - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - - if (data.success === true) { - expect(data).toHaveProperty('output') - } else { - expect(data).toHaveProperty('error') - } - }) - - it('should process workspace file URLs with context query param', async () => { - setupFileApiMocks({ - cloudEnabled: true, - storageProvider: 's3', - authenticated: true, - }) - - const req = createMockRequest('POST', { - filePath: - '/api/files/serve/s3/fa8e96e6-7482-4e3c-a0e8-ea083b28af55-be56ca4f-83c2-4559-a6a4-e25eb4ab8ee2_1761691045516-1ie5q86-Confirmation.pdf?context=workspace', - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - - if (data.success === true) { - expect(data).toHaveProperty('output') - } else { - expect(data).toHaveProperty('error') - } - }) - - it('should handle S3 access errors gracefully', async () => { - setupFileApiMocks({ - cloudEnabled: true, - storageProvider: 's3', - authenticated: true, - }) - - storageServiceMockFns.mockDownloadFile.mockRejectedValue(new Error('Access denied')) - storageServiceMockFns.mockHasCloudStorage.mockReturnValue(true) - - const req = new NextRequest('http://localhost:3000/api/files/parse', { - method: 'POST', - body: JSON.stringify({ - filePath: '/api/files/serve/s3/test-file.txt', - }), - }) - - const response = await POST(req) - const data = await response.json() - - expect(data).toBeDefined() - expect(typeof data).toBe('object') - }) - - it('should handle access errors gracefully', async () => { - setupFileApiMocks({ - cloudEnabled: false, - storageProvider: 'local', - authenticated: true, - }) - - mockFsAccess.mockRejectedValue(new Error('ENOENT: no such file')) - - const req = createMockRequest('POST', { - filePath: 'nonexistent.txt', - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data).toHaveProperty('success') - expect(data).toHaveProperty('error') - }) -}) - -describe('Files Parse API - Path Traversal Security', () => { - beforeEach(() => { - vi.clearAllMocks() - setupFileApiMocks({ - authenticated: true, - }) - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue({ canView: true }) - }) - - describe('Path Traversal Prevention', () => { - it('should reject path traversal attempts with .. segments', async () => { - const maliciousRequests = [ - '../../../etc/passwd', - '/api/files/serve/../../../etc/passwd', - '/api/files/serve/../../app.js', - '/api/files/serve/../.env', - 'uploads/../../../etc/hosts', - ] - - for (const maliciousPath of maliciousRequests) { - const request = new NextRequest('http://localhost:3000/api/files/parse', { - method: 'POST', - body: JSON.stringify({ - filePath: maliciousPath, - }), - }) - - const response = await POST(request) - const result = await response.json() - - expect(result.success).toBe(false) - expect(result.error).toMatch( - /Access denied|Invalid path|Path outside allowed directory|Unauthorized/ - ) - } - }) - - it('should reject paths with tilde characters', async () => { - const maliciousPaths = [ - '~/../../etc/passwd', - '/api/files/serve/~/secret.txt', - '~root/.ssh/id_rsa', - ] - - for (const maliciousPath of maliciousPaths) { - const request = new NextRequest('http://localhost:3000/api/files/parse', { - method: 'POST', - body: JSON.stringify({ - filePath: maliciousPath, - }), - }) - - const response = await POST(request) - const result = await response.json() - - expect(result.success).toBe(false) - expect(result.error).toMatch(/Access denied|Invalid path|Unauthorized/) - } - }) - - it('should reject absolute paths outside upload directory', async () => { - const maliciousPaths = [ - '/etc/passwd', - '/root/.bashrc', - '/app/.env', - '/var/log/auth.log', - 'C:\\Windows\\System32\\drivers\\etc\\hosts', - ] - - for (const maliciousPath of maliciousPaths) { - const request = new NextRequest('http://localhost:3000/api/files/parse', { - method: 'POST', - body: JSON.stringify({ - filePath: maliciousPath, - }), - }) - - const response = await POST(request) - const result = await response.json() - - expect(result.success).toBe(false) - expect(result.error).toMatch(/Access denied|Path outside allowed directory|Unauthorized/) - } - }) - - it('should allow valid paths within upload directory', async () => { - const validPaths = [ - '/api/files/serve/document.txt', - '/api/files/serve/folder/file.pdf', - '/api/files/serve/subfolder/image.png', - ] - - for (const validPath of validPaths) { - const request = new NextRequest('http://localhost:3000/api/files/parse', { - method: 'POST', - body: JSON.stringify({ - filePath: validPath, - }), - }) - - const response = await POST(request) - const result = await response.json() - - if (result.error) { - expect(result.error).not.toMatch( - /Access denied|Path outside allowed directory|Invalid path/ - ) - } - } - }) - - it('should not treat .. inside external URLs as path traversal', async () => { - inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ - isValid: true, - resolvedIP: '203.0.113.10', - }) - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValue( - new Response('slack file content', { - status: 200, - headers: { 'content-type': 'text/plain' }, - }) - ) - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('write') - - // Slack truncates long titles with a literal ellipsis, so the slug contains `..` - const slackUrl = - 'https://files.slack.com/files-pri/T08-F0B/_other__no_invitation_messages_get_sent_-_sim_on_railway...txt' - - const request = new NextRequest('http://localhost:3000/api/files/parse', { - method: 'POST', - body: JSON.stringify({ filePath: slackUrl, workspaceId: 'workspace-id' }), - }) - - const response = await POST(request) - const result = await response.json() - - expect(result.success).toBe(true) - // The URL reaching the pinned fetch proves it passed validation and routed - // to external-URL handling rather than being rejected as a local path. - expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenCalledWith( - slackUrl, - '203.0.113.10', - expect.any(Object) - ) - }) - - it('should still reject traversal in https URLs that look like internal serve URLs', async () => { - inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ - isValid: true, - resolvedIP: '203.0.113.10', - }) - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValue( - new Response('should never be fetched', { status: 200 }) - ) - - // Absolute https URL containing `/api/files/serve/` matches isInternalFileUrl and would - // route to handleCloudFile — so it must keep traversal protection, not be waved through - // as an external URL. - const request = new NextRequest('http://localhost:3000/api/files/parse', { - method: 'POST', - body: JSON.stringify({ - filePath: 'https://attacker.com/api/files/serve/../../../etc/passwd', - }), - }) - - const response = await POST(request) - const result = await response.json() - - expect(result.success).toBe(false) - expect(result.error).toMatch(/Access denied: path traversal detected/) - expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() - }) - - it('should handle encoded path traversal attempts', async () => { - const encodedMaliciousPaths = [ - '/api/files/serve/%2e%2e%2f%2e%2e%2fetc%2fpasswd', // ../../../etc/passwd - '/api/files/serve/..%2f..%2f..%2fetc%2fpasswd', - '/api/files/serve/%2e%2e/%2e%2e/etc/passwd', - ] - - for (const maliciousPath of encodedMaliciousPaths) { - const request = new NextRequest('http://localhost:3000/api/files/parse', { - method: 'POST', - body: JSON.stringify({ - filePath: decodeURIComponent(maliciousPath), - }), - }) - - const response = await POST(request) - const result = await response.json() - - expect(result.success).toBe(false) - expect(result.error).toMatch( - /Access denied|Invalid path|Path outside allowed directory|Unauthorized/ - ) - } - }) - - it('should handle null byte injection attempts', async () => { - const nullBytePaths = [ - '/api/files/serve/file.txt\0../../etc/passwd', - 'file.txt\0/etc/passwd', - '/api/files/serve/document.pdf\0/var/log/auth.log', - ] - - for (const maliciousPath of nullBytePaths) { - const request = new NextRequest('http://localhost:3000/api/files/parse', { - method: 'POST', - body: JSON.stringify({ - filePath: maliciousPath, - }), - }) - - const response = await POST(request) - const result = await response.json() - - expect(result.success).toBe(false) - } - }) - }) - - describe('Edge Cases', () => { - it('should handle empty file paths', async () => { - const request = new NextRequest('http://localhost:3000/api/files/parse', { - method: 'POST', - body: JSON.stringify({ - filePath: '', - }), - }) - - const response = await POST(request) - const result = await response.json() - - expect(response.status).toBe(400) - expect(result.error).toBe('No file path provided') - }) - - it('should handle missing filePath parameter', async () => { - const request = new NextRequest('http://localhost:3000/api/files/parse', { - method: 'POST', - body: JSON.stringify({}), - }) - - const response = await POST(request) - const result = await response.json() - - expect(response.status).toBe(400) - expect(result.error).toBe('No file path provided') - }) - }) -}) diff --git a/apps/sim/app/api/files/parse/route.ts b/apps/sim/app/api/files/parse/route.ts deleted file mode 100644 index f86218641e5..00000000000 --- a/apps/sim/app/api/files/parse/route.ts +++ /dev/null @@ -1,1175 +0,0 @@ -import { Buffer, isUtf8 } from 'buffer' -import { createHash } from 'crypto' -import fsPromises from 'fs/promises' -import path from 'path' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateShortId } from '@sim/utils/id' -import binaryExtensionsList from 'binary-extensions' -import { type NextRequest, NextResponse } from 'next/server' -import { fileParseContract } from '@/lib/api/contracts/storage-transfer' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { sanitizeUrlForLog } from '@/lib/core/utils/logging' -import { assertKnownSizeWithinLimit, isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { isSupportedFileType, parseFile } from '@/lib/file-parsers' -import { isFileParserError } from '@/lib/file-parsers/errors' -import { isUsingCloudStorage, StorageService } from '@/lib/uploads' -import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' -import { - ExternalUrlValidationError, - fetchExternalUrlToWorkspace, -} from '@/lib/uploads/contexts/workspace' -import { UPLOAD_DIR_SERVER } from '@/lib/uploads/core/setup.server' -import { getFileMetadataByKey } from '@/lib/uploads/server/metadata' -import { isWorkspaceScopedContext } from '@/lib/uploads/shared/types' -import { - extractCleanFilename, - extractStorageKey, - extractWorkspaceIdFromExecutionKey, - getMimeTypeFromExtension, - getViewerUrl, - inferContextFromKey, - isInternalFileUrl, -} from '@/lib/uploads/utils/file-utils' -import { verifyFileAccess } from '@/app/api/files/authorization' -import type { UserFile } from '@/executor/types' -import '@/lib/uploads/core/setup.server' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('FilesParseAPI') - -const MAX_DOWNLOAD_SIZE_BYTES = 100 * 1024 * 1024 // 100 MB -const DOWNLOAD_TIMEOUT_MS = 30000 // 30 seconds -const MAX_FILE_REFERENCE_LENGTH = 4096 -const MAX_MULTI_FILE_PARSE_OUTPUT_BYTES = 5 * 1024 * 1024 -const BINARY_EXTENSIONS = new Set(binaryExtensionsList) - -function isLikelyTextBuffer(fileBuffer: Buffer): boolean { - return isUtf8(fileBuffer) && !fileBuffer.includes(0) -} - -interface ExecutionContext { - workspaceId: string - workflowId: string - executionId: string -} - -interface ParseResult { - success: boolean - content?: string - error?: string - filePath: string - originalName?: string // Original filename from database (for workspace files) - viewerUrl?: string | null // Viewer URL for the file if available - userFile?: UserFile // UserFile object for the raw file - metadata?: { - fileType: string - size: number - hash: string - processingTime: number - } -} - -function getContentBytes(content: unknown): number { - return typeof content === 'string' ? Buffer.byteLength(content, 'utf8') : 0 -} - -/** - * Main API route handler - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - const startTime = Date.now() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: true }) - - if (!authResult.success) { - logger.warn('Unauthorized file parse request', { - error: authResult.error || 'Authentication failed', - }) - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - if (!authResult.userId) { - logger.warn('File parse request missing userId', { - authType: authResult.authType, - }) - return NextResponse.json({ success: false, error: 'User context required' }, { status: 401 }) - } - - const userId = authResult.userId - - const parsed = await parseRequest( - fileParseContract, - request, - {}, - { - validationErrorResponse: (error) => { - const message = getValidationErrorMessage(error, 'Invalid request data') - return NextResponse.json( - { - success: false, - error: message, - filePath: '', - }, - { status: message.includes('At most 10 files') ? 413 : 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - - const { filePath, fileType, headers, workspaceId, workflowId, executionId } = parsed.data.body - - if (!filePath || (typeof filePath === 'string' && filePath.trim() === '')) { - return NextResponse.json({ success: false, error: 'No file path provided' }, { status: 400 }) - } - - // Build execution context if all required fields are present - const executionContext: ExecutionContext | undefined = - workspaceId && workflowId && executionId - ? { workspaceId, workflowId, executionId } - : undefined - - logger.info('File parse request received:', { - filePath, - fileType, - workspaceId, - userId, - hasExecutionContext: !!executionContext, - hasHeaders: Boolean(headers && Object.keys(headers).length > 0), - }) - - if (Array.isArray(filePath)) { - const results = [] - let totalOutputBytes = 0 - - for (const singlePath of filePath) { - if (!singlePath || (typeof singlePath === 'string' && singlePath.trim() === '')) { - results.push({ - success: false, - error: 'Empty file path in array', - filePath: singlePath || '', - }) - continue - } - - const remainingOutputBytes = MAX_MULTI_FILE_PARSE_OUTPUT_BYTES - totalOutputBytes - if (remainingOutputBytes <= 0) { - return parsedOutputTooLargeResponse(results) - } - - const result = await parseFileSingle( - singlePath, - fileType, - workspaceId, - userId, - executionContext, - headers, - request.signal, - MAX_DOWNLOAD_SIZE_BYTES, - remainingOutputBytes - ) - if (result.metadata) { - result.metadata.processingTime = Date.now() - startTime - } - - if (result.success) { - totalOutputBytes += getContentBytes(result.content) - if (totalOutputBytes > MAX_MULTI_FILE_PARSE_OUTPUT_BYTES) { - return parsedOutputTooLargeResponse(results) - } - - const displayName = - result.originalName || extractCleanFilename(result.filePath) || 'unknown' - results.push({ - success: true, - output: { - content: result.content, - name: displayName, - fileType: result.metadata?.fileType || 'application/octet-stream', - size: result.metadata?.size || 0, - binary: false, - file: result.userFile, - }, - filePath: result.filePath, - viewerUrl: result.viewerUrl, - }) - continue - } - - if (result.error?.startsWith('Parsed file output is too large')) { - return parsedOutputTooLargeResponse(results) - } - - results.push(result) - } - - return NextResponse.json({ - success: true, - results, - }) - } - - const result = await parseFileSingle( - filePath, - fileType, - workspaceId, - userId, - executionContext, - headers, - request.signal - ) - - if (result.metadata) { - result.metadata.processingTime = Date.now() - startTime - } - - if (result.success) { - const displayName = result.originalName || extractCleanFilename(result.filePath) || 'unknown' - return NextResponse.json({ - success: true, - output: { - content: result.content, - name: displayName, - fileType: result.metadata?.fileType || 'application/octet-stream', - size: result.metadata?.size || 0, - binary: false, - file: result.userFile, - }, - filePath: result.filePath, - viewerUrl: result.viewerUrl, - }) - } - - return NextResponse.json(result) - } catch (error) { - logger.error('Error in file parse API:', error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - filePath: '', - }, - { status: 500 } - ) - } -}) - -/** - * Parse a single file and return its content - */ -async function parseFileSingle( - filePath: string, - fileType: string, - workspaceId: string, - userId: string, - executionContext?: ExecutionContext, - headers?: Record, - signal?: AbortSignal, - maxDownloadBytes = MAX_DOWNLOAD_SIZE_BYTES, - maxParsedOutputBytes?: number -): Promise { - logger.info('Parsing file:', filePath) - - if (!filePath || filePath.trim() === '') { - return { - success: false, - error: 'Empty file path provided', - filePath: filePath || '', - } - } - - const referenceValidation = validateFileReferenceShape(filePath) - if (!referenceValidation.isValid) { - return { - success: false, - error: referenceValidation.error || 'Invalid file reference', - filePath, - } - } - - const pathValidation = validateFilePath(filePath) - if (!pathValidation.isValid) { - return { - success: false, - error: pathValidation.error || 'Invalid path', - filePath, - } - } - - if (isInternalFileUrl(filePath)) { - return handleCloudFile( - filePath, - fileType, - userId, - executionContext, - maxDownloadBytes, - maxParsedOutputBytes - ) - } - - if (filePath.startsWith('http://') || filePath.startsWith('https://')) { - return handleExternalUrl( - filePath, - fileType, - workspaceId, - userId, - executionContext, - headers, - signal, - maxDownloadBytes, - maxParsedOutputBytes - ) - } - - if (isUsingCloudStorage()) { - return handleCloudFile( - filePath, - fileType, - userId, - executionContext, - maxDownloadBytes, - maxParsedOutputBytes - ) - } - - return handleLocalFile( - filePath, - fileType, - userId, - executionContext, - maxDownloadBytes, - maxParsedOutputBytes - ) -} - -function validateFileReferenceShape(filePath: string): { isValid: boolean; error?: string } { - const trimmed = filePath.trim() - if ( - trimmed.startsWith('http://') || - trimmed.startsWith('https://') || - isInternalFileUrl(trimmed) - ) { - return { isValid: true } - } - - if (trimmed.startsWith('data:')) { - return { - isValid: false, - error: 'File input must be a URL or uploaded file reference, not inline file content', - } - } - - if (filePath.length > MAX_FILE_REFERENCE_LENGTH) { - return { - isValid: false, - error: 'File reference is too long; provide a file URL or upload the file instead', - } - } - - if (/[\x00-\x08\x0B\x0C\x0E-\x1F]/.test(filePath)) { - return { - isValid: false, - error: - 'File reference contains binary content; provide a file URL or upload the file instead', - } - } - - const newlineCount = filePath.match(/\r\n|\r|\n/g)?.length ?? 0 - if (newlineCount > 2) { - return { - isValid: false, - error: - 'File reference looks like inline file content; provide a file URL or upload the file instead', - } - } - - return { isValid: true } -} - -function parsedOutputTooLargeResponse(results?: unknown[]): NextResponse { - const hasPartialResults = Boolean(results && results.length > 0) - return NextResponse.json( - { - success: hasPartialResults, - error: `Parsed file output is too large to return safely. Maximum combined parsed output is ${prettySize( - MAX_MULTI_FILE_PARSE_OUTPUT_BYTES - )}.`, - ...(results && results.length > 0 ? { results } : {}), - }, - { status: hasPartialResults ? 200 : 413 } - ) -} - -function getParsedOutputTooLargeMessage(maxBytes: number): string { - return `Parsed file output is too large to return safely. Maximum parsed output is ${prettySize( - maxBytes - )}.` -} - -function assertParsedContentWithinLimit(content: string, maxBytes?: number): string { - if (maxBytes !== undefined) { - assertKnownSizeWithinLimit(Buffer.byteLength(content, 'utf8'), maxBytes, 'parsed file output') - } - return content -} - -/** - * Validate file path for security - prevents null byte injection and path traversal attacks. - * - * External URLs (`http`/`https`) are fetched over HTTP — with SSRF protection applied - * downstream in `fetchExternalUrlToWorkspace` (DNS resolution + private/reserved IP blocking) - * — and are never resolved against the filesystem, so `..`/`~` are legal URL content and must - * not be rejected. Providers such as Slack routinely emit slugs containing a literal `...`. - * - * Internal file URLs (`/api/files/serve/...`) ARE resolved to storage keys and filesystem - * paths via `extractStorageKey`, so they keep full traversal protection. The external - * short-circuit explicitly excludes them: `parseFileSingle` routes anything matching - * `isInternalFileUrl` to `handleCloudFile` (even an absolute `https://host/api/files/serve/...`), - * so such inputs must stay subject to the `..`/`~` checks rather than being waved through as - * external URLs. Only the leading-`/` "outside allowed directory" check is relaxed for them, - * since that prefix is expected. - */ -function validateFilePath(filePath: string): { isValid: boolean; error?: string } { - if (filePath.includes('\0')) { - return { isValid: false, error: 'Invalid path: null byte detected' } - } - - if ( - (filePath.startsWith('http://') || filePath.startsWith('https://')) && - !isInternalFileUrl(filePath) - ) { - return { isValid: true } - } - - if (filePath.includes('..')) { - return { isValid: false, error: 'Access denied: path traversal detected' } - } - - if (filePath.includes('~')) { - return { isValid: false, error: 'Invalid path: tilde character not allowed' } - } - - if (filePath.startsWith('/') && !isInternalFileUrl(filePath)) { - return { isValid: false, error: 'Path outside allowed directory' } - } - - if (/^[A-Za-z]:\\/.test(filePath)) { - return { isValid: false, error: 'Path outside allowed directory' } - } - - return { isValid: true } -} - -/** - * Handle external URL. - * - * Always fetches the URL fresh — there is no filename-based dedup. Distinct URLs - * commonly share a path tail (e.g. every Slack clipboard paste is `image.png`), - * so keying a cache by filename returns stale bytes. `fetchExternalUrlToWorkspace` - * delegates to `uploadWorkspaceFile`, which suffix-disambiguates collisions on save. - * - * Workspace save is skipped when the URL already points at our execution-files - * bucket (re-uploading our own bytes is wasteful and would generate `image (1).png` - * style aliases for files we already own). - */ -async function handleExternalUrl( - url: string, - fileType: string, - workspaceId: string, - userId: string, - executionContext?: ExecutionContext, - headers?: Record, - signal?: AbortSignal, - maxDownloadBytes = MAX_DOWNLOAD_SIZE_BYTES, - maxParsedOutputBytes?: number -): Promise { - try { - logger.info('Fetching external URL:', url) - - const { getStorageConfig, USE_S3_STORAGE, USE_BLOB_STORAGE, USE_GCS_STORAGE } = await import( - '@/lib/uploads/config' - ) - const executionConfig = getStorageConfig('execution') - - let isExecutionFile = false - try { - const parsedUrl = new URL(url) - - if (USE_S3_STORAGE && executionConfig.bucket) { - const bucketInHost = parsedUrl.hostname.startsWith(executionConfig.bucket) - const bucketInPath = parsedUrl.pathname.startsWith(`/${executionConfig.bucket}/`) - isExecutionFile = bucketInHost || bucketInPath - } else if (USE_BLOB_STORAGE && executionConfig.containerName) { - isExecutionFile = url.includes(`/${executionConfig.containerName}/`) - } else if (USE_GCS_STORAGE && executionConfig.bucket) { - const bucketInHost = parsedUrl.hostname.startsWith(`${executionConfig.bucket}.`) - const bucketInPath = parsedUrl.pathname.startsWith(`/${executionConfig.bucket}/`) - isExecutionFile = bucketInHost || bucketInPath - } - } catch (error) { - logger.warn('Failed to parse URL for execution file check:', error) - isExecutionFile = false - } - - const { filename, buffer, mimeType } = await fetchExternalUrlToWorkspace({ - url, - userId, - workspaceId: workspaceId || undefined, - saveToWorkspace: Boolean(workspaceId) && !isExecutionFile, - headers, - signal, - maxDownloadBytes, - timeoutMs: DOWNLOAD_TIMEOUT_MS, - }) - const extension = path.extname(filename).toLowerCase().substring(1) - - logger.info(`Downloaded file from URL: ${url}, size: ${buffer.length} bytes`) - - let userFile: UserFile | undefined - if (executionContext) { - try { - userFile = await uploadExecutionFile(executionContext, buffer, filename, mimeType, userId) - logger.info(`Stored file in execution storage: ${filename}`, { key: userFile.key }) - } catch (uploadError) { - logger.warn('Failed to store file in execution storage:', uploadError) - } - } - - let parseResult: ParseResult - if (extension === 'pdf') { - parseResult = await handlePdfBuffer(buffer, filename, fileType, url, maxParsedOutputBytes) - } else if (extension === 'csv') { - parseResult = await handleCsvBuffer(buffer, filename, fileType, url, maxParsedOutputBytes) - } else if (isSupportedFileType(extension)) { - parseResult = await handleGenericTextBuffer( - buffer, - filename, - extension, - fileType, - url, - maxParsedOutputBytes - ) - } else { - parseResult = handleGenericBuffer(buffer, filename, extension, fileType, maxParsedOutputBytes) - } - - // Attach userFile to the result - if (userFile) { - parseResult.userFile = userFile - } - - return parseResult - } catch (error) { - logger.error(`Error handling external URL ${sanitizeUrlForLog(url)}:`, error) - if (isPayloadSizeLimitError(error)) { - logger.warn('Rejected oversized external file parse payload', { - maxBytes: error.maxBytes, - observedBytes: error.observedBytes, - label: error.label, - url: sanitizeUrlForLog(url), - }) - return { - success: false, - error: - error.label === 'parsed file output' - ? getParsedOutputTooLargeMessage(error.maxBytes) - : `File is too large to parse safely. Maximum supported download size is ${prettySize( - error.maxBytes - )}.`, - filePath: url, - } - } - - if (error instanceof ExternalUrlValidationError) { - logger.warn(`Blocked external URL request: ${error.message}`) - return { - success: false, - error: error.message, - filePath: url, - } - } - - return { - success: false, - error: `Error fetching URL: ${(error as Error).message}`, - filePath: url, - } - } -} - -/** - * Handle file stored in cloud storage - * If executionContext is provided and file is not already from execution storage, - * copies the file to execution storage and returns UserFile - */ -async function handleCloudFile( - filePath: string, - fileType: string, - userId: string, - executionContext?: ExecutionContext, - maxDownloadBytes = MAX_DOWNLOAD_SIZE_BYTES, - maxParsedOutputBytes?: number -): Promise { - try { - const cloudKey = extractStorageKey(filePath) - - logger.info('Extracted cloud key:', cloudKey) - - const context = inferContextFromKey(cloudKey) - - const hasAccess = await verifyFileAccess( - cloudKey, - userId, - undefined, // customConfig - context, // context - false // isLocal - ) - - if (!hasAccess) { - logger.warn('Unauthorized cloud file parse attempt', { userId, key: cloudKey, context }) - return { - success: false, - error: 'File not found', - filePath, - } - } - - let originalFilename: string | undefined - // Not filtered to `context = 'workspace'`: a chat attachment carries the same key - // prefix and has an `originalName` worth recovering too, and without it the parse - // result is labelled with the raw storage segment. Access was authorized above; - // this only recovers a display name. - if (isWorkspaceScopedContext(context)) { - try { - const fileRecord = await getFileMetadataByKey(cloudKey) - - if (fileRecord) { - originalFilename = fileRecord.originalName - logger.debug(`Found original filename for workspace file: ${originalFilename}`) - } - } catch (dbError) { - logger.debug(`Failed to lookup original filename for ${cloudKey}:`, dbError) - } - } - - const fileBuffer = await StorageService.downloadFile({ - key: cloudKey, - context, - maxBytes: maxDownloadBytes, - }) - logger.info( - `Downloaded file from ${context} storage: ${cloudKey}, size: ${fileBuffer.length} bytes` - ) - - const filename = originalFilename || cloudKey.split('/').pop() || cloudKey - const extension = path.extname(filename).toLowerCase().substring(1) - const mimeType = getMimeTypeFromExtension(extension) - - const normalizedFilePath = `/api/files/serve/${encodeURIComponent(cloudKey)}?context=${context}` - let workspaceIdFromKey: string | undefined - - if (context === 'execution') { - workspaceIdFromKey = extractWorkspaceIdFromExecutionKey(cloudKey) || undefined - } else if (context === 'workspace') { - const segments = cloudKey.split('/') - if (segments.length >= 2 && /^[a-f0-9-]{36}$/.test(segments[0])) { - workspaceIdFromKey = segments[0] - } - } - - const viewerUrl = getViewerUrl(cloudKey, workspaceIdFromKey) - - // Store file in execution storage if executionContext is provided - let userFile: UserFile | undefined - - if (executionContext) { - // If file is already from execution context, create UserFile reference without re-uploading - if (context === 'execution') { - userFile = { - id: `file_${Date.now()}_${generateShortId(7)}`, - name: filename, - url: normalizedFilePath, - size: fileBuffer.length, - type: mimeType, - key: cloudKey, - context: 'execution', - } - logger.info(`Created UserFile reference for existing execution file: ${filename}`) - } else { - // Copy from workspace/other storage to execution storage - try { - userFile = await uploadExecutionFile( - executionContext, - fileBuffer, - filename, - mimeType, - userId - ) - logger.info(`Copied file to execution storage: ${filename}`, { key: userFile.key }) - } catch (uploadError) { - logger.warn(`Failed to copy file to execution storage:`, uploadError) - } - } - } - - let parseResult: ParseResult - if (extension === 'pdf') { - parseResult = await handlePdfBuffer( - fileBuffer, - filename, - fileType, - normalizedFilePath, - maxParsedOutputBytes - ) - } else if (extension === 'csv') { - parseResult = await handleCsvBuffer( - fileBuffer, - filename, - fileType, - normalizedFilePath, - maxParsedOutputBytes - ) - } else if (isSupportedFileType(extension)) { - parseResult = await handleGenericTextBuffer( - fileBuffer, - filename, - extension, - fileType, - normalizedFilePath, - maxParsedOutputBytes - ) - } else { - parseResult = handleGenericBuffer( - fileBuffer, - filename, - extension, - fileType, - maxParsedOutputBytes - ) - parseResult.filePath = normalizedFilePath - } - - if (originalFilename) { - parseResult.originalName = originalFilename - } - - parseResult.viewerUrl = viewerUrl - - // Attach userFile to the result - if (userFile) { - parseResult.userFile = userFile - } - - return parseResult - } catch (error) { - logger.error(`Error handling cloud file ${filePath}:`, error) - - const errorMessage = (error as Error).message - if (isPayloadSizeLimitError(error)) { - logger.warn('Rejected oversized cloud file parse payload', { - maxBytes: error.maxBytes, - observedBytes: error.observedBytes, - label: error.label, - filePath, - }) - return { - success: false, - error: - error.label === 'parsed file output' - ? getParsedOutputTooLargeMessage(error.maxBytes) - : `File is too large to parse safely. Maximum supported download size is ${prettySize( - error.maxBytes - )}.`, - filePath, - } - } - - if (errorMessage.includes('Access denied') || errorMessage.includes('Forbidden')) { - throw new Error(`Error accessing file from cloud storage: ${errorMessage}`) - } - - return { - success: false, - error: `Error accessing file from cloud storage: ${errorMessage}`, - filePath, - } - } -} - -/** - * Handle local file - */ -async function handleLocalFile( - filePath: string, - fileType: string, - userId: string, - executionContext?: ExecutionContext, - maxDownloadBytes = MAX_DOWNLOAD_SIZE_BYTES, - maxParsedOutputBytes?: number -): Promise { - try { - const storageKey = isInternalFileUrl(filePath) ? extractStorageKey(filePath) : filePath - const filename = storageKey.split('/').pop() || storageKey - - const context = inferContextFromKey(storageKey) - const hasAccess = await verifyFileAccess( - storageKey, - userId, - undefined, // customConfig - context, // context - true // isLocal - ) - - if (!hasAccess) { - logger.warn('Unauthorized local file parse attempt', { userId, filename }) - return { - success: false, - error: 'File not found', - filePath, - } - } - - const fullPath = path.join(UPLOAD_DIR_SERVER, storageKey) - - logger.info('Processing local file:', fullPath) - - try { - await fsPromises.access(fullPath) - } catch { - throw new Error(`File not found: ${filename}`) - } - - const stats = await fsPromises.stat(fullPath) - assertKnownSizeWithinLimit(stats.size, maxDownloadBytes, 'local file') - - const result = await parseFile(fullPath) - const content = assertParsedContentWithinLimit(result.content, maxParsedOutputBytes) - const fileBuffer = await fsPromises.readFile(fullPath) - const hash = createHash('md5').update(fileBuffer).digest('hex') - - const extension = path.extname(filename).toLowerCase().substring(1) - const mimeType = fileType || getMimeTypeFromExtension(extension) - - // Store file in execution storage if executionContext is provided - let userFile: UserFile | undefined - if (executionContext) { - try { - userFile = await uploadExecutionFile( - executionContext, - fileBuffer, - filename, - mimeType, - userId - ) - logger.info(`Stored local file in execution storage: ${filename}`, { key: userFile.key }) - } catch (uploadError) { - logger.warn(`Failed to store local file in execution storage:`, uploadError) - } - } - - return { - success: true, - content, - filePath, - userFile, - metadata: { - fileType: mimeType, - size: stats.size, - hash, - processingTime: 0, - }, - } - } catch (error) { - logger.error(`Error handling local file ${filePath}:`, error) - if (isPayloadSizeLimitError(error)) { - logger.warn('Rejected oversized local file parse payload', { - maxBytes: error.maxBytes, - observedBytes: error.observedBytes, - label: error.label, - filePath, - }) - return { - success: false, - error: - error.label === 'parsed file output' - ? getParsedOutputTooLargeMessage(error.maxBytes) - : `File is too large to parse safely. Maximum supported local file size is ${prettySize( - error.maxBytes - )}.`, - filePath, - } - } - - return { - success: false, - error: `Error processing local file: ${(error as Error).message}`, - filePath, - } - } -} - -/** - * Handle a PDF buffer directly in memory - */ -async function handlePdfBuffer( - fileBuffer: Buffer, - filename: string, - fileType?: string, - originalPath?: string, - maxParsedOutputBytes?: number -): Promise { - try { - logger.info(`Parsing PDF in memory: ${filename}`) - - const result = await parseBufferAsPdf(fileBuffer) - - const content = - result.content || - createPdfFallbackMessage(result.metadata?.pageCount || 0, fileBuffer.length, originalPath) - const limitedContent = assertParsedContentWithinLimit(content, maxParsedOutputBytes) - - return { - success: true, - content: limitedContent, - filePath: originalPath || filename, - metadata: { - fileType: fileType || 'application/pdf', - size: fileBuffer.length, - hash: createHash('md5').update(fileBuffer).digest('hex'), - processingTime: 0, - }, - } - } catch (error) { - if (isPayloadSizeLimitError(error)) throw error - - logger.error('Failed to parse PDF in memory:', error) - - const content = createPdfFailureMessage( - 0, - fileBuffer.length, - originalPath || filename, - (error as Error).message - ) - - return { - success: true, - content, - filePath: originalPath || filename, - metadata: { - fileType: fileType || 'application/pdf', - size: fileBuffer.length, - hash: createHash('md5').update(fileBuffer).digest('hex'), - processingTime: 0, - }, - } - } -} - -/** - * Handle a CSV buffer directly in memory - */ -async function handleCsvBuffer( - fileBuffer: Buffer, - filename: string, - fileType?: string, - originalPath?: string, - maxParsedOutputBytes?: number -): Promise { - try { - logger.info(`Parsing CSV in memory: ${filename}`) - - const { parseBuffer } = await import('@/lib/file-parsers') - const result = await parseBuffer(fileBuffer, 'csv') - - return { - success: true, - content: assertParsedContentWithinLimit(result.content, maxParsedOutputBytes), - filePath: originalPath || filename, - metadata: { - fileType: fileType || 'text/csv', - size: fileBuffer.length, - hash: createHash('md5').update(fileBuffer).digest('hex'), - processingTime: 0, - }, - } - } catch (error) { - if (isPayloadSizeLimitError(error)) throw error - - logger.error('Failed to parse CSV in memory:', error) - return { - success: false, - error: `Failed to parse CSV: ${(error as Error).message}`, - filePath: originalPath || filename, - metadata: { - fileType: 'text/csv', - size: 0, - hash: '', - processingTime: 0, - }, - } - } -} - -/** - * Handle a generic text file buffer in memory - */ -async function handleGenericTextBuffer( - fileBuffer: Buffer, - filename: string, - extension: string, - fileType?: string, - originalPath?: string, - maxParsedOutputBytes?: number -): Promise { - try { - logger.info(`Parsing text file in memory: ${filename}`) - - try { - const { parseBuffer, isSupportedFileType } = await import('@/lib/file-parsers') - - if (isSupportedFileType(extension)) { - const result = await parseBuffer(fileBuffer, extension) - - return { - success: true, - content: assertParsedContentWithinLimit(result.content, maxParsedOutputBytes), - filePath: originalPath || filename, - metadata: { - fileType: fileType || getMimeTypeFromExtension(extension), - size: fileBuffer.length, - hash: createHash('md5').update(fileBuffer).digest('hex'), - processingTime: 0, - }, - } - } - } catch (parserError) { - if (isPayloadSizeLimitError(parserError)) throw parserError - if (isFileParserError(parserError) && parserError.code === 'complexity_limit') { - throw parserError - } - - logger.warn('Specialized parser failed, falling back to generic parsing:', parserError) - } - - const content = fileBuffer.toString('utf-8') - const limitedContent = assertParsedContentWithinLimit(content, maxParsedOutputBytes) - - return { - success: true, - content: limitedContent, - filePath: originalPath || filename, - metadata: { - fileType: fileType || getMimeTypeFromExtension(extension), - size: fileBuffer.length, - hash: createHash('md5').update(fileBuffer).digest('hex'), - processingTime: 0, - }, - } - } catch (error) { - if (isPayloadSizeLimitError(error)) throw error - - logger.error('Failed to parse text file in memory:', error) - return { - success: false, - error: `Failed to parse file: ${(error as Error).message}`, - filePath: originalPath || filename, - metadata: { - fileType: 'text/plain', - size: 0, - hash: '', - processingTime: 0, - }, - } - } -} - -/** - * Handle a generic binary buffer - */ -function handleGenericBuffer( - fileBuffer: Buffer, - filename: string, - extension: string, - fileType?: string, - maxParsedOutputBytes?: number -): ParseResult { - const normalizedExtension = extension.toLowerCase() - const content = - !BINARY_EXTENSIONS.has(normalizedExtension) && isLikelyTextBuffer(fileBuffer) - ? assertParsedContentWithinLimit(fileBuffer.toString('utf-8'), maxParsedOutputBytes) - : `[Binary ${normalizedExtension.toUpperCase()} file - ${fileBuffer.length} bytes]` - - return { - success: true, - content, - filePath: filename, - metadata: { - fileType: fileType || getMimeTypeFromExtension(extension), - size: fileBuffer.length, - hash: createHash('md5').update(fileBuffer).digest('hex'), - processingTime: 0, - }, - } -} - -/** - * Parse a PDF buffer - */ -async function parseBufferAsPdf(buffer: Buffer) { - try { - const { PdfParser } = await import('@/lib/file-parsers/pdf-parser') - const parser = new PdfParser() - logger.info('Using main PDF parser for buffer') - - return await parser.parseBuffer(buffer) - } catch (error) { - throw new Error(`PDF parsing failed: ${(error as Error).message}`) - } -} - -/** - * Format bytes to human readable size - */ -function prettySize(bytes: number): string { - if (bytes === 0) return '0 Bytes' - - const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'] - const i = Math.floor(Math.log(bytes) / Math.log(1024)) - - return `${Number.parseFloat((bytes / 1024 ** i).toFixed(2))} ${sizes[i]}` -} - -/** - * Create a formatted message for PDF content - */ -function createPdfFallbackMessage(pageCount: number, size: number, path?: string): string { - const formattedPath = path || 'Unknown path' - - return `PDF document - ${pageCount} page(s), ${prettySize(size)} -Path: ${formattedPath} - -This file appears to be a PDF document that could not be fully processed as text. -Please use a PDF viewer for best results.` -} - -/** - * Create error message for PDF parsing failure and make it more readable - */ -function createPdfFailureMessage( - pageCount: number, - size: number, - path: string, - error: string -): string { - return `PDF document - Processing failed, ${prettySize(size)} -Path: ${path} -Error: ${error} - -This file appears to be a PDF document that could not be processed. -Please use a PDF viewer for best results.` -} diff --git a/apps/sim/app/api/function/execute/route.test.ts b/apps/sim/app/api/function/execute/route.test.ts deleted file mode 100644 index 19d4f5e7000..00000000000 --- a/apps/sim/app/api/function/execute/route.test.ts +++ /dev/null @@ -1,3199 +0,0 @@ -/** - * Tests for function execution API route - * - * @vitest-environment node - */ -import { spawnSync } from 'node:child_process' -import { readFileSync } from 'node:fs' -import { resolve } from 'node:path' -import { - createMockRequest, - envFlagsMock, - hybridAuthMockFns, - resetEnvFlagsMock, - workflowsUtilsMock, -} from '@sim/testing' -import { NextRequest } from 'next/server' -import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' -import { INTERNAL_EXECUTION_DEADLINE_HEADER } from '@/lib/execution/execution-deadline-header' -import { - MOUNTED_WORKSPACE_FILES_PROVENANCE_KEY, - PRIVATE_SECRET_PROVENANCE_BUNDLE_V1, - PRIVATE_SECRET_PROVENANCE_FIELD, - PRIVATE_SECRET_PROVENANCE_HEADER, -} from '@/lib/execution/private-tool-metadata' -import { - MAX_SANDBOX_OUTPUT_BYTES, - SandboxOutputFileError, - SandboxOutputLimitError, -} from '@/lib/execution/remote-sandbox/output-limits' - -function grantedAccess(workspaceId: string) { - return { - exists: true, - hasAccess: true, - canWrite: true, - canAdmin: false, - workspace: { id: workspaceId }, - permission: 'admin', - } -} - -const { - mockExecuteInSandbox, - mockExecuteInIsolatedVM, - mockExecuteShellInSandbox, - mockFetchWorkspaceFileBuffer, - mockDecryptSecret, - mockEncryptSecret, - mockGetWorkspaceFile, - mockResolveWorkspaceFileReference, - mockUpdateWorkspaceFileContent, - mockUploadFile, - mockValidateWorkspaceFileWriteTarget, - mockWriteWorkspaceFileByPath, - mockCheckWorkspaceAccess, - mockResolveWorkspaceAccess, -} = vi.hoisted(() => ({ - mockExecuteInSandbox: vi.fn(), - mockExecuteInIsolatedVM: vi.fn(), - mockExecuteShellInSandbox: vi.fn(), - mockFetchWorkspaceFileBuffer: vi.fn(), - mockDecryptSecret: vi.fn(async (value: string) => ({ - decrypted: value === 'encrypted:mounted-secret' ? 'mounted-secret' : value, - })), - mockEncryptSecret: vi.fn(async (value: string) => ({ - encrypted: `encrypted:${value}`, - iv: 'iv', - })), - mockGetWorkspaceFile: vi.fn(), - mockResolveWorkspaceFileReference: vi.fn(), - mockUpdateWorkspaceFileContent: vi.fn(), - mockUploadFile: vi.fn(), - mockValidateWorkspaceFileWriteTarget: vi.fn(), - mockWriteWorkspaceFileByPath: vi.fn(), - mockCheckWorkspaceAccess: vi.fn(), - mockResolveWorkspaceAccess: vi.fn(), -})) - -vi.mock('@/lib/workspaces/permissions/utils', () => ({ - checkWorkspaceAccess: mockCheckWorkspaceAccess, - resolveWorkspaceAccess: mockResolveWorkspaceAccess, -})) - -vi.mock('@/lib/core/security/encryption', () => ({ - decryptSecret: mockDecryptSecret, - encryptSecret: mockEncryptSecret, -})) - -vi.mock('@/lib/execution/isolated-vm', () => ({ - executeInIsolatedVM: mockExecuteInIsolatedVM, -})) - -vi.mock('@/lib/execution/remote-sandbox', () => ({ - executeInSandbox: mockExecuteInSandbox, - executeShellInSandbox: mockExecuteShellInSandbox, - SIM_RESULT_PREFIX: '__SIM_RESULT__=', -})) - -vi.mock('@/lib/copilot/request/tools/files', () => ({ - FORMAT_TO_CONTENT_TYPE: { - json: 'application/json', - csv: 'text/csv', - txt: 'text/plain', - md: 'text/markdown', - html: 'text/html', - }, - normalizeOutputWorkspaceFileName: vi.fn((p: string) => p.replace(/^files\//, '')), - resolveOutputFormat: vi.fn(() => 'json'), - getOutputFileDeclarations: vi.fn((params: Record) => { - if (Array.isArray(params.outputs?.files)) { - return params.outputs.files.map((file: Record) => ({ - path: file.path, - mode: file.mode === 'overwrite' ? 'overwrite' : 'create', - sandboxPath: file.sandboxPath, - mimeType: file.mimeType, - format: file.format, - })) - } - return params.outputPath - ? [ - { - path: params.overwriteFileId || params.outputPath, - mode: params.overwriteFileId ? 'overwrite' : 'create', - sandboxPath: params.outputSandboxPath, - mimeType: params.outputMimeType, - format: params.outputFormat, - formatPath: params.outputPath, - overwriteFileId: params.overwriteFileId, - }, - ] - : [] - }), -})) - -vi.mock('@/lib/copilot/vfs/resource-writer', () => ({ - validateWorkspaceFileWriteTarget: mockValidateWorkspaceFileWriteTarget, - writeWorkspaceFileByPath: mockWriteWorkspaceFileByPath, -})) - -vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ - fetchWorkspaceFileBuffer: mockFetchWorkspaceFileBuffer, - getWorkspaceFile: mockGetWorkspaceFile, - resolveWorkspaceFileReference: mockResolveWorkspaceFileReference, - updateWorkspaceFileContent: mockUpdateWorkspaceFileContent, - uploadWorkspaceFile: vi.fn(), -})) - -vi.mock('@/lib/workspace-files/application/resolve-workspace-file-reference', () => ({ - resolveWorkspaceFileReference: mockResolveWorkspaceFileReference, -})) - -vi.mock('@/lib/workspace-files/application/read-workspace-file-content', () => ({ - readWorkspaceFileContent: { - execute: vi.fn(async () => ({ content: await mockFetchWorkspaceFileBuffer() })), - }, -})) - -vi.mock('@/lib/uploads', () => ({ - StorageService: { - uploadFile: mockUploadFile, - }, -})) - -vi.mock('@/lib/workflows/utils', () => workflowsUtilsMock) - -import { validateProxyUrl } from '@/lib/core/security/input-validation' -import { clearLargeValueCacheForTests } from '@/lib/execution/payloads/cache' -import { isLargeArrayManifest } from '@/lib/execution/payloads/large-array-manifest-metadata' -import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref' -import { POST } from '@/app/api/function/execute/route' - -afterAll(resetEnvFlagsMock) - -describe('Function Execute API Route', () => { - beforeEach(() => { - vi.clearAllMocks() - envFlagsMock.isRemoteSandboxEnabled = false - envFlagsMock.isMothershipSandboxEnabled = false - - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-123', - authType: 'internal_jwt', - }) - - mockCheckWorkspaceAccess.mockImplementation(async (id: string) => grantedAccess(id)) - mockResolveWorkspaceAccess.mockImplementation(async (id: string) => grantedAccess(id)) - - mockExecuteInIsolatedVM.mockResolvedValue({ result: 'test', stdout: '' }) - mockUploadFile.mockImplementation(async ({ customKey }) => ({ key: customKey })) - clearLargeValueCacheForTests() - - mockExecuteInSandbox.mockResolvedValue({ - result: 'e2b success', - stdout: 'e2b output', - sandboxId: 'test-sandbox-id', - }) - mockExecuteShellInSandbox.mockResolvedValue({ - result: null, - stdout: '', - sandboxId: 'test-shell-sandbox-id', - }) - mockGetWorkspaceFile.mockResolvedValue({ - id: 'wf_existing', - name: 'existing.png', - size: 10, - type: 'image/png', - url: '/api/files/view/existing', - key: 'workspace/existing.png', - }) - mockUpdateWorkspaceFileContent.mockResolvedValue({ - id: 'wf_existing', - name: 'existing.png', - size: 20, - type: 'image/png', - url: '/api/files/view/existing', - key: 'workspace/existing.png', - }) - mockResolveWorkspaceFileReference.mockResolvedValue({ - id: 'wf_existing', - workspaceId: 'workspace-1', - name: 'existing.txt', - size: 0, - key: 'workspace/existing.txt', - }) - mockFetchWorkspaceFileBuffer.mockResolvedValue(Buffer.alloc(0)) - mockValidateWorkspaceFileWriteTarget.mockImplementation(async ({ target }) => ({ - mode: target.mode, - vfsPath: target.path, - })) - mockWriteWorkspaceFileByPath.mockImplementation(async ({ target, buffer }) => ({ - id: `wf_${String(target.path).split('/').pop()?.replace(/\W+/g, '_') || 'file'}`, - name: String(target.path).split('/').pop() || 'file', - vfsPath: target.path, - downloadUrl: `/api/files/view/${encodeURIComponent(target.path)}`, - mode: target.mode, - size: buffer.length, - contentType: target.mimeType || 'application/octet-stream', - })) - }) - - describe('Security Tests', () => { - it('should reject unauthorized requests', async () => { - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValueOnce({ - success: false, - error: 'Unauthorized', - }) - - const req = createMockRequest('POST', { - code: 'return "test"', - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data).toHaveProperty('error', 'Unauthorized') - }) - - it('rejects a body-supplied workspaceId the acting user is not a member of', async () => { - mockCheckWorkspaceAccess.mockResolvedValue({ - exists: true, - hasAccess: false, - canWrite: false, - canAdmin: false, - workspace: { id: 'workspace-victim' }, - permission: null, - }) - - const req = createMockRequest('POST', { - code: 'return "test"', - workspaceId: 'workspace-victim', - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(403) - expect(data).toHaveProperty('error', 'Workspace access denied') - expect(mockCheckWorkspaceAccess).toHaveBeenCalledWith('workspace-victim', 'user-123') - expect(mockExecuteInIsolatedVM).not.toHaveBeenCalled() - }) - - it('rejects a sandbox output export into a workspace the acting user cannot write to', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockResolvedValueOnce({ - result: 'done', - stdout: 'ok', - sandboxId: 'sandbox-123', - exportedFiles: { '/tmp/out.txt': 'owned by attacker' }, - }) - const readOnly = { - exists: true, - hasAccess: true, - canWrite: false, - canAdmin: false, - workspace: { id: 'workspace-victim' }, - permission: 'read', - } - mockCheckWorkspaceAccess.mockResolvedValue(readOnly) - mockResolveWorkspaceAccess.mockResolvedValue(readOnly) - - const req = createMockRequest('POST', { - code: 'print("done")', - language: 'python', - workspaceId: 'workspace-victim', - outputs: { - files: [{ path: 'files/README.md', mode: 'overwrite', sandboxPath: '/tmp/out.txt' }], - }, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(403) - expect(data).toHaveProperty('error', 'Workspace access denied') - expect(mockValidateWorkspaceFileWriteTarget).not.toHaveBeenCalled() - expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() - }) - - it('rejects an export whose workspace is derived from a body-supplied workflowId', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockResolvedValueOnce({ - result: 'done', - stdout: 'ok', - sandboxId: 'sandbox-123', - exportedFiles: { '/tmp/out.txt': 'owned by attacker' }, - }) - workflowsUtilsMock.getWorkflowById.mockResolvedValueOnce({ - id: 'workflow-victim', - workspaceId: 'workspace-victim', - }) - mockResolveWorkspaceAccess.mockResolvedValue({ - exists: true, - hasAccess: false, - canWrite: false, - canAdmin: false, - workspace: { id: 'workspace-victim' }, - permission: null, - }) - - const req = createMockRequest('POST', { - code: 'print("done")', - language: 'python', - workflowId: 'workflow-victim', - outputs: { - files: [{ path: 'files/README.md', mode: 'overwrite', sandboxPath: '/tmp/out.txt' }], - }, - }) - - const response = await POST(req) - - expect(response.status).toBe(403) - expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() - }) - - it('runs import-free JavaScript in isolated-vm without a remote provider', async () => { - const req = createMockRequest('POST', { - code: 'return "test"', - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.output.result).toBe('test') - expect(mockExecuteInIsolatedVM).toHaveBeenCalledTimes(1) - expect(mockExecuteInSandbox).not.toHaveBeenCalled() - expect(mockExecuteShellInSandbox).not.toHaveBeenCalled() - }) - - it('does not accept a Mothership sandbox profile from the request body', async () => { - const req = createMockRequest('POST', { - code: 'return "test"', - sandboxProfile: 'mothership', - }) - - const response = await POST(req) - - expect(response.status).toBe(200) - expect(mockExecuteInIsolatedVM).toHaveBeenCalledTimes(1) - expect(mockExecuteInSandbox).not.toHaveBeenCalled() - }) - - it('fails closed when a trusted Mothership call has no configured image', async () => { - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValueOnce({ - success: true, - userId: 'user-123', - authType: 'internal_jwt', - sandboxProfile: 'mothership', - }) - - const response = await POST( - createMockRequest('POST', { code: 'return "test"', language: 'javascript' }) - ) - - expect(response.status).toBe(503) - await expect(response.json()).resolves.toMatchObject({ - success: false, - error: 'Mothership code sandbox is not configured', - }) - expect(mockExecuteInIsolatedVM).not.toHaveBeenCalled() - expect(mockExecuteInSandbox).not.toHaveBeenCalled() - }) - - it.each([ - { language: 'javascript', code: 'return 42' }, - { language: 'python', code: '__sim_result__ = 42' }, - ])( - 'runs trusted Mothership $language in the Mothership sandbox image', - async ({ language, code }) => { - envFlagsMock.isMothershipSandboxEnabled = true - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValueOnce({ - success: true, - userId: 'user-123', - authType: 'internal_jwt', - sandboxProfile: 'mothership', - }) - - const response = await POST(createMockRequest('POST', { code, language })) - - expect(response.status).toBe(200) - expect(mockExecuteInSandbox).toHaveBeenCalledWith( - expect.objectContaining({ - language, - sandboxKind: 'mothership', - }) - ) - expect(mockExecuteInIsolatedVM).not.toHaveBeenCalled() - } - ) - - it('runs trusted Mothership Shell in the Mothership sandbox image', async () => { - envFlagsMock.isMothershipSandboxEnabled = true - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValueOnce({ - success: true, - userId: 'user-123', - authType: 'internal_jwt', - sandboxProfile: 'mothership', - }) - - const response = await POST( - createMockRequest('POST', { code: 'echo ready', language: 'shell' }) - ) - - expect(response.status).toBe(200) - expect(mockExecuteShellInSandbox).toHaveBeenCalledWith( - expect.objectContaining({ sandboxKind: 'mothership' }) - ) - }) - - it.each([ - { language: 'javascript', code: 'return 42' }, - { language: 'python', code: '__sim_result__ = 42' }, - ])( - 'runs trusted Mothership $language in the selected Function-based Sim sandbox', - async ({ language, code }) => { - envFlagsMock.isRemoteSandboxEnabled = true - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValueOnce({ - success: true, - userId: 'user-123', - authType: 'internal_jwt', - sandboxProfile: 'mothership', - }) - - const response = await POST( - createMockRequest('POST', { - code, - language, - workspaceId: 'workspace-1', - sandboxId: 'sandbox-1', - }) - ) - - expect(response.status).toBe(200) - const request = mockExecuteInSandbox.mock.calls.at(-1)?.[0] - expect(request).toMatchObject({ - language, - workspaceId: 'workspace-1', - sandboxId: 'sandbox-1', - }) - expect(request).not.toHaveProperty('sandboxKind') - } - ) - - it('runs trusted Mothership Shell in the selected Sim sandbox', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValueOnce({ - success: true, - userId: 'user-123', - authType: 'internal_jwt', - sandboxProfile: 'mothership', - }) - - const response = await POST( - createMockRequest('POST', { - code: 'kubectl version --client', - language: 'shell', - workspaceId: 'workspace-1', - sandboxId: 'sandbox-1', - }) - ) - - expect(response.status).toBe(200) - const request = mockExecuteShellInSandbox.mock.calls.at(-1)?.[0] - expect(request).toMatchObject({ - workspaceId: 'workspace-1', - sandboxId: 'sandbox-1', - }) - expect(request).not.toHaveProperty('sandboxKind') - }) - - it('does not treat the Mothership base as a fallback for a selected Sim sandbox', async () => { - envFlagsMock.isMothershipSandboxEnabled = true - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValueOnce({ - success: true, - userId: 'user-123', - authType: 'internal_jwt', - sandboxProfile: 'mothership', - }) - - const response = await POST( - createMockRequest('POST', { - code: 'return 42', - language: 'javascript', - workspaceId: 'workspace-1', - sandboxId: 'sandbox-1', - }) - ) - - expect(response.status).toBe(503) - await expect(response.json()).resolves.toMatchObject({ - error: 'The Function code sandbox is not configured', - }) - expect(mockExecuteInSandbox).not.toHaveBeenCalled() - expect(mockExecuteInIsolatedVM).not.toHaveBeenCalled() - }) - - it('forces import-free JavaScript into the remote runtime when a Sim sandbox is selected', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - - const response = await POST( - createMockRequest('POST', { - code: 'return 42', - language: 'javascript', - workspaceId: 'workspace-1', - sandboxId: 'sandbox-1', - }) - ) - - expect(response.status).toBe(200) - expect(mockExecuteInSandbox).toHaveBeenCalledWith( - expect.objectContaining({ - language: 'javascript', - workspaceId: 'workspace-1', - sandboxId: 'sandbox-1', - }) - ) - expect(mockExecuteInIsolatedVM).not.toHaveBeenCalled() - }) - - it('should prevent VM escape via constructor chain', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: undefined, stdout: '' }) - - const req = createMockRequest('POST', { - code: 'return this.constructor.constructor("return process")().env', - }) - - const response = await POST(req) - const data = await response.json() - - if (response.status === 422 || response.status === 500) { - expect(data.success).toBe(false) - } else { - const result = data.output?.result - expect(result === undefined || result === null).toBe(true) - } - }) - - it.concurrent('should prevent access to require via constructor chain', async () => { - const req = createMockRequest('POST', { - code: ` - const proc = this.constructor.constructor("return process")(); - const fs = proc.mainModule.require("fs"); - return fs.readFileSync("/etc/passwd", "utf8"); - `, - }) - - const response = await POST(req) - const data = await response.json() - - if (response.status === 200) { - const result = data.output?.result - if (result !== undefined && result !== null && typeof result === 'string') { - expect(result).not.toContain('root:') - } - } - }) - - it('should not expose process object', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'undefined', stdout: '' }) - - const req = createMockRequest('POST', { - code: 'return typeof process', - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.output.result).toBe('undefined') - }) - - it('should not expose require function', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'undefined', stdout: '' }) - - const req = createMockRequest('POST', { - code: 'return typeof require', - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.output.result).toBe('undefined') - }) - - it.concurrent('should block SSRF attacks through secure fetch wrapper', async () => { - expect(validateProxyUrl('http://169.254.169.254/latest/meta-data/').isValid).toBe(false) - expect(validateProxyUrl('http://127.0.0.1:8080/admin').isValid).toBe(true) - expect(validateProxyUrl('http://192.168.1.1/config').isValid).toBe(false) - expect(validateProxyUrl('http://10.0.0.1/internal').isValid).toBe(false) - }) - - it.concurrent('should allow legitimate external URLs', async () => { - expect(validateProxyUrl('https://api.github.com/user').isValid).toBe(true) - expect(validateProxyUrl('https://httpbin.org/get').isValid).toBe(true) - expect(validateProxyUrl('https://example.com/api').isValid).toBe(true) - }) - - it.concurrent('should block dangerous protocols', async () => { - expect(validateProxyUrl('file:///etc/passwd').isValid).toBe(false) - expect(validateProxyUrl('ftp://internal.server/files').isValid).toBe(false) - expect(validateProxyUrl('gopher://old.server/menu').isValid).toBe(false) - }) - }) - - describe('Basic Function Execution', () => { - it.concurrent('should execute simple JavaScript code successfully', async () => { - const req = createMockRequest('POST', { - code: 'return "Hello World"', - timeout: 5000, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.output).toHaveProperty('result') - expect(data.output).toHaveProperty('executionTime') - }) - - it('compacts large array result fields to manifests when execution context is durable', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ - result: { - rows: Array.from({ length: 120_000 }, (_, index) => ({ - key: `SIM-${index}`, - payload: 'x'.repeat(100), - })), - }, - stdout: '', - }) - - const req = createMockRequest('POST', { - code: 'return rows', - workflowId: 'workflow-1', - workspaceId: 'workspace-1', - executionId: 'execution-1', - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(isLargeArrayManifest(data.output.result.rows)).toBe(true) - expect(data.output.result.rows).toMatchObject({ - __simLargeArrayManifest: true, - kind: 'array', - totalCount: 120_000, - }) - }) - - it('keeps large string result fields as generic large value refs', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ - result: { - text: 'x'.repeat(9 * 1024 * 1024), - }, - stdout: '', - }) - - const req = createMockRequest('POST', { - code: 'return text', - workflowId: 'workflow-1', - workspaceId: 'workspace-1', - executionId: 'execution-1', - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(isLargeValueRef(data.output.result.text)).toBe(true) - }) - - it('captures secret provenance before a large result is compacted', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ - result: { text: `${'x'.repeat(9 * 1024 * 1024)}secret-at-the-end` }, - stdout: '', - }) - - const response = await POST( - createMockRequest( - 'POST', - { - code: 'return {{API_KEY}}', - envVars: { API_KEY: 'secret-at-the-end' }, - workflowId: 'workflow-1', - workspaceId: 'workspace-1', - executionId: 'execution-1', - }, - { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } - ) - ) - const data = await response.json() - - expect(isLargeValueRef(data.output.result.text)).toBe(true) - expect(data.__resolvedSecretNames).toEqual(['API_KEY']) - }) - - it('exports multiple declared sandbox output files', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockResolvedValueOnce({ - result: 'done', - stdout: 'ok', - sandboxId: 'sandbox-123', - exportedFiles: { - '/home/user/chart.png': 'iVBORw0KGgo=', - '/home/user/summary.json': '{"ok":true}', - }, - }) - - const req = createMockRequest('POST', { - code: 'print("done")', - language: 'python', - workspaceId: 'workspace-1', - outputs: { - files: [ - { - path: 'files/reports/chart.png', - mode: 'create', - sandboxPath: '/home/user/chart.png', - mimeType: 'image/png', - }, - { - path: 'files/reports/summary.json', - mode: 'overwrite', - sandboxPath: '/home/user/summary.json', - mimeType: 'application/json', - }, - ], - }, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(mockExecuteInSandbox).toHaveBeenCalledWith( - expect.objectContaining({ - outputSandboxPaths: ['/home/user/chart.png', '/home/user/summary.json'], - }) - ) - expect(mockValidateWorkspaceFileWriteTarget).toHaveBeenCalledTimes(2) - expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledTimes(2) - expect(mockWriteWorkspaceFileByPath).toHaveBeenNthCalledWith( - 1, - expect.objectContaining({ - target: expect.objectContaining({ path: 'files/reports/chart.png', mode: 'create' }), - }) - ) - expect(mockWriteWorkspaceFileByPath).toHaveBeenNthCalledWith( - 2, - expect.objectContaining({ - target: expect.objectContaining({ - path: 'files/reports/summary.json', - mode: 'overwrite', - }), - }) - ) - expect(data.output.result.files).toHaveLength(2) - expect(data.resources).toEqual([ - expect.objectContaining({ path: 'files/reports/chart.png' }), - expect.objectContaining({ path: 'files/reports/summary.json' }), - ]) - }) - - it('atomically classifies text exports and acknowledges the durable v2 capability', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockResolvedValueOnce({ - result: 'done', - stdout: '', - sandboxId: 'sandbox-123', - exportedFiles: { '/home/user/secret.txt': 'Bearer secret-value' }, - }) - - const response = await POST( - createMockRequest( - 'POST', - { - code: 'print("{{API_KEY}}")', - language: 'python', - workspaceId: 'workspace-1', - envVars: { API_KEY: 'secret-value' }, - outputs: { - files: [ - { - path: 'files/secret.txt', - sandboxPath: '/home/user/secret.txt', - mimeType: 'text/plain', - }, - ], - }, - }, - { - 'x-sim-request-private-tool-metadata': 'resolved-secret-names-durable-files-v2', - } - ) - ) - const data = await response.json() - - expect(response.status).toBe(200) - expect(response.headers.get('x-sim-private-tool-metadata')).toBe( - 'resolved-secret-names-durable-files-v2' - ) - expect(data).not.toHaveProperty('__resolvedSecretFileNames') - expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( - expect.objectContaining({ - secretProvenance: { - status: 'exact', - entries: [ - { - name: 'API_KEY', - encryptedValue: 'encrypted:secret-value', - sourceUserId: 'user-123', - sourceWorkspaceId: 'workspace-1', - }, - ], - }, - }) - ) - }) - - it('classifies exports exact-empty when the only compiled secret is exempt, still reporting its name', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockResolvedValueOnce({ - result: 'done', - stdout: '', - sandboxId: 'sandbox-123', - exportedFiles: { - '/home/user/secret.txt': 'Bearer secret-value', - '/home/user/small.jpg': '/9j/4AAQ', - }, - }) - - const response = await POST( - createMockRequest( - 'POST', - { - code: 'print("{{API_KEY}}")', - language: 'python', - workspaceId: 'workspace-1', - envVars: { API_KEY: 'secret-value' }, - unredactedSecretNames: ['API_KEY'], - outputs: { - files: [ - { - path: 'files/secret.txt', - sandboxPath: '/home/user/secret.txt', - mimeType: 'text/plain', - }, - { - path: 'files/small.jpg', - sandboxPath: '/home/user/small.jpg', - mimeType: 'image/jpeg', - }, - ], - }, - }, - { - 'x-sim-request-private-tool-metadata': 'resolved-secret-names-durable-files-v2', - } - ) - ) - const data = await response.json() - - expect(response.status).toBe(200) - // The text export carries the exempt plaintext yet records no entry for it. - expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( - expect.objectContaining({ - target: expect.objectContaining({ path: 'files/secret.txt' }), - secretProvenance: { status: 'exact', entries: [] }, - }) - ) - // With only exempt material in scope the binary export must not lock as unknown. - expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( - expect.objectContaining({ - target: expect.objectContaining({ path: 'files/small.jpg' }), - secretProvenance: { status: 'exact', entries: [] }, - }) - ) - // The exemption changes file classification only — the usage trail still sees the name. - expect(data.__resolvedSecretNames).toEqual(['API_KEY']) - }) - - it('keeps recording the non-exempt owner when an exempt name shares its plaintext', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockResolvedValueOnce({ - result: 'done', - stdout: '', - sandboxId: 'sandbox-123', - exportedFiles: { '/home/user/secret.txt': 'Bearer shared-value' }, - }) - - const response = await POST( - createMockRequest('POST', { - code: 'print("{{EXEMPT_KEY}}", "{{OTHER_KEY}}")', - language: 'python', - workspaceId: 'workspace-1', - envVars: { EXEMPT_KEY: 'shared-value', OTHER_KEY: 'shared-value' }, - unredactedSecretNames: ['EXEMPT_KEY'], - outputs: { - files: [ - { - path: 'files/secret.txt', - sandboxPath: '/home/user/secret.txt', - mimeType: 'text/plain', - }, - ], - }, - }) - ) - - expect(response.status).toBe(200) - expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( - expect.objectContaining({ - secretProvenance: { - status: 'exact', - entries: [ - { - name: 'OTHER_KEY', - encryptedValue: 'encrypted:shared-value', - sourceUserId: 'user-123', - sourceWorkspaceId: 'workspace-1', - }, - ], - }, - }) - ) - }) - - it('classifies text exports against private mounted-file provenance', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockResolvedValueOnce({ - result: 'done', - stdout: '', - sandboxId: 'sandbox-123', - exportedFiles: { '/home/user/copied.txt': 'Bearer mounted-secret' }, - }) - - const response = await POST( - createMockRequest( - 'POST', - { - code: 'print("done")', - language: 'python', - workspaceId: 'workspace-1', - outputs: { - files: [ - { - path: 'files/copied.txt', - sandboxPath: '/home/user/copied.txt', - mimeType: 'text/plain', - }, - ], - }, - [PRIVATE_SECRET_PROVENANCE_FIELD]: { - version: 1, - complete: true, - selections: [ - { - key: MOUNTED_WORKSPACE_FILES_PROVENANCE_KEY, - provenance: { - version: 1, - complete: true, - entries: [{ encryptedValue: 'encrypted:mounted-secret' }], - scope: { userId: 'user-123', workspaceId: 'workspace-1' }, - }, - }, - ], - }, - }, - { - 'x-sim-request-private-tool-metadata': 'resolved-secret-names-durable-files-v2', - [PRIVATE_SECRET_PROVENANCE_HEADER]: PRIVATE_SECRET_PROVENANCE_BUNDLE_V1, - } - ) - ) - - expect(response.status).toBe(200) - expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( - expect.objectContaining({ - secretProvenance: { - status: 'exact', - entries: [ - { - name: 'MOUNTED_FILE_SECRET', - encryptedValue: 'encrypted:mounted-secret', - sourceUserId: 'user-123', - sourceWorkspaceId: 'workspace-1', - }, - ], - }, - }) - ) - }) - - it('rejects a partial mounted-file provenance envelope before execution', async () => { - const response = await POST( - createMockRequest('POST', { - code: 'return 1', - [PRIVATE_SECRET_PROVENANCE_FIELD]: { - version: 1, - complete: true, - selections: [ - { - key: MOUNTED_WORKSPACE_FILES_PROVENANCE_KEY, - provenance: { version: 1, complete: true, entries: [] }, - }, - ], - }, - }) - ) - - expect(response.status).toBe(400) - expect(await response.json()).toEqual({ - success: false, - error: 'Mounted file secret provenance is invalid', - }) - expect(mockExecuteInIsolatedVM).not.toHaveBeenCalled() - expect(mockExecuteInSandbox).not.toHaveBeenCalled() - }) - - it('runs with authenticated incomplete mount provenance and marks exported bytes unknown', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockResolvedValueOnce({ - result: 'raw result', - stdout: '', - sandboxId: 'sandbox-123', - exportedFiles: { '/home/user/output.txt': 'raw output' }, - }) - - const response = await POST( - createMockRequest( - 'POST', - { - code: 'print("done")', - language: 'python', - workspaceId: 'workspace-1', - outputs: { - files: [ - { - path: 'files/output.txt', - sandboxPath: '/home/user/output.txt', - mimeType: 'text/plain', - }, - ], - }, - [PRIVATE_SECRET_PROVENANCE_FIELD]: { - version: 1, - complete: false, - selections: [], - }, - }, - { [PRIVATE_SECRET_PROVENANCE_HEADER]: PRIVATE_SECRET_PROVENANCE_BUNDLE_V1 } - ) - ) - - expect(response.status).toBe(200) - expect((await response.json()).output.result).toEqual( - expect.objectContaining({ fileId: 'wf_output_txt', vfsPath: 'files/output.txt' }) - ) - expect(mockExecuteInSandbox).toHaveBeenCalledOnce() - expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( - expect.objectContaining({ - buffer: Buffer.from('raw output'), - secretProvenance: { status: 'unknown' }, - }) - ) - }) - - it('does not rewrite a static export path that happens to equal a resolved secret', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockResolvedValueOnce({ - result: 'done', - stdout: '', - sandboxId: 'sandbox-123', - exportedFiles: { '/home/user/report.txt': 'safe content' }, - }) - - const response = await POST( - createMockRequest('POST', { - code: 'print("done")', - language: 'python', - workspaceId: 'workspace-1', - envVars: { API_KEY: 'secret-value' }, - outputs: { - files: [ - { - path: 'files/report-secret-value.txt', - sandboxPath: '/home/user/report.txt', - mimeType: 'text/plain', - }, - ], - }, - }) - ) - const data = await response.json() - - expect(response.status).toBe(200) - expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( - expect.objectContaining({ - target: expect.objectContaining({ path: 'files/report-secret-value.txt' }), - secretProvenance: { status: 'exact', entries: [] }, - }) - ) - expect(JSON.stringify(data)).toContain('files/report-secret-value.txt') - }) - - it('classifies a binary export exact-empty when no secret was in scope', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockResolvedValueOnce({ - result: 'done', - stdout: '', - sandboxId: 'sandbox-123', - exportedFiles: { '/home/user/small.jpg': '/9j/4AAQ' }, - }) - - const response = await POST( - createMockRequest('POST', { - code: 'print("done")', - language: 'python', - workspaceId: 'workspace-1', - outputs: { - files: [ - { - path: 'files/small.jpg', - sandboxPath: '/home/user/small.jpg', - mimeType: 'image/jpeg', - }, - ], - }, - }) - ) - - expect(response.status).toBe(200) - expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( - expect.objectContaining({ secretProvenance: { status: 'exact', entries: [] } }) - ) - }) - - it('classifies a binary export exact-empty when ordinary files were mounted without secret provenance', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockResolvedValueOnce({ - result: 'done', - stdout: '', - sandboxId: 'sandbox-123', - exportedFiles: { '/home/user/small.jpg': '/9j/4AAQ' }, - }) - - const response = await POST( - createMockRequest('POST', { - code: 'print("done")', - language: 'python', - workspaceId: 'workspace-1', - _sandboxFiles: [{ path: '/home/user/in.bin', content: 'mounted bytes' }], - outputs: { - files: [ - { - path: 'files/small.jpg', - sandboxPath: '/home/user/small.jpg', - mimeType: 'image/jpeg', - }, - ], - }, - }) - ) - - expect(response.status).toBe(200) - expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( - expect.objectContaining({ secretProvenance: { status: 'exact', entries: [] } }) - ) - }) - - it('keeps a binary export unknown when a mounted input file carried a secret', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockResolvedValueOnce({ - result: 'done', - stdout: '', - sandboxId: 'sandbox-123', - exportedFiles: { '/home/user/small.jpg': '/9j/4AAQ' }, - }) - - const response = await POST( - createMockRequest( - 'POST', - { - code: 'print("done")', - language: 'python', - workspaceId: 'workspace-1', - outputs: { - files: [ - { - path: 'files/small.jpg', - sandboxPath: '/home/user/small.jpg', - mimeType: 'image/jpeg', - }, - ], - }, - [PRIVATE_SECRET_PROVENANCE_FIELD]: { - version: 1, - complete: true, - selections: [ - { - key: MOUNTED_WORKSPACE_FILES_PROVENANCE_KEY, - provenance: { - version: 1, - complete: true, - entries: [{ encryptedValue: 'encrypted:mounted-secret' }], - scope: { userId: 'user-123', workspaceId: 'workspace-1' }, - }, - }, - ], - }, - }, - { - [PRIVATE_SECRET_PROVENANCE_HEADER]: PRIVATE_SECRET_PROVENANCE_BUNDLE_V1, - } - ) - ) - - expect(response.status).toBe(200) - expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( - expect.objectContaining({ secretProvenance: { status: 'unknown' } }) - ) - }) - - it('marks binary exports unknown without failing the Function execution', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockResolvedValueOnce({ - result: 'done', - stdout: '', - sandboxId: 'sandbox-123', - exportedFiles: { '/home/user/archive.zip': 'UEsDBA==' }, - }) - - const response = await POST( - createMockRequest('POST', { - code: 'print("{{API_KEY}}")', - language: 'python', - workspaceId: 'workspace-1', - envVars: { API_KEY: 'secret-value' }, - outputs: { - files: [ - { - path: 'files/archive.zip', - sandboxPath: '/home/user/archive.zip', - mimeType: 'application/zip', - }, - ], - }, - }) - ) - - expect(response.status).toBe(200) - expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( - expect.objectContaining({ secretProvenance: { status: 'unknown' } }) - ) - }) - - it('rejects one oversized sandbox output before creating a workspace file buffer', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockResolvedValueOnce({ - result: 'done', - stdout: 'ok', - sandboxId: 'sandbox-123', - exportedFiles: { - '/home/user/report.json': 'x'.repeat(MAX_SANDBOX_OUTPUT_BYTES + 1), - }, - }) - - const req = createMockRequest('POST', { - code: 'print("done")', - language: 'python', - workspaceId: 'workspace-1', - outputs: { - files: [ - { - path: 'files/report.json', - sandboxPath: '/home/user/report.json', - mimeType: 'application/json', - }, - ], - }, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toBe(`Sandbox output files exceed ${MAX_SANDBOX_OUTPUT_BYTES} bytes total`) - expect(mockValidateWorkspaceFileWriteTarget).not.toHaveBeenCalled() - expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() - }) - - it('rejects cumulative sandbox output size before validating workspace destinations', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - const fileSize = MAX_SANDBOX_OUTPUT_BYTES / 2 + 1 - mockExecuteInSandbox.mockResolvedValueOnce({ - result: 'done', - stdout: 'ok', - sandboxId: 'sandbox-123', - exportedFiles: { - '/home/user/first.json': 'x'.repeat(fileSize), - '/home/user/second.json': 'y'.repeat(fileSize), - }, - }) - - const req = createMockRequest('POST', { - code: 'print("done")', - language: 'python', - workspaceId: 'workspace-1', - outputs: { - files: [ - { - path: 'files/first.json', - sandboxPath: '/home/user/first.json', - mimeType: 'application/json', - }, - { - path: 'files/second.json', - sandboxPath: '/home/user/second.json', - mimeType: 'application/json', - }, - ], - }, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toBe(`Sandbox output files exceed ${MAX_SANDBOX_OUTPUT_BYTES} bytes total`) - expect(mockValidateWorkspaceFileWriteTarget).not.toHaveBeenCalled() - expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() - }) - - it('preserves output-limit classification from provider-side size inspection', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockRejectedValueOnce( - new SandboxOutputLimitError(MAX_SANDBOX_OUTPUT_BYTES + 1) - ) - - const req = createMockRequest('POST', { - code: 'print("done")', - language: 'python', - workspaceId: 'workspace-1', - outputs: { - files: [ - { - path: 'files/report.json', - sandboxPath: '/home/user/report.json', - }, - ], - }, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toBe(`Sandbox output files exceed ${MAX_SANDBOX_OUTPUT_BYTES} bytes total`) - expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() - }) - - it('rejects non-regular sandbox output paths as a client error', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockRejectedValueOnce(new SandboxOutputFileError('/out/link.json')) - - const response = await POST( - createMockRequest('POST', { - code: 'print("done")', - language: 'python', - workspaceId: 'workspace-1', - outputs: { - files: [{ path: 'files/report.json', sandboxPath: '/out/link.json' }], - }, - }) - ) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toContain('must reference a regular file') - expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() - }) - - it('prevalidates all sandbox output destinations before writing any files', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockResolvedValueOnce({ - result: 'done', - stdout: 'ok', - sandboxId: 'sandbox-123', - exportedFiles: { - '/home/user/first.json': '{"first":true}', - '/home/user/second.json': '{"second":true}', - }, - }) - mockValidateWorkspaceFileWriteTarget - .mockResolvedValueOnce({ mode: 'create', vfsPath: 'files/first.json' }) - .mockRejectedValueOnce(new Error('Directory not yet created: files/missing')) - - const req = createMockRequest('POST', { - code: 'print("done")', - language: 'python', - workspaceId: 'workspace-1', - outputs: { - files: [ - { - path: 'files/first.json', - mode: 'create', - sandboxPath: '/home/user/first.json', - }, - { - path: 'files/missing/second.json', - mode: 'create', - sandboxPath: '/home/user/second.json', - }, - ], - }, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.success).toBe(false) - expect(data.error).toContain('Directory not yet created') - expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() - }) - - it('rejects duplicate sandbox output destinations before writing files', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockResolvedValueOnce({ - result: 'done', - stdout: 'ok', - sandboxId: 'sandbox-123', - exportedFiles: { - '/home/user/first.json': '{"first":true}', - '/home/user/second.json': '{"second":true}', - }, - }) - mockValidateWorkspaceFileWriteTarget.mockResolvedValue({ - mode: 'create', - vfsPath: 'files/dupe.json', - }) - - const req = createMockRequest('POST', { - code: 'print("done")', - language: 'python', - workspaceId: 'workspace-1', - outputs: { - files: [ - { - path: 'files/dupe.json', - mode: 'create', - sandboxPath: '/home/user/first.json', - }, - { - path: 'files/dupe.json', - mode: 'create', - sandboxPath: '/home/user/second.json', - }, - ], - }, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.success).toBe(false) - expect(data.error).toContain('Duplicate sandbox output destination') - expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() - }) - - it('returns a targeted error when a declared sandbox output is missing', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockResolvedValueOnce({ - result: 'done', - stdout: 'ok', - sandboxId: 'sandbox-123', - exportedFiles: {}, - }) - - const req = createMockRequest('POST', { - code: 'print("done")', - language: 'python', - workspaceId: 'workspace-1', - outputs: { - files: [ - { - path: 'files/missing.json', - mode: 'create', - sandboxPath: '/home/user/missing.json', - }, - ], - }, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(500) - expect(data.success).toBe(false) - expect(data.error).toContain('Sandbox file "/home/user/missing.json" was not found') - expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() - }) - - it('rejects sandboxPath outputs when the call would run in isolated-vm (E2B enabled, JS without imports)', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - - const req = createMockRequest('POST', { - code: 'return "content"', - language: 'javascript', - workspaceId: 'workspace-1', - outputs: { - files: [ - { - path: 'files/doc.md', - mode: 'overwrite', - sandboxPath: '/home/user/doc.md', - }, - ], - }, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(422) - expect(data.success).toBe(false) - expect(data.error).toContain('no sandbox filesystem') - expect(mockExecuteInIsolatedVM).not.toHaveBeenCalled() - expect(mockExecuteInSandbox).not.toHaveBeenCalled() - expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() - }) - - it('rejects sandbox file mounts when the call would run in isolated-vm', async () => { - const req = createMockRequest('POST', { - code: 'return 1', - language: 'javascript', - workspaceId: 'workspace-1', - _sandboxFiles: [{ path: '/home/user/files/data.csv', content: 'a,b\n1,2' }], - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(422) - expect(data.success).toBe(false) - // No remote sandbox is enabled in this test, so the remediation must name - // that cause instead of suggesting python (which would also fail without one). - expect(data.error).toContain('No remote code sandbox is enabled') - expect(mockExecuteInIsolatedVM).not.toHaveBeenCalled() - }) - - it('flags an overwrite export whose bytes are identical to the current file content as unchanged', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - const staleContent = '# doc\nunchanged mounted content\n' - mockExecuteInSandbox.mockResolvedValueOnce({ - result: 'done', - stdout: 'ok', - sandboxId: 'sandbox-123', - exportedFiles: { '/home/user/doc.md': staleContent }, - }) - mockResolveWorkspaceFileReference.mockResolvedValue({ - id: 'wf_doc', - name: 'doc.md', - size: Buffer.byteLength(staleContent, 'utf-8'), - key: 'workspace/doc.md', - }) - mockFetchWorkspaceFileBuffer.mockResolvedValue(Buffer.from(staleContent, 'utf-8')) - - const req = createMockRequest('POST', { - code: 'print("done")', - language: 'python', - workspaceId: 'workspace-1', - outputs: { - files: [ - { - path: 'files/doc.md', - mode: 'overwrite', - sandboxPath: '/home/user/doc.md', - mimeType: 'text/markdown', - }, - ], - }, - }) - - const response = await POST(req) - const data = await response.json() - - // Idempotent overwrites (retries, unchanged regenerations) must not fail; - // the write proceeds and the receipt carries the loud unchanged signal so - // the model can tell its "new content" never reached the sandbox file. - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledTimes(1) - expect(data.output.result.unchanged).toBe(true) - expect(data.output.result.message).toContain('byte-identical to the previous version') - expect(data.output.result.message).toContain('/home/user/doc.md') - }) - - it('continues an overwrite when the advisory comparison fails', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - const newContent = '# doc\nnew content\n' - mockExecuteInSandbox.mockResolvedValueOnce({ - result: 'done', - stdout: 'ok', - sandboxId: 'sandbox-123', - exportedFiles: { '/home/user/doc.md': newContent }, - }) - mockResolveWorkspaceFileReference.mockRejectedValueOnce( - new Error('comparison storage unavailable') - ) - - const response = await POST( - createMockRequest('POST', { - code: 'print("done")', - language: 'python', - workspaceId: 'workspace-1', - outputs: { - files: [ - { - path: 'files/doc.md', - mode: 'overwrite', - sandboxPath: '/home/user/doc.md', - mimeType: 'text/markdown', - }, - ], - }, - }) - ) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledTimes(1) - expect(data.output.result).toMatchObject({ unchanged: false }) - expect(data.output.result).not.toHaveProperty('previousSize') - }) - - it('reports size, previousSize, and sha256 receipts on a successful overwrite export', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - const newContent = '# doc\nnew content\n' - mockExecuteInSandbox.mockResolvedValueOnce({ - result: 'done', - stdout: 'ok', - sandboxId: 'sandbox-123', - exportedFiles: { '/home/user/doc.md': newContent }, - }) - mockResolveWorkspaceFileReference.mockResolvedValue({ - id: 'wf_doc', - name: 'doc.md', - size: 36728, - key: 'workspace/doc.md', - }) - - const req = createMockRequest('POST', { - code: 'print("done")', - language: 'python', - workspaceId: 'workspace-1', - outputs: { - files: [ - { - path: 'files/doc.md', - mode: 'overwrite', - sandboxPath: '/home/user/doc.md', - mimeType: 'text/markdown', - }, - ], - }, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - // Sizes differ, so the current content is never downloaded for comparison. - expect(mockFetchWorkspaceFileBuffer).not.toHaveBeenCalled() - expect(data.output.result.size).toBe(Buffer.byteLength(newContent, 'utf-8')) - expect(data.output.result.previousSize).toBe(36728) - expect(data.output.result.sha256).toMatch(/^[0-9a-f]{64}$/) - expect(data.output.result.unchanged).toBe(false) - expect(data.output.result.message).toContain('replaced 36728 bytes') - expect(data.output.result.message).toContain('sha256:') - // The python wrapper prints the marker with a leading \n so it always - // starts a fresh line even after non-newline-terminated user output. - const e2bCode = mockExecuteInSandbox.mock.calls[0][0].code as string - expect(e2bCode).toContain("print('\\n__SIM_RESULT__=' + json.dumps(__sim_result__))") - }) - - it('runs complete Python modules without nesting their main guard inside a function', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - const source = [ - 'import subprocess', - '', - 'def main():', - ' subprocess.run(["bq", "version"], check=True)', - '', - 'if __name__ == "__main__":', - ' main()', - ].join('\n') - - const response = await POST( - createMockRequest('POST', { - code: source, - language: 'python', - workspaceId: 'workspace-1', - }) - ) - - expect(response.status).toBe(200) - const e2bCode = mockExecuteInSandbox.mock.calls[0][0].code as string - expect(e2bCode).toContain('compile(__sim_source__, "", "exec")') - expect(e2bCode).toContain('__sim_exec_globals__["__name__"] = "__main__"') - expect(e2bCode).toContain(JSON.stringify(source)) - expect(e2bCode).not.toContain('def __sim_main__():\n import subprocess') - }) - - it('supports a Fellows-style Python module that invokes bq and exports a deterministic archive', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - const archiveBase64 = - 'UEsDBBQAAAAIAAAAIQAcWyFBIAAAAB8AAAAMAAAAcHJldmlldy5odG1ss8kwtHNLzcnJLy9WcM4vzUvOzFEIT03Nzqm00QdKAQBQSwECFAMUAAAACAAAACEAHFshQSAAAAAfAAAADAAAAAAAAAAAAAAAgAEAAAAAcHJldmlldy5odG1sUEsFBgAAAAABAAEAOgAAAEoAAAAAAA==' - const source = readFileSync( - resolve(process.cwd(), 'lib/execution/remote-sandbox/fixtures/fellows-council-weekly.py'), - 'utf8' - ) - mockExecuteInSandbox.mockResolvedValueOnce({ - result: null, - stdout: 'generated 1 preview', - sandboxId: 'sandbox-123', - exportedFiles: { '/tmp/fellows-previews.zip': archiveBase64 }, - }) - - const response = await POST( - createMockRequest('POST', { - code: source, - language: 'python', - workspaceId: 'workspace-1', - sandboxId: 'fellows-sandbox', - envVars: { - AIRTABLE_PAT: 'stub-airtable-token', - ANTHROPIC_API_KEY: 'stub-anthropic-key', - GOOGLE_SERVICE_ACCOUNT_JSON: - '{"type":"service_account","project_id":"fixture-project"}', - NCBI_API_KEY: 'stub-ncbi-key', - }, - outputs: { - files: [ - { - path: 'files/fellows-previews.zip', - sandboxPath: '/tmp/fellows-previews.zip', - mimeType: 'application/zip', - }, - ], - }, - }) - ) - - expect(response.status).toBe(200) - const sandboxRequest = mockExecuteInSandbox.mock.calls[0][0] - expect(sandboxRequest.code).toContain("['bq', 'query'") - expect(sandboxRequest.code).toContain('__sim_exec_globals__["__name__"] = "__main__"') - expect(sandboxRequest.sandboxId).toBe('fellows-sandbox') - expect(sandboxRequest.outputSandboxPaths).toEqual(['/tmp/fellows-previews.zip']) - expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( - expect.objectContaining({ - buffer: Buffer.from(archiveBase64, 'base64'), - target: expect.objectContaining({ path: 'files/fellows-previews.zip' }), - }) - ) - }) - - it('retains Function-body return semantics for Python snippets', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - - await POST( - createMockRequest('POST', { - code: 'value = 41\nreturn value + 1', - language: 'python', - workspaceId: 'workspace-1', - }) - ) - - const e2bCode = mockExecuteInSandbox.mock.calls[0][0].code as string - expect(e2bCode).toContain('"outside function" not in str(__sim_compile_error__)') - expect(e2bCode).toContain('__sim_result__ = __sim_exec_globals__["__sim_main__"]()') - }) - - it.each([ - { reason: 'timeout', status: 408, message: 'timed out' }, - { reason: 'user', status: 499, message: 'cancelled' }, - ])('keeps $reason aborts distinct', async ({ reason, status, message }) => { - envFlagsMock.isRemoteSandboxEnabled = true - const controller = new AbortController() - const req = new NextRequest('http://localhost:3000/api/function/execute', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - code: 'print("running")', - language: 'python', - workspaceId: 'workspace-1', - timeout: 30_000, - }), - signal: controller.signal, - }) - mockExecuteInSandbox.mockImplementationOnce(async () => { - controller.abort(new DOMException(reason, 'AbortError')) - throw controller.signal.reason - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(status) - expect(data.error).toContain(message) - }) - - it.each([ - { - termination: 'timeout' as const, - errorName: 'TimeoutError', - status: 408, - message: 'timed out', - }, - { - termination: 'cancelled' as const, - errorName: 'AbortError', - status: 499, - message: 'cancelled', - }, - ])( - 'classifies trusted isolated-vm $termination results consistently with remote runtimes', - async ({ termination, errorName, status, message }) => { - const partialStdout = `partial output before ${termination}` - mockExecuteInIsolatedVM.mockResolvedValueOnce({ - result: null, - stdout: partialStdout, - error: { name: errorName, message: `${errorName} from isolated-vm` }, - termination, - }) - - const response = await POST( - createMockRequest('POST', { - code: 'return true', - language: 'javascript', - timeout: 30_000, - }) - ) - const data = await response.json() - - expect(response.status).toBe(status) - expect(data.error).toContain(message) - expect(data.output.stdout).toBe(partialStdout) - } - ) - - it.each(['TimeoutError', 'AbortError'])( - 'keeps a user-thrown %s as an ordinary code error', - async (errorName) => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ - result: null, - stdout: 'partial output before user error', - error: { name: errorName, message: `User threw ${errorName}` }, - }) - - const response = await POST( - createMockRequest('POST', { - code: `const error = new Error('user error'); error.name = '${errorName}'; throw error`, - language: 'javascript', - timeout: 30_000, - }) - ) - const data = await response.json() - - expect(response.status).toBe(422) - expect(data.output.stdout).toBe('partial output before user error') - expect(data.debug.errorType).toBe(errorName) - } - ) - - it('enforces the explicit Function timeout with a server-owned abort signal', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockImplementationOnce( - ({ signal }: { signal: AbortSignal }) => - new Promise((_resolve, reject) => { - signal.addEventListener('abort', () => reject(signal.reason), { once: true }) - }) - ) - - const response = await POST( - createMockRequest('POST', { - code: 'print("running")', - language: 'python', - workspaceId: 'workspace-1', - timeout: 1, - }) - ) - const data = await response.json() - - expect(response.status).toBe(408) - expect(data.error).toContain('timed out after 1ms') - }) - - it('uses the remaining workflow deadline when no block timeout is supplied', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - const remainingBudgetMs = 10 * 60_000 - const req = new NextRequest('http://localhost:3000/api/function/execute', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - [INTERNAL_EXECUTION_DEADLINE_HEADER]: String(Date.now() + remainingBudgetMs), - }, - body: JSON.stringify({ - code: 'print("running")', - language: 'python', - workspaceId: 'workspace-1', - }), - }) - - const response = await POST(req) - - expect(response.status).toBe(200) - const sandboxRequest = mockExecuteInSandbox.mock.calls[0][0] - expect(sandboxRequest.timeoutMs).toBeGreaterThan(9 * 60_000) - expect(sandboxRequest.timeoutMs).toBeLessThanOrEqual(remainingBudgetMs) - }) - - it('classifies a client abort at the propagated execution deadline as a timeout', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - const controller = new AbortController() - const req = new NextRequest('http://localhost:3000/api/function/execute', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - [INTERNAL_EXECUTION_DEADLINE_HEADER]: String(Date.now() - 1_000), - }, - body: JSON.stringify({ - code: 'print("running")', - language: 'python', - workspaceId: 'workspace-1', - timeout: 30_000, - }), - signal: controller.signal, - }) - mockExecuteInSandbox.mockImplementationOnce(async (sandboxRequest) => { - expect(sandboxRequest.timeoutMs).toBe(1) - controller.abort(new DOMException('The operation was aborted.', 'AbortError')) - throw controller.signal.reason - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(408) - expect(data.error).toContain('timed out') - }) - - it('should return computed result for multi-line code', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 10, stdout: '' }) - - const req = createMockRequest('POST', { - code: 'const a = 1;\nconst b = 2;\nconst c = 3;\nconst d = 4;\nreturn a + b + c + d;', - timeout: 5000, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.output.result).toBe(10) - }) - - it.concurrent('should handle missing code parameter', async () => { - const req = createMockRequest('POST', { - timeout: 5000, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data).toHaveProperty('error') - }) - - it.concurrent('should use default timeout when not provided', async () => { - const req = createMockRequest('POST', { - code: 'return "test"', - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - }) - - it('rejects large refs in runtimes without ref-native helpers', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - const req = createMockRequest('POST', { - code: 'echo "$__blockRef_0"', - language: 'shell', - contextVariables: { - __blockRef_0: { - __simLargeValueRef: true, - version: 1, - id: 'lv_ABCDEFGHIJKL', - kind: 'array', - size: 12 * 1024 * 1024, - executionId: 'execution-1', - }, - }, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(500) - expect(data.success).toBe(false) - expect(data.error).toContain( - 'Large execution values require the JavaScript isolated-vm runtime' - ) - }) - - it('registers manifest array read broker for isolated-vm execution', async () => { - const req = createMockRequest('POST', { - code: 'return await sim.values.readArray(__blockRef_0)', - language: 'javascript', - contextVariables: { - __blockRef_0: { - __simLargeArrayManifest: true, - version: 2, - kind: 'array', - totalCount: 1, - chunkCount: 1, - byteSize: 16, - chunks: [ - { - ref: { - __simLargeValueRef: true, - version: 1, - id: 'lv_ABCDEFGHIJKL', - kind: 'array', - size: 16, - executionId: 'execution-1', - }, - count: 1, - byteSize: 16, - }, - ], - preview: [{ id: 1 }], - }, - }, - }) - - const response = await POST(req) - const data = await response.json() - const [, options] = mockExecuteInIsolatedVM.mock.calls.at(-1) ?? [] - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(options?.brokers).toHaveProperty('sim.values.readArray') - }) - }) - - describe('Template Variable Resolution', () => { - it('should resolve environment variables with {{var_name}} syntax', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'secret-key-123', stdout: '' }) - const req = createMockRequest( - 'POST', - { - code: 'return {{API_KEY}}', - envVars: { - API_KEY: 'secret-key-123', - }, - }, - { - 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', - } - ) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.__resolvedSecretNames).toEqual(['API_KEY']) - }) - - it('keeps an exact-name/exact-value JavaScript secret out of source and returns its raw runtime value with private provenance', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'Test', stdout: '' }) - - const response = await POST( - createMockRequest( - 'POST', - { - code: 'return {{Test}}', - language: 'javascript', - envVars: { Test: 'Test' }, - }, - { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } - ) - ) - - const data = await response.json() - const [request] = mockExecuteInIsolatedVM.mock.calls.at(-1) ?? [] - const bindingEntries = Object.entries(request.contextVariables) - - expect(response.status).toBe(200) - expect(request.code).not.toContain('Test') - expect(request.code).not.toContain('{{Test}}') - expect(request.code).not.toContain('__var_') - expect(bindingEntries).toHaveLength(1) - expect(bindingEntries[0]?.[0]).toMatch(/^__sim_code_\d+_binding_\d+$/) - expect(bindingEntries[0]?.[1]).toBe('Test') - expect(data.output.result).toBe('Test') - expect(data.__resolvedSecretNames).toEqual(['Test']) - expect(JSON.stringify(data)).not.toContain('__sim_code_') - expect(JSON.stringify(data)).not.toContain('__var_') - }) - - it('keeps an exact-name/exact-value Python secret out of source and supplies it only through private runtime input', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteInSandbox.mockResolvedValueOnce({ - result: 'Test', - stdout: '', - sandboxId: 'test-sandbox-id', - }) - - const response = await POST( - createMockRequest( - 'POST', - { - code: 'return {{Test}}', - language: 'python', - envVars: { Test: 'Test' }, - }, - { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } - ) - ) - - const data = await response.json() - const [request] = mockExecuteInSandbox.mock.calls.at(-1) ?? [] - const runtimeInput = request.privateInputs.find( - (input: { environmentVariable: string }) => - input.environmentVariable === '__SIM_RUNTIME_PAYLOAD_PATH' - ) - const runtimePayload = JSON.parse(runtimeInput?.content ?? '{}') - const secretBinding = runtimePayload.contextVariables.find( - (entry: { value?: unknown }) => entry.value === 'Test' - ) - - expect(response.status).toBe(200) - expect(request.code).not.toContain('Test') - expect(request.code).not.toContain('{{Test}}') - expect(request.code).not.toContain('__var_') - expect(runtimePayload.environmentVariables).toEqual({ Test: 'Test' }) - expect(secretBinding).toMatchObject({ kind: 'json', value: 'Test' }) - expect(secretBinding.name).toMatch(/^__sim_code_\d+_binding_\d+__$/) - expect(request.code).toContain(secretBinding.name) - expect(data.output.result).toBe('Test') - expect(data.__resolvedSecretNames).toEqual(['Test']) - expect(JSON.stringify(data)).not.toContain('__sim_code_') - expect(JSON.stringify(data)).not.toContain('__var_') - }) - - it('compiles legacy bare and quoted Custom Tool placeholders into opaque VM bindings', async () => { - const secret = 'quote" slash\\ newline\n{{OTHER}} true 123' - mockExecuteInIsolatedVM.mockResolvedValueOnce({ - result: [secret, secret, `Bearer ${secret}`], - stdout: '', - }) - const response = await POST( - createMockRequest( - 'POST', - { - code: [ - 'const bare = {{API_KEY}}', - 'const quoted = "{{API_KEY}}"', - 'return [bare, quoted, "Bearer {{API_KEY}}"]', - ].join('\n'), - isCustomTool: true, - envVars: { API_KEY: secret, OTHER: 'must-not-resolve' }, - }, - { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } - ) - ) - - const [request] = mockExecuteInIsolatedVM.mock.calls.at(-1) ?? [] - expect(response.status).toBe(200) - expect((await response.json()).__resolvedSecretNames).toEqual(['API_KEY']) - expect(request.code).not.toContain(secret) - expect(request.code).not.toContain('__var_') - expect(request.code).toContain('__sim_code_') - expect(request.code).not.toContain('globalThis[') - expect(Object.values(request.contextVariables)).toContain(secret) - expect(Object.keys(request.contextVariables)).not.toContain('API_KEY') - }) - - it('installs regex constructors as opaque runtime bindings before isolated user code', async () => { - const response = await POST( - createMockRequest('POST', { - code: [ - 'RegExp.prototype.constructor = null', - 'return /^{{PATTERN}}$/.test("candidate")', - ].join('\n'), - envVars: { PATTERN: 'secret' }, - }) - ) - - const [request] = mockExecuteInIsolatedVM.mock.calls.at(-1) ?? [] - const [runtimeBinding] = request.runtimeBindings - expect(response.status).toBe(200) - expect(runtimeBinding.kind).toBe('javascript-runtime') - expect(request.code).toContain(`new ${runtimeBinding.name}.RegExp`) - expect(request.code).not.toContain('secret') - }) - - it('captures regex constructors in the remote preload before static imports execute', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - const response = await POST( - createMockRequest('POST', { - code: ['import "side-effect-module"', 'return /^{{PATTERN}}$/.test("candidate")'].join( - '\n' - ), - language: 'javascript', - envVars: { PATTERN: 'secret' }, - }) - ) - - const [sandboxRequest] = mockExecuteInSandbox.mock.calls.at(-1) ?? [] - const runtimeBindingName = /new (__sim_code_\d+_runtime_\d+)\.RegExp/.exec( - sandboxRequest.code - )?.[1] - expect(response.status).toBe(200) - expect(runtimeBindingName).toBeDefined() - expect(sandboxRequest.runtimeBindings).toContainEqual({ - name: runtimeBindingName, - kind: 'javascript-runtime', - }) - expect(JSON.stringify(sandboxRequest.runtimeBindings)).not.toContain('secret') - }) - - it('allocates remote runtime helpers against decoded JavaScript identifiers', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - const escapedAlias = String.raw`\u005f\u005fsim_runtime_read_0` - - const response = await POST( - createMockRequest('POST', { - code: [ - `import { basename as ${escapedAlias} } from "node:path"`, - `return ["{{KEY}}", ${escapedAlias}("/tmp/file.txt")]`, - ].join('\n'), - language: 'javascript', - envVars: { KEY: 'secret' }, - }) - ) - - const [sandboxRequest] = mockExecuteInSandbox.mock.calls.at(-1) ?? [] - const syntaxCheck = spawnSync(process.execPath, ['--input-type=module', '--check'], { - encoding: 'utf8', - input: sandboxRequest.code, - }) - expect(response.status).toBe(200) - expect(sandboxRequest.code).toContain('readFileSync as __sim_runtime_read_1') - expect(sandboxRequest.code).not.toContain('readFileSync as __sim_runtime_read_0') - expect(syntaxCheck.stderr).toBe('') - expect(syntaxCheck.status).toBe(0) - }) - - it('keeps comments and missing placeholders unchanged without secret provenance', async () => { - const response = await POST( - createMockRequest( - 'POST', - { - code: '// {{COMMENT_ONLY}}\nreturn "{{MISSING}}"', - envVars: { COMMENT_ONLY: 'must-not-bind' }, - }, - { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } - ) - ) - - const [request] = mockExecuteInIsolatedVM.mock.calls.at(-1) ?? [] - expect(response.status).toBe(200) - expect((await response.json()).__resolvedSecretNames).toEqual([]) - expect(request.code).toContain('// {{COMMENT_ONLY}}') - expect(request.code).toContain('"{{MISSING}}"') - expect(Object.values(request.contextVariables)).not.toContain('must-not-bind') - }) - - it('does not infer provenance from an unused low-entropy environment value', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'Box eSign', stdout: '' }) - - const response = await POST( - createMockRequest( - 'POST', - { - code: 'return "Box eSign"', - envVars: { SERVICENOW_PASSWORD: 'x' }, - }, - { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } - ) - ) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.output.result).toBe('Box eSign') - expect(data.__resolvedSecretNames).toEqual([]) - }) - - it('does not build provenance matchers for unused oversized environment values', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'safe', stdout: '' }) - - const response = await POST( - createMockRequest( - 'POST', - { - code: 'return "safe"', - envVars: { UNUSED: 'x'.repeat(65 * 1024) }, - }, - { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } - ) - ) - - expect(response.status).toBe(200) - expect((await response.json()).__resolvedSecretNames).toEqual([]) - }) - - it('conservatively reports only compiled secrets when bounded output classification is exceeded', async () => { - const result = Array.from({ length: 100_001 }, () => 'ordinary') - mockExecuteInIsolatedVM.mockResolvedValueOnce({ result, stdout: '' }) - - const response = await POST( - createMockRequest( - 'POST', - { - code: 'const key = {{API_KEY}}; return params.items', - params: { items: result }, - envVars: { API_KEY: 'secret-value', UNUSED: 'x' }, - }, - { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } - ) - ) - const data = await response.json() - - expect(response.status).toBe(200) - expect(response.headers.get('x-sim-private-tool-metadata')).toBe('resolved-secret-names-v1') - expect(data.output.result).toHaveLength(100_001) - expect(data.output.result[0]).toBe('ordinary') - expect(data.__resolvedSecretNames).toEqual(['API_KEY']) - }) - - it('conservatively reports a compiled secret whose value exceeds matcher capacity', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'ordinary', stdout: '' }) - - const response = await POST( - createMockRequest( - 'POST', - { - code: 'const key = {{OVERSIZED_SECRET}}; return "ordinary"', - envVars: { OVERSIZED_SECRET: 's'.repeat(64 * 1024 + 1), UNUSED: 'x' }, - }, - { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } - ) - ) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.output.result).toBe('ordinary') - expect(data.__resolvedSecretNames).toEqual(['OVERSIZED_SECRET']) - }) - - it('tracks only compiled names when configured secrets share the same value', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'true', stdout: '' }) - const oneResponse = await POST( - createMockRequest( - 'POST', - { - code: 'return {{SECOND}}', - envVars: { FIRST: 'true', SECOND: 'true' }, - }, - { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } - ) - ) - - mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'true', stdout: '' }) - const bothResponse = await POST( - createMockRequest( - 'POST', - { - code: 'const first = {{FIRST}}; return {{SECOND}}', - envVars: { FIRST: 'true', SECOND: 'true' }, - }, - { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } - ) - ) - - expect((await oneResponse.json()).__resolvedSecretNames).toEqual(['SECOND']) - expect((await bothResponse.json()).__resolvedSecretNames).toEqual(['FIRST', 'SECOND']) - }) - - it('lowers missing shell placeholders while preserving comments and heredoc delimiters', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - const response = await POST( - createMockRequest( - 'POST', - { - code: [ - '# {{COMMENT_ONLY}}', - 'printf \'%s\\n\' "before{{MISSING}}after"', - "cat <<'{{DELIMITER}}'", - 'literal body', - '{{DELIMITER}}', - ].join('\n'), - language: 'shell', - envVars: { COMMENT_ONLY: 'must-not-bind' }, - }, - { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } - ) - ) - - const [request] = mockExecuteShellInSandbox.mock.calls.at(-1) ?? [] - expect(response.status).toBe(200) - expect((await response.json()).__resolvedSecretNames).toEqual([]) - expect(request.code).toContain('# {{COMMENT_ONLY}}') - expect(request.code).toContain('"beforeafter"') - expect(request.code).toContain("cat <<'{{DELIMITER}}'") - expect(request.code).toContain('\n{{DELIMITER}}') - expect(request.code).not.toContain('{{MISSING}}') - }) - - it.each([ - { - language: 'javascript', - code: 'import path from "node:path"\nreturn "{{API_KEY}}"', - }, - { language: 'python', code: 'return "{{API_KEY}}"' }, - ])( - 'keeps $language runtime values out of remote generated source', - async ({ language, code }) => { - envFlagsMock.isRemoteSandboxEnabled = true - const secret = 'remote"\\\nsecret' - - const response = await POST( - createMockRequest('POST', { - code, - language, - envVars: { API_KEY: secret }, - params: { input: 'value' }, - contextVariables: { __blockRef_0: 'context' }, - }) - ) - - const [sandboxRequest] = mockExecuteInSandbox.mock.calls.at(-1) ?? [] - expect(response.status).toBe(200) - expect(sandboxRequest.code).not.toContain(secret) - expect(sandboxRequest.code).not.toContain('__var_') - expect(sandboxRequest.privateInputs).toHaveLength(1) - const payload = JSON.parse(sandboxRequest.privateInputs[0].content) - expect(payload.environmentVariables.API_KEY).toBe(secret) - expect(payload.params.input).toBe('value') - expect(payload.contextVariables).toContainEqual({ - name: '__blockRef_0', - kind: 'json', - value: 'context', - }) - } - ) - - it('routes quoted shell heredocs through private sandbox input files', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - const secret = 'shell"\\\n{{OTHER}}' - mockExecuteShellInSandbox.mockResolvedValueOnce({ - result: null, - stdout: `Bearer ${secret}\n$UNRELATED \`touch /tmp/nope\``, - sandboxId: 'test-shell-sandbox-id', - }) - - const response = await POST( - createMockRequest( - 'POST', - { - code: [ - "cat <<'PAYLOAD'", - 'Bearer {{API_KEY}}', - '$UNRELATED `touch /tmp/nope`', - 'PAYLOAD', - ].join('\n'), - language: 'shell', - envVars: { API_KEY: secret, OTHER: 'must-not-resolve' }, - }, - { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } - ) - ) - - const [sandboxRequest] = mockExecuteShellInSandbox.mock.calls.at(-1) ?? [] - expect(response.status).toBe(200) - expect((await response.json()).__resolvedSecretNames).toEqual(['API_KEY']) - expect(sandboxRequest.code).not.toContain(secret) - expect(sandboxRequest.code).not.toContain('$UNRELATED') - expect(sandboxRequest.privateInputs).toHaveLength(1) - expect(sandboxRequest.privateInputs[0].content).toContain(secret) - expect(sandboxRequest.privateInputs[0].content).toContain('$UNRELATED `touch /tmp/nope`') - }) - - /** - * The founding scenario of the usage trail: code that reads a secret and emits it only in - * transformed form. No output ever matches the value, so an output-gated report said - * "never used" for exactly the run an admin needs to see. A referenced secret reports - * whether or not its value surfaces. - */ - it('reports a secret exfiltrated character by character', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ - result: 's|e|c|r|e|t|-|v|a|l|u|e|-|1|2|3|4', - stdout: '', - }) - const response = await POST( - createMockRequest( - 'POST', - { - code: "const k = '{{API_KEY}}'; return k.split('').join('|')", - envVars: { API_KEY: 'secret-value-1234' }, - }, - { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } - ) - ) - - expect(response.status).toBe(200) - expect((await response.json()).__resolvedSecretNames).toEqual(['API_KEY']) - }) - - /** The ordinary silent use: the key authenticates a call and never appears in output. */ - it('reports a secret used without appearing in the output', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: { status: 200 }, stdout: '' }) - const response = await POST( - createMockRequest( - 'POST', - { - code: "await fetch('https://api.example.com', { headers: { auth: environmentVariables['API_KEY'] } }); return { status: 200 }", - envVars: { API_KEY: 'secret-value-1234' }, - }, - { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } - ) - ) - - expect(response.status).toBe(200) - expect((await response.json()).__resolvedSecretNames).toEqual(['API_KEY']) - }) - - it('does not report a reference when validation rejects before code resolution', async () => { - const response = await POST( - createMockRequest( - 'POST', - { - code: 'return {{API_KEY}}', - envVars: { API_KEY: 'secret-value' }, - outputs: { - files: Array.from({ length: 21 }, (_, index) => ({ - path: `files/output-${index}.json`, - sandboxPath: `/home/user/output-${index}.json`, - })), - }, - }, - { - 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', - } - ) - ) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toContain('Too many sandbox output files requested') - expect(data.__resolvedSecretNames).toEqual([]) - expect(mockExecuteInIsolatedVM).not.toHaveBeenCalled() - expect(mockExecuteInSandbox).not.toHaveBeenCalled() - }) - - /** - * A direct read is a factual reference to the environment binding, not the value-coincidence - * inference #6374 removed — that one claimed a secret because its plaintext happened to equal - * an unrelated output. Reporting it is what activates execution-log masking for the value. - */ - it('reports secrets reached through placeholders and through direct environment reads', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ - result: 'secret-valueother-secret', - stdout: '', - }) - const envResponse = await POST( - createMockRequest( - 'POST', - { - code: 'return {{SHARED}} + {{ENV_ONLY}} + {{MISSING}}', - params: { SHARED: 'param-value', MISSING: 'ordinary-param' }, - envVars: { SHARED: 'secret-value', ENV_ONLY: 'other-secret' }, - }, - { - 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', - } - ) - ) - const envData = await envResponse.json() - - mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'secret-value', stdout: '' }) - const directResponse = await POST( - createMockRequest( - 'POST', - { - code: 'return environmentVariables.API_KEY + params.API_KEY', - params: { API_KEY: 'ordinary-param' }, - envVars: { API_KEY: 'secret-value' }, - }, - { - 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', - } - ) - ) - const directData = await directResponse.json() - - expect(envData.__resolvedSecretNames).toEqual(['ENV_ONLY', 'SHARED']) - expect(directData.output.result).toBe('secret-value') - expect(directData.__resolvedSecretNames).toEqual(['API_KEY']) - }) - - it.each([ - { name: 'numeric', secret: '123', result: 123 }, - { name: 'boolean', secret: 'true', result: true }, - ])( - 'preserves a typed $name value returned through a direct environment read while reporting it', - async ({ secret, result }) => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ result, stdout: '' }) - - const response = await POST( - createMockRequest( - 'POST', - { - code: 'return environmentVariables.API_KEY', - envVars: { API_KEY: secret }, - }, - { - 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', - } - ) - ) - const data = await response.json() - - /** The typed value survives: a secret this short is never substitutable. */ - expect(data.output.result).toBe(result) - expect(data.__resolvedSecretNames).toEqual(['API_KEY']) - } - ) - - it('reports placeholder output and a shell environment expansion alike', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - mockExecuteShellInSandbox.mockResolvedValueOnce({ - result: null, - stdout: 'secret-value', - sandboxId: 'test-shell-sandbox-id', - }) - - const referencedResponse = await POST( - createMockRequest( - 'POST', - { - code: 'printf "%s" "{{API_KEY}}"', - language: 'shell', - envVars: { API_KEY: 'secret-value' }, - }, - { - 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', - } - ) - ) - const referencedData = await referencedResponse.json() - - mockExecuteShellInSandbox.mockResolvedValueOnce({ - result: null, - stdout: 'secret-value', - sandboxId: 'test-shell-sandbox-id', - }) - const directResponse = await POST( - createMockRequest( - 'POST', - { - code: 'printf "%s" "$API_KEY"', - language: 'shell', - envVars: { API_KEY: 'secret-value' }, - }, - { - 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', - } - ) - ) - const directData = await directResponse.json() - - expect(referencedData.__resolvedSecretNames).toEqual(['API_KEY']) - expect(directData.output.stdout).toBe('secret-value') - expect(directData.__resolvedSecretNames).toEqual(['API_KEY']) - }) - - it('returns nonzero shell stderr as a visible 422 error and diagnostic output', async () => { - envFlagsMock.isRemoteSandboxEnabled = true - const stderr = "error: unknown flag: --short\nSee 'kubectl version --help' for usage." - mockExecuteShellInSandbox.mockResolvedValueOnce({ - result: null, - stdout: stderr, - error: stderr, - sandboxId: 'test-shell-sandbox-id', - }) - - const response = await POST( - createMockRequest('POST', { - code: 'kubectl version --client --short', - language: 'shell', - }) - ) - const data = await response.json() - - expect(response.status).toBe(422) - expect(data).toMatchObject({ - success: false, - error: stderr, - output: { result: null, stdout: stderr }, - }) - }) - - it('keeps execution available when the scoped catalog exceeds provenance matcher bounds', async () => { - const response = await POST( - createMockRequest( - 'POST', - { - code: 'return "ok"', - envVars: { OVERSIZED_SECRET: 's'.repeat(64 * 1024 + 1) }, - }, - { - 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', - } - ) - ) - - expect(response.status).toBe(200) - expect((await response.json()).__resolvedSecretNames).toEqual([]) - expect(response.headers.get('x-sim-private-tool-metadata')).toBe('resolved-secret-names-v1') - expect(mockExecuteInIsolatedVM).toHaveBeenCalled() - }) - - it('reports only substitutions allowed by the Function secret scope', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'allowed-secret', stdout: '' }) - const response = await POST( - createMockRequest( - 'POST', - { - code: 'return {{ALLOWED}} + {{BLOCKED}}', - envVars: { ALLOWED: 'allowed-secret', BLOCKED: 'blocked-secret' }, - secretScope: 'selected', - mountedSecrets: ['ALLOWED'], - }, - { - 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', - } - ) - ) - - expect((await response.json()).__resolvedSecretNames).toEqual(['ALLOWED']) - }) - - it('resolves a selected __proto__ secret as an own environment key', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'secret-value', stdout: '' }) - const response = await POST( - createMockRequest( - 'POST', - { - code: 'return "{{__proto__}}"', - envVars: Object.fromEntries([['__proto__', 'secret-value']]), - secretScope: 'selected', - mountedSecrets: ['__proto__'], - }, - { - 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', - } - ) - ) - - expect(response.status).toBe(200) - expect((await response.json()).__resolvedSecretNames).toEqual(['__proto__']) - }) - - /** - * Previously asserted the inverse: a referenced secret whose value stayed out of the - * result reported nothing. That gate made the trail miss silent use — the ordinary - * API-call case and the transformed-exfiltration case alike — so activation now follows - * the referenced set. The value never appearing costs nothing downstream; the masking - * matcher simply never fires on it. - */ - it('activates a referenced secret even when its value never crosses the result', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'safe-result', stdout: '' }) - - const response = await POST( - createMockRequest( - 'POST', - { - code: 'const key = {{API_KEY}}; return "safe-result"', - envVars: { API_KEY: 'secret-value' }, - }, - { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } - ) - ) - - expect((await response.json()).__resolvedSecretNames).toEqual(['API_KEY']) - }) - - it.concurrent('should resolve tag variables with syntax', async () => { - const req = createMockRequest('POST', { - code: 'return ', - blockData: { - 'block-123': { id: '123', subject: 'Test Email' }, - }, - blockNameMapping: { - email: 'block-123', - }, - }) - - const response = await POST(req) - - expect(response.status).toBe(200) - }) - - it.concurrent('should NOT treat email addresses as template variables', async () => { - const req = createMockRequest('POST', { - code: 'return "Email sent to user"', - params: { - email: { - from: 'Dr. Shaw ', - to: 'User ', - }, - }, - }) - - const response = await POST(req) - - expect(response.status).toBe(200) - }) - - it.concurrent('should only match valid variable names in angle brackets', async () => { - const req = createMockRequest('POST', { - code: 'return + "" + ', - blockData: { - 'block-1': 'hello', - 'block-2': 'world', - }, - blockNameMapping: { - validvar: 'block-1', - another_valid: 'block-2', - }, - }) - - const response = await POST(req) - - expect(response.status).toBe(200) - }) - }) - - describe('Gmail Email Data Handling', () => { - it.concurrent( - 'should handle Gmail webhook data with email addresses containing angle brackets', - async () => { - const emailData = { - id: '123', - from: 'Dr. Shaw ', - to: 'User ', - subject: 'Test Email', - bodyText: 'Hello world', - } - - const req = createMockRequest('POST', { - code: 'return ', - blockData: { - 'block-email': emailData, - }, - blockNameMapping: { - email: 'block-email', - }, - }) - - const response = await POST(req) - - expect(response.status).toBe(200) - const data = await response.json() - expect(data.success).toBe(true) - } - ) - - it.concurrent( - 'should properly serialize complex email objects with special characters', - async () => { - const emailData = { - from: 'Test User ', - bodyHtml: '
HTML content with "quotes" and \'apostrophes\'
', - bodyText: 'Text with\nnewlines\tand\ttabs', - } - - const req = createMockRequest('POST', { - code: 'return ', - blockData: { - 'block-email': emailData, - }, - blockNameMapping: { - email: 'block-email', - }, - }) - - const response = await POST(req) - - expect(response.status).toBe(200) - } - ) - }) - - describe('Custom Tools', () => { - it.concurrent('should handle custom tool execution with direct parameter access', async () => { - const req = createMockRequest('POST', { - code: 'return location + " weather is sunny"', - params: { - location: 'San Francisco', - }, - isCustomTool: true, - }) - - const response = await POST(req) - - expect(response.status).toBe(200) - }) - }) - - describe('Security and Edge Cases', () => { - it.concurrent('should handle malformed JSON in request body', async () => { - const req = new NextRequest('http://localhost:3000/api/function/execute', { - method: 'POST', - body: 'invalid json{', - headers: { 'Content-Type': 'application/json' }, - }) - - const response = await POST(req) - - expect(response.status).toBe(400) - }) - - it.concurrent('should handle timeout parameter', async () => { - const req = createMockRequest('POST', { - code: 'return "test"', - timeout: 10000, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(mockExecuteInIsolatedVM).toHaveBeenCalledWith( - expect.objectContaining({ timeoutMs: 10000 }), - expect.any(Object) - ) - }) - - it.concurrent('should handle empty parameters object', async () => { - const req = createMockRequest('POST', { - code: 'return "no params"', - params: {}, - }) - - const response = await POST(req) - - expect(response.status).toBe(200) - }) - }) - - describe('Enhanced Error Handling', () => { - it('should provide detailed syntax error with line content', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ - result: null, - stdout: '', - error: { message: 'Unexpected end of input', name: 'SyntaxError' }, - }) - - const req = createMockRequest('POST', { - code: 'const obj = {\n name: "test",\n description: "This has a missing closing quote\n};\nreturn obj;', - timeout: 5000, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(422) - expect(data.success).toBe(false) - expect(data.error).toBeTruthy() - }) - - it('should provide detailed runtime error with line and column', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ - result: null, - stdout: '', - error: { - message: "Cannot read properties of null (reading 'someMethod')", - name: 'TypeError', - }, - }) - - const req = createMockRequest('POST', { - code: 'const obj = null;\nreturn obj.someMethod();', - timeout: 5000, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(422) - expect(data.success).toBe(false) - expect(data.error).toContain('Type Error') - expect(data.error).toContain('Cannot read properties of null') - }) - - it('should handle ReferenceError with enhanced details', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ - result: null, - stdout: '', - error: { message: 'undefinedVariable is not defined', name: 'ReferenceError' }, - }) - - const req = createMockRequest('POST', { - code: 'const x = 42;\nreturn undefinedVariable + x;', - timeout: 5000, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(422) - expect(data.success).toBe(false) - expect(data.error).toContain('Reference Error') - expect(data.error).toContain('undefinedVariable is not defined') - }) - - it('should show original source code when resolved block references cause syntax errors', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ - result: null, - stdout: '', - error: { - message: 'Unexpected identifier "globalThis"', - name: 'SyntaxError', - line: 1, - column: 7, - lineContent: 'retur globalThis["__blockRef_0"]', - }, - }) - - const req = createMockRequest('POST', { - code: 'retur globalThis["__blockRef_0"]', - sourceCode: 'retur ', - contextVariables: { __blockRef_0: 'value' }, - timeout: 5000, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(422) - expect(data.success).toBe(false) - expect(data.error).toContain('Line 1: `retur `') - expect(data.error).not.toContain('globalThis') - expect(data.debug.lineContent).toBe('retur ') - }) - - it('should handle thrown errors gracefully', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ - result: null, - stdout: '', - error: { message: 'Custom error message', name: 'Error' }, - }) - - const req = createMockRequest('POST', { - code: 'throw new Error("Custom error message");', - timeout: 5000, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(422) - expect(data.success).toBe(false) - expect(data.error).toContain('Custom error message') - }) - - it('should provide helpful suggestions for common syntax errors', async () => { - mockExecuteInIsolatedVM.mockResolvedValueOnce({ - result: null, - stdout: '', - error: { message: 'Unexpected end of input', name: 'SyntaxError' }, - }) - - const req = createMockRequest('POST', { - code: 'const obj = {\n name: "test"\n// Missing closing brace', - timeout: 5000, - }) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(422) - expect(data.success).toBe(false) - expect(data.error).toBeTruthy() - }) - }) - - describe('Utility Functions', () => { - it.concurrent('should properly escape regex special characters', async () => { - const req = createMockRequest('POST', { - code: 'return {{special.chars+*?}}', - envVars: { - 'special.chars+*?': 'escaped-value', - }, - }) - - const response = await POST(req) - - expect(response.status).toBe(200) - }) - - it.concurrent('should handle JSON serialization edge cases', async () => { - const complexData = { - special: 'chars"with\'quotes', - unicode: '🎉 Unicode content', - nested: { - deep: { - value: 'test', - }, - }, - } - - const req = createMockRequest('POST', { - code: 'return ', - blockData: { - 'block-complex': complexData, - }, - blockNameMapping: { - complexdata: 'block-complex', - }, - }) - - const response = await POST(req) - - expect(response.status).toBe(200) - }) - }) -}) diff --git a/apps/sim/app/api/function/execute/route.ts b/apps/sim/app/api/function/execute/route.ts deleted file mode 100644 index 59164125214..00000000000 --- a/apps/sim/app/api/function/execute/route.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { type NextRequest, NextResponse } from 'next/server' -import { functionExecuteContract } from '@/lib/api/contracts' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - executeFunctionRequest, - projectFunctionValidationResponse, -} from '@/lib/function-execution/execute-request' - -export const dynamic = 'force-dynamic' -export const runtime = 'nodejs' -/** Static host ceiling; the trusted workflow deadline applies the smaller per-call budget. */ -export const maxDuration = 604800 - -/** - * Accepts legacy Function requests from pre-direct-execution tasks during a blue/green rollout. - * Remove this adapter only after those tasks have drained past the maximum execution window. - */ -export const POST = withRouteHandler(async (req: NextRequest) => { - const auth = await checkInternalAuth(req) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - const parsed = await parseRequest(functionExecuteContract, req, {}) - if (!parsed.success) return projectFunctionValidationResponse(req, parsed.response) - return executeFunctionRequest(req, parsed.data.body, { - userId: auth.userId, - ...(auth.sandboxProfile === 'mothership' ? { sandboxProfile: 'mothership' } : {}), - }) -}) diff --git a/apps/sim/app/api/guardrails/validate/route.test.ts b/apps/sim/app/api/guardrails/validate/route.test.ts deleted file mode 100644 index 797e7852af0..00000000000 --- a/apps/sim/app/api/guardrails/validate/route.test.ts +++ /dev/null @@ -1,466 +0,0 @@ -/** - * @vitest-environment node - */ -import { createMockRequest, hybridAuthMockFns, workflowAuthzMockFns } from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { - mockAuthorizeCredentialUse, - mockCheckActorUsageLimits, - mockCheckAttributedUsageLimits, - mockRequireBillingAttributionHeader, - mockResolveBillingAttribution, - mockToBillingContext, - mockValidateHallucination, - mockRecordUsage, - mockCheckAndBillPayerOverageThreshold, - mockPrepareCopilotEnvironmentContext, - mockImportProvenance, - mockRegistryIsComplete, -} = vi.hoisted(() => ({ - mockAuthorizeCredentialUse: vi.fn(), - mockCheckActorUsageLimits: vi.fn(), - mockCheckAttributedUsageLimits: vi.fn(), - mockRequireBillingAttributionHeader: vi.fn(), - mockResolveBillingAttribution: vi.fn(), - mockToBillingContext: vi.fn(), - mockValidateHallucination: vi.fn(), - mockRecordUsage: vi.fn(), - mockCheckAndBillPayerOverageThreshold: vi.fn(), - mockPrepareCopilotEnvironmentContext: vi.fn(), - mockImportProvenance: vi.fn(), - mockRegistryIsComplete: vi.fn(), -})) - -vi.mock('@/lib/auth/credential-access', () => ({ - authorizeCredentialUse: mockAuthorizeCredentialUse, -})) - -vi.mock('@/lib/billing/calculations/usage-monitor', () => ({ - checkActorUsageLimits: mockCheckActorUsageLimits, -})) - -vi.mock('@/lib/billing/core/billing-attribution', () => ({ - checkAttributedUsageLimits: mockCheckAttributedUsageLimits, - requireBillingAttributionHeader: mockRequireBillingAttributionHeader, - resolveBillingAttribution: mockResolveBillingAttribution, - toBillingContext: mockToBillingContext, -})) - -vi.mock('@/lib/billing/core/usage-log', () => ({ - recordUsage: mockRecordUsage, -})) - -vi.mock('@/lib/billing/threshold-billing', () => ({ - checkAndBillPayerOverageThreshold: mockCheckAndBillPayerOverageThreshold, -})) - -vi.mock('@/lib/guardrails/validate_hallucination', () => ({ - validateHallucination: mockValidateHallucination, -})) - -vi.mock('@/lib/copilot/environment-context', () => ({ - prepareCopilotEnvironmentContext: mockPrepareCopilotEnvironmentContext, -})) - -vi.mock('@/lib/guardrails/validate_json', () => ({ - validateJson: vi.fn(() => ({ passed: true })), -})) - -vi.mock('@/lib/guardrails/validate_pii', () => ({ - validatePII: vi.fn(() => ({ passed: true })), -})) - -vi.mock('@/lib/guardrails/validate_regex', () => ({ - validateRegex: vi.fn(() => ({ passed: true })), -})) - -vi.mock('@/ee/access-control/utils/permission-check', () => ({ - assertPermissionsAllowed: vi.fn(), - ModelNotAllowedError: class ModelNotAllowedError extends Error {}, - ProviderNotAllowedError: class ProviderNotAllowedError extends Error {}, -})) - -import { POST } from '@/app/api/guardrails/validate/route' - -describe('POST /api/guardrails/validate', () => { - beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'session', - }) - workflowAuthzMockFns.mockAuthorizeWorkflowByWorkspacePermission.mockResolvedValue({ - allowed: true, - workflow: { id: 'wf-1', workspaceId: 'ws-1' }, - }) - mockCheckActorUsageLimits.mockResolvedValue({ isExceeded: false }) - mockCheckAttributedUsageLimits.mockResolvedValue({ isExceeded: false }) - mockResolveBillingAttribution.mockResolvedValue({ - actorUserId: 'user-1', - workspaceId: 'ws-1', - billingEntity: { type: 'organization', id: 'org-1' }, - }) - mockRequireBillingAttributionHeader.mockReturnValue({ - actorUserId: 'user-1', - workspaceId: 'ws-1', - billingEntity: { type: 'organization', id: 'org-1' }, - }) - mockToBillingContext.mockReturnValue({ - billingEntity: { type: 'organization', id: 'org-1' }, - billingPeriod: { - start: new Date('2026-07-01T00:00:00.000Z'), - end: new Date('2026-08-01T00:00:00.000Z'), - }, - }) - mockValidateHallucination.mockResolvedValue({ passed: true, score: 8 }) - mockImportProvenance.mockResolvedValue({ success: true, matched: true }) - mockRegistryIsComplete.mockReturnValue(true) - mockPrepareCopilotEnvironmentContext.mockResolvedValue({ - resolvedSecretTraceRegistry: { - importProvenanceForValueAtInputPath: mockImportProvenance, - isComplete: mockRegistryIsComplete, - }, - }) - }) - - it('rejects a vertexCredential the caller does not have access to before calling validateHallucination', async () => { - mockAuthorizeCredentialUse.mockResolvedValue({ - ok: false, - error: 'You do not have access to this credential.', - }) - - const res = await POST( - createMockRequest('POST', { - validationType: 'hallucination', - input: 'test input', - knowledgeBaseId: 'kb-1', - model: 'vertex/gemini-2.5-pro', - workflowId: 'wf-1', - vertexCredential: 'someone-elses-account-id', - }) - ) - - expect(res.status).toBe(401) - expect(mockAuthorizeCredentialUse).toHaveBeenCalledWith( - expect.anything(), - expect.objectContaining({ - credentialId: 'someone-elses-account-id', - workflowId: 'wf-1', - requireWorkflowIdForInternal: false, - }) - ) - expect(mockValidateHallucination).not.toHaveBeenCalled() - }) - - it('proceeds with hallucination validation when the caller has access to the vertexCredential', async () => { - mockAuthorizeCredentialUse.mockResolvedValue({ ok: true }) - - const res = await POST( - createMockRequest('POST', { - validationType: 'hallucination', - input: 'test input', - knowledgeBaseId: 'kb-1', - model: 'vertex/gemini-2.5-pro', - workflowId: 'wf-1', - vertexCredential: 'my-own-account-id', - }) - ) - - expect(res.status).toBe(200) - const json = await res.json() - expect(json.output.passed).toBe(true) - expect(mockAuthorizeCredentialUse).toHaveBeenCalledWith( - expect.anything(), - expect.objectContaining({ credentialId: 'my-own-account-id' }) - ) - expect(mockValidateHallucination).toHaveBeenCalled() - }) - - it('does not gate on vertexCredential for non-hallucination validation types', async () => { - const res = await POST( - createMockRequest('POST', { - validationType: 'json', - input: '{"a":1}', - }) - ) - - expect(res.status).toBe(200) - expect(mockAuthorizeCredentialUse).not.toHaveBeenCalled() - }) - - it('does not gate hallucination validation when no vertexCredential is supplied', async () => { - const res = await POST( - createMockRequest('POST', { - validationType: 'hallucination', - input: 'test input', - knowledgeBaseId: 'kb-1', - model: 'gpt-4o', - workflowId: 'wf-1', - }) - ) - - expect(res.status).toBe(200) - expect(mockAuthorizeCredentialUse).not.toHaveBeenCalled() - expect(mockValidateHallucination).toHaveBeenCalledWith( - expect.objectContaining({ - actorUserId: 'user-1', - billingAttribution: expect.objectContaining({ - actorUserId: 'user-1', - workspaceId: 'ws-1', - }), - }) - ) - }) - - it('imports transported active provenance before hallucination model egress', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - const provenance = { - version: 1, - complete: true, - entries: [{ encryptedValue: 'encrypted-secret', name: 'TOKEN' }], - scope: { userId: 'user-1', workspaceId: 'ws-1' }, - } - const res = await POST( - createMockRequest( - 'POST', - { - validationType: 'hallucination', - input: 'secret value', - knowledgeBaseId: 'kb-1', - model: 'gpt-4o', - workflowId: 'wf-1', - __resolvedSecretTraceProvenance: provenance, - }, - { 'x-sim-private-model-input-provenance': 'resolved-secret-provenance-v1' } - ) - ) - - expect(res.status).toBe(200) - expect(mockImportProvenance).toHaveBeenCalledWith(provenance, 'secret value', ['input'], { - trusted: true, - origin: 'guardrailsRoute.inputProvenance', - }) - }) - - it('rejects private provenance supplied through session authentication', async () => { - const provenance = { version: 1, complete: true, entries: [] } - - const res = await POST( - createMockRequest( - 'POST', - { - validationType: 'hallucination', - input: 'test input', - knowledgeBaseId: 'kb-1', - model: 'gpt-4o', - workflowId: 'wf-1', - __resolvedSecretTraceProvenance: provenance, - }, - { 'x-sim-private-model-input-provenance': 'resolved-secret-provenance-v1' } - ) - ) - - expect(res.status).toBe(400) - expect(mockImportProvenance).not.toHaveBeenCalled() - expect(mockValidateHallucination).not.toHaveBeenCalled() - }) - - it('rejects a partial hallucination provenance envelope before model egress', async () => { - const res = await POST( - createMockRequest( - 'POST', - { - validationType: 'hallucination', - input: 'test input', - knowledgeBaseId: 'kb-1', - model: 'gpt-4o', - workflowId: 'wf-1', - }, - { 'x-sim-private-model-input-provenance': 'resolved-secret-provenance-v1' } - ) - ) - - expect(res.status).toBe(400) - expect(mockValidateHallucination).not.toHaveBeenCalled() - }) - - it('does not gate on a leftover vertexCredential when the resolved model is not vertex', async () => { - const res = await POST( - createMockRequest('POST', { - validationType: 'hallucination', - input: 'test input', - knowledgeBaseId: 'kb-1', - model: 'gpt-4o', - workflowId: 'wf-1', - vertexCredential: 'someone-elses-account-id', - }) - ) - - expect(res.status).toBe(200) - expect(mockAuthorizeCredentialUse).not.toHaveBeenCalled() - expect(mockValidateHallucination).toHaveBeenCalled() - }) - - it('bills a hosted hallucination check against the exact workspace payer', async () => { - mockValidateHallucination.mockResolvedValue({ passed: true, score: 8, cost: 0.01 }) - - const res = await POST( - createMockRequest('POST', { - validationType: 'hallucination', - input: 'test input', - knowledgeBaseId: 'kb-1', - model: 'gpt-4o', - workflowId: 'wf-1', - }) - ) - - expect(res.status).toBe(200) - expect(mockRecordUsage).toHaveBeenCalledWith( - expect.objectContaining({ - userId: 'user-1', - workspaceId: 'ws-1', - billingEntity: { type: 'organization', id: 'org-1' }, - }) - ) - expect(mockCheckAndBillPayerOverageThreshold).toHaveBeenCalledWith({ - type: 'organization', - id: 'org-1', - }) - }) - - it('requires and forwards immutable attribution for internal hallucination checks', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - - const request = createMockRequest( - 'POST', - { - validationType: 'hallucination', - input: 'test input', - knowledgeBaseId: 'kb-1', - model: 'gpt-4o', - workflowId: 'wf-1', - __resolvedSecretTraceProvenance: { version: 1, complete: true, entries: [] }, - }, - { 'x-sim-private-model-input-provenance': 'resolved-secret-provenance-v1' } - ) - const res = await POST(request) - - expect(res.status).toBe(200) - expect(mockRequireBillingAttributionHeader).toHaveBeenCalledWith(request.headers, { - actorUserId: 'user-1', - workspaceId: 'ws-1', - }) - expect(mockResolveBillingAttribution).not.toHaveBeenCalled() - expect(mockValidateHallucination).toHaveBeenCalledWith( - expect.objectContaining({ - actorUserId: 'user-1', - billingAttribution: expect.objectContaining({ - actorUserId: 'user-1', - workspaceId: 'ws-1', - }), - }) - ) - }) - - it('preserves a headerless legacy internal hallucination check', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - - const res = await POST( - createMockRequest('POST', { - validationType: 'hallucination', - input: 'test input', - knowledgeBaseId: 'kb-1', - model: 'gpt-4o', - workflowId: 'wf-1', - }) - ) - - expect(res.status).toBe(200) - expect(mockImportProvenance).not.toHaveBeenCalled() - expect(mockValidateHallucination).toHaveBeenCalled() - }) - - it('rejects invalid internal billing attribution as a protocol error', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - mockRequireBillingAttributionHeader.mockImplementationOnce(() => { - throw new Error('Billing attribution header is required') - }) - - const res = await POST( - createMockRequest('POST', { - validationType: 'hallucination', - input: 'test input', - knowledgeBaseId: 'kb-1', - model: 'gpt-4o', - workflowId: 'wf-1', - }) - ) - - expect(res.status).toBe(400) - await expect(res.json()).resolves.toEqual({ error: 'Invalid billing attribution' }) - expect(mockValidateHallucination).not.toHaveBeenCalled() - }) - - it('surfaces workspace billing resolution failures as infrastructure errors', async () => { - mockResolveBillingAttribution.mockRejectedValueOnce(new Error('Database unavailable')) - - const res = await POST( - createMockRequest('POST', { - validationType: 'hallucination', - input: 'test input', - knowledgeBaseId: 'kb-1', - model: 'gpt-4o', - workflowId: 'wf-1', - }) - ) - - expect(res.status).toBe(500) - await expect(res.json()).resolves.toEqual({ error: 'Failed to resolve billing attribution' }) - expect(mockValidateHallucination).not.toHaveBeenCalled() - }) - - /** - * The signal now reaches the scoring model, so cancellation is reachable here. - * `passed: false` would read to a consumer as the guardrail rejecting the content, - * blocking a run that was abandoned rather than judged. - */ - it('reports a cancelled run as cancellation rather than a failed guardrail', async () => { - mockAuthorizeCredentialUse.mockResolvedValue({ ok: true }) - mockValidateHallucination.mockRejectedValueOnce( - Object.assign(new Error('The operation was aborted.'), { name: 'AbortError' }) - ) - - const res = await POST( - createMockRequest('POST', { - validationType: 'hallucination', - input: 'test input', - knowledgeBaseId: 'kb-1', - model: 'openai/gpt-4o', - workflowId: 'wf-1', - }) - ) - - expect(res.status).toBe(499) - const json = await res.json() - expect(json.success).toBe(false) - expect(json.output?.passed).toBeUndefined() - }) -}) diff --git a/apps/sim/app/api/guardrails/validate/route.ts b/apps/sim/app/api/guardrails/validate/route.ts deleted file mode 100644 index d6edea5b4ef..00000000000 --- a/apps/sim/app/api/guardrails/validate/route.ts +++ /dev/null @@ -1,518 +0,0 @@ -import { createLogger } from '@sim/logger' -import { authorizeWorkflowByWorkspacePermission } from '@sim/platform-authz/workflow' -import { type NextRequest, NextResponse } from 'next/server' -import { guardrailsValidateContract } from '@/lib/api/contracts' -import { parseRequest } from '@/lib/api/server' -import { authorizeCredentialUse } from '@/lib/auth/credential-access' -import { AuthType, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { - type BillingAttributionSnapshot, - checkAttributedUsageLimits, - requireBillingAttributionHeader, - resolveBillingAttribution, - toBillingContext, -} from '@/lib/billing/core/billing-attribution' -import { checkAndBillPayerOverageThreshold } from '@/lib/billing/threshold-billing' -import { prepareCopilotEnvironmentContext } from '@/lib/copilot/environment-context' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { inspectModelInputProvenanceRequest } from '@/lib/execution/model-input-provenance' -import type { CustomPiiPattern } from '@/lib/guardrails/pii-entities' -import { validateHallucination } from '@/lib/guardrails/validate_hallucination' -import { validateJson } from '@/lib/guardrails/validate_json' -import { validatePII } from '@/lib/guardrails/validate_pii' -import { validateRegex } from '@/lib/guardrails/validate_regex' -import { - assertPermissionsAllowed, - ModelNotAllowedError, - ProviderNotAllowedError, -} from '@/ee/access-control/utils/permission-check' -import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' -import { isAbortError } from '@/providers/streaming-tool-loop-shared' -import { getProviderFromModel } from '@/providers/utils' - -const logger = createLogger('GuardrailsValidateAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - logger.info(`[${requestId}] Guardrails validation request received`) - - try { - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(guardrailsValidateContract, request, {}) - if (!parsed.success) return parsed.response - const { body } = parsed.data - const { - validationType, - input, - regex, - knowledgeBaseId, - threshold, - topK, - model, - apiKey, - azureEndpoint, - azureApiVersion, - vertexProject, - vertexLocation, - vertexCredential, - bedrockAccessKeyId, - bedrockSecretKey, - bedrockRegion, - workflowId, - piiEntityTypes, - piiMode, - piiLanguage, - piiCustomPatterns, - } = body - - if (!validationType) { - return NextResponse.json({ - success: true, - output: { - passed: false, - validationType: 'unknown', - input: input || '', - error: 'Missing required field: validationType', - }, - }) - } - - if (input === undefined || input === null) { - return NextResponse.json({ - success: true, - output: { - passed: false, - validationType, - input: '', - error: 'Input is missing or undefined', - }, - }) - } - - if ( - validationType !== 'json' && - validationType !== 'regex' && - validationType !== 'hallucination' && - validationType !== 'pii' - ) { - return NextResponse.json({ - success: true, - output: { - passed: false, - validationType, - input: input || '', - error: 'Invalid validationType. Must be "json", "regex", "hallucination", or "pii"', - }, - }) - } - - if (validationType === 'regex' && !regex) { - return NextResponse.json({ - success: true, - output: { - passed: false, - validationType, - input: input || '', - error: 'Regex pattern is required for regex validation', - }, - }) - } - - if (validationType === 'hallucination' && !model) { - return NextResponse.json({ - success: true, - output: { - passed: false, - validationType, - input: input || '', - error: 'Model is required for hallucination validation', - }, - }) - } - - let resolvedWorkspaceId: string | undefined - let billingAttribution: BillingAttributionSnapshot | undefined - let resolvedSecretTraceRegistry: ResolvedSecretTraceRegistry | undefined - - if (validationType === 'hallucination' && model) { - if (!workflowId || typeof workflowId !== 'string') { - return NextResponse.json({ - success: true, - output: { - passed: false, - validationType, - input: input || '', - error: - 'Workflow context is required for hallucination validation. Call this endpoint via a workflow execution, not directly.', - }, - }) - } - - const authorization = await authorizeWorkflowByWorkspacePermission({ - workflowId, - userId: auth.userId, - action: 'read', - }) - - if (!authorization.allowed || !authorization.workflow?.workspaceId) { - return NextResponse.json({ - success: true, - output: { - passed: false, - validationType, - input: input || '', - error: authorization.message || 'Workflow not found or access denied.', - }, - }) - } - - resolvedWorkspaceId = authorization.workflow.workspaceId - resolvedSecretTraceRegistry = ( - await prepareCopilotEnvironmentContext(auth.userId, resolvedWorkspaceId) - ).resolvedSecretTraceRegistry - try { - billingAttribution = - auth.authType === AuthType.INTERNAL_JWT - ? requireBillingAttributionHeader(request.headers, { - actorUserId: auth.userId, - workspaceId: resolvedWorkspaceId, - }) - : await resolveBillingAttribution({ - actorUserId: auth.userId, - workspaceId: resolvedWorkspaceId, - }) - } catch (error) { - const isInternalRequest = auth.authType === AuthType.INTERNAL_JWT - logger.error(`[${requestId}] Failed to establish billing attribution`, { error }) - return NextResponse.json( - { - error: isInternalRequest - ? 'Invalid billing attribution' - : 'Failed to resolve billing attribution', - }, - { status: isInternalRequest ? 400 : 500 } - ) - } - - try { - await assertPermissionsAllowed({ - userId: auth.userId, - workspaceId: resolvedWorkspaceId, - model, - }) - } catch (err) { - if (err instanceof ProviderNotAllowedError || err instanceof ModelNotAllowedError) { - return NextResponse.json({ - success: true, - output: { - passed: false, - validationType, - input: input || '', - error: err.message, - }, - }) - } - throw err - } - - // Gate the actor's usage before incurring hosted LLM + RAG cost. In a normal - // workflow run this already passed at preprocessing; this also blocks direct - // calls to this route by an over-limit or frozen actor. - const usage = await checkAttributedUsageLimits(billingAttribution) - if (usage.isExceeded) { - return NextResponse.json( - { error: usage.message || 'Usage limit exceeded. Please upgrade your plan to continue.' }, - { status: 402 } - ) - } - - if (vertexCredential && getProviderFromModel(model) === 'vertex') { - const vertexCredAccess = await authorizeCredentialUse(request, { - credentialId: vertexCredential, - workflowId, - requireWorkflowIdForInternal: false, - }) - if (!vertexCredAccess.ok) { - logger.warn(`[${requestId}] Vertex credential access denied`, { - error: vertexCredAccess.error, - credentialId: vertexCredential, - }) - return NextResponse.json( - { error: vertexCredAccess.error || 'Unauthorized' }, - { status: 401 } - ) - } - } - } - - const inputStr = convertInputToString(input) - - if (validationType === 'hallucination' && resolvedSecretTraceRegistry) { - const provenanceInspection = inspectModelInputProvenanceRequest(request.headers, body) - if (provenanceInspection.status === 'invalid') { - return NextResponse.json({ error: 'Invalid model input provenance' }, { status: 400 }) - } - if (provenanceInspection.status === 'verified' && auth.authType !== AuthType.INTERNAL_JWT) { - return NextResponse.json({ error: 'Invalid model input provenance' }, { status: 400 }) - } - const provenanceReady = - provenanceInspection.status === 'verified' - ? ( - await resolvedSecretTraceRegistry.importProvenanceForValueAtInputPath( - provenanceInspection.value, - inputStr, - ['input'], - { trusted: true, origin: 'guardrailsRoute.inputProvenance' } - ) - ).success - : true - if (!provenanceReady || !resolvedSecretTraceRegistry.isComplete()) { - return NextResponse.json( - { error: 'Model input provenance is unavailable' }, - { status: 400 } - ) - } - } - - logger.info(`[${requestId}] Executing validation locally`, { - validationType, - inputType: typeof input, - }) - const validationResult = await executeValidation( - validationType, - inputStr, - regex, - knowledgeBaseId, - threshold, - topK, - model, - apiKey, - { - azureEndpoint, - azureApiVersion, - vertexProject, - vertexLocation, - vertexCredential, - bedrockAccessKeyId, - bedrockSecretKey, - bedrockRegion, - }, - workflowId, - resolvedWorkspaceId, - piiEntityTypes, - piiMode, - piiLanguage, - piiCustomPatterns, - auth.userId, - billingAttribution, - requestId, - resolvedSecretTraceRegistry, - request.signal - ) - - /** - * Hallucination scoring records the caller as actor and the workflow - * workspace as payer. BYOK and non-hosted validation resolve to zero cost. - */ - if ( - resolvedWorkspaceId && - billingAttribution && - typeof validationResult.cost === 'number' && - validationResult.cost > 0 - ) { - const { recordUsage } = await import('@/lib/billing/core/usage-log') - try { - await recordUsage({ - userId: auth.userId, - workspaceId: resolvedWorkspaceId, - ...toBillingContext(billingAttribution), - entries: [ - { - category: 'model', - source: 'workflow', - description: `guardrail-hallucination:${model ?? 'unknown'}`, - cost: validationResult.cost, - sourceReference: `guardrail:${workflowId ?? 'unknown'}:${requestId}`, - }, - ], - }) - await checkAndBillPayerOverageThreshold(billingAttribution.billingEntity) - } catch (billingError) { - logger.error(`[${requestId}] Failed to record guardrail usage`, { error: billingError }) - } - } - - logger.info(`[${requestId}] Validation completed`, { - passed: validationResult.passed, - hasError: !!validationResult.error, - score: validationResult.score, - }) - - return NextResponse.json({ - success: true, - output: { - passed: validationResult.passed, - validationType, - input, - error: validationResult.error, - score: validationResult.score, - reasoning: validationResult.reasoning, - detectedEntities: validationResult.detectedEntities, - maskedText: validationResult.maskedText, - }, - }) - } catch (error: any) { - /** - * A cancelled run must not be reshaped into a verdict. `passed: false` here reads - * to a consumer as the guardrail rejecting the content, so an abandoned run would - * block content that was never actually judged. 499 matches the convention the - * workflow execute route already uses for a client-cancelled request. - */ - if (isAbortError(error)) { - logger.info(`[${requestId}] Guardrails validation cancelled by client`) - return NextResponse.json( - { success: false, error: 'Client cancelled request' }, - { status: 499 } - ) - } - logger.error(`[${requestId}] Guardrails validation failed`, { error }) - return NextResponse.json({ - success: true, - output: { - passed: false, - validationType: 'unknown', - input: '', - error: error.message || 'Validation failed due to unexpected error', - }, - }) - } -}) - -/** - * Convert input to string for validation - */ -function convertInputToString(input: any): string { - if (typeof input === 'string') { - return input - } - if (input === null || input === undefined) { - return '' - } - if (typeof input === 'object') { - return JSON.stringify(input) - } - return String(input) -} - -/** - * Execute validation using TypeScript validators - */ -async function executeValidation( - validationType: string, - inputStr: string, - regex: string | undefined, - knowledgeBaseId: string | undefined, - threshold: string | undefined, - topK: string | undefined, - model: string | undefined, - apiKey: string | undefined, - providerCredentials: { - azureEndpoint?: string - azureApiVersion?: string - vertexProject?: string - vertexLocation?: string - vertexCredential?: string - bedrockAccessKeyId?: string - bedrockSecretKey?: string - bedrockRegion?: string - }, - workflowId: string | undefined, - workspaceId: string | undefined, - piiEntityTypes: string[] | undefined, - piiMode: string | undefined, - piiLanguage: string | undefined, - piiCustomPatterns: CustomPiiPattern[] | undefined, - actorUserId: string, - billingAttribution: BillingAttributionSnapshot | undefined, - requestId: string, - resolvedSecretTraceRegistry: ResolvedSecretTraceRegistry | undefined, - abortSignal: AbortSignal | undefined -): Promise<{ - passed: boolean - error?: string - score?: number - reasoning?: string - detectedEntities?: any[] - maskedText?: string - cost?: number -}> { - // Use TypeScript validators for all validation types - if (validationType === 'json') { - return validateJson(inputStr) - } - if (validationType === 'regex') { - if (!regex) { - return { - passed: false, - error: 'Regex pattern is required', - } - } - return validateRegex(inputStr, regex) - } - if (validationType === 'hallucination') { - if (!knowledgeBaseId) { - return { - passed: false, - error: 'Knowledge base ID is required for hallucination check', - } - } - if (!model) { - return { - passed: false, - error: 'Model is required for hallucination validation', - } - } - if (!resolvedSecretTraceRegistry) { - throw new Error('Secret projection context is unavailable for hallucination validation') - } - if (!billingAttribution) { - throw new Error('Billing attribution is unavailable for hallucination validation') - } - - return await validateHallucination({ - userInput: inputStr, - knowledgeBaseId, - threshold: threshold != null ? Number.parseFloat(threshold) : 3, // Default threshold is 3 (confidence score, scores < 3 fail) - topK: topK ? Number.parseInt(topK) : 10, // Default topK is 10 - model: model, - apiKey, - providerCredentials, - workflowId, - workspaceId, - actorUserId, - billingAttribution, - requestId, - resolvedSecretTraceRegistry, - abortSignal, - }) - } - if (validationType === 'pii') { - return await validatePII({ - text: inputStr, - entityTypes: piiEntityTypes || [], // Empty array = detect all PII types - mode: (piiMode as 'block' | 'mask') || 'block', // Default to block mode - language: piiLanguage || 'en', - customPatterns: piiCustomPatterns, - requestId, - }) - } - return { - passed: false, - error: 'Unknown validation type', - } -} diff --git a/apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/[chunkId]/route.ts b/apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/[chunkId]/route.ts index a5d4fdb84d4..407f9b48d2f 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/[chunkId]/route.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/[chunkId]/route.ts @@ -8,24 +8,24 @@ import { import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' import { OrchestrationError } from '@/lib/core/orchestration/types' import { - internalKnowledgeActorUserId, internalKnowledgeAuthType, + internalKnowledgeProvenanceUserId, toInternalKnowledgeChunk, } from '@/lib/knowledge/api/internal-route' import { internalKnowledgeErrorPolicies, internalKnowledgeSessionOrExecutorAuth, } from '@/lib/knowledge/api/route-policies' +import { + finalizeKnowledgePersistedResponse, + resolveKnowledgeWriteSecretProvenance, +} from '@/lib/knowledge/api/secret-provenance' import { deleteKnowledgeChunk, readKnowledgeChunk, updateKnowledgeChunk, } from '@/lib/knowledge/application/chunks' import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { - finalizeKnowledgePersistedResponse, - resolveKnowledgeWriteSecretProvenance, -} from '@/app/api/knowledge/secret-provenance' function resolveContentProvenance( request: NextRequest, @@ -35,10 +35,10 @@ function resolveContentProvenance( includeContent: boolean ) { const resolved = resolveKnowledgeWriteSecretProvenance({ - request, + headers: request.headers, payload, authType: internalKnowledgeAuthType(principal), - userId: internalKnowledgeActorUserId(principal), + userId: internalKnowledgeProvenanceUserId(request.headers, principal, workspaceId), ...(workspaceId ? { workspaceId } : {}), selectionKeys: includeContent ? ['chunk-content'] : [], }) @@ -63,9 +63,9 @@ export const GET = defineInternalJsonRoute({ present: ({ chunk }) => ({ success: true as const, data: toInternalKnowledgeChunk(chunk) }), finalizeResponse: ({ request, principal, result, body }) => finalizeKnowledgePersistedResponse({ - request, + headers: request.headers, authType: internalKnowledgeAuthType(principal), - userId: internalKnowledgeActorUserId(principal), + userId: internalKnowledgeProvenanceUserId(request.headers, principal, result.workspaceId), workspaceId: result.workspaceId, body, chunks: [ @@ -100,9 +100,9 @@ export const PUT = defineInternalJsonRoute({ present: ({ chunk }) => ({ success: true as const, data: toInternalKnowledgeChunk(chunk) }), finalizeResponse: ({ request, principal, result, body }) => finalizeKnowledgePersistedResponse({ - request, + headers: request.headers, authType: internalKnowledgeAuthType(principal), - userId: internalKnowledgeActorUserId(principal), + userId: internalKnowledgeProvenanceUserId(request.headers, principal, result.workspaceId), workspaceId: result.workspaceId, body, chunks: [ diff --git a/apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/route.test.ts b/apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/route.test.ts index 152186137e7..a655c2c290b 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/route.test.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/route.test.ts @@ -26,7 +26,7 @@ vi.mock('@/lib/knowledge/application/chunks', () => ({ }, })) -vi.mock('@/app/api/knowledge/secret-provenance', () => ({ +vi.mock('@/lib/knowledge/api/secret-provenance', () => ({ finalizeKnowledgePersistedResponse: vi.fn(), finalizeKnowledgeProvenanceResponse: vi.fn(), resolveKnowledgeWriteSecretProvenance: vi.fn(), diff --git a/apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/route.ts b/apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/route.ts index d805898f287..8123a11cc1a 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/route.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/route.ts @@ -8,25 +8,25 @@ import { import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' import { OrchestrationError } from '@/lib/core/orchestration/types' import { - internalKnowledgeActorUserId, internalKnowledgeAuthType, + internalKnowledgeProvenanceUserId, toInternalKnowledgeChunk, } from '@/lib/knowledge/api/internal-route' import { internalKnowledgeErrorPolicies, internalKnowledgeSessionOrExecutorAuth, } from '@/lib/knowledge/api/route-policies' +import { + finalizeKnowledgePersistedResponse, + finalizeKnowledgeProvenanceResponse, + resolveKnowledgeWriteSecretProvenance, +} from '@/lib/knowledge/api/secret-provenance' import { bulkUpdateKnowledgeChunks, createKnowledgeChunk, listKnowledgeChunks, } from '@/lib/knowledge/application/chunks' import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { - finalizeKnowledgePersistedResponse, - finalizeKnowledgeProvenanceResponse, - resolveKnowledgeWriteSecretProvenance, -} from '@/app/api/knowledge/secret-provenance' function resolveContentProvenance( request: NextRequest, @@ -36,10 +36,10 @@ function resolveContentProvenance( includeContent: boolean ) { const resolved = resolveKnowledgeWriteSecretProvenance({ - request, + headers: request.headers, payload, authType: internalKnowledgeAuthType(principal), - userId: internalKnowledgeActorUserId(principal), + userId: internalKnowledgeProvenanceUserId(request.headers, principal, workspaceId), ...(workspaceId ? { workspaceId } : {}), selectionKeys: includeContent ? ['chunk-content'] : [], }) @@ -68,9 +68,9 @@ export const GET = defineInternalJsonRoute({ }), finalizeResponse: ({ request, principal, result, body }) => finalizeKnowledgePersistedResponse({ - request, + headers: request.headers, authType: internalKnowledgeAuthType(principal), - userId: internalKnowledgeActorUserId(principal), + userId: internalKnowledgeProvenanceUserId(request.headers, principal, result.workspaceId), workspaceId: result.workspaceId, body, chunks: result.chunks.map((chunk) => ({ @@ -102,7 +102,7 @@ export const POST = defineInternalJsonRoute({ present: ({ chunk }) => ({ success: true as const, data: toInternalKnowledgeChunk(chunk) }), finalizeResponse: ({ request, principal, result, body }) => finalizeKnowledgeProvenanceResponse({ - request, + headers: request.headers, authType: internalKnowledgeAuthType(principal), userId: result.userId, workspaceId: result.workspaceId, diff --git a/apps/sim/app/api/knowledge/[id]/documents/[documentId]/route.ts b/apps/sim/app/api/knowledge/[id]/documents/[documentId]/route.ts index 26cea872efa..9bc8d8999ad 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/[documentId]/route.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/[documentId]/route.ts @@ -5,9 +5,9 @@ import { } from '@/lib/api/contracts/knowledge' import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' import { - internalKnowledgeActorUserId, internalKnowledgeAnalytics, internalKnowledgeAuthType, + internalKnowledgeProvenanceUserId, resolveInternalKnowledgeBillingAttribution, toInternalKnowledgeDocument, } from '@/lib/knowledge/api/internal-route' @@ -15,6 +15,7 @@ import { internalKnowledgeErrorPolicies, internalKnowledgeSessionOrExecutorAuth, } from '@/lib/knowledge/api/route-policies' +import { finalizeKnowledgePersistedResponse } from '@/lib/knowledge/api/secret-provenance' import { deleteKnowledgeDocument, readKnowledgeDocument, @@ -22,7 +23,6 @@ import { } from '@/lib/knowledge/application/documents' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { createKnowledgeDocumentSourceValue } from '@/lib/knowledge/secret-provenance' -import { finalizeKnowledgePersistedResponse } from '@/app/api/knowledge/secret-provenance' export const GET = defineInternalJsonRoute({ contract: getKnowledgeDocumentContract, @@ -43,9 +43,9 @@ export const GET = defineInternalJsonRoute({ }), finalizeResponse: ({ request, principal, result, body }) => finalizeKnowledgePersistedResponse({ - request, + headers: request.headers, authType: internalKnowledgeAuthType(principal), - userId: internalKnowledgeActorUserId(principal), + userId: internalKnowledgeProvenanceUserId(request.headers, principal, result.workspaceId), workspaceId: result.workspaceId, body, documents: [ diff --git a/apps/sim/app/api/knowledge/[id]/documents/route.ts b/apps/sim/app/api/knowledge/[id]/documents/route.ts index ae6bb49230b..ce22bfe43ac 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/route.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/route.ts @@ -1,34 +1,26 @@ import { bulkKnowledgeDocumentsContract, - createKnowledgeDocumentsContract, listKnowledgeDocumentsContract, parseDocumentTagFiltersParam, } from '@/lib/api/contracts/knowledge' import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' import { OrchestrationError } from '@/lib/core/orchestration/types' import { - internalKnowledgeActorUserId, - internalKnowledgeAnalytics, internalKnowledgeAuthType, - resolveInternalKnowledgeBillingAttribution, + internalKnowledgeProvenanceUserId, toInternalKnowledgeDocument, } from '@/lib/knowledge/api/internal-route' import { internalKnowledgeErrorPolicies, internalKnowledgeSessionOrExecutorAuth, } from '@/lib/knowledge/api/route-policies' +import { finalizeKnowledgePersistedResponse } from '@/lib/knowledge/api/secret-provenance' import { bulkUpdateKnowledgeDocuments, - createKnowledgeDocuments, listKnowledgeDocuments, } from '@/lib/knowledge/application/documents' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { createKnowledgeDocumentSourceValue } from '@/lib/knowledge/secret-provenance' -import { - finalizeKnowledgePersistedResponse, - finalizeKnowledgeProvenanceResponse, - resolveKnowledgeDocumentWriteSecretProvenance, -} from '@/app/api/knowledge/secret-provenance' export const GET = defineInternalJsonRoute({ contract: listKnowledgeDocumentsContract, @@ -66,9 +58,9 @@ export const GET = defineInternalJsonRoute({ }), finalizeResponse: ({ request, principal, result, body }) => finalizeKnowledgePersistedResponse({ - request, + headers: request.headers, authType: internalKnowledgeAuthType(principal), - userId: internalKnowledgeActorUserId(principal), + userId: internalKnowledgeProvenanceUserId(request.headers, principal, result.workspaceId), workspaceId: result.workspaceId, body, documents: result.documents.map((document) => ({ @@ -79,61 +71,6 @@ export const GET = defineInternalJsonRoute({ }), }) -export const POST = defineInternalJsonRoute({ - contract: createKnowledgeDocumentsContract, - auth: internalKnowledgeSessionOrExecutorAuth, - operation: knowledgeOperations.uploadDocument, - rateLimit: internalRateLimits.none({ - reason: 'Preserve existing internal document-create behavior', - }), - errorPolicy: internalKnowledgeErrorPolicies.uploads, - mapInput: ({ params, body }, { principal, request }) => { - const documents = body.bulk ? body.documents : [body] - return { - knowledgeBaseId: params.id, - documents, - bulk: body.bulk, - processingOptions: body.bulk ? body.processingOptions : undefined, - resolveBillingAttribution: (workspaceId: string) => - resolveInternalKnowledgeBillingAttribution(request, principal, workspaceId), - resolveSecretProvenances: ({ userId, workspaceId }) => { - const resolution = resolveKnowledgeDocumentWriteSecretProvenance({ - request, - payload: body, - authType: internalKnowledgeAuthType(principal), - userId, - workspaceId, - documents, - }) - if (!resolution.success) { - throw new OrchestrationError('validation', 'Invalid knowledge secret provenance') - } - return resolution.provenances - }, - source: 'ui' as const, - } - }, - useCase: createKnowledgeDocuments, - onSuccess: internalKnowledgeAnalytics.documentsUploaded, - present: (result) => ({ - success: true as const, - data: result.kind === 'bulk' ? result.data : toInternalKnowledgeDocument(result.data), - }), - finalizeResponse: ({ request, principal, result, body }) => - finalizeKnowledgeProvenanceResponse({ - request, - authType: internalKnowledgeAuthType(principal), - userId: result.userId, - workspaceId: result.workspaceId, - provenances: - result.secretProvenances?.flatMap((provenance) => [ - provenance.filename, - ...provenance.tags.map((tag) => tag.provenance), - ]) ?? [], - body, - }), -}) - export const PATCH = defineInternalJsonRoute({ contract: bulkKnowledgeDocumentsContract, auth: internalKnowledgeSessionOrExecutorAuth, diff --git a/apps/sim/app/api/knowledge/[id]/documents/upsert/route.ts b/apps/sim/app/api/knowledge/[id]/documents/upsert/route.ts deleted file mode 100644 index 2da21e78cd9..00000000000 --- a/apps/sim/app/api/knowledge/[id]/documents/upsert/route.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { upsertKnowledgeDocumentContract } from '@/lib/api/contracts/knowledge' -import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' -import { OrchestrationError } from '@/lib/core/orchestration/types' -import { - internalKnowledgeAnalytics, - internalKnowledgeAuthType, - resolveInternalKnowledgeBillingAttribution, -} from '@/lib/knowledge/api/internal-route' -import { - internalKnowledgeErrorPolicies, - internalKnowledgeSessionOrExecutorAuth, -} from '@/lib/knowledge/api/route-policies' -import { upsertKnowledgeDocument } from '@/lib/knowledge/application/documents' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { - finalizeKnowledgeProvenanceResponse, - resolveKnowledgeDocumentWriteSecretProvenance, -} from '@/app/api/knowledge/secret-provenance' - -export const POST = defineInternalJsonRoute({ - contract: upsertKnowledgeDocumentContract, - auth: internalKnowledgeSessionOrExecutorAuth, - operation: knowledgeOperations.uploadDocument, - rateLimit: internalRateLimits.none({ - reason: 'Preserve existing internal document-upsert behavior', - }), - errorPolicy: internalKnowledgeErrorPolicies.upsert, - parseOptions: { maxBodyBytes: 2 * 1024 * 1024 }, - mapInput: ({ params, body }, { principal, request }) => ({ - knowledgeBaseId: params.id, - documentId: body.documentId, - filename: body.filename, - fileUrl: body.fileUrl, - fileSize: body.fileSize, - mimeType: body.mimeType, - documentTagsData: body.documentTagsData, - processingOptions: body.processingOptions, - resolveBillingAttribution: (workspaceId: string) => - resolveInternalKnowledgeBillingAttribution(request, principal, workspaceId), - resolveSecretProvenances: ({ - userId, - workspaceId, - }: { - userId: string - workspaceId: string - }) => { - const resolved = resolveKnowledgeDocumentWriteSecretProvenance({ - request, - payload: body, - authType: internalKnowledgeAuthType(principal), - userId, - workspaceId, - documents: [body], - }) - if (!resolved.success) { - throw new OrchestrationError('validation', 'Invalid knowledge secret provenance') - } - return resolved.provenances - }, - }), - useCase: upsertKnowledgeDocument, - onSuccess: internalKnowledgeAnalytics.documentUpserted, - present: ({ document, isUpdate, previousDocumentId, processingConfig }) => ({ - success: true as const, - data: { - documentsCreated: [ - { - documentId: document.documentId, - filename: document.filename, - status: 'pending' as const, - }, - ], - isUpdate, - previousDocumentId, - processingMethod: 'background' as const, - processingConfig, - }, - }), - finalizeResponse: ({ request, principal, result, body }) => - finalizeKnowledgeProvenanceResponse({ - request, - authType: internalKnowledgeAuthType(principal), - userId: result.userId, - workspaceId: result.workspaceId, - body, - provenances: - result.secretProvenances?.flatMap((provenance) => [ - provenance.filename, - ...provenance.tags.map((tag) => tag.provenance), - ]) ?? [], - }), -}) diff --git a/apps/sim/app/api/knowledge/migrated-routes.test.ts b/apps/sim/app/api/knowledge/migrated-routes.test.ts index 77f97469793..2917e9b3830 100644 --- a/apps/sim/app/api/knowledge/migrated-routes.test.ts +++ b/apps/sim/app/api/knowledge/migrated-routes.test.ts @@ -8,12 +8,10 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ useCase: (id: string, execute: ReturnType) => ({ operation: { id }, execute }), listDocuments: vi.fn(), - createDocuments: vi.fn(), bulkDocuments: vi.fn(), readDocument: vi.fn(), updateDocument: vi.fn(), deleteDocument: vi.fn(), - upsertDocument: vi.fn(), listConnectors: vi.fn(), createConnector: vi.fn(), readConnector: vi.fn(), @@ -22,15 +20,11 @@ const mocks = vi.hoisted(() => ({ syncConnector: vi.fn(), listConnectorDocuments: vi.fn(), updateConnectorDocuments: vi.fn(), - search: vi.fn(), createUpload: vi.fn(), issueParts: vi.fn(), completeUpload: vi.fn(), cancelUpload: vi.fn(), persistedResponse: vi.fn(), - provenanceResponse: vi.fn(), - registryResponse: vi.fn(), - resolveDocumentProvenance: vi.fn(), readKnowledgeBase: vi.fn(), updateKnowledgeBase: vi.fn(), deleteKnowledgeBase: vi.fn(), @@ -41,12 +35,10 @@ const mocks = vi.hoisted(() => ({ vi.mock('@/lib/knowledge/application/documents', () => ({ listKnowledgeDocuments: mocks.useCase('knowledge.documents.list', mocks.listDocuments), - createKnowledgeDocuments: mocks.useCase('knowledge.documents.upload', mocks.createDocuments), bulkUpdateKnowledgeDocuments: mocks.useCase('knowledge.documents.bulk', mocks.bulkDocuments), readKnowledgeDocument: mocks.useCase('knowledge.documents.read', mocks.readDocument), updateKnowledgeDocument: mocks.useCase('knowledge.documents.update', mocks.updateDocument), deleteKnowledgeDocument: mocks.useCase('knowledge.documents.delete', mocks.deleteDocument), - upsertKnowledgeDocument: mocks.useCase('knowledge.documents.upload', mocks.upsertDocument), })) vi.mock('@/lib/knowledge/application/knowledge-bases', () => ({ @@ -78,7 +70,6 @@ vi.mock('@/lib/knowledge/application/connectors', () => ({ vi.mock('@/lib/knowledge/application/search', () => ({ KnowledgeSearchProvenanceUnavailableError: class extends Error {}, - searchKnowledge: mocks.useCase('knowledge.search', mocks.search), })) vi.mock('@/lib/knowledge/application/upload-sessions', () => ({ @@ -101,11 +92,8 @@ vi.mock('@/lib/knowledge/application/upload-sessions', () => ({ ), })) -vi.mock('@/app/api/knowledge/secret-provenance', () => ({ +vi.mock('@/lib/knowledge/api/secret-provenance', () => ({ finalizeKnowledgePersistedResponse: mocks.persistedResponse, - finalizeKnowledgeProvenanceResponse: mocks.provenanceResponse, - finalizeKnowledgeRegistryResponse: mocks.registryResponse, - resolveKnowledgeDocumentWriteSecretProvenance: mocks.resolveDocumentProvenance, })) vi.mock('@/lib/core/telemetry', () => ({ @@ -115,7 +103,6 @@ vi.mock('@/lib/core/telemetry', () => ({ vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: mocks.capture })) import { OrchestrationError } from '@/lib/core/orchestration/types' -import { KnowledgeUsageLimitExceededError } from '@/lib/knowledge/application/billing' import { GET as listConnectorDocuments, PATCH as updateConnectorDocuments, @@ -123,18 +110,15 @@ import { import { PUT as updateDocument } from '@/app/api/knowledge/[id]/documents/[documentId]/route' import { PATCH as bulkDocuments, - POST as createDocuments, GET as listDocuments, } from '@/app/api/knowledge/[id]/documents/route' import { POST as completeUpload } from '@/app/api/knowledge/[id]/documents/uploads/[uploadId]/complete/route' -import { POST as upsertDocument } from '@/app/api/knowledge/[id]/documents/upsert/route' import { POST as restoreKnowledgeBase } from '@/app/api/knowledge/[id]/restore/route' import { DELETE as deleteKnowledgeBase, GET as readKnowledgeBase, PUT as updateKnowledgeBase, } from '@/app/api/knowledge/[id]/route' -import { POST as search } from '@/app/api/knowledge/search/route' const session = { user: { id: 'user-1', email: 'user@example.com', name: 'User' }, @@ -179,25 +163,6 @@ describe('migrated internal Knowledge routes', () => { vi.clearAllMocks() authMockFns.mockGetSession.mockResolvedValue(session) mocks.persistedResponse.mockResolvedValue({}) - mocks.provenanceResponse.mockResolvedValue({}) - mocks.registryResponse.mockReturnValue({}) - mocks.resolveDocumentProvenance.mockReturnValue({ success: true }) - }) - - it('authenticates before parsing malformed document JSON', async () => { - authMockFns.mockGetSession.mockResolvedValueOnce(null) - const request = new NextRequest('http://localhost/api/knowledge/knowledge-1/documents', { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: '{', - }) - - const response = await createDocuments(request, { - params: Promise.resolve({ id: 'knowledge-1' }), - }) - - expect(response.status).toBe(401) - expect(mocks.createDocuments).not.toHaveBeenCalled() }) it('authenticates before parsing malformed knowledge base JSON', async () => { @@ -301,93 +266,6 @@ describe('migrated internal Knowledge routes', () => { }) }) - it('keeps document upsert admission behind auth and preserves its response', async () => { - mocks.upsertDocument.mockResolvedValue({ - document: { documentId: 'document-2', filename: 'new.txt' }, - knowledgeBaseId: 'knowledge-1', - isUpdate: false, - previousDocumentId: null, - processingConfig: { maxConcurrentDocuments: 5, batchSize: 10 }, - workspaceId: 'workspace-1', - userId: 'user-1', - }) - const response = await upsertDocument( - createMockRequest('POST', { - filename: 'new.txt', - fileUrl: 'data:text/plain;base64,aGVsbG8=', - fileSize: 5, - mimeType: 'text/plain', - }), - { params: Promise.resolve({ id: 'knowledge-1' }) } - ) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toEqual({ - success: true, - data: expect.objectContaining({ - documentsCreated: [{ documentId: 'document-2', filename: 'new.txt', status: 'pending' }], - isUpdate: false, - previousDocumentId: null, - processingMethod: 'background', - }), - }) - expect(mocks.platformUpload).toHaveBeenCalledWith( - expect.objectContaining({ knowledgeBaseId: 'knowledge-1', documentsCount: 1 }) - ) - }) - - it('runs internal document analytics only after application success', async () => { - mocks.createDocuments.mockResolvedValue({ - kind: 'single', - data: document, - workspaceId: 'workspace-1', - userId: 'user-1', - }) - const request = createMockRequest('POST', { - bulk: false, - filename: document.filename, - fileUrl: document.fileUrl, - fileSize: document.fileSize, - mimeType: document.mimeType, - }) - - const response = await createDocuments(request, { - params: Promise.resolve({ id: 'knowledge-1' }), - }) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - success: true, - data: { processingStatus: 'pending' }, - }) - expect(mocks.platformUpload).toHaveBeenCalledOnce() - expect(mocks.capture).toHaveBeenCalledOnce() - expect(mocks.createDocuments.mock.invocationCallOrder[0]).toBeLessThan( - mocks.platformUpload.mock.invocationCallOrder[0] - ) - }) - - it('preserves payment-required for document usage admission', async () => { - mocks.createDocuments.mockRejectedValueOnce( - new KnowledgeUsageLimitExceededError('Usage limit exceeded') - ) - - const response = await createDocuments( - createMockRequest('POST', { - bulk: false, - filename: document.filename, - fileUrl: document.fileUrl, - fileSize: document.fileSize, - mimeType: document.mimeType, - }), - { params: Promise.resolve({ id: 'knowledge-1' }) } - ) - - expect(response.status).toBe(402) - await expect(response.json()).resolves.toEqual({ error: 'Usage limit exceeded' }) - expect(mocks.capture).not.toHaveBeenCalled() - }) - it('returns not found when a bulk document selection has no active matches', async () => { mocks.bulkDocuments.mockRejectedValueOnce( new OrchestrationError('not_found', 'No valid documents found to update') @@ -407,24 +285,6 @@ describe('migrated internal Knowledge routes', () => { }) }) - it('rejects oversized document-create arrays at the contract boundary', async () => { - const response = await createDocuments( - createMockRequest('POST', { - bulk: true, - documents: Array.from({ length: 101 }, (_, index) => ({ - filename: `document-${index}.txt`, - fileUrl: `https://example.com/document-${index}.txt`, - fileSize: 1, - mimeType: 'text/plain', - })), - }), - { params: Promise.resolve({ id: 'knowledge-1' }) } - ) - - expect(response.status).toBe(400) - expect(mocks.createDocuments).not.toHaveBeenCalled() - }) - it('preserves connector-document list and mutation envelopes', async () => { mocks.listConnectorDocuments.mockResolvedValue({ documents: [ @@ -504,56 +364,6 @@ describe('migrated internal Knowledge routes', () => { expect(mocks.updateConnectorDocuments).not.toHaveBeenCalled() }) - it('preserves search cost shape and sanitizes infrastructure errors', async () => { - const registry = {} - mocks.search.mockResolvedValue({ - results: [ - { - embeddingId: 'embedding-1', - documentId: 'document-1', - documentName: 'Guide', - sourceUrl: null, - content: 'hello', - chunkIndex: 0, - metadata: {}, - similarity: 0.9, - }, - ], - query: 'hello', - knowledgeBaseIds: ['knowledge-1'], - knowledgeBaseId: 'knowledge-1', - topK: 10, - totalResults: 1, - workspaceId: 'workspace-1', - userId: 'user-1', - resultSecretRegistry: registry, - cost: { - input: 0.1, - output: 0, - total: 0.1, - tokens: { prompt: 1, completion: 0, total: 1 }, - model: 'text-embedding-3-small', - pricing: { input: 0.1, output: 0 }, - }, - }) - const response = await search( - createMockRequest('POST', { - knowledgeBaseIds: ['knowledge-1'], - query: 'hello', - }) - ) - const body = await response.json() - expect(body.data.results[0]).not.toHaveProperty('embeddingId') - expect(body.data.cost).toEqual(expect.objectContaining({ total: 0.1 })) - - mocks.search.mockRejectedValueOnce(new Error('database host secret.internal')) - const failure = await search( - createMockRequest('POST', { knowledgeBaseIds: ['knowledge-1'], query: 'hello' }) - ) - expect(failure.status).toBe(500) - await expect(failure.json()).resolves.toEqual({ error: 'Failed to perform vector search' }) - }) - it('runs upload analytics only after a newly-created completion', async () => { const completed = { session: { diff --git a/apps/sim/app/api/knowledge/search/route.ts b/apps/sim/app/api/knowledge/search/route.ts deleted file mode 100644 index c9a99ffa3de..00000000000 --- a/apps/sim/app/api/knowledge/search/route.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { internalKnowledgeSearchContract } from '@/lib/api/contracts/knowledge' -import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' -import { OrchestrationError } from '@/lib/core/orchestration/types' -import { - internalKnowledgeAuthType, - resolveInternalKnowledgeBillingAttribution, -} from '@/lib/knowledge/api/internal-route' -import { - internalKnowledgeErrorPolicies, - internalKnowledgeSessionOrExecutorAuth, -} from '@/lib/knowledge/api/route-policies' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { searchKnowledge } from '@/lib/knowledge/application/search' -import { prepareKnowledgeModelInputProvenance } from '@/lib/knowledge/model-input-provenance' -import { finalizeKnowledgeRegistryResponse } from '@/app/api/knowledge/secret-provenance' - -export const POST = defineInternalJsonRoute({ - contract: internalKnowledgeSearchContract, - auth: internalKnowledgeSessionOrExecutorAuth, - operation: knowledgeOperations.search, - rateLimit: internalRateLimits.none({ - reason: 'Preserve existing internal Knowledge-search behavior', - }), - errorPolicy: internalKnowledgeErrorPolicies.search, - parseOptions: { maxBodyBytes: 2 * 1024 * 1024 }, - mapInput: ({ body }, { principal, request }) => ({ - knowledgeBaseIds: Array.isArray(body.knowledgeBaseIds) - ? body.knowledgeBaseIds - : [body.knowledgeBaseIds], - query: body.query, - topK: body.topK, - tagFilters: body.tagFilters, - searchMode: body.searchMode, - rerankerEnabled: body.rerankerEnabled, - rerankerModel: body.rerankerModel, - rerankerInputCount: body.rerankerInputCount, - rerankerApiKey: body.rerankerApiKey, - skipUsageBilling: body.skipUsageBilling, - resolveBillingAttribution: (workspaceId: string) => - resolveInternalKnowledgeBillingAttribution(request, principal, workspaceId), - prepareModelInputProvenance: async ({ - userId, - workspaceId, - }: { - userId: string - workspaceId: string - }) => { - const prepared = await prepareKnowledgeModelInputProvenance({ - headers: request.headers, - payload: body, - isInternalRequest: principal.kind === 'delegated', - userId, - workspaceId, - modelInput: body.query, - }) - if (!prepared.success) throw new OrchestrationError('validation', prepared.error) - return prepared.registry - }, - }), - useCase: searchKnowledge, - present: (result) => ({ - success: true as const, - data: { - results: result.results.map(({ embeddingId: _embeddingId, ...item }) => item), - query: result.query, - knowledgeBaseIds: result.knowledgeBaseIds, - knowledgeBaseId: result.knowledgeBaseId, - topK: result.topK, - totalResults: result.totalResults, - ...(result.cost ? { cost: result.cost } : {}), - }, - }), - finalizeResponse: ({ request, principal, result, body }) => { - if (!result.resultSecretRegistry) { - throw new Error('Internal Knowledge search did not produce a provenance registry') - } - return finalizeKnowledgeRegistryResponse({ - request, - authType: internalKnowledgeAuthType(principal), - body, - registry: result.resultSecretRegistry, - }) - }, -}) diff --git a/apps/sim/app/api/knowledge/secret-provenance.test.ts b/apps/sim/app/api/knowledge/secret-provenance.test.ts index 305a65926e1..a1ecf3383d7 100644 --- a/apps/sim/app/api/knowledge/secret-provenance.test.ts +++ b/apps/sim/app/api/knowledge/secret-provenance.test.ts @@ -13,7 +13,7 @@ import { import { resolveKnowledgeDocumentWriteSecretProvenance, resolveKnowledgeWriteSecretProvenance, -} from '@/app/api/knowledge/secret-provenance' +} from '@/lib/knowledge/api/secret-provenance' const PRIVATE_PROVENANCE_SCOPE = { userId: 'user-1', @@ -62,7 +62,7 @@ describe('knowledge write secret provenance', () => { const payload = { content: 'manual content' } const result = resolveKnowledgeWriteSecretProvenance({ - request: createHeaderlessRequest(payload), + headers: createHeaderlessRequest(payload).headers, payload, authType: AuthType.API_KEY, userId: 'user-1', @@ -82,7 +82,7 @@ describe('knowledge write secret provenance', () => { } const result = resolveKnowledgeDocumentWriteSecretProvenance({ - request: createHeaderlessRequest(payload), + headers: createHeaderlessRequest(payload).headers, payload, authType: AuthType.SESSION, userId: 'user-1', @@ -106,7 +106,7 @@ describe('knowledge write secret provenance', () => { const payload = { content: 'legacy workflow content' } const result = resolveKnowledgeWriteSecretProvenance({ - request: createHeaderlessRequest(payload), + headers: createHeaderlessRequest(payload).headers, payload, authType: AuthType.INTERNAL_JWT, userId: 'user-1', @@ -121,7 +121,7 @@ describe('knowledge write secret provenance', () => { const payload = privateChunkPayload(PRIVATE_PROVENANCE_SCOPE) const result = resolveKnowledgeWriteSecretProvenance({ - request: createRequest(payload), + headers: createRequest(payload).headers, payload, authType: AuthType.INTERNAL_JWT, userId: 'user-1', @@ -142,7 +142,7 @@ describe('knowledge write secret provenance', () => { expect( resolveKnowledgeWriteSecretProvenance({ - request: createRequest(payload), + headers: createRequest(payload).headers, payload, authType: AuthType.INTERNAL_JWT, userId: 'billing-actor', @@ -173,7 +173,7 @@ describe('knowledge write secret provenance', () => { workspaceId: 'workspace-2', }) const result = resolveKnowledgeWriteSecretProvenance({ - request: createRequest(payload), + headers: createRequest(payload).headers, payload, authType: AuthType.INTERNAL_JWT, userId: 'billing-actor', @@ -188,7 +188,7 @@ describe('knowledge write secret provenance', () => { it('keeps workspace-less knowledge writes isolated to the authenticated user', () => { const payload = privateChunkPayload({ userId: 'workflow-owner' }) const result = resolveKnowledgeWriteSecretProvenance({ - request: createRequest(payload), + headers: createRequest(payload).headers, payload, authType: AuthType.INTERNAL_JWT, userId: 'billing-actor', @@ -218,7 +218,7 @@ describe('knowledge write secret provenance', () => { const payload = { content: 'external content', [PRIVATE_SECRET_PROVENANCE_FIELD]: bundle } const result = resolveKnowledgeWriteSecretProvenance({ - request: createRequest(payload), + headers: createRequest(payload).headers, payload, authType: AuthType.API_KEY, userId: 'user-1', @@ -249,7 +249,7 @@ describe('knowledge write secret provenance', () => { const payload = { [PRIVATE_SECRET_PROVENANCE_FIELD]: bundle } const result = resolveKnowledgeWriteSecretProvenance({ - request: createRequest(payload), + headers: createRequest(payload).headers, payload, authType: AuthType.INTERNAL_JWT, userId: 'user-1', @@ -270,7 +270,7 @@ describe('knowledge write secret provenance', () => { } const result = resolveKnowledgeDocumentWriteSecretProvenance({ - request: createRequest(payload), + headers: createRequest(payload).headers, payload, authType: AuthType.INTERNAL_JWT, userId: 'user-1', @@ -330,7 +330,7 @@ describe('knowledge write secret provenance', () => { } const result = resolveKnowledgeDocumentWriteSecretProvenance({ - request: createRequest(payload), + headers: createRequest(payload).headers, payload, authType: AuthType.INTERNAL_JWT, userId: 'user-1', diff --git a/apps/sim/app/api/knowledge/secret-provenance.ts b/apps/sim/app/api/knowledge/secret-provenance.ts deleted file mode 100644 index 27210da78ba..00000000000 --- a/apps/sim/app/api/knowledge/secret-provenance.ts +++ /dev/null @@ -1,260 +0,0 @@ -import { type NextRequest, NextResponse } from 'next/server' -import type { InternalJsonResponseFinalization } from '@/lib/api/server/routes/internal-json-route' -import { AuthType, type AuthTypeValue } from '@/lib/auth/hybrid' -import { OrchestrationError } from '@/lib/core/orchestration/types' -import { - createDurableSecretProvenanceRegistry, - type DurableSecretProvenance, - durableSecretProvenanceFromPrivateBundle, - EXACT_EMPTY_DURABLE_SECRET_PROVENANCE, -} from '@/lib/execution/durable-secret-provenance' -import { - inspectPrivateSecretProvenanceRequest, - isPrivateSecretProvenanceBundleV1, -} from '@/lib/execution/model-input-provenance' -import { - negotiatePrivateToolMetadataResponse, - RESOLVED_SECRET_PROVENANCE_FIELD, - RESOLVED_SECRET_PROVENANCE_METADATA_V1, - serializePrivateToolMetadataResponseEnvelope, -} from '@/lib/execution/private-tool-metadata' -import { - importKnowledgePersistedResponseSecretProvenance, - type KnowledgeDocumentSourceValue, - type KnowledgeDocumentWriteSecretProvenance, -} from '@/lib/knowledge/secret-provenance' -import { - knowledgeDocumentContentSelectionKey, - knowledgeDocumentFilenameSelectionKey, - knowledgeDocumentTagValueSelectionKey, - parseKnowledgeDocumentTagProvenanceTargets, -} from '@/lib/knowledge/secret-provenance-selection' -import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' - -function invalidKnowledgeProvenanceResponse(): NextResponse { - return NextResponse.json({ error: 'Invalid knowledge secret provenance' }, { status: 400 }) -} - -function rejectInvalidKnowledgeProvenance(): never { - throw new OrchestrationError('validation', 'Invalid knowledge secret provenance') -} - -function finalizeKnowledgeMetadataEnvelope( - envelope: ReturnType -): InternalJsonResponseFinalization { - return { - bodyFields: { - [RESOLVED_SECRET_PROVENANCE_FIELD]: envelope.body[RESOLVED_SECRET_PROVENANCE_FIELD], - }, - headers: envelope.headers, - } -} - -type KnowledgeWriteProvenanceResolution = - | { success: true; provenances?: DurableSecretProvenance[] } - | { success: false; response: NextResponse } - -/** Resolves private document/chunk write selections after auth and workspace authorization. */ -export function resolveKnowledgeWriteSecretProvenance(options: { - request: NextRequest - payload: unknown - authType: AuthTypeValue | undefined - userId: string - workspaceId?: string - selectionKeys: readonly string[] -}): KnowledgeWriteProvenanceResolution { - const { request } = options - const inspection = inspectPrivateSecretProvenanceRequest(request.headers, options.payload) - if (inspection.status === 'unsupported') { - return options.authType === AuthType.INTERNAL_JWT - ? { success: true } - : { - success: true, - provenances: options.selectionKeys.map(() => EXACT_EMPTY_DURABLE_SECRET_PROVENANCE), - } - } - if (inspection.status !== 'verified' || options.authType !== AuthType.INTERNAL_JWT) { - return { success: false, response: invalidKnowledgeProvenanceResponse() } - } - if (!isPrivateSecretProvenanceBundleV1(inspection.value)) { - return { success: false, response: invalidKnowledgeProvenanceResponse() } - } - if (!inspection.value.complete) { - return { - success: true, - provenances: options.selectionKeys.map(() => ({ status: 'unknown' })), - } - } - if (inspection.value.selections.length !== options.selectionKeys.length) { - return { success: false, response: invalidKnowledgeProvenanceResponse() } - } - const provenances = options.selectionKeys.map((selectionKey) => - durableSecretProvenanceFromPrivateBundle(inspection.value, selectionKey, { - userId: options.userId, - ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), - }) - ) - if (provenances.some((provenance) => provenance === undefined)) { - return { success: false, response: invalidKnowledgeProvenanceResponse() } - } - return { success: true, provenances: provenances as DurableSecretProvenance[] } -} - -type KnowledgeDocumentWriteProvenanceResolution = - | { success: true; provenances?: KnowledgeDocumentWriteSecretProvenance[] } - | { success: false; response: NextResponse } - -/** Resolves provenance for durable document fields; persisted tag names remain raw and untracked. */ -export function resolveKnowledgeDocumentWriteSecretProvenance(options: { - request: NextRequest - payload: unknown - authType: AuthTypeValue | undefined - userId: string - workspaceId?: string - documents: readonly { documentTagsData?: string }[] -}): KnowledgeDocumentWriteProvenanceResolution { - const tagTargets = options.documents.map((document) => - parseKnowledgeDocumentTagProvenanceTargets(document.documentTagsData) - ) - const selectionKeys = options.documents.flatMap((_document, documentIndex) => [ - knowledgeDocumentFilenameSelectionKey(documentIndex), - knowledgeDocumentContentSelectionKey(documentIndex), - ...tagTargets[documentIndex].map((_tag, tagIndex) => - knowledgeDocumentTagValueSelectionKey(documentIndex, tagIndex) - ), - ]) - const resolved = resolveKnowledgeWriteSecretProvenance({ - request: options.request, - payload: options.payload, - authType: options.authType, - userId: options.userId, - ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), - selectionKeys, - }) - if (!resolved.success) return resolved - if (!resolved.provenances) return { success: true } - - let provenanceIndex = 0 - const provenances: KnowledgeDocumentWriteSecretProvenance[] = [] - for (const tags of tagTargets) { - const filename = resolved.provenances[provenanceIndex++] - const content = resolved.provenances[provenanceIndex++] - const tagProvenances: KnowledgeDocumentWriteSecretProvenance['tags'][number][] = [] - for (const tag of tags) { - const tagValue = resolved.provenances[provenanceIndex++] - tagProvenances.push({ tagName: tag.tagName, provenance: tagValue }) - } - provenances.push({ filename, content, tags: tagProvenances }) - } - return { success: true, provenances } -} - -/** Finalizes private provenance after the functional Knowledge response passes its contract. */ -export async function finalizeKnowledgeProvenanceResponse(options: { - request: NextRequest - authType: AuthTypeValue | undefined - userId: string - workspaceId?: string - body: Record - provenances: readonly DurableSecretProvenance[] -}): Promise { - const { request } = options - const negotiation = negotiatePrivateToolMetadataResponse( - request.headers, - RESOLVED_SECRET_PROVENANCE_METADATA_V1, - options.authType === AuthType.INTERNAL_JWT - ) - if (negotiation.status === 'not-requested') return {} - if (negotiation.status === 'rejected') rejectInvalidKnowledgeProvenance() - const registry = new ResolvedSecretTraceRegistry([], { - userId: options.userId, - ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), - }) - for (const provenance of options.provenances) { - if (provenance.status === 'unknown') { - registry.markIncomplete('durable-provenance-unknown') - break - } - const sourceRegistry = await createDurableSecretProvenanceRegistry(provenance, { - userId: options.userId, - ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), - }) - if (sourceRegistry) registry.mergeToolCallRegistry(sourceRegistry) - } - const envelope = serializePrivateToolMetadataResponseEnvelope( - options.body, - RESOLVED_SECRET_PROVENANCE_METADATA_V1, - registry.exportCommittedProvenanceForValue(options.body) - ) - return finalizeKnowledgeMetadataEnvelope(envelope) -} - -/** Serializes an already-populated request registry as private response metadata. */ -export function finalizeKnowledgeRegistryResponse(options: { - request: NextRequest - authType: AuthTypeValue | undefined - body: Record - registry: ResolvedSecretTraceRegistry -}): InternalJsonResponseFinalization { - const { request } = options - const negotiation = negotiatePrivateToolMetadataResponse( - request.headers, - RESOLVED_SECRET_PROVENANCE_METADATA_V1, - options.authType === AuthType.INTERNAL_JWT - ) - if (negotiation.status === 'not-requested') return {} - if (negotiation.status === 'rejected') rejectInvalidKnowledgeProvenance() - const envelope = serializePrivateToolMetadataResponseEnvelope( - options.body, - RESOLVED_SECRET_PROVENANCE_METADATA_V1, - options.registry.exportCommittedProvenanceForValue(options.body) - ) - return finalizeKnowledgeMetadataEnvelope(envelope) -} - -/** Emits private response provenance for a bounded exact snapshot of persisted KB rows. */ -export async function finalizeKnowledgePersistedResponse(options: { - request: NextRequest - authType: AuthTypeValue | undefined - userId: string - workspaceId?: string - body: Record - documents?: readonly { - id: string - source: KnowledgeDocumentSourceValue - value: unknown - }[] - chunks?: readonly { - id: string - documentId: string - content: string - value: unknown - }[] -}): Promise { - const { request } = options - const negotiation = negotiatePrivateToolMetadataResponse( - request.headers, - RESOLVED_SECRET_PROVENANCE_METADATA_V1, - options.authType === AuthType.INTERNAL_JWT - ) - if (negotiation.status === 'not-requested') return {} - if (negotiation.status === 'rejected') rejectInvalidKnowledgeProvenance() - - const registry = new ResolvedSecretTraceRegistry([], { - userId: options.userId, - ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), - }) - await importKnowledgePersistedResponseSecretProvenance({ - registry, - documents: options.documents, - chunks: options.chunks, - ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), - actorUserId: options.userId, - }) - return finalizeKnowledgeRegistryResponse({ - request: options.request, - authType: options.authType, - body: options.body, - registry, - }) -} diff --git a/apps/sim/app/api/logs/[id]/route.ts b/apps/sim/app/api/logs/[id]/route.ts index 5c0acd33e08..5f3dcc17421 100644 --- a/apps/sim/app/api/logs/[id]/route.ts +++ b/apps/sim/app/api/logs/[id]/route.ts @@ -1,39 +1,31 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' import { getLogDetailContract } from '@/lib/api/contracts/logs' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { fetchLogDetail } from '@/lib/logs/fetch-log-detail' +import { + defineInternalJsonRoute, + internalErrorResponse, + internalOrchestrationErrorPolicy, + internalRateLimits, +} from '@/lib/api/server/routes' +import { internalLogsSessionOrExecutorAuth } from '@/lib/logs/api/route-policies' +import { logOperations } from '@/lib/logs/application/operations' +import { readLogDetailUseCase } from '@/lib/logs/application/read-log-detail' -const logger = createLogger('LogDetailsByIdAPI') +const errorPolicy = { + ...internalOrchestrationErrorPolicy, + unhandled: () => internalErrorResponse(500, { error: 'Failed to fetch log' }), +} -export const GET = withRouteHandler( - async (request: NextRequest, context: { params: Promise<{ id: string }> }) => { - const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json( - { error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(getLogDetailContract, request, context) - if (!parsed.success) return parsed.response - - const { id } = parsed.data.params - const { workspaceId } = parsed.data.query - - const data = await fetchLogDetail({ - userId: authResult.userId, - workspaceId, - lookupColumn: 'id', - lookupValue: id, - }) - - if (!data) return NextResponse.json({ error: 'Not found' }, { status: 404 }) - - logger.debug('Fetched log detail', { id, workspaceId }) - return NextResponse.json({ data }) - } -) +export const GET = defineInternalJsonRoute({ + contract: getLogDetailContract, + auth: internalLogsSessionOrExecutorAuth, + operation: logOperations.readDetail, + rateLimit: internalRateLimits.none({ reason: 'Preserve existing internal log detail behavior' }), + errorPolicy, + mapInput: ({ params, query }, { principal, request }) => ({ + workspaceId: principal.kind === 'delegated' ? principal.workspaceId : query.workspaceId, + lookupColumn: 'id' as const, + lookupValue: params.id, + signal: request.signal, + }), + useCase: readLogDetailUseCase, + present: ({ detail }) => ({ data: detail }), +}) diff --git a/apps/sim/app/api/logs/by-execution/[executionId]/route.ts b/apps/sim/app/api/logs/by-execution/[executionId]/route.ts index bab53092456..28182c6939f 100644 --- a/apps/sim/app/api/logs/by-execution/[executionId]/route.ts +++ b/apps/sim/app/api/logs/by-execution/[executionId]/route.ts @@ -1,39 +1,31 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' import { getLogByExecutionIdContract } from '@/lib/api/contracts/logs' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { fetchLogDetail } from '@/lib/logs/fetch-log-detail' +import { + defineInternalJsonRoute, + internalErrorResponse, + internalOrchestrationErrorPolicy, + internalRateLimits, +} from '@/lib/api/server/routes' +import { internalLogsSessionOrExecutorAuth } from '@/lib/logs/api/route-policies' +import { logOperations } from '@/lib/logs/application/operations' +import { readLogDetailUseCase } from '@/lib/logs/application/read-log-detail' -const logger = createLogger('LogDetailsByExecutionAPI') +const errorPolicy = { + ...internalOrchestrationErrorPolicy, + unhandled: () => internalErrorResponse(500, { error: 'Failed to fetch log' }), +} -export const GET = withRouteHandler( - async (request: NextRequest, context: { params: Promise<{ executionId: string }> }) => { - const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json( - { error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(getLogByExecutionIdContract, request, context) - if (!parsed.success) return parsed.response - - const { executionId } = parsed.data.params - const { workspaceId } = parsed.data.query - - const data = await fetchLogDetail({ - userId: authResult.userId, - workspaceId, - lookupColumn: 'executionId', - lookupValue: executionId, - }) - - if (!data) return NextResponse.json({ error: 'Not found' }, { status: 404 }) - - logger.debug('Fetched log by execution id', { executionId, workspaceId }) - return NextResponse.json({ data }) - } -) +export const GET = defineInternalJsonRoute({ + contract: getLogByExecutionIdContract, + auth: internalLogsSessionOrExecutorAuth, + operation: logOperations.readDetail, + rateLimit: internalRateLimits.none({ reason: 'Preserve existing execution log detail behavior' }), + errorPolicy, + mapInput: ({ params, query }, { principal, request }) => ({ + workspaceId: principal.kind === 'delegated' ? principal.workspaceId : query.workspaceId, + lookupColumn: 'executionId' as const, + lookupValue: params.executionId, + signal: request.signal, + }), + useCase: readLogDetailUseCase, + present: ({ detail }) => ({ data: detail }), +}) diff --git a/apps/sim/app/api/logs/execution/[executionId]/route.ts b/apps/sim/app/api/logs/execution/[executionId]/route.ts index 56de7000f7e..7130d9802eb 100644 --- a/apps/sim/app/api/logs/execution/[executionId]/route.ts +++ b/apps/sim/app/api/logs/execution/[executionId]/route.ts @@ -1,190 +1,29 @@ -import { db } from '@sim/db' +import { getExecutionSnapshotContract } from '@/lib/api/contracts/logs' import { - jobExecutionLogs, - workflow, - workflowExecutionLogs, - workflowExecutionSnapshots, -} from '@sim/db/schema' -import { createLogger } from '@sim/logger' -import { eq, inArray } from 'drizzle-orm' -import { type NextRequest, NextResponse } from 'next/server' -import { executionIdParamsSchema } from '@/lib/api/contracts/logs' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { hydrateChildTraces } from '@/lib/logs/execution/hydrate-child-traces' -import { materializeExecutionData } from '@/lib/logs/execution/trace-store' -import type { TraceSpan, WorkflowExecutionLog } from '@/lib/logs/types' -import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils' - -const logger = createLogger('LogsByExecutionIdAPI') - -export const GET = withRouteHandler( - async (request: NextRequest, { params }: { params: Promise<{ executionId: string }> }) => { - const requestId = generateRequestId() - - try { - const { executionId } = executionIdParamsSchema.parse(await params) - - const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized execution data access attempt for: ${executionId}`) - return NextResponse.json( - { error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const authenticatedUserId = authResult.userId - - const [workflowLog] = await db - .select({ - id: workflowExecutionLogs.id, - workflowId: workflowExecutionLogs.workflowId, - workspaceId: workflowExecutionLogs.workspaceId, - executionId: workflowExecutionLogs.executionId, - stateSnapshotId: workflowExecutionLogs.stateSnapshotId, - trigger: workflowExecutionLogs.trigger, - startedAt: workflowExecutionLogs.startedAt, - endedAt: workflowExecutionLogs.endedAt, - totalDurationMs: workflowExecutionLogs.totalDurationMs, - costTotal: workflowExecutionLogs.costTotal, - executionData: workflowExecutionLogs.executionData, - }) - .from(workflowExecutionLogs) - .leftJoin(workflow, eq(workflowExecutionLogs.workflowId, workflow.id)) - .where(eq(workflowExecutionLogs.executionId, executionId)) - .limit(1) - - if ( - workflowLog && - !(await checkWorkspaceAccess(workflowLog.workspaceId, authenticatedUserId)).hasAccess - ) { - logger.warn(`[${requestId}] Execution access denied: ${executionId}`) - return NextResponse.json({ error: 'Workflow execution not found' }, { status: 404 }) - } - - // Fallback: check job_execution_logs - if (!workflowLog) { - const [jobLog] = await db - .select({ - id: jobExecutionLogs.id, - workspaceId: jobExecutionLogs.workspaceId, - executionId: jobExecutionLogs.executionId, - trigger: jobExecutionLogs.trigger, - startedAt: jobExecutionLogs.startedAt, - endedAt: jobExecutionLogs.endedAt, - totalDurationMs: jobExecutionLogs.totalDurationMs, - cost: jobExecutionLogs.cost, - executionData: jobExecutionLogs.executionData, - }) - .from(jobExecutionLogs) - .where(eq(jobExecutionLogs.executionId, executionId)) - .limit(1) - - if ( - !jobLog || - !(await checkWorkspaceAccess(jobLog.workspaceId, authenticatedUserId)).hasAccess - ) { - logger.warn(`[${requestId}] Execution not found or access denied: ${executionId}`) - return NextResponse.json({ error: 'Workflow execution not found' }, { status: 404 }) - } - - return NextResponse.json({ - executionId, - workflowId: null, - workflowState: null, - childWorkflowSnapshots: {}, - executionMetadata: { - trigger: jobLog.trigger, - startedAt: jobLog.startedAt.toISOString(), - endedAt: jobLog.endedAt?.toISOString(), - totalDurationMs: jobLog.totalDurationMs, - cost: jobLog.cost || null, - }, - }) - } - - const [snapshot] = await db - .select() - .from(workflowExecutionSnapshots) - .where(eq(workflowExecutionSnapshots.id, workflowLog.stateSnapshotId)) - .limit(1) - - if (!snapshot) { - logger.warn( - `[${requestId}] Workflow state snapshot not found for execution: ${executionId}` - ) - return NextResponse.json({ error: 'Workflow state snapshot not found' }, { status: 404 }) - } - - const executionData = (await materializeExecutionData( - workflowLog.executionData as Record | null, - { - workspaceId: workflowLog.workspaceId, - workflowId: workflowLog.workflowId, - executionId: workflowLog.executionId, - } - )) as WorkflowExecutionLog['executionData'] - const traceSpans = (executionData?.traceSpans as TraceSpan[]) || [] - - // Join any custom-block child runs first: the spans they contribute carry - // their own `childWorkflowSnapshotId`s, so the collection below picks them - // up and canvas drill-down works across the workspace boundary too. - if (traceSpans.length > 0) { - await hydrateChildTraces(traceSpans, { viewerUserId: authenticatedUserId }) - } - - const childSnapshotIds = new Set() - const collectSnapshotIds = (spans: TraceSpan[]) => { - spans.forEach((span) => { - const snapshotId = span.childWorkflowSnapshotId - if (typeof snapshotId === 'string') { - childSnapshotIds.add(snapshotId) - } - if (span.children?.length) { - collectSnapshotIds(span.children) - } - }) - } - if (traceSpans.length > 0) { - collectSnapshotIds(traceSpans) - } - - const childWorkflowSnapshots = - childSnapshotIds.size > 0 - ? await db - .select() - .from(workflowExecutionSnapshots) - .where(inArray(workflowExecutionSnapshots.id, Array.from(childSnapshotIds))) - : [] - - const childSnapshotMap = childWorkflowSnapshots.reduce>( - (acc, snap) => { - acc[snap.id] = snap.stateData - return acc - }, - {} - ) - - const response = { - executionId, - workflowId: workflowLog.workflowId, - workflowState: snapshot.stateData, - childWorkflowSnapshots: childSnapshotMap, - executionMetadata: { - trigger: workflowLog.trigger, - startedAt: workflowLog.startedAt.toISOString(), - endedAt: workflowLog.endedAt?.toISOString(), - totalDurationMs: workflowLog.totalDurationMs, - cost: workflowLog.costTotal != null ? { total: Number(workflowLog.costTotal) } : null, - }, - } - - return NextResponse.json(response) - } catch (error) { - logger.error(`[${requestId}] Error fetching execution data:`, error) - return NextResponse.json({ error: 'Failed to fetch execution data' }, { status: 500 }) - } - } -) + defineInternalJsonRoute, + internalErrorResponse, + internalOrchestrationErrorPolicy, + internalRateLimits, +} from '@/lib/api/server/routes' +import { internalLogsSessionOrExecutorAuth } from '@/lib/logs/api/route-policies' +import { logOperations } from '@/lib/logs/application/operations' +import { readExecutionSnapshotUseCase } from '@/lib/logs/application/read-execution-snapshot' + +const errorPolicy = { + ...internalOrchestrationErrorPolicy, + unhandled: () => internalErrorResponse(500, { error: 'Failed to fetch execution data' }), +} + +export const GET = defineInternalJsonRoute({ + contract: getExecutionSnapshotContract, + auth: internalLogsSessionOrExecutorAuth, + operation: logOperations.readExecutionSnapshot, + rateLimit: internalRateLimits.none({ reason: 'Preserve existing execution snapshot behavior' }), + errorPolicy, + mapInput: ({ params }, { request }) => ({ + executionId: params.executionId, + signal: request.signal, + }), + useCase: readExecutionSnapshotUseCase, + present: (result) => result, +}) diff --git a/apps/sim/app/api/logs/route.ts b/apps/sim/app/api/logs/route.ts index 9286efd9b8c..102dc979ad7 100644 --- a/apps/sim/app/api/logs/route.ts +++ b/apps/sim/app/api/logs/route.ts @@ -1,37 +1,30 @@ -import { createLogger } from '@sim/logger' -import type { NextRequest } from 'next/server' -import { NextResponse } from 'next/server' import { listLogsContract } from '@/lib/api/contracts/logs' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { listLogs } from '@/lib/logs/list-logs' +import { + defineInternalJsonRoute, + internalErrorResponse, + internalOrchestrationErrorPolicy, + internalRateLimits, +} from '@/lib/api/server/routes' +import { internalLogsSessionOrExecutorAuth } from '@/lib/logs/api/route-policies' +import { listLogsUseCase } from '@/lib/logs/application/list-logs' +import { logOperations } from '@/lib/logs/application/operations' -const logger = createLogger('LogsAPI') +const errorPolicy = { + ...internalOrchestrationErrorPolicy, + unhandled: () => internalErrorResponse(500, { error: 'Failed to list logs' }), +} -export const GET = withRouteHandler(async (request: NextRequest) => { - const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json( - { error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - const userId = authResult.userId - - const parsed = await parseRequest(listLogsContract, request, {}) - if (!parsed.success) return parsed.response - - const params = parsed.data.query - const result = await listLogs(params, userId) - - logger.debug('Listed logs', { - workspaceId: params.workspaceId, - count: result.data.length, - hasMore: result.nextCursor !== null, - sortBy: params.sortBy, - sortOrder: params.sortOrder, - }) - - return NextResponse.json(result) +export const GET = defineInternalJsonRoute({ + contract: listLogsContract, + auth: internalLogsSessionOrExecutorAuth, + operation: logOperations.list, + rateLimit: internalRateLimits.none({ reason: 'Preserve existing internal logs list behavior' }), + errorPolicy, + mapInput: ({ query }, { principal, request }) => ({ + ...query, + workspaceId: principal.kind === 'delegated' ? principal.workspaceId : query.workspaceId, + signal: request.signal, + }), + useCase: listLogsUseCase, + present: (result) => result, }) diff --git a/apps/sim/app/api/mcp/tools/execute/route.test.ts b/apps/sim/app/api/mcp/tools/execute/route.test.ts deleted file mode 100644 index 24d9b20fe91..00000000000 --- a/apps/sim/app/api/mcp/tools/execute/route.test.ts +++ /dev/null @@ -1,253 +0,0 @@ -/** - * @vitest-environment node - */ -import { NextRequest } from 'next/server' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { - mockCapExecutionTimeoutMs, - mockDiscoverServerTools, - mockExecuteTool, - mockGetExecutionTimeout, -} = vi.hoisted(() => ({ - mockCapExecutionTimeoutMs: vi.fn((_policy: number, requested?: number) => requested ?? 0), - mockDiscoverServerTools: vi.fn(), - mockExecuteTool: vi.fn(), - mockGetExecutionTimeout: vi.fn(() => 0), -})) - -vi.mock('@/lib/mcp/middleware', () => ({ - withMcpAuth: - () => - ( - handler: ( - request: NextRequest, - context: { - userId: string - workspaceId: string - requestId: string - authType: 'internal_jwt' | 'session' - }, - routeContext: { params: Promise> } - ) => Promise - ) => - (request: NextRequest) => - handler( - request, - { - userId: 'user-1', - workspaceId: 'workspace-1', - requestId: 'request-1', - authType: request.headers.has('x-test-session') ? 'session' : 'internal_jwt', - }, - { params: Promise.resolve({}) } - ), - readMcpJsonBodyWithLimit: (request: NextRequest) => request.json(), - mcpBodyReadErrorResponse: () => null, -})) - -vi.mock('@/lib/mcp/service', () => ({ - mcpService: { - discoverServerTools: mockDiscoverServerTools, - executeTool: mockExecuteTool, - }, -})) - -vi.mock('@/lib/billing/core/billing-attribution', () => ({ - requireBillingAttributionHeader: () => ({ payerSubscription: { plan: 'pro' } }), - resolveBillingAttribution: async () => ({ payerSubscription: { plan: 'pro' } }), -})) - -vi.mock('@/lib/core/execution-limits', () => ({ - DEFAULT_EXECUTION_TIMEOUT_MS: 30_000, - capExecutionTimeoutMs: mockCapExecutionTimeoutMs, - getExecutionTimeout: mockGetExecutionTimeout, -})) - -vi.mock('@/ee/access-control/utils/permission-check', () => ({ - assertPermissionsAllowed: async () => {}, - McpToolsNotAllowedError: class McpToolsNotAllowedError extends Error {}, -})) - -vi.mock('@/lib/core/telemetry', () => ({ - PlatformEvents: { mcpToolExecuted: vi.fn() }, -})) - -import { POST } from '@/app/api/mcp/tools/execute/route' - -const URL = 'http://localhost/api/mcp/tools/execute' -const REQUEST_BODY = { - workspaceId: 'workspace-1', - serverId: 'server-1', - toolName: 'example_tool', - arguments: {}, -} - -function createRequest(headers: Record = {}): NextRequest { - return new NextRequest(URL, { - method: 'POST', - headers: { 'content-type': 'application/json', ...headers }, - body: JSON.stringify(REQUEST_BODY), - }) -} - -describe('MCP tool execution private secret provenance', () => { - beforeEach(() => { - vi.clearAllMocks() - mockDiscoverServerTools.mockResolvedValue([{ name: 'example_tool', inputSchema: {} }]) - mockExecuteTool.mockResolvedValue({ content: [{ type: 'text', text: 'ok' }] }) - }) - - it('returns provenance activated by this MCP transport call', async () => { - mockDiscoverServerTools.mockImplementationOnce( - async ( - _userId: string, - _serverId: string, - _workspaceId: string, - _forceRefresh: boolean, - recordProvenance?: (provenance: unknown) => void - ) => { - recordProvenance?.({ - version: 1, - complete: true, - entries: [{ name: 'MCP_TOKEN', encryptedValue: 'encrypted-mcp-token' }], - scope: { userId: 'user-1', workspaceId: 'workspace-1' }, - }) - return [{ name: 'example_tool', inputSchema: {} }] - } - ) - const request = createRequest({ - 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1', - }) - - const response = await POST(request, {}) - const body = (await response.json()) as Record - - expect(response.headers.get('x-sim-private-tool-metadata')).toBe( - 'resolved-secret-provenance-v1' - ) - expect(body.__resolvedSecretTraceProvenance).toEqual({ - version: 1, - complete: true, - entries: [{ name: 'MCP_TOKEN', encryptedValue: 'encrypted-mcp-token' }], - scope: { userId: 'user-1', workspaceId: 'workspace-1' }, - }) - expect(mockDiscoverServerTools.mock.calls[0]?.[4]).toEqual(expect.any(Function)) - expect(mockExecuteTool.mock.calls[0]?.[5]).toEqual(expect.any(Function)) - }) - - it('does not expose private provenance metadata to a session caller', async () => { - const request = createRequest({ - 'x-test-session': 'true', - 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1', - }) - - const response = await POST(request, {}) - const body = (await response.json()) as Record - - expect(response.headers.has('x-sim-private-tool-metadata')).toBe(false) - expect(body).not.toHaveProperty('__resolvedSecretTraceProvenance') - expect(mockDiscoverServerTools.mock.calls[0]?.[4]).toBeUndefined() - expect(mockExecuteTool.mock.calls[0]?.[5]).toBeUndefined() - }) - - it('preserves MCP error status and message when attaching private provenance', async () => { - mockExecuteTool.mockResolvedValueOnce({ - isError: true, - content: [{ type: 'text', text: 'Provider rejected the request' }], - }) - const request = createRequest({ - 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1', - }) - - const response = await POST(request, {}) - const body = (await response.json()) as Record - - expect(response.status).toBe(400) - expect(response.headers.get('x-sim-private-tool-metadata')).toBe( - 'resolved-secret-provenance-v1' - ) - expect(body).toMatchObject({ - success: false, - error: 'Provider rejected the request', - __resolvedSecretTraceProvenance: { - version: 1, - complete: true, - entries: [], - scope: { userId: 'user-1', workspaceId: 'workspace-1' }, - }, - }) - }) - - it('attaches private provenance without imposing a second functional response limit', async () => { - const largeText = 'x'.repeat(10 * 1024 * 1024 + 1) - mockExecuteTool.mockResolvedValueOnce({ - content: [{ type: 'text', text: largeText }], - }) - const request = createRequest({ - 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1', - }) - - const response = await (async () => { - const responseJsonSpy = vi.spyOn(Response.prototype, 'json') - try { - const result = await POST(request, {}) - expect(responseJsonSpy).not.toHaveBeenCalled() - return result - } finally { - responseJsonSpy.mockRestore() - } - })() - const body = (await response.json()) as Record - - expect(response.status).toBe(200) - expect(response.ok).toBe(true) - expect(response.headers.get('x-sim-private-tool-metadata')).toBe( - 'resolved-secret-provenance-v1' - ) - expect(body.__resolvedSecretTraceProvenance).toEqual({ - version: 1, - complete: true, - entries: [], - scope: { userId: 'user-1', workspaceId: 'workspace-1' }, - }) - expect(body).toMatchObject({ - success: true, - data: { - success: true, - output: { content: [{ type: 'text' }] }, - }, - }) - expect( - (body.data as { output: { content: Array<{ text?: unknown }> } }).output.content[0]?.text - ).toBe(largeText) - }) - - it('uses the remaining workflow deadline for trusted internal tool calls', async () => { - const request = createRequest({ - 'x-sim-execution-deadline-ms': String(Date.now() + 10_000), - }) - - const response = await POST(request, {}) - - expect(response.status).toBe(200) - expect(mockGetExecutionTimeout).toHaveBeenCalledWith('pro', 'async', undefined) - expect(mockCapExecutionTimeoutMs).toHaveBeenCalledWith(0, expect.any(Number)) - const remainingMs = mockCapExecutionTimeoutMs.mock.calls.at(-1)?.[1] - expect(remainingMs).toBeGreaterThan(0) - expect(remainingMs).toBeLessThanOrEqual(10_000) - }) - - it('ignores the internal deadline header for session callers', async () => { - const request = createRequest({ - 'x-test-session': 'true', - 'x-sim-execution-deadline-ms': String(Date.now() + 10_000), - }) - - const response = await POST(request, {}) - - expect(response.status).toBe(200) - expect(mockGetExecutionTimeout).toHaveBeenCalledWith('pro', 'sync') - expect(mockCapExecutionTimeoutMs).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/app/api/mcp/tools/execute/route.ts b/apps/sim/app/api/mcp/tools/execute/route.ts deleted file mode 100644 index cd371580e42..00000000000 --- a/apps/sim/app/api/mcp/tools/execute/route.ts +++ /dev/null @@ -1,437 +0,0 @@ -import { UnauthorizedError } from '@modelcontextprotocol/sdk/client/auth.js' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import type { NextRequest } from 'next/server' -import { NextResponse } from 'next/server' -import { mcpToolExecutionBodySchema } from '@/lib/api/contracts/mcp' -import { AuthType } from '@/lib/auth/hybrid' -import { - requireBillingAttributionHeader, - resolveBillingAttribution, -} from '@/lib/billing/core/billing-attribution' -import { capExecutionTimeoutMs, getExecutionTimeout } from '@/lib/core/execution-limits' -import type { SubscriptionPlan } from '@/lib/core/rate-limiter/types' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { SIM_VIA_HEADER } from '@/lib/execution/call-chain' -import { parseRemainingExecutionDeadlineMs } from '@/lib/execution/execution-deadline-header' -import { - RESOLVED_SECRET_PROVENANCE_METADATA_V1, - requestsPrivateToolMetadata, - serializePrivateToolMetadataResponseEnvelope, -} from '@/lib/execution/private-tool-metadata' -import { - mcpBodyReadErrorResponse, - readMcpJsonBodyWithLimit, - withMcpAuth, -} from '@/lib/mcp/middleware' -import { McpOauthRedirectRequired } from '@/lib/mcp/oauth' -import { mcpService } from '@/lib/mcp/service' -import { - McpOauthAuthorizationRequiredError, - type McpTool, - type McpToolCall, - type McpToolResult, -} from '@/lib/mcp/types' -import { categorizeError } from '@/lib/mcp/utils' -import { - assertPermissionsAllowed, - McpToolsNotAllowedError, -} from '@/ee/access-control/utils/permission-check' -import { - ResolvedSecretTraceProvenanceAccumulator, - type ResolvedSecretTraceProvenanceV1, -} from '@/executor/utils/resolved-secret-trace-registry' - -const logger = createLogger('McpToolExecutionAPI') - -export const dynamic = 'force-dynamic' - -interface SchemaProperty { - type: 'string' | 'number' | 'integer' | 'boolean' | 'object' | 'array' - description?: string - enum?: unknown[] - format?: string - items?: SchemaProperty - properties?: Record -} - -interface ToolExecutionResult { - success: boolean - output?: McpToolResult - error?: string -} - -function hasType(prop: unknown): prop is SchemaProperty { - return typeof prop === 'object' && prop !== null && 'type' in prop -} - -function createToolExecutionResponse( - body: Record, - status: number, - provenance: ResolvedSecretTraceProvenanceAccumulator | undefined -): NextResponse { - if (!provenance) { - return NextResponse.json(body, { status }) - } - - const envelope = serializePrivateToolMetadataResponseEnvelope( - body, - RESOLVED_SECRET_PROVENANCE_METADATA_V1, - provenance.exportProvenance() - ) - return NextResponse.json(envelope.body, { status, headers: envelope.headers }) -} - -/** - * POST - Execute a tool on an MCP server - */ -export const POST = withRouteHandler( - withMcpAuth('read')( - async (request: NextRequest, { userId, workspaceId, requestId, authType }) => { - let serverId: string | undefined - const includePrivateProvenance = - authType === AuthType.INTERNAL_JWT && - requestsPrivateToolMetadata(request.headers, RESOLVED_SECRET_PROVENANCE_METADATA_V1) - const resolvedSecretTraceProvenance = includePrivateProvenance - ? new ResolvedSecretTraceProvenanceAccumulator({ userId, workspaceId }) - : undefined - const recordProvenance = resolvedSecretTraceProvenance - ? (provenance: ResolvedSecretTraceProvenanceV1): void => { - resolvedSecretTraceProvenance.record(provenance) - } - : undefined - const errorResponse = (message: string, status: number): NextResponse => - createToolExecutionResponse( - { success: false, error: message }, - status, - resolvedSecretTraceProvenance - ) - const successResponse = (data: T, status = 200): NextResponse => - createToolExecutionResponse({ success: true, data }, status, resolvedSecretTraceProvenance) - - return (async (): Promise => { - try { - const rawBody = await readMcpJsonBodyWithLimit(request) - const parsedBody = mcpToolExecutionBodySchema.safeParse(rawBody) - - if (!parsedBody.success) { - return errorResponse('Invalid request format', 400) - } - - const body = parsedBody.data - - logger.info(`[${requestId}] MCP tool execution request received`, { - hasAuthHeader: !!request.headers.get('authorization'), - bodyKeys: Object.keys(body), - serverId: body.serverId, - toolName: body.toolName, - hasWorkflowId: !!body.workflowId, - workflowId: body.workflowId, - userId: userId, - }) - - const { toolName, arguments: rawArgs } = body - serverId = body.serverId - const args = rawArgs || {} - - try { - await assertPermissionsAllowed({ - userId, - workspaceId, - toolKind: 'mcp', - }) - } catch (err) { - if (err instanceof McpToolsNotAllowedError) { - return errorResponse(err.message, 403) - } - throw err - } - - logger.info( - `[${requestId}] Executing tool ${toolName} on server ${serverId} for user ${userId} in workspace ${workspaceId}` - ) - - let tool: McpTool | null = null - try { - const tools = await mcpService.discoverServerTools( - userId, - serverId, - workspaceId, - 'cache-aside', - recordProvenance - ) - tool = tools.find((t) => t.name === toolName) ?? null - - if (!tool) { - logger.warn(`[${requestId}] Tool ${toolName} not found on server ${serverId}`, { - availableTools: tools.map((t) => t.name), - }) - return errorResponse('Tool not found on the specified server', 404) - } - - if (tool.inputSchema?.properties) { - for (const [paramName, paramSchema] of Object.entries(tool.inputSchema.properties)) { - const schema = hasType(paramSchema) ? paramSchema : null - if (!schema) continue - const value = args[paramName] - - if (value === undefined || value === null) { - continue - } - - if ( - (schema.type === 'number' || schema.type === 'integer') && - typeof value === 'string' - ) { - const numValue = - schema.type === 'integer' ? Number.parseInt(value) : Number.parseFloat(value) - if (!Number.isNaN(numValue)) { - args[paramName] = numValue - } - } else if (schema.type === 'boolean' && typeof value === 'string') { - if (value.toLowerCase() === 'true') { - args[paramName] = true - } else if (value.toLowerCase() === 'false') { - args[paramName] = false - } - } else if (schema.type === 'array' && typeof value === 'string') { - const stringValue = value.trim() - if (stringValue) { - try { - const parsed = JSON.parse(stringValue) - if (Array.isArray(parsed)) { - args[paramName] = parsed - } else { - args[paramName] = [parsed] - } - } catch { - if (stringValue.includes(',')) { - args[paramName] = stringValue - .split(',') - .map((item) => item.trim()) - .filter((item) => item) - } else { - args[paramName] = [stringValue] - } - } - } else { - args[paramName] = [] - } - } - } - } - } catch (error) { - logger.warn( - `[${requestId}] Failed to discover tools for validation, proceeding without schema`, - error - ) - } - - if (tool) { - const validationError = validateToolArguments(tool, args) - if (validationError) { - logger.warn(`[${requestId}] Tool validation failed: ${validationError}`) - return errorResponse('Invalid tool arguments', 400) - } - } - - const toolCall: McpToolCall = { - name: toolName, - arguments: args, - } - - const billingAttribution = - authType === AuthType.INTERNAL_JWT - ? requireBillingAttributionHeader(request.headers, { - actorUserId: userId, - workspaceId, - }) - : await resolveBillingAttribution({ actorUserId: userId, workspaceId }) - const remainingWorkflowDeadlineMs = - authType === AuthType.INTERNAL_JWT - ? parseRemainingExecutionDeadlineMs(request.headers) - : undefined - const executionTimeout = - remainingWorkflowDeadlineMs === undefined - ? getExecutionTimeout( - billingAttribution.payerSubscription?.plan as SubscriptionPlan | undefined, - 'sync' - ) - : capExecutionTimeoutMs( - getExecutionTimeout( - billingAttribution.payerSubscription?.plan as SubscriptionPlan | undefined, - 'async', - billingAttribution.payerSubscription?.enterpriseWorkflowExecutionTimeoutSeconds - ), - remainingWorkflowDeadlineMs - ) - - const simViaHeader = request.headers.get(SIM_VIA_HEADER) - const extraHeaders: Record = {} - if (simViaHeader) { - extraHeaders[SIM_VIA_HEADER] = simViaHeader - } - - let timeoutHandle: ReturnType | undefined - const executePromise = mcpService.executeTool( - userId, - serverId, - toolCall, - workspaceId, - extraHeaders, - recordProvenance - ) - // A zero timeout means "no timeout" (billing-disabled deployments). - const result = await (executionTimeout > 0 - ? Promise.race([ - executePromise, - new Promise((_, reject) => { - timeoutHandle = setTimeout( - () => reject(new Error('Tool execution timeout')), - executionTimeout - ) - }), - ]) - : executePromise - ).finally(() => { - if (timeoutHandle !== undefined) clearTimeout(timeoutHandle) - }) - - const transformedResult = transformToolResult(result) - - if (result.isError) { - logger.warn( - `[${requestId}] Tool execution returned error for ${toolName} on ${serverId}` - ) - return errorResponse(transformedResult.error || 'Tool execution failed', 400) - } - logger.info(`[${requestId}] Successfully executed tool ${toolName} on server ${serverId}`) - - try { - const { PlatformEvents } = await import('@/lib/core/telemetry') - PlatformEvents.mcpToolExecuted({ - serverId, - toolName, - status: 'success', - workspaceId, - }) - } catch (error) { - logger.warn('Failed to record MCP tool execution telemetry', { - error: getErrorMessage(error), - serverId, - toolName, - workspaceId, - }) - } - - return successResponse(transformedResult) - } catch (error) { - if (getErrorMessage(error) === 'Tool execution timeout') { - resolvedSecretTraceProvenance?.markIncomplete('mcp-tool-execution-timeout') - } - const bodyErrorResponse = mcpBodyReadErrorResponse(error, request) - if (bodyErrorResponse) return bodyErrorResponse - if ( - error instanceof McpOauthAuthorizationRequiredError || - error instanceof McpOauthRedirectRequired || - error instanceof UnauthorizedError - ) { - const errorServerId = - error instanceof McpOauthAuthorizationRequiredError ? error.serverId : serverId - logger.warn(`[${requestId}] OAuth re-authorization required for MCP tool execution`, { - serverId: errorServerId, - }) - return createToolExecutionResponse( - { - success: false, - error: 'OAuth re-authorization required', - code: 'reauth_required', - serverId: errorServerId, - }, - 401, - resolvedSecretTraceProvenance - ) - } - - logger.error(`[${requestId}] Error executing MCP tool:`, error) - - const { message, status } = categorizeError(error) - return errorResponse(message, status) - } - })() - } - ) -) - -function validateToolArguments(tool: McpTool, args: Record): string | null { - if (!tool.inputSchema) { - return null - } - - const schema = tool.inputSchema - - if (schema.required && Array.isArray(schema.required)) { - for (const requiredProp of schema.required) { - if (!(requiredProp in (args || {}))) { - return `Missing required property: ${requiredProp}` - } - } - } - - if (schema.properties && args) { - for (const [propName, propSchema] of Object.entries(schema.properties)) { - const propValue = args[propName] - if (propValue !== undefined && hasType(propSchema)) { - const expectedType = propSchema.type - const actualType = typeof propValue - - if (expectedType === 'string' && actualType !== 'string') { - return `Property ${propName} must be a string` - } - if (expectedType === 'number' && actualType !== 'number') { - return `Property ${propName} must be a number` - } - if ( - expectedType === 'integer' && - (actualType !== 'number' || !Number.isInteger(propValue)) - ) { - return `Property ${propName} must be an integer` - } - if (expectedType === 'boolean' && actualType !== 'boolean') { - return `Property ${propName} must be a boolean` - } - if ( - expectedType === 'object' && - (actualType !== 'object' || propValue === null || Array.isArray(propValue)) - ) { - return `Property ${propName} must be an object` - } - if (expectedType === 'array' && !Array.isArray(propValue)) { - return `Property ${propName} must be an array` - } - } - } - } - - return null -} - -function transformToolResult(result: McpToolResult): ToolExecutionResult { - if (result.isError) { - const firstContent = Array.isArray(result.content) ? result.content[0] : undefined - const errorText = - firstContent && typeof firstContent === 'object' && typeof firstContent.text === 'string' - ? firstContent.text - : undefined - - return { - success: false, - error: errorText && errorText.trim().length > 0 ? errorText : 'Tool execution failed', - } - } - - return { - success: true, - output: result, - } -} diff --git a/apps/sim/app/api/memory/[id]/route.test.ts b/apps/sim/app/api/memory/[id]/route.test.ts deleted file mode 100644 index bae4aa0043d..00000000000 --- a/apps/sim/app/api/memory/[id]/route.test.ts +++ /dev/null @@ -1,166 +0,0 @@ -/** - * @vitest-environment node - */ - -import { memory } from '@sim/db/schema' -import { - createMockRequest, - dbChainMockFns, - hybridAuthMockFns, - queueTableRows, - resetDbChainMock, -} from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' -import { AuthType } from '@/lib/auth/hybrid' -import { - PRIVATE_TOOL_METADATA_REQUEST_HEADER, - PRIVATE_TOOL_METADATA_RESPONSE_HEADER, - RESOLVED_SECRET_PROVENANCE_FIELD, - RESOLVED_SECRET_PROVENANCE_METADATA_V1, -} from '@/lib/execution/private-tool-metadata' - -const { mockCheckWorkspaceAccess, mockReplaceMemorySecretProvenanceInTx } = vi.hoisted(() => ({ - mockCheckWorkspaceAccess: vi.fn(), - mockReplaceMemorySecretProvenanceInTx: vi.fn(), -})) - -vi.mock('@/lib/workspaces/permissions/utils', () => ({ - checkWorkspaceAccess: mockCheckWorkspaceAccess, -})) - -vi.mock('@/lib/memory/secret-provenance', () => ({ - readBoundMemorySecretProvenance: vi.fn(() => ({ status: 'unknown' })), - replaceMemorySecretProvenanceInTx: mockReplaceMemorySecretProvenanceInTx, -})) - -import { GET, PUT } from '@/app/api/memory/[id]/route' - -const WORKSPACE_ID = '11111111-1111-4111-8111-111111111111' -const CONTEXT = { params: Promise.resolve({ id: 'missing-conversation' }) } - -describe('GET /api/memory/[id]', () => { - beforeEach(() => { - vi.clearAllMocks() - resetDbChainMock() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: AuthType.INTERNAL_JWT, - }) - mockCheckWorkspaceAccess.mockResolvedValue({ exists: true, hasAccess: true }) - queueTableRows(memory, []) - }) - - it('returns verified exact-empty metadata when a tool lookup has no matching memory', async () => { - const response = await GET( - createMockRequest( - 'GET', - undefined, - { - [PRIVATE_TOOL_METADATA_REQUEST_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1, - }, - `http://localhost:3000/api/memory/missing-conversation?workspaceId=${WORKSPACE_ID}` - ), - CONTEXT - ) - - expect(response.status).toBe(200) - expect(response.headers.get(PRIVATE_TOOL_METADATA_RESPONSE_HEADER)).toBe( - RESOLVED_SECRET_PROVENANCE_METADATA_V1 - ) - expect(await response.json()).toEqual({ - success: true, - data: null, - [RESOLVED_SECRET_PROVENANCE_FIELD]: { - version: 1, - complete: true, - entries: [], - scope: { userId: 'user-1', workspaceId: WORKSPACE_ID }, - }, - }) - }) - - it('preserves the existing headerless empty response for ordinary API callers', async () => { - const response = await GET( - createMockRequest( - 'GET', - undefined, - {}, - `http://localhost:3000/api/memory/missing-conversation?workspaceId=${WORKSPACE_ID}` - ), - CONTEXT - ) - - expect(response.status).toBe(200) - expect(response.headers.get(PRIVATE_TOOL_METADATA_RESPONSE_HEADER)).toBeNull() - expect(await response.json()).toEqual({ success: true, data: null }) - }) -}) - -describe('PUT /api/memory/[id]', () => { - const CONVERSATION_ID = 'conversation-1' - const PUT_CONTEXT = { params: Promise.resolve({ id: CONVERSATION_ID }) } - const MESSAGE = { role: 'user', content: 'hi' } as const - - beforeEach(() => { - vi.clearAllMocks() - resetDbChainMock() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: AuthType.API_KEY, - }) - mockCheckWorkspaceAccess.mockResolvedValue({ - exists: true, - hasAccess: true, - canWrite: true, - }) - mockReplaceMemorySecretProvenanceInTx.mockResolvedValue(undefined) - dbChainMockFns.returning.mockResolvedValue([{ id: 'memory-row-1' }]) - queueTableRows(memory, [ - { id: 'memory-row-1', key: CONVERSATION_ID, data: [], secretProvenanceVersion: null }, - ]) - queueTableRows(memory, [ - { id: 'memory-row-1', key: CONVERSATION_ID, data: [MESSAGE], secretProvenanceVersion: 1 }, - ]) - }) - - it('persists the updated message wrapped in an array so reads keep the POST-written shape', async () => { - const response = await PUT( - createMockRequest( - 'PUT', - { data: MESSAGE, workspaceId: WORKSPACE_ID }, - {}, - `http://localhost:3000/api/memory/${CONVERSATION_ID}` - ), - PUT_CONTEXT - ) - - expect(response.status).toBe(200) - expect(dbChainMockFns.set).toHaveBeenCalledWith( - expect.objectContaining({ data: [{ role: 'user', content: 'hi' }] }) - ) - }) - - it('hashes provenance over the exact persisted value', async () => { - await PUT( - createMockRequest( - 'PUT', - { data: MESSAGE, workspaceId: WORKSPACE_ID }, - {}, - `http://localhost:3000/api/memory/${CONVERSATION_ID}` - ), - PUT_CONTEXT - ) - - expect(mockReplaceMemorySecretProvenanceInTx).toHaveBeenCalledWith( - expect.anything(), - 'memory-row-1', - [{ role: 'user', content: 'hi' }], - expect.anything() - ) - const [setPayload] = dbChainMockFns.set.mock.calls[0] as [{ data: unknown }] - const [, , provenanceValue] = mockReplaceMemorySecretProvenanceInTx.mock.calls[0] - expect(provenanceValue).toBe(setPayload.data) - }) -}) diff --git a/apps/sim/app/api/memory/[id]/route.ts b/apps/sim/app/api/memory/[id]/route.ts deleted file mode 100644 index 55e49dffda5..00000000000 --- a/apps/sim/app/api/memory/[id]/route.ts +++ /dev/null @@ -1,320 +0,0 @@ -import { db } from '@sim/db' -import { memory } from '@sim/db/schema' -import { createLogger } from '@sim/logger' -import { and, eq, isNull } from 'drizzle-orm' -import { type NextRequest, NextResponse } from 'next/server' -import { - agentMemoryDataSchemaContract, - deleteMemoryByIdContract, - getMemoryByIdContract, - updateMemoryByIdContract, -} from '@/lib/api/contracts/memory' -import { parseRequest } from '@/lib/api/server' -import { type AuthTypeValue, checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { replaceMemorySecretProvenanceInTx } from '@/lib/memory/secret-provenance' -import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils' -import { - createMemoryResponse, - resolveMemoryWriteSecretProvenance, -} from '@/app/api/memory/secret-provenance' - -const logger = createLogger('MemoryByIdAPI') - -interface MemoryRouteContext { - params: Promise<{ id: string }> -} - -function memoryEnvelopeError(message: string, status: number) { - return NextResponse.json({ success: false, error: { message } }, { status }) -} - -async function validateMemoryAccess( - request: NextRequest, - workspaceId: string, - requestId: string, - action: 'read' | 'write' -): Promise<{ userId: string; authType?: AuthTypeValue } | { error: NextResponse }> { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized memory ${action} attempt`) - return { error: memoryEnvelopeError('Authentication required', 401) } - } - - const access = await checkWorkspaceAccess(workspaceId, authResult.userId) - if (!access.exists || !access.hasAccess) { - return { error: memoryEnvelopeError('Workspace not found', 404) } - } - - if (action === 'write' && !access.canWrite) { - return { error: memoryEnvelopeError('Write access denied', 403) } - } - - return { userId: authResult.userId, authType: authResult.authType } -} - -export const dynamic = 'force-dynamic' -export const runtime = 'nodejs' - -export const GET = withRouteHandler(async (request: NextRequest, context: MemoryRouteContext) => { - const requestId = generateRequestId() - const { id } = await context.params - - try { - const validation = await parseRequest(getMemoryByIdContract, request, context, { - validationErrorResponse: (error) => - memoryEnvelopeError( - error.issues.map((err) => `${err.path.join('.')}: ${err.message}`).join(', '), - 400 - ), - }) - if (!validation.success) return validation.response - const { workspaceId: validatedWorkspaceId } = validation.data.query - - const accessCheck = await validateMemoryAccess(request, validatedWorkspaceId, requestId, 'read') - if ('error' in accessCheck) { - return accessCheck.error - } - - const memories = await db - .select() - .from(memory) - .where( - and( - eq(memory.key, id), - eq(memory.workspaceId, validatedWorkspaceId), - isNull(memory.deletedAt) - ) - ) - .orderBy(memory.createdAt) - .limit(1) - - if (memories.length === 0) { - return createMemoryResponse({ - request, - authType: accessCheck.authType, - userId: accessCheck.userId, - workspaceId: validatedWorkspaceId, - body: { success: true, data: null }, - memories: [], - }) - } - - const mem = memories[0] - - logger.info(`[${requestId}] Memory retrieved: ${id} for workspace: ${validatedWorkspaceId}`) - return createMemoryResponse({ - request, - authType: accessCheck.authType, - userId: accessCheck.userId, - workspaceId: validatedWorkspaceId, - body: { success: true, data: { conversationId: mem.key, data: mem.data } }, - memories: [ - { - id: mem.id, - data: mem.data, - secretProvenanceVersion: mem.secretProvenanceVersion, - }, - ], - }) - } catch (error: any) { - logger.error(`[${requestId}] Error retrieving memory`, { error }) - return memoryEnvelopeError(error.message || 'Failed to retrieve memory', 500) - } -}) - -export const DELETE = withRouteHandler( - async (request: NextRequest, context: MemoryRouteContext) => { - const requestId = generateRequestId() - const { id } = await context.params - - try { - const validation = await parseRequest(deleteMemoryByIdContract, request, context, { - validationErrorResponse: (error) => - memoryEnvelopeError( - error.issues.map((err) => `${err.path.join('.')}: ${err.message}`).join(', '), - 400 - ), - }) - if (!validation.success) return validation.response - const { workspaceId: validatedWorkspaceId } = validation.data.query - - const accessCheck = await validateMemoryAccess( - request, - validatedWorkspaceId, - requestId, - 'write' - ) - if ('error' in accessCheck) { - return accessCheck.error - } - - const existingMemory = await db - .select({ id: memory.id }) - .from(memory) - .where( - and( - eq(memory.key, id), - eq(memory.workspaceId, validatedWorkspaceId), - isNull(memory.deletedAt) - ) - ) - .limit(1) - - if (existingMemory.length === 0) { - return memoryEnvelopeError('Memory not found', 404) - } - - await db - .delete(memory) - .where( - and( - eq(memory.key, id), - eq(memory.workspaceId, validatedWorkspaceId), - isNull(memory.deletedAt) - ) - ) - - logger.info(`[${requestId}] Memory deleted: ${id} for workspace: ${validatedWorkspaceId}`) - return NextResponse.json( - { success: true, data: { message: 'Memory deleted successfully' } }, - { status: 200 } - ) - } catch (error: any) { - logger.error(`[${requestId}] Error deleting memory`, { error }) - return memoryEnvelopeError(error.message || 'Failed to delete memory', 500) - } - } -) - -export const PUT = withRouteHandler(async (request: NextRequest, context: MemoryRouteContext) => { - const requestId = generateRequestId() - const { id } = await context.params - - try { - const validation = await parseRequest(updateMemoryByIdContract, request, context, { - validationErrorResponse: (error) => - memoryEnvelopeError( - `Invalid request body: ${error.issues.map((err) => `${err.path.join('.')}: ${err.message}`).join(', ')}`, - 400 - ), - invalidJsonResponse: () => memoryEnvelopeError('Invalid JSON in request body', 400), - }) - if (!validation.success) return validation.response - const { data: validatedData, workspaceId: validatedWorkspaceId } = validation.data.body - - const accessCheck = await validateMemoryAccess( - request, - validatedWorkspaceId, - requestId, - 'write' - ) - if ('error' in accessCheck) { - return accessCheck.error - } - - const existingMemories = await db - .select() - .from(memory) - .where( - and( - eq(memory.key, id), - eq(memory.workspaceId, validatedWorkspaceId), - isNull(memory.deletedAt) - ) - ) - .limit(1) - - if (existingMemories.length === 0) { - return memoryEnvelopeError('Memory not found', 404) - } - - const agentValidation = agentMemoryDataSchemaContract.safeParse(validatedData) - if (!agentValidation.success) { - const errorMessage = agentValidation.error.issues - .map((err) => `${err.path.join('.')}: ${err.message}`) - .join(', ') - return memoryEnvelopeError(`Invalid agent memory data: ${errorMessage}`, 400) - } - - /** - * POST stores memory data as an array (`route.ts`), and its `onConflictDoUpdate` relies on - * jsonb append semantics that require an array on both sides. Every persisted row is therefore - * an array, and `tools/memory/types.ts` declares `data: AgentMemoryData[]`. Wrap here so an - * update never leaves behind a bare object that reads like `data[0]` would break on. - */ - const nextData = Array.isArray(validatedData) ? validatedData : [validatedData] - - const now = new Date() - const writeProvenance = resolveMemoryWriteSecretProvenance({ - request, - payload: validation.data.body, - authType: accessCheck.authType, - userId: accessCheck.userId, - workspaceId: validatedWorkspaceId, - }) - if (!writeProvenance.success) return writeProvenance.response - await db.transaction(async (tx) => { - const [updated] = await tx - .update(memory) - .set({ - data: nextData, - secretProvenanceVersion: writeProvenance.provenance - ? 1 - : existingMemories[0].secretProvenanceVersion, - updatedAt: now, - }) - .where( - and( - eq(memory.key, id), - eq(memory.workspaceId, validatedWorkspaceId), - isNull(memory.deletedAt) - ) - ) - .returning({ id: memory.id }) - if (!updated) throw new Error('Memory not found') - if (writeProvenance.provenance) { - await replaceMemorySecretProvenanceInTx( - tx, - updated.id, - nextData, - writeProvenance.provenance - ) - } - }) - - const updatedMemories = await db - .select() - .from(memory) - .where( - and( - eq(memory.key, id), - eq(memory.workspaceId, validatedWorkspaceId), - isNull(memory.deletedAt) - ) - ) - .limit(1) - - const mem = updatedMemories[0] - - logger.info(`[${requestId}] Memory updated: ${id} for workspace: ${validatedWorkspaceId}`) - return createMemoryResponse({ - request, - authType: accessCheck.authType, - userId: accessCheck.userId, - workspaceId: validatedWorkspaceId, - body: { success: true, data: { conversationId: mem.key, data: mem.data } }, - memories: [ - { - id: mem.id, - data: mem.data, - secretProvenanceVersion: mem.secretProvenanceVersion, - }, - ], - }) - } catch (error: any) { - logger.error(`[${requestId}] Error updating memory`, { error }) - return memoryEnvelopeError(error.message || 'Failed to update memory', 500) - } -}) diff --git a/apps/sim/app/api/memory/route.ts b/apps/sim/app/api/memory/route.ts deleted file mode 100644 index 963b08eb42e..00000000000 --- a/apps/sim/app/api/memory/route.ts +++ /dev/null @@ -1,411 +0,0 @@ -import { db } from '@sim/db' -import { memory, memorySecretProvenance } from '@sim/db/schema' -import { createLogger } from '@sim/logger' -import { getPostgresErrorCode } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { and, eq, isNull, like } from 'drizzle-orm' -import { type NextRequest, NextResponse } from 'next/server' -import { - createMemoryContract, - deleteMemoryByQueryContract, - listMemoriesContract, - memoryMessageSchema, -} from '@/lib/api/contracts/memory' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { mergeDurableSecretProvenance } from '@/lib/execution/durable-secret-provenance' -import { - readBoundMemorySecretProvenance, - replaceMemorySecretProvenanceInTx, -} from '@/lib/memory/secret-provenance' -import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils' -import { - createMemoryResponse, - resolveMemoryWriteSecretProvenance, -} from '@/app/api/memory/secret-provenance' - -const logger = createLogger('MemoryAPI') - -export const dynamic = 'force-dynamic' -export const runtime = 'nodejs' - -export const GET = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request) - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized memory access attempt`) - return NextResponse.json( - { success: false, error: { message: authResult.error || 'Authentication required' } }, - { status: 401 } - ) - } - - const validation = await parseRequest(listMemoriesContract, request, {}) - if (!validation.success) return validation.response - const { workspaceId, query: searchQuery, limit } = validation.data.query - - if (!workspaceId) { - return NextResponse.json( - { success: false, error: { message: 'workspaceId parameter is required' } }, - { status: 400 } - ) - } - - const access = await checkWorkspaceAccess(workspaceId, authResult.userId) - if (!access.exists) { - return NextResponse.json( - { success: false, error: { message: 'Workspace not found' } }, - { status: 404 } - ) - } - if (!access.hasAccess) { - return NextResponse.json( - { success: false, error: { message: 'Access denied to this workspace' } }, - { status: 403 } - ) - } - - const conditions = [isNull(memory.deletedAt), eq(memory.workspaceId, workspaceId)] - - if (searchQuery) { - conditions.push(like(memory.key, `%${searchQuery}%`)) - } - - const rawMemories = await db - .select() - .from(memory) - .where(and(...conditions)) - .orderBy(memory.createdAt) - .limit(limit) - - const enrichedMemories = rawMemories.map((mem) => ({ - conversationId: mem.key, - data: mem.data, - })) - - logger.info( - `[${requestId}] Found ${enrichedMemories.length} memories for workspace: ${workspaceId}` - ) - return createMemoryResponse({ - request, - authType: authResult.authType, - userId: authResult.userId, - workspaceId, - body: { success: true, data: { memories: enrichedMemories } }, - memories: rawMemories.map((record) => ({ - id: record.id, - data: record.data, - secretProvenanceVersion: record.secretProvenanceVersion, - })), - }) - } catch (error: any) { - logger.error(`[${requestId}] Error searching memories`, { error }) - return NextResponse.json( - { success: false, error: { message: error.message || 'Failed to search memories' } }, - { status: 500 } - ) - } -}) - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request) - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized memory creation attempt`) - return NextResponse.json( - { success: false, error: { message: authResult.error || 'Authentication required' } }, - { status: 401 } - ) - } - - const validation = await parseRequest(createMemoryContract, request, {}) - if (!validation.success) return validation.response - const { key, data, workspaceId } = validation.data.body - - if (!key) { - return NextResponse.json( - { success: false, error: { message: 'Memory key is required' } }, - { status: 400 } - ) - } - - if (!data) { - return NextResponse.json( - { success: false, error: { message: 'Memory data is required' } }, - { status: 400 } - ) - } - - if (!workspaceId) { - return NextResponse.json( - { success: false, error: { message: 'workspaceId is required' } }, - { status: 400 } - ) - } - - const access = await checkWorkspaceAccess(workspaceId, authResult.userId) - if (!access.exists) { - return NextResponse.json( - { success: false, error: { message: 'Workspace not found' } }, - { status: 404 } - ) - } - if (!access.hasAccess) { - return NextResponse.json( - { success: false, error: { message: 'Access denied to this workspace' } }, - { status: 403 } - ) - } - - if (!access.canWrite) { - return NextResponse.json( - { success: false, error: { message: 'Write access denied to this workspace' } }, - { status: 403 } - ) - } - - const dataToValidate = Array.isArray(data) ? data : [data] - - for (const msg of dataToValidate) { - const parsedMessage = memoryMessageSchema.safeParse(msg) - if (!parsedMessage.success) { - const role = - msg && typeof msg === 'object' && 'role' in msg - ? (msg as { role?: unknown }).role - : undefined - const invalidRole = Boolean(role) && !['user', 'assistant', 'system'].includes(String(role)) - return NextResponse.json( - { - success: false, - error: { - message: invalidRole - ? 'Message role must be user, assistant, or system' - : 'Memory requires messages with role and content', - }, - }, - { status: 400 } - ) - } - } - - const initialData = Array.isArray(data) ? data : [data] - const now = new Date() - const id = `mem_${generateId().replace(/-/g, '')}` - const writeProvenance = resolveMemoryWriteSecretProvenance({ - request, - payload: validation.data.body, - authType: authResult.authType, - userId: authResult.userId, - workspaceId, - }) - if (!writeProvenance.success) return writeProvenance.response - - const { sql } = await import('drizzle-orm') - - await db.transaction(async (tx) => { - const [existing] = await tx - .select({ - id: memory.id, - data: memory.data, - updatedAt: memory.updatedAt, - secretProvenanceVersion: memory.secretProvenanceVersion, - }) - .from(memory) - .where(and(eq(memory.workspaceId, workspaceId), eq(memory.key, key))) - .limit(1) - .for('update') - let previousProvenance - if (existing && writeProvenance.provenance) { - const [sidecar] = await tx - .select() - .from(memorySecretProvenance) - .where(eq(memorySecretProvenance.memoryId, existing.id)) - .limit(1) - previousProvenance = readBoundMemorySecretProvenance({ - secretProvenanceVersion: existing.secretProvenanceVersion, - data: existing.data, - provenanceContentHash: sidecar?.contentHash ?? null, - status: sidecar?.status ?? null, - entries: sidecar?.entries, - }) - } - - const [written] = await tx - .insert(memory) - .values({ - id, - workspaceId, - key, - data: initialData, - secretProvenanceVersion: writeProvenance.provenance ? 1 : null, - createdAt: now, - updatedAt: now, - }) - .onConflictDoUpdate({ - target: [memory.workspaceId, memory.key], - set: { - data: sql`${memory.data} || ${JSON.stringify(initialData)}::jsonb`, - secretProvenanceVersion: writeProvenance.provenance - ? 1 - : (existing?.secretProvenanceVersion ?? null), - updatedAt: now, - }, - }) - .returning({ id: memory.id, data: memory.data }) - if (writeProvenance.provenance) { - await replaceMemorySecretProvenanceInTx( - tx, - written.id, - written.data, - previousProvenance - ? mergeDurableSecretProvenance(previousProvenance, writeProvenance.provenance) - : writeProvenance.provenance - ) - } - }) - - logger.info(`[${requestId}] Memory operation successful: ${key} for workspace: ${workspaceId}`) - - const allMemories = await db - .select() - .from(memory) - .where( - and(eq(memory.key, key), eq(memory.workspaceId, workspaceId), isNull(memory.deletedAt)) - ) - .orderBy(memory.createdAt) - - if (allMemories.length === 0) { - return NextResponse.json( - { success: false, error: { message: 'Failed to retrieve memory after creation/update' } }, - { status: 500 } - ) - } - - const memoryRecord = allMemories[0] - - return createMemoryResponse({ - request, - authType: authResult.authType, - userId: authResult.userId, - workspaceId, - body: { - success: true, - data: { conversationId: memoryRecord.key, data: memoryRecord.data }, - }, - memories: [ - { - id: memoryRecord.id, - data: memoryRecord.data, - secretProvenanceVersion: memoryRecord.secretProvenanceVersion, - }, - ], - }) - } catch (error: any) { - if (getPostgresErrorCode(error) === '23505') { - return NextResponse.json( - { success: false, error: { message: 'Memory with this key already exists' } }, - { status: 409 } - ) - } - - logger.error(`[${requestId}] Error creating memory`, { error }) - return NextResponse.json( - { success: false, error: { message: 'Failed to create memory' } }, - { status: 500 } - ) - } -}) - -export const DELETE = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request) - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized memory deletion attempt`) - return NextResponse.json( - { success: false, error: { message: authResult.error || 'Authentication required' } }, - { status: 401 } - ) - } - - const validation = await parseRequest(deleteMemoryByQueryContract, request, {}) - if (!validation.success) return validation.response - const { workspaceId, conversationId } = validation.data.query - - if (!workspaceId) { - return NextResponse.json( - { success: false, error: { message: 'workspaceId parameter is required' } }, - { status: 400 } - ) - } - - if (!conversationId) { - return NextResponse.json( - { success: false, error: { message: 'conversationId must be provided' } }, - { status: 400 } - ) - } - - const access = await checkWorkspaceAccess(workspaceId, authResult.userId) - if (!access.exists) { - return NextResponse.json( - { success: false, error: { message: 'Workspace not found' } }, - { status: 404 } - ) - } - if (!access.hasAccess) { - return NextResponse.json( - { success: false, error: { message: 'Access denied to this workspace' } }, - { status: 403 } - ) - } - - if (!access.canWrite) { - return NextResponse.json( - { success: false, error: { message: 'Write access denied to this workspace' } }, - { status: 403 } - ) - } - - const result = await db - .delete(memory) - .where( - and( - eq(memory.key, conversationId), - eq(memory.workspaceId, workspaceId), - isNull(memory.deletedAt) - ) - ) - .returning({ id: memory.id }) - - const deletedCount = result.length - - logger.info(`[${requestId}] Deleted ${deletedCount} memories for workspace: ${workspaceId}`) - return NextResponse.json( - { - success: true, - data: { - message: - deletedCount > 0 - ? `Successfully deleted ${deletedCount} memories` - : 'No memories found matching the criteria', - deletedCount, - }, - }, - { status: 200 } - ) - } catch (error: any) { - logger.error(`[${requestId}] Error deleting memories`, { error }) - return NextResponse.json( - { success: false, error: { message: error.message || 'Failed to delete memories' } }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/memory/secret-provenance.test.ts b/apps/sim/app/api/memory/secret-provenance.test.ts deleted file mode 100644 index 8132ed5e5ad..00000000000 --- a/apps/sim/app/api/memory/secret-provenance.test.ts +++ /dev/null @@ -1,352 +0,0 @@ -/** - * @vitest-environment node - */ - -import { memorySecretProvenance } from '@sim/db/schema' -import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' -import { NextRequest } from 'next/server' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { mockIsEnforced, mockReport } = vi.hoisted(() => ({ - mockIsEnforced: vi.fn(() => false), - mockReport: vi.fn(), -})) - -vi.mock('@/lib/execution/durable-secret-provenance-enforcement', () => ({ - DURABLE_SECRET_PROVENANCE_SURFACES: ['memory', 'table-row', 'knowledge'], - isDurableSecretProvenanceEnforced: mockIsEnforced, - reportUnrecordedDurableProvenance: mockReport, -})) - -import { memoryListQuerySchema } from '@/lib/api/contracts/memory' -import { AuthType } from '@/lib/auth/hybrid' -import { - PRIVATE_SECRET_PROVENANCE_BUNDLE_V1, - PRIVATE_SECRET_PROVENANCE_FIELD, - PRIVATE_SECRET_PROVENANCE_HEADER, - PRIVATE_TOOL_METADATA_REQUEST_HEADER, - RESOLVED_SECRET_PROVENANCE_FIELD, - RESOLVED_SECRET_PROVENANCE_METADATA_V1, -} from '@/lib/execution/private-tool-metadata' -import { - createMemoryResponse, - resolveMemoryWriteSecretProvenance, -} from '@/app/api/memory/secret-provenance' - -function privateMemoryWrite( - scope: { userId: string; workspaceId?: string }, - entries: Array<{ name: string; encryptedValue: string }> = [] -) { - const payload = { - [PRIVATE_SECRET_PROVENANCE_FIELD]: { - version: 1 as const, - complete: true, - selections: [ - { - key: 'data', - provenance: { version: 1 as const, complete: true, entries, scope }, - }, - ], - }, - } - const request = new NextRequest('http://localhost/api/memory', { - method: 'POST', - headers: { [PRIVATE_SECRET_PROVENANCE_HEADER]: PRIVATE_SECRET_PROVENANCE_BUNDLE_V1 }, - body: JSON.stringify(payload), - }) - return { payload, request } -} - -describe('memory write secret provenance', () => { - beforeEach(() => { - resetDbChainMock() - mockReport.mockClear() - mockIsEnforced.mockReturnValue(false) - }) - it('classifies a headerless external write as exact-empty', () => { - const request = new NextRequest('http://localhost/api/memory', { method: 'POST' }) - const result = resolveMemoryWriteSecretProvenance({ - request, - payload: {}, - authType: AuthType.SESSION, - userId: 'user-1', - workspaceId: 'workspace-1', - }) - - expect(result).toEqual({ - success: true, - provenance: { status: 'exact', entries: [] }, - }) - }) - - it('keeps a headerless internal write on the legacy untracked path', () => { - const request = new NextRequest('http://localhost/api/memory', { method: 'POST' }) - const result = resolveMemoryWriteSecretProvenance({ - request, - payload: {}, - authType: AuthType.INTERNAL_JWT, - userId: 'user-1', - workspaceId: 'workspace-1', - }) - - expect(result).toEqual({ success: true }) - }) - - it('persists authenticated unavailable selection lineage as unknown', () => { - const bundle = { - version: 1 as const, - complete: true, - selections: [ - { - key: 'data', - provenance: { - version: 1 as const, - complete: false, - entries: [], - scope: { userId: 'user-1', workspaceId: 'workspace-1' }, - }, - }, - ], - } - const payload = { [PRIVATE_SECRET_PROVENANCE_FIELD]: bundle } - const request = new NextRequest('http://localhost/api/memory', { - method: 'POST', - headers: { [PRIVATE_SECRET_PROVENANCE_HEADER]: PRIVATE_SECRET_PROVENANCE_BUNDLE_V1 }, - body: JSON.stringify(payload), - }) - - const result = resolveMemoryWriteSecretProvenance({ - request, - payload, - authType: AuthType.INTERNAL_JWT, - userId: 'user-1', - workspaceId: 'workspace-1', - }) - - expect(result).toEqual({ success: true, provenance: { status: 'unknown' } }) - }) - - it('persists an authenticated incomplete bundle as unknown', () => { - const payload = { - [PRIVATE_SECRET_PROVENANCE_FIELD]: { - version: 1 as const, - complete: false, - selections: [], - }, - } - const request = new NextRequest('http://localhost/api/memory', { - method: 'POST', - headers: { [PRIVATE_SECRET_PROVENANCE_HEADER]: PRIVATE_SECRET_PROVENANCE_BUNDLE_V1 }, - body: JSON.stringify(payload), - }) - - expect( - resolveMemoryWriteSecretProvenance({ - request, - payload, - authType: AuthType.INTERNAL_JWT, - userId: 'user-1', - workspaceId: 'workspace-1', - }) - ).toEqual({ success: true, provenance: { status: 'unknown' } }) - }) - - it('accepts exact-empty provenance from the workflow owner in the actor workspace', () => { - const { payload, request } = privateMemoryWrite({ - userId: 'workflow-owner', - workspaceId: 'workspace-1', - }) - - expect( - resolveMemoryWriteSecretProvenance({ - request, - payload, - authType: AuthType.INTERNAL_JWT, - userId: 'billing-actor', - workspaceId: 'workspace-1', - }) - ).toEqual({ success: true, provenance: { status: 'exact', entries: [] } }) - }) - - it('preserves the workflow owner as the source of same-workspace provenance', () => { - const { payload, request } = privateMemoryWrite( - { userId: 'workflow-owner', workspaceId: 'workspace-1' }, - [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }] - ) - - expect( - resolveMemoryWriteSecretProvenance({ - request, - payload, - authType: AuthType.INTERNAL_JWT, - userId: 'billing-actor', - workspaceId: 'workspace-1', - }) - ).toEqual({ - success: true, - provenance: { - status: 'exact', - entries: [ - { - name: 'TOKEN', - encryptedValue: 'encrypted-token', - sourceUserId: 'workflow-owner', - sourceWorkspaceId: 'workspace-1', - }, - ], - }, - }) - }) - - it('rejects provenance from another workspace', () => { - const { payload, request } = privateMemoryWrite({ - userId: 'workflow-owner', - workspaceId: 'workspace-2', - }) - const result = resolveMemoryWriteSecretProvenance({ - request, - payload, - authType: AuthType.INTERNAL_JWT, - userId: 'billing-actor', - workspaceId: 'workspace-1', - }) - - expect(result.success).toBe(false) - if (!result.success) expect(result.response.status).toBe(400) - }) - - /** - * A read of this width used to be refused outright on the record count alone. How many memories - * crossed said nothing about whether their provenance could be established, so the read now - * vouches for them and the page size is what keeps the statement count bounded. - */ - it('vouches for a very wide crossing instead of refusing on the record count', async () => { - const request = new NextRequest('http://localhost/api/memory', { - headers: { - [PRIVATE_TOOL_METADATA_REQUEST_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1, - }, - }) - const response = await createMemoryResponse({ - request, - authType: AuthType.INTERNAL_JWT, - userId: 'user-1', - workspaceId: 'workspace-1', - body: { success: true }, - memories: Array.from({ length: 10_001 }, (_, index) => ({ - id: `memory-${index}`, - data: { value: index }, - secretProvenanceVersion: null, - })), - }) - - await expect(response.json()).resolves.toMatchObject({ - [RESOLVED_SECRET_PROVENANCE_FIELD]: { version: 1, complete: true, entries: [] }, - }) - /** Eleven pages of a thousand, not one statement per handful of memories. */ - expect(dbChainMockFns.select.mock.calls.length).toBeLessThanOrEqual(11) - }) - - /** - * A sidecar too large to carry reads as unrecorded, not as a reason to fail the read. The run - * keeps its other provenance and proceeds without this memory's — best effort, with the risk - * recorded — rather than refusing every projection for the rest of the run. - */ - it('treats a sidecar too large to carry as unrecorded rather than failing the read', async () => { - queueTableRows(memorySecretProvenance, [ - { - memoryId: 'memory-1', - contentHash: 'irrelevant-after-budget-check', - status: 'exact', - entries: Array.from({ length: 10_001 }, (_, index) => ({ - name: `SECRET_${index}`, - encryptedValue: `encrypted-${index}`, - })), - updatedAt: new Date(), - }, - ]) - const request = new NextRequest('http://localhost/api/memory', { - headers: { - [PRIVATE_TOOL_METADATA_REQUEST_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1, - }, - }) - const response = await createMemoryResponse({ - request, - authType: AuthType.INTERNAL_JWT, - userId: 'user-1', - workspaceId: 'workspace-1', - body: { success: true, memories: [{ id: 'memory-1', data: 'value' }] }, - memories: [{ id: 'memory-1', data: 'value', secretProvenanceVersion: 1 }], - }) - - await expect(response.json()).resolves.toMatchObject({ - [RESOLVED_SECRET_PROVENANCE_FIELD]: { version: 1, complete: true, entries: [] }, - }) - }) - /** - * One entry for the read, not one per record: the per-record import knows no workspace, so its - * report can only ever be a log line, and passing the workspace down instead would write - * thousands of audit rows for a single event. - */ - it('reports one aggregated entry for a read that proceeded unvouched', async () => { - const request = new NextRequest('http://localhost/api/memory', { - headers: { - [PRIVATE_TOOL_METADATA_REQUEST_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1, - }, - }) - - await createMemoryResponse({ - request, - authType: AuthType.INTERNAL_JWT, - userId: 'user-1', - workspaceId: 'workspace-1', - body: { success: true }, - memories: [ - { id: 'memory-1', data: 'value', secretProvenanceVersion: 1 }, - { id: 'memory-2', data: 'value', secretProvenanceVersion: 1 }, - ], - }) - - expect(mockReport).toHaveBeenCalledTimes(1) - expect(mockReport).toHaveBeenCalledWith( - expect.objectContaining({ - surface: 'memory', - cause: 'durable-provenance-unknown', - affectedCount: 2, - workspaceId: 'workspace-1', - }) - ) - }) - - /** - * Under enforcement the import fails the registry closed rather than proceeding, so there is no - * fail-open read to record. Counting those records anyway would audit something that never - * happened, in the one trail whose whole purpose is to say a read went ahead unvouched. - */ - it('records nothing when the surface is enforced and the read fails closed', async () => { - mockIsEnforced.mockReturnValue(true) - const request = new NextRequest('http://localhost/api/memory', { - headers: { - [PRIVATE_TOOL_METADATA_REQUEST_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1, - }, - }) - - await createMemoryResponse({ - request, - authType: AuthType.INTERNAL_JWT, - userId: 'user-1', - workspaceId: 'workspace-1', - body: { success: true }, - memories: [{ id: 'memory-1', data: 'value', secretProvenanceVersion: 1 }], - }) - - expect(mockReport).not.toHaveBeenCalled() - }) -}) - -describe('memory list query contract', () => { - it('rejects a limit past the page ceiling and keeps the default below it', () => { - expect(memoryListQuerySchema.safeParse({ limit: '2000' }).success).toBe(false) - expect(memoryListQuerySchema.parse({})).toMatchObject({ limit: 50 }) - expect(memoryListQuerySchema.parse({ limit: '1000' })).toMatchObject({ limit: 1000 }) - }) -}) diff --git a/apps/sim/app/api/memory/secret-provenance.ts b/apps/sim/app/api/memory/secret-provenance.ts deleted file mode 100644 index b2439c7be95..00000000000 --- a/apps/sim/app/api/memory/secret-provenance.ts +++ /dev/null @@ -1,184 +0,0 @@ -import { db } from '@sim/db' -import { memorySecretProvenance } from '@sim/db/schema' -import { inArray } from 'drizzle-orm' -import { type NextRequest, NextResponse } from 'next/server' -import { AuthType, type AuthTypeValue } from '@/lib/auth/hybrid' -import { - type DurableSecretProvenance, - durableSecretProvenanceFromPrivateBundle, - EXACT_EMPTY_DURABLE_SECRET_PROVENANCE, - importDurableSecretProvenance, -} from '@/lib/execution/durable-secret-provenance' -import { - isDurableSecretProvenanceEnforced, - reportUnrecordedDurableProvenance, -} from '@/lib/execution/durable-secret-provenance-enforcement' -import { - inspectPrivateSecretProvenanceRequest, - isPrivateSecretProvenanceBundleV1, -} from '@/lib/execution/model-input-provenance' -import { - negotiatePrivateToolMetadataResponse, - RESOLVED_SECRET_PROVENANCE_METADATA_V1, - serializePrivateToolMetadataResponseEnvelope, -} from '@/lib/execution/private-tool-metadata' -import { readBoundMemorySecretProvenance } from '@/lib/memory/secret-provenance' -import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' - -/** - * Ids per sidecar lookup, matching how the rest of the codebase chunks an `inArray`. - * - * It was 8, which only worked because a cap refused any read over ten thousand memories — at that - * width the page loop would otherwise have issued more than a thousand sequential statements. The - * cap is gone because refusing on a record count told us nothing about the data, so the page size - * now has to be the thing that keeps the read bounded. - */ -const PRIVATE_MEMORY_QUERY_CHUNK_SIZE = 1_000 - -interface MemoryCrossing { - id: string - data: unknown - secretProvenanceVersion: number | null -} - -function invalidMemoryProvenanceResponse(): NextResponse { - return NextResponse.json({ error: 'Invalid memory secret provenance' }, { status: 400 }) -} - -/** Resolves write provenance while retaining headerless internal calls as legacy/untracked. */ -export function resolveMemoryWriteSecretProvenance(options: { - request: NextRequest - payload: unknown - authType: AuthTypeValue | undefined - userId: string - workspaceId: string -}): - | { success: true; provenance?: DurableSecretProvenance } - | { success: false; response: NextResponse } { - const { request } = options - const inspection = inspectPrivateSecretProvenanceRequest(request.headers, options.payload) - if (inspection.status === 'unsupported') { - return options.authType === AuthType.INTERNAL_JWT - ? { success: true } - : { success: true, provenance: EXACT_EMPTY_DURABLE_SECRET_PROVENANCE } - } - if (inspection.status !== 'verified' || options.authType !== AuthType.INTERNAL_JWT) { - return { success: false, response: invalidMemoryProvenanceResponse() } - } - if (!isPrivateSecretProvenanceBundleV1(inspection.value)) { - return { success: false, response: invalidMemoryProvenanceResponse() } - } - if (!inspection.value.complete) { - return { success: true, provenance: { status: 'unknown' } } - } - if (inspection.value.selections.length !== 1) { - return { success: false, response: invalidMemoryProvenanceResponse() } - } - const provenance = durableSecretProvenanceFromPrivateBundle(inspection.value, 'data', { - userId: options.userId, - workspaceId: options.workspaceId, - }) - return provenance - ? { success: true, provenance } - : { success: false, response: invalidMemoryProvenanceResponse() } -} - -/** Adds private response provenance only when an authenticated tool caller requests it. */ -export async function createMemoryResponse(options: { - request: NextRequest - authType: AuthTypeValue | undefined - userId: string - workspaceId: string - body: Record - memories: MemoryCrossing[] -}): Promise { - const { request } = options - const negotiation = negotiatePrivateToolMetadataResponse( - request.headers, - RESOLVED_SECRET_PROVENANCE_METADATA_V1, - options.authType === AuthType.INTERNAL_JWT - ) - if (negotiation.status === 'not-requested') return NextResponse.json(options.body) - if (negotiation.status === 'rejected') return invalidMemoryProvenanceResponse() - - const registry = new ResolvedSecretTraceRegistry([], { - userId: options.userId, - workspaceId: options.workspaceId, - }) - /** - * No cap on how many memories may cross, and no separate accounting of what they carry. - * - * There was both: a refusal past ten thousand records, and a running total of every sidecar's - * entries. The first said nothing about the data. The second summed entries *before* they were - * folded, so a page of memories sharing a handful of secrets counted once per mention and could - * refuse a read whose envelope would have held a dozen entries. - * - * Neither is needed, because the registry already bounds the only thing that has a real limit — - * the serialized envelope — as each entry is added, at the granularity it actually dedupes on. - * A second estimate in front of it could only ever be wrong in one of two directions. - */ - const ids = [...new Set(options.memories.map((record) => record.id))] - const memoriesById = new Map() - for (const memory of options.memories) { - const matching = memoriesById.get(memory.id) ?? [] - matching.push(memory) - memoriesById.set(memory.id, matching) - } - /** - * Counted only while the surface is open. Under enforcement the import fails the registry closed - * instead of proceeding, so counting those records would audit a fail-open read that never - * happened — and this entry exists precisely to say a read went ahead unvouched. - */ - const memoryEnforced = isDurableSecretProvenanceEnforced('memory') - let unrecordedMemoryCount = 0 - for (let index = 0; index < ids.length; index += PRIVATE_MEMORY_QUERY_CHUNK_SIZE) { - const pageIds = ids.slice(index, index + PRIVATE_MEMORY_QUERY_CHUNK_SIZE) - const sidecars = await db - .select() - .from(memorySecretProvenance) - .where(inArray(memorySecretProvenance.memoryId, pageIds)) - const sidecarById = new Map(sidecars.map((sidecar) => [sidecar.memoryId, sidecar])) - for (const memoryId of pageIds) { - for (const record of memoriesById.get(memoryId) ?? []) { - const sidecar = sidecarById.get(record.id) - const provenance = readBoundMemorySecretProvenance({ - secretProvenanceVersion: record.secretProvenanceVersion, - data: record.data, - provenanceContentHash: sidecar?.contentHash ?? null, - status: sidecar?.status ?? null, - entries: sidecar?.entries, - }) - if (provenance.status === 'unknown' && !memoryEnforced) unrecordedMemoryCount += 1 - await importDurableSecretProvenance(registry, provenance, record.data, 'memory', { - reportUnrecorded: false, - }) - } - } - } - - /** - * Counted here and reported once, rather than left to the per-record import. - * - * That import reports without a workspace, so the workspace-visible half of the trail never - * reached the people it concerns — the audit entry is skipped when it cannot name one. Passing - * the workspace down instead would have written one row per record, which on a wide read is - * thousands of fire-and-forget inserts for a single event. One read is one thing that happened, - * so it is one entry carrying how many records it covered — the shape the table surface uses. - */ - if (unrecordedMemoryCount > 0) { - reportUnrecordedDurableProvenance({ - surface: 'memory', - cause: 'durable-provenance-unknown', - affectedCount: unrecordedMemoryCount, - workspaceId: options.workspaceId, - actorUserId: options.userId, - }) - } - - const envelope = serializePrivateToolMetadataResponseEnvelope( - options.body, - RESOLVED_SECRET_PROVENANCE_METADATA_V1, - registry.exportCommittedProvenanceForValue(options.body) - ) - return NextResponse.json(envelope.body, { headers: envelope.headers }) -} diff --git a/apps/sim/app/api/organizations/[id]/usage/breakdown/route.ts b/apps/sim/app/api/organizations/[id]/usage/breakdown/route.ts new file mode 100644 index 00000000000..7af435c43da --- /dev/null +++ b/apps/sim/app/api/organizations/[id]/usage/breakdown/route.ts @@ -0,0 +1,39 @@ +import { getOrganizationUsageBreakdownContract } from '@/lib/api/contracts/organization-usage' +import { + defineInternalJsonRoute, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { getOrganizationUsageBreakdown } from '@/lib/billing/application/organization-usage/get-organization-usage-breakdown' +import { organizationUsageOperations } from '@/lib/billing/application/organization-usage/operations' +import { organizationUsageErrorPolicy } from '@/app/api/organizations/[id]/usage/error-policy' + +export const dynamic = 'force-dynamic' + +/** + * One route for all five dimensions: they share a scope, a window, a row shape, and + * authorization, so five routes would be five copies of the same mapping. Separate + * from the summary because three of the five heap-scan the ledger. + */ +export const GET = defineInternalJsonRoute({ + contract: getOrganizationUsageBreakdownContract, + auth: internalSessionAuth, + operation: organizationUsageOperations.readBreakdown, + rateLimit: internalRateLimits.none({ + reason: + 'Authenticated org-admin settings read, gated on enterprise entitlement and billing authority', + }), + errorPolicy: organizationUsageErrorPolicy, + mapInput: ({ params, query }) => ({ + organizationId: params.id, + dimension: query.dimension, + workspaceId: query.workspaceId, + preset: query.preset, + startDate: query.startDate ? new Date(query.startDate) : undefined, + endDate: query.endDate ? new Date(query.endDate) : undefined, + timezone: query.timezone, + limit: query.limit, + }), + useCase: getOrganizationUsageBreakdown, + present: (result) => result, +}) diff --git a/apps/sim/app/api/organizations/[id]/usage/error-policy.ts b/apps/sim/app/api/organizations/[id]/usage/error-policy.ts new file mode 100644 index 00000000000..862fc6dd554 --- /dev/null +++ b/apps/sim/app/api/organizations/[id]/usage/error-policy.ts @@ -0,0 +1,27 @@ +import { + extendInternalErrorPolicy, + internalErrorResponse, + internalOrchestrationErrorPolicy, +} from '@/lib/api/server/routes' +import { + UsageWindowRangeInvertedError, + UsageWindowRangeTooLargeError, +} from '@/lib/billing/core/usage-analytics' + +/** + * The window resolver throws when a custom range exceeds its cap or ends before it +begins, both of which are + * caller-fixable input error rather than a fault. Without this it fell through to + * the orchestration policy's `unhandled` branch and every over-long range answered + * `500 Internal server error`, so the client could neither surface the real reason + * nor tell the two apart. + * + * Shared by all four usage routes so they cannot classify the same throw differently. + */ +export const organizationUsageErrorPolicy = extendInternalErrorPolicy( + internalOrchestrationErrorPolicy, + (error) => + error instanceof UsageWindowRangeTooLargeError || error instanceof UsageWindowRangeInvertedError + ? internalErrorResponse(400, { error: error.message }) + : null +) diff --git a/apps/sim/app/api/organizations/[id]/usage/events/route.ts b/apps/sim/app/api/organizations/[id]/usage/events/route.ts new file mode 100644 index 00000000000..59d5c28de45 --- /dev/null +++ b/apps/sim/app/api/organizations/[id]/usage/events/route.ts @@ -0,0 +1,39 @@ +import { listOrganizationUsageEventsContract } from '@/lib/api/contracts/organization-usage' +import { + defineInternalJsonRoute, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { listOrganizationUsageEvents } from '@/lib/billing/application/organization-usage/list-organization-usage-events' +import { organizationUsageOperations } from '@/lib/billing/application/organization-usage/operations' +import { organizationUsageErrorPolicy } from '@/app/api/organizations/[id]/usage/error-policy' + +export const dynamic = 'force-dynamic' + +/** + * The raw ledger, paged. Separate from the summary because it owns a cursor + * lifecycle and its own staleness — folding it in would re-run the headline + * aggregate on every scroll. + */ +export const GET = defineInternalJsonRoute({ + contract: listOrganizationUsageEventsContract, + auth: internalSessionAuth, + operation: organizationUsageOperations.listEvents, + rateLimit: internalRateLimits.none({ + reason: + 'Authenticated org-admin settings read, gated on enterprise entitlement and billing authority', + }), + errorPolicy: organizationUsageErrorPolicy, + mapInput: ({ params, query }) => ({ + organizationId: params.id, + preset: query.preset, + startDate: query.startDate ? new Date(query.startDate) : undefined, + endDate: query.endDate ? new Date(query.endDate) : undefined, + timezone: query.timezone, + source: query.source, + limit: query.limit, + cursor: query.cursor, + }), + useCase: listOrganizationUsageEvents, + present: (result) => result, +}) diff --git a/apps/sim/app/api/organizations/[id]/usage/export/route.ts b/apps/sim/app/api/organizations/[id]/usage/export/route.ts new file mode 100644 index 00000000000..c4b77332e17 --- /dev/null +++ b/apps/sim/app/api/organizations/[id]/usage/export/route.ts @@ -0,0 +1,107 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { type NextRequest, NextResponse } from 'next/server' +import { exportOrganizationUsageContract } from '@/lib/api/contracts/organization-usage' +import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' +import { getSession } from '@/lib/auth' +import { + exportOrganizationUsageEvents, + type OrganizationUsageExportRow, +} from '@/lib/billing/application/organization-usage/export-organization-usage-events' +import { + UsageWindowRangeInvertedError, + UsageWindowRangeTooLargeError, +} from '@/lib/billing/core/usage-analytics' +import { ForbiddenOperationError } from '@/lib/core/application' +import { formatCsvValue, toCsvRow } from '@/lib/core/utils/csv' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' + +const logger = createLogger('OrganizationUsageExportAPI') + +const CSV_HEADER = toCsvRow(['Date', 'Source', 'Description', 'Workflow', 'Credits']) + +/** + * A bare number, to four decimals, with trailing zeros trimmed. + * + * Not `formatCreditsLabel`: that renders `"0 credits"` for any charge under half a + * credit, so a real cost vanished from the export, and the `"N credits"` text made + * the column unsummable in a spreadsheet — which is most of what a CSV is for. + */ +function formatExportCredits(credits: number): string { + return String(Number(credits.toFixed(4))) +} + +/** `formatCsvValue` neutralizes formula injection — model and workflow names are user-controlled. */ +function toCsvLine(row: OrganizationUsageExportRow): string { + return toCsvRow([ + formatCsvValue(row.createdAt), + formatCsvValue(row.source), + formatCsvValue(row.description), + formatCsvValue(row.workflowName ?? ''), + formatCsvValue(formatExportCredits(row.credits)), + ]) +} + +/** + * A raw handler rather than a JSON builder: the body is `text/csv`, and the response + * carries `X-Export-Truncated` so the client can tell the user their range was capped + * rather than silently handing them a partial file. + */ +export const GET = withRouteHandler(async (request: NextRequest, context) => { + try { + const session = await getSession() + const sessionId = session?.session?.id + if (!session?.user?.id || !sessionId) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } + + const parsed = await parseRequest(exportOrganizationUsageContract, request, context, { + validationErrorResponse: (error) => + NextResponse.json( + { error: getValidationErrorMessage(error, 'Invalid query parameters') }, + { status: 400 } + ), + }) + if (!parsed.success) return parsed.response + + const { query, params } = parsed.data + const result = await exportOrganizationUsageEvents.execute({ + principal: { kind: 'session', userId: session.user.id, sessionId }, + input: { + organizationId: params.id, + preset: query.preset, + startDate: query.startDate ? new Date(query.startDate) : undefined, + endDate: query.endDate ? new Date(query.endDate) : undefined, + timezone: query.timezone, + source: query.source, + }, + }) + + const csv = [CSV_HEADER, ...result.rows.map(toCsvLine)].join('\n') + return new NextResponse(csv, { + headers: { + 'Content-Type': 'text/csv; charset=utf-8', + // Every member's spend for one organization, behind session auth. Without + // this a browser or shared intermediary may serve it again after the + // viewer's access to that organization has been revoked. + 'Cache-Control': 'no-store', + 'Content-Disposition': `attachment; filename="organization-usage-${params.id}.csv"`, + ...(result.truncated ? { 'X-Export-Truncated': '1' } : {}), + }, + }) + } catch (error) { + if (error instanceof ForbiddenOperationError) { + return NextResponse.json({ error: error.message }, { status: 403 }) + } + // A range over the cap, or inverted, is the caller's input — the same + // classification the three JSON routes make through `organizationUsageErrorPolicy`. + if ( + error instanceof UsageWindowRangeTooLargeError || + error instanceof UsageWindowRangeInvertedError + ) { + return NextResponse.json({ error: error.message }, { status: 400 }) + } + logger.error('Failed to export organization usage', { error: getErrorMessage(error) }) + return NextResponse.json({ error: 'Failed to export usage' }, { status: 500 }) + } +}) diff --git a/apps/sim/app/api/organizations/[id]/usage/summary/route.ts b/apps/sim/app/api/organizations/[id]/usage/summary/route.ts new file mode 100644 index 00000000000..ba7abb7372a --- /dev/null +++ b/apps/sim/app/api/organizations/[id]/usage/summary/route.ts @@ -0,0 +1,36 @@ +import { getOrganizationUsageSummaryContract } from '@/lib/api/contracts/organization-usage' +import { + defineInternalJsonRoute, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { getOrganizationUsageSummary } from '@/lib/billing/application/organization-usage/get-organization-usage-summary' +import { organizationUsageOperations } from '@/lib/billing/application/organization-usage/operations' +import { organizationUsageErrorPolicy } from '@/app/api/organizations/[id]/usage/error-policy' + +export const dynamic = 'force-dynamic' + +/** + * Everything above the fold in one round trip. Kept separate from the breakdown + * route because every read here is index-covered, and folding in a dimension that + * heap-scans would put that cost on first paint. + */ +export const GET = defineInternalJsonRoute({ + contract: getOrganizationUsageSummaryContract, + auth: internalSessionAuth, + operation: organizationUsageOperations.readSummary, + rateLimit: internalRateLimits.none({ + reason: + 'Authenticated org-admin settings read, gated on enterprise entitlement and billing authority', + }), + errorPolicy: organizationUsageErrorPolicy, + mapInput: ({ params, query }) => ({ + organizationId: params.id, + preset: query.preset, + startDate: query.startDate ? new Date(query.startDate) : undefined, + endDate: query.endDate ? new Date(query.endDate) : undefined, + timezone: query.timezone, + }), + useCase: getOrganizationUsageSummary, + present: (result) => result, +}) diff --git a/apps/sim/app/api/providers/openrouter/embeddings/models/route.test.ts b/apps/sim/app/api/providers/openrouter/embeddings/models/route.test.ts index bc27283d580..6807800c062 100644 --- a/apps/sim/app/api/providers/openrouter/embeddings/models/route.test.ts +++ b/apps/sim/app/api/providers/openrouter/embeddings/models/route.test.ts @@ -32,18 +32,15 @@ describe('GET /api/providers/openrouter/embeddings/models', () => { }) it('returns every unique embedding model with the OpenRouter prefix', async () => { - mockFetch.mockResolvedValue({ - ok: true, - status: 200, - statusText: 'OK', - json: async () => ({ + mockFetch.mockResolvedValue( + Response.json({ data: [ { id: 'qwen/qwen3-embedding-8b', context_length: 32768 }, { id: 'openai/text-embedding-3-small', context_length: 8192 }, { id: 'qwen/qwen3-embedding-8b', context_length: 32768 }, ], - }), - }) + }) + ) const response = await GET(request(), undefined as never) @@ -68,11 +65,9 @@ describe('GET /api/providers/openrouter/embeddings/models', () => { }) it('fails fast when OpenRouter rejects the model-list request', async () => { - mockFetch.mockResolvedValue({ - ok: false, - status: 503, - statusText: 'Service Unavailable', - }) + mockFetch.mockResolvedValue( + new Response(null, { status: 503, statusText: 'Service Unavailable' }) + ) const response = await GET(request(), undefined as never) diff --git a/apps/sim/app/api/providers/route.test.ts b/apps/sim/app/api/providers/route.test.ts deleted file mode 100644 index 2113b6ac213..00000000000 --- a/apps/sim/app/api/providers/route.test.ts +++ /dev/null @@ -1,408 +0,0 @@ -/** - * @vitest-environment node - */ -import { createMockRequest, hybridAuthMockFns } from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' -import { - PRIVATE_MODEL_INPUT_STATE_HEADER, - PROJECTED_MODEL_INPUT_PATHS_V1, -} from '@/lib/execution/model-input-provenance' - -const { - mockExecuteProviderRequest, - mockRequireBillingAttributionHeader, - mockCheckWorkspaceAccess, - mockAuthorizeCredentialUse, - mockPrepareCopilotEnvironmentContext, - mockImportProvenance, - mockRegistryIsComplete, - mockProjectResolvedSecretModelContent, -} = vi.hoisted(() => ({ - mockExecuteProviderRequest: vi.fn(), - mockRequireBillingAttributionHeader: vi.fn(), - mockCheckWorkspaceAccess: vi.fn(), - mockAuthorizeCredentialUse: vi.fn(), - mockPrepareCopilotEnvironmentContext: vi.fn(), - mockImportProvenance: vi.fn(), - mockRegistryIsComplete: vi.fn(), - mockProjectResolvedSecretModelContent: vi.fn(), -})) - -vi.mock('@/providers', () => ({ - executeProviderRequest: mockExecuteProviderRequest, -})) - -vi.mock('@/lib/billing/core/billing-attribution', () => ({ - BILLING_ATTRIBUTION_HEADER: 'x-sim-billing-attribution', - requireBillingAttributionHeader: mockRequireBillingAttributionHeader, -})) - -vi.mock('@/lib/workspaces/permissions/utils', () => ({ - checkWorkspaceAccess: mockCheckWorkspaceAccess, -})) - -vi.mock('@/lib/auth/credential-access', () => ({ - authorizeCredentialUse: mockAuthorizeCredentialUse, -})) - -vi.mock('@/lib/copilot/environment-context', () => ({ - prepareCopilotEnvironmentContext: mockPrepareCopilotEnvironmentContext, -})) - -vi.mock('@/executor/utils/resolved-secret-content-projection', () => ({ - projectResolvedSecretModelContent: mockProjectResolvedSecretModelContent, -})) - -vi.mock('@/lib/oauth/credential-service', () => ({ - getServiceAccountToken: vi.fn(), - refreshTokenIfNeeded: vi.fn(), - resolveOAuthAccountId: vi.fn(), -})) - -vi.mock('@/ee/access-control/utils/permission-check', () => ({ - assertPermissionsAllowed: vi.fn(), - IntegrationNotAllowedError: class IntegrationNotAllowedError extends Error {}, - ModelNotAllowedError: class ModelNotAllowedError extends Error {}, - ProviderNotAllowedError: class ProviderNotAllowedError extends Error {}, -})) - -import { POST } from '@/app/api/providers/route' - -const BILLING_ATTRIBUTION = { - actorUserId: 'user-1', - workspaceId: 'ws-1', - organizationId: 'org-1', - billedAccountUserId: 'owner-1', - billingEntity: { type: 'organization', id: 'org-1' }, - billingPeriod: { - start: '2026-07-01T00:00:00.000Z', - end: '2026-08-01T00:00:00.000Z', - }, - payerSubscription: null, -} - -function createProviderRequest( - body: Record, - headers: Record = {} -) { - return createMockRequest( - 'POST', - { - ...body, - __resolvedSecretTraceProvenance: { version: 1, complete: true, entries: [] }, - }, - { - 'x-sim-private-model-input-provenance': 'resolved-secret-provenance-v1', - [PRIVATE_MODEL_INPUT_STATE_HEADER]: PROJECTED_MODEL_INPUT_PATHS_V1, - ...headers, - } - ) -} - -describe('POST /api/providers', () => { - beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - mockCheckWorkspaceAccess.mockResolvedValue({ hasAccess: true }) - mockRequireBillingAttributionHeader.mockReturnValue(BILLING_ATTRIBUTION) - mockExecuteProviderRequest.mockResolvedValue({ - content: 'hello', - model: 'gpt-4o', - tokens: { input: 1, output: 1, total: 2 }, - }) - mockImportProvenance.mockResolvedValue(true) - mockRegistryIsComplete.mockReturnValue(true) - mockProjectResolvedSecretModelContent.mockImplementation((value) => ({ - safe: true, - value, - })) - mockPrepareCopilotEnvironmentContext.mockResolvedValue({ - resolvedSecretTraceRegistry: { - importProvenance: mockImportProvenance, - isComplete: mockRegistryIsComplete, - }, - }) - }) - - it('validates the attribution header and forwards it to executeProviderRequest', async () => { - const res = await POST( - createProviderRequest( - { provider: 'openai', model: 'gpt-4o', workspaceId: 'ws-1' }, - { 'x-sim-billing-attribution': 'encoded-attribution' } - ) - ) - - expect(res.status).toBe(200) - expect(mockRequireBillingAttributionHeader).toHaveBeenCalledWith(expect.anything(), { - actorUserId: 'user-1', - workspaceId: 'ws-1', - }) - expect(mockExecuteProviderRequest).toHaveBeenCalledWith( - 'openai', - expect.objectContaining({ billingAttribution: BILLING_ATTRIBUTION }), - expect.objectContaining({ - resolvedSecretTraceRegistry: expect.anything(), - }) - ) - }) - - it('executes without attribution when the header is absent', async () => { - const res = await POST( - createProviderRequest({ provider: 'openai', model: 'gpt-4o', workspaceId: 'ws-1' }) - ) - - expect(res.status).toBe(200) - expect(mockRequireBillingAttributionHeader).not.toHaveBeenCalled() - expect(mockExecuteProviderRequest).toHaveBeenCalledWith( - 'openai', - expect.objectContaining({ billingAttribution: undefined }), - expect.objectContaining({ - resolvedSecretTraceRegistry: expect.anything(), - }) - ) - }) - - it('passes caller environment values and model-egress context to the provider boundary', async () => { - const res = await POST( - createProviderRequest({ - provider: 'openai', - model: 'gpt-4o', - workspaceId: 'ws-1', - environmentVariables: { RUNTIME_TOKEN: 'runtime-secret' }, - }) - ) - - expect(res.status).toBe(200) - expect(mockPrepareCopilotEnvironmentContext).toHaveBeenCalledWith('user-1', 'ws-1') - expect(mockExecuteProviderRequest).toHaveBeenCalledWith( - 'openai', - expect.objectContaining({ - environmentVariables: { RUNTIME_TOKEN: 'runtime-secret' }, - }), - expect.objectContaining({ resolvedSecretTraceRegistry: expect.anything() }) - ) - }) - - it('imports authenticated active provenance', async () => { - const provenance = { - version: 1, - complete: true, - entries: [{ encryptedValue: 'encrypted-secret', name: 'TOKEN' }], - scope: { userId: 'user-1', workspaceId: 'ws-1' }, - } - const res = await POST( - createMockRequest( - 'POST', - { - provider: 'openai', - model: 'gpt-4o', - workspaceId: 'ws-1', - __resolvedSecretTraceProvenance: provenance, - }, - { 'x-sim-private-model-input-provenance': 'resolved-secret-provenance-v1' } - ) - ) - - expect(res.status).toBe(200) - expect(mockImportProvenance).toHaveBeenCalledWith(provenance, { - trusted: true, - origin: 'providersRoute.requestProvenance', - }) - }) - - it('projects legacy private prompt provenance on the provider-facing copy', async () => { - mockProjectResolvedSecretModelContent.mockReturnValue({ - safe: true, - value: { - systemPrompt: 'Use {{TOKEN}} safely', - context: '[{"role":"user","content":"{{TOKEN}}"}]', - }, - }) - - const res = await POST( - createMockRequest( - 'POST', - { - provider: 'openai', - model: 'gpt-4o', - workspaceId: 'ws-1', - systemPrompt: 'Use secret-value safely', - context: '[{"role":"user","content":"secret-value"}]', - __resolvedSecretTraceProvenance: { version: 1, complete: true, entries: [] }, - }, - { 'x-sim-private-model-input-provenance': 'resolved-secret-provenance-v1' } - ) - ) - - expect(res.status).toBe(200) - expect(mockExecuteProviderRequest).toHaveBeenCalledWith( - 'openai', - expect.objectContaining({ - systemPrompt: 'Use {{TOKEN}} safely', - context: '[{"role":"user","content":"{{TOKEN}}"}]', - }), - expect.anything() - ) - }) - - it('does not re-project an explicitly projected private request', async () => { - mockProjectResolvedSecretModelContent.mockReturnValue({ - safe: true, - value: { systemPrompt: 'Bo{{TOKEN}}', context: undefined }, - }) - - const res = await POST( - createProviderRequest({ - provider: 'openai', - model: 'gpt-4o', - workspaceId: 'ws-1', - systemPrompt: 'Box', - }) - ) - - expect(res.status).toBe(200) - expect(mockProjectResolvedSecretModelContent).not.toHaveBeenCalled() - expect(mockExecuteProviderRequest).toHaveBeenCalledWith( - 'openai', - expect.objectContaining({ systemPrompt: 'Box' }), - expect.anything() - ) - }) - - it('rejects a projected marker without a private provenance envelope', async () => { - const res = await POST( - createMockRequest( - 'POST', - { provider: 'openai', model: 'gpt-4o', workspaceId: 'ws-1' }, - { [PRIVATE_MODEL_INPUT_STATE_HEADER]: PROJECTED_MODEL_INPUT_PATHS_V1 } - ) - ) - - expect(res.status).toBe(400) - expect(mockExecuteProviderRequest).not.toHaveBeenCalled() - }) - - it('rejects an unknown private projection marker', async () => { - const res = await POST( - createProviderRequest( - { provider: 'openai', model: 'gpt-4o', workspaceId: 'ws-1' }, - { [PRIVATE_MODEL_INPUT_STATE_HEADER]: 'unknown-projection' } - ) - ) - - expect(res.status).toBe(400) - expect(mockExecuteProviderRequest).not.toHaveBeenCalled() - }) - - it('rejects a partial private provenance envelope', async () => { - const res = await POST( - createMockRequest( - 'POST', - { provider: 'openai', model: 'gpt-4o', workspaceId: 'ws-1' }, - { 'x-sim-private-model-input-provenance': 'resolved-secret-provenance-v1' } - ) - ) - - expect(res.status).toBe(400) - expect(mockExecuteProviderRequest).not.toHaveBeenCalled() - }) - - it('preserves legacy internal requests without the private provenance envelope', async () => { - const res = await POST( - createMockRequest('POST', { - provider: 'openai', - model: 'gpt-4o', - workspaceId: 'ws-1', - }) - ) - - expect(res.status).toBe(200) - expect(mockImportProvenance).not.toHaveBeenCalled() - expect(mockExecuteProviderRequest).toHaveBeenCalled() - }) - - it('omits provisional stream output from the execution header', async () => { - mockExecuteProviderRequest.mockResolvedValue({ - streamFormat: 'agent-events-v1', - stream: new ReadableStream({ - start(controller) { - controller.enqueue({ type: 'text_delta', text: 'hello', turn: 'final' }) - controller.close() - }, - }), - execution: { - success: true, - output: { - content: '', - model: 'gpt-4o', - tokens: { input: 0, output: 0, total: 0 }, - cost: { input: 0, output: 0, total: 0 }, - providerTiming: { - startTime: '2026-07-01T00:00:00.000Z', - endTime: '2026-07-01T00:00:00.000Z', - duration: 0, - }, - }, - logs: [], - metadata: { - startTime: '2026-07-01T00:00:00.000Z', - endTime: '2026-07-01T00:00:00.000Z', - duration: 0, - }, - isStreaming: true, - }, - }) - - const res = await POST( - createProviderRequest({ - provider: 'openai', - model: 'gpt-4o', - workspaceId: 'ws-1', - stream: true, - }) - ) - - expect(res.status).toBe(200) - expect(await res.text()).toBe('hello') - const executionHeader = JSON.parse(res.headers.get('X-Execution-Data') ?? '{}') - expect(executionHeader.output).toEqual({ model: 'gpt-4o' }) - expect(executionHeader.metadata).toEqual({ startTime: '2026-07-01T00:00:00.000Z' }) - }) - - it('rejects an attribution header when the body has no workspaceId to validate against', async () => { - const res = await POST( - createProviderRequest( - { provider: 'openai', model: 'gpt-4o' }, - { 'x-sim-billing-attribution': 'encoded-attribution' } - ) - ) - - expect(res.status).toBe(400) - expect(mockRequireBillingAttributionHeader).not.toHaveBeenCalled() - expect(mockExecuteProviderRequest).not.toHaveBeenCalled() - }) - - it('rejects with 400 when the attribution header does not match the authenticated scope', async () => { - mockRequireBillingAttributionHeader.mockImplementation(() => { - throw new Error('Billing attribution header does not match the authenticated request scope') - }) - - const res = await POST( - createProviderRequest( - { provider: 'openai', model: 'gpt-4o', workspaceId: 'ws-1' }, - { 'x-sim-billing-attribution': 'encoded-attribution' } - ) - ) - - expect(res.status).toBe(400) - const body = await res.json() - expect(body.error).toBe( - 'Billing attribution header does not match the authenticated request scope' - ) - expect(mockExecuteProviderRequest).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/app/api/providers/route.ts b/apps/sim/app/api/providers/route.ts deleted file mode 100644 index f61c7f58d06..00000000000 --- a/apps/sim/app/api/providers/route.ts +++ /dev/null @@ -1,548 +0,0 @@ -import { db } from '@sim/db' -import { account } from '@sim/db/schema' -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { isPlainRecord } from '@sim/utils/object' -import { eq } from 'drizzle-orm' -import { type NextRequest, NextResponse } from 'next/server' -import { executeProviderContract } from '@/lib/api/contracts/providers' -import { parseRequest } from '@/lib/api/server' -import { authorizeCredentialUse } from '@/lib/auth/credential-access' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - BILLING_ATTRIBUTION_HEADER, - type BillingAttributionSnapshot, - requireBillingAttributionHeader, -} from '@/lib/billing/core/billing-attribution' -import { prepareCopilotEnvironmentContext } from '@/lib/copilot/environment-context' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - inspectModelInputProjectionState, - inspectModelInputProvenanceRequest, -} from '@/lib/execution/model-input-provenance' -import { - getServiceAccountToken, - refreshTokenIfNeeded, - resolveOAuthAccountId, -} from '@/lib/oauth/credential-service' -import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils' -import { - assertPermissionsAllowed, - IntegrationNotAllowedError, - ModelNotAllowedError, - ProviderNotAllowedError, -} from '@/ee/access-control/utils/permission-check' -import type { StreamingExecution } from '@/executor/types' -import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection' -import { executeProviderRequest } from '@/providers' -import { projectStreamingExecutionToByteStream } from '@/providers/stream-pump' -import type { ProviderRequest } from '@/providers/types' - -const logger = createLogger('ProvidersAPI') - -export const dynamic = 'force-dynamic' - -/** - * Server-side proxy for provider requests - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - const startTime = Date.now() - - try { - const auth = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - logger.info(`[${requestId}] Provider API request started`, { - timestamp: new Date().toISOString(), - userAgent: request.headers.get('User-Agent'), - contentType: request.headers.get('Content-Type'), - }) - - const validation = await parseRequest( - executeProviderContract, - request, - {}, - { - validationErrorResponse: () => - NextResponse.json({ error: 'Invalid request body' }, { status: 400 }), - invalidJsonResponse: () => - NextResponse.json({ error: 'Invalid request body' }, { status: 400 }), - } - ) - if (!validation.success) return validation.response - - const body = validation.data.body - const { - provider, - model, - systemPrompt, - context, - tools, - temperature, - maxTokens, - apiKey, - azureEndpoint, - azureApiVersion, - vertexProject, - vertexLocation, - vertexCredential, - bedrockAccessKeyId, - bedrockSecretKey, - bedrockRegion, - responseFormat, - workflowId, - workspaceId, - stream, - messages, - environmentVariables, - workflowVariables, - blockData, - blockNameMapping, - reasoningEffort, - verbosity, - } = body - - logger.info(`[${requestId}] Provider request details`, { - provider, - model, - hasSystemPrompt: !!systemPrompt, - hasContext: !!context, - hasTools: !!tools?.length, - toolCount: tools?.length || 0, - hasApiKey: !!apiKey, - hasAzureEndpoint: !!azureEndpoint, - hasAzureApiVersion: !!azureApiVersion, - hasVertexProject: !!vertexProject, - hasVertexLocation: !!vertexLocation, - hasVertexCredential: !!vertexCredential, - hasBedrockAccessKeyId: !!bedrockAccessKeyId, - hasBedrockSecretKey: !!bedrockSecretKey, - hasBedrockRegion: !!bedrockRegion, - hasResponseFormat: !!responseFormat, - workflowId, - stream: !!stream, - hasMessages: !!messages?.length, - messageCount: messages?.length || 0, - hasEnvironmentVariables: - !!environmentVariables && Object.keys(environmentVariables).length > 0, - hasWorkflowVariables: !!workflowVariables && Object.keys(workflowVariables).length > 0, - reasoningEffort, - verbosity, - }) - - if (workspaceId) { - const workspaceAccess = await checkWorkspaceAccess(workspaceId, auth.userId) - if (!workspaceAccess.hasAccess) { - return NextResponse.json({ error: 'Forbidden' }, { status: 403 }) - } - - try { - await assertPermissionsAllowed({ - userId: auth.userId, - workspaceId, - model, - }) - } catch (err) { - if ( - err instanceof ProviderNotAllowedError || - err instanceof ModelNotAllowedError || - err instanceof IntegrationNotAllowedError - ) { - return NextResponse.json({ error: err.message }, { status: 403 }) - } - throw err - } - } - - let finalApiKey: string | undefined = apiKey - try { - if (provider === 'vertex' && vertexCredential) { - const vertexCredAccess = await authorizeCredentialUse(request, { - credentialId: vertexCredential, - workflowId: workflowId || undefined, - requireWorkflowIdForInternal: false, - }) - if (!vertexCredAccess.ok) { - logger.warn(`[${requestId}] Vertex credential access denied`, { - error: vertexCredAccess.error, - credentialId: vertexCredential, - }) - return NextResponse.json( - { error: vertexCredAccess.error || 'Unauthorized' }, - { status: 401 } - ) - } - finalApiKey = await resolveVertexCredential(requestId, vertexCredential) - } - } catch (error) { - logger.error(`[${requestId}] Failed to resolve Vertex credential:`, { - provider, - model, - error: toError(error).message, - hasVertexCredential: !!vertexCredential, - }) - return NextResponse.json( - { error: getErrorMessage(error, 'Credential error') }, - { status: 400 } - ) - } - - /** - * Nested tool calls made by the LLM (e.g. knowledge search) hit internal - * routes that require the upstream billing decision. This route is - * internal-JWT-only, so the caller's attribution arrives as a header; - * validate it against the authenticated scope and thread it through. A - * header the route cannot validate is a caller protocol error (400), never - * silently dropped. - */ - let billingAttribution: BillingAttributionSnapshot | undefined - if (request.headers.get(BILLING_ATTRIBUTION_HEADER)) { - if (!workspaceId) { - return NextResponse.json( - { error: 'workspaceId is required when billing attribution is supplied' }, - { status: 400 } - ) - } - try { - billingAttribution = requireBillingAttributionHeader(request.headers, { - actorUserId: auth.userId, - workspaceId, - }) - } catch (error) { - return NextResponse.json( - { error: getErrorMessage(error, 'Invalid billing attribution header') }, - { status: 400 } - ) - } - } - - logger.info(`[${requestId}] Executing provider request`, { - provider, - model, - workflowId, - hasApiKey: !!finalApiKey, - hasBillingAttribution: !!billingAttribution, - }) - - let providerRequest: ProviderRequest = { - model, - systemPrompt, - context, - tools, - temperature, - maxTokens, - apiKey: finalApiKey, - azureEndpoint, - azureApiVersion, - vertexProject, - vertexLocation, - bedrockAccessKeyId, - bedrockSecretKey, - bedrockRegion, - responseFormat, - workflowId, - workspaceId, - userId: auth.userId, - stream, - messages, - environmentVariables, - workflowVariables, - blockData, - blockNameMapping, - billingAttribution, - reasoningEffort, - verbosity, - } - const provenanceInspection = inspectModelInputProvenanceRequest(request.headers, body) - const projectionState = inspectModelInputProjectionState(request.headers) - if ( - provenanceInspection.status === 'invalid' || - projectionState === 'invalid' || - (projectionState === 'projected' && provenanceInspection.status !== 'verified') - ) { - return NextResponse.json({ error: 'Invalid model input provenance' }, { status: 400 }) - } - - const providerRuntimeContext = await prepareCopilotEnvironmentContext(auth.userId, workspaceId) - if (provenanceInspection.status === 'verified') { - const provenanceReady = - await providerRuntimeContext.resolvedSecretTraceRegistry.importProvenance( - provenanceInspection.value, - { trusted: true, origin: 'providersRoute.requestProvenance' } - ) - if (!provenanceReady || !providerRuntimeContext.resolvedSecretTraceRegistry.isComplete()) { - return NextResponse.json( - { error: 'Model input provenance is unavailable' }, - { status: 400 } - ) - } - - if (projectionState === 'unmarked') { - const projection = projectResolvedSecretModelContent( - { systemPrompt: providerRequest.systemPrompt, context: providerRequest.context }, - providerRuntimeContext.resolvedSecretTraceRegistry - ) - if (!projection.safe || !isPlainRecord(projection.value)) { - return NextResponse.json( - { error: 'Model input provenance is unavailable' }, - { status: 400 } - ) - } - const projectedSystemPrompt = projection.value.systemPrompt - const projectedContext = projection.value.context - if ( - (projectedSystemPrompt !== undefined && typeof projectedSystemPrompt !== 'string') || - (projectedContext !== undefined && typeof projectedContext !== 'string') - ) { - return NextResponse.json({ error: 'Invalid model input provenance' }, { status: 400 }) - } - providerRequest = { - ...providerRequest, - systemPrompt: projectedSystemPrompt, - context: projectedContext, - } - } - } - - const response = await executeProviderRequest(provider, providerRequest, providerRuntimeContext) - - const executionTime = Date.now() - startTime - logger.info(`[${requestId}] Provider request completed successfully`, { - provider, - model, - workflowId, - executionTime, - responseType: - response instanceof ReadableStream - ? 'stream' - : response && typeof response === 'object' && 'stream' in response - ? 'streaming-execution' - : 'json', - }) - - // Check if the response is a StreamingExecution - if ( - response && - typeof response === 'object' && - 'stream' in response && - 'execution' in response - ) { - const streamingExec = response as StreamingExecution - logger.info(`[${requestId}] Received StreamingExecution from provider`) - - const executionData = streamingExec.execution - const byteStream = projectStreamingExecutionToByteStream(streamingExec) - - let executionDataHeader - try { - const outputContent = executionData.output?.content - const outputTokens = executionData.output?.tokens - const hasSettledOutput = - Boolean(outputContent) || - Boolean(outputTokens?.total) || - Boolean(executionData.output?.toolCalls) - const safeExecutionData = { - success: executionData.success, - output: { - model: executionData.output?.model, - ...(hasSettledOutput - ? { - content: String(outputContent ?? '').replace(/[\u0080-\uFFFF]/g, ''), - tokens: outputTokens, - toolCalls: executionData.output?.toolCalls - ? sanitizeToolCalls(executionData.output.toolCalls) - : undefined, - providerTiming: executionData.output?.providerTiming, - cost: executionData.output?.cost, - } - : {}), - }, - error: executionData.error, - logs: [], - metadata: { - startTime: executionData.metadata?.startTime, - ...(hasSettledOutput - ? { - endTime: executionData.metadata?.endTime, - duration: executionData.metadata?.duration, - } - : {}), - }, - isStreaming: true, - blockId: executionData.logs?.[0]?.blockId, - blockName: executionData.logs?.[0]?.blockName, - blockType: executionData.logs?.[0]?.blockType, - } - executionDataHeader = JSON.stringify(safeExecutionData) - } catch (error) { - logger.error(`[${requestId}] Failed to serialize execution data:`, error) - executionDataHeader = JSON.stringify({ - success: executionData.success, - error: 'Failed to serialize full execution data', - }) - } - - return new Response(byteStream, { - headers: { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - 'X-Execution-Data': executionDataHeader, - }, - }) - } - - // Check if the response is a ReadableStream for streaming - if (response instanceof ReadableStream) { - logger.info(`[${requestId}] Streaming response from provider`) - return new Response(response, { - headers: { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - }, - }) - } - - // Return regular JSON response for non-streaming - return NextResponse.json(response) - } catch (error) { - const executionTime = Date.now() - startTime - logger.error(`[${requestId}] Provider request failed:`, { - error: toError(error).message, - errorName: error instanceof Error ? error.name : 'Unknown', - errorStack: error instanceof Error ? error.stack : undefined, - executionTime, - timestamp: new Date().toISOString(), - }) - - return NextResponse.json({ error: toError(error).message }, { status: 500 }) - } -}) - -/** - * Helper function to sanitize tool calls to remove Unicode characters - */ -function sanitizeToolCalls(toolCalls: any) { - // If it's an object with a list property, sanitize the list - if (toolCalls && typeof toolCalls === 'object' && Array.isArray(toolCalls.list)) { - return { - ...toolCalls, - list: toolCalls.list.map(sanitizeToolCall), - } - } - - // If it's an array, sanitize each item - if (Array.isArray(toolCalls)) { - return toolCalls.map(sanitizeToolCall) - } - - return toolCalls -} - -/** - * Sanitize a single tool call to remove Unicode characters - */ -function sanitizeToolCall(toolCall: any) { - if (!toolCall || typeof toolCall !== 'object') return toolCall - - // Create a sanitized copy - const sanitized = { ...toolCall } - - // Sanitize any string fields that might contain Unicode - if (typeof sanitized.name === 'string') { - sanitized.name = sanitized.name.replace(/[\u0080-\uFFFF]/g, '') - } - - // Sanitize input/arguments - if (sanitized.input && typeof sanitized.input === 'object') { - sanitized.input = sanitizeObject(sanitized.input) - } - - if (sanitized.arguments && typeof sanitized.arguments === 'object') { - sanitized.arguments = sanitizeObject(sanitized.arguments) - } - - // Sanitize output/result - if (sanitized.output && typeof sanitized.output === 'object') { - sanitized.output = sanitizeObject(sanitized.output) - } - - if (sanitized.result && typeof sanitized.result === 'object') { - sanitized.result = sanitizeObject(sanitized.result) - } - - // Sanitize error message - if (typeof sanitized.error === 'string') { - sanitized.error = sanitized.error.replace(/[\u0080-\uFFFF]/g, '') - } - - return sanitized -} - -/** - * Recursively sanitize an object to remove Unicode characters from strings - */ -function sanitizeObject(obj: any): any { - if (!obj || typeof obj !== 'object') return obj - - // Handle arrays - if (Array.isArray(obj)) { - return obj.map((item) => sanitizeObject(item)) - } - - // Handle objects - const result: any = {} - for (const [key, value] of Object.entries(obj)) { - if (typeof value === 'string') { - result[key] = value.replace(/[\u0080-\uFFFF]/g, '') - } else if (typeof value === 'object' && value !== null) { - result[key] = sanitizeObject(value) - } else { - result[key] = value - } - } - - return result -} - -/** - * Resolves a Vertex AI OAuth credential to an access token - */ -async function resolveVertexCredential(requestId: string, credentialId: string): Promise { - logger.info(`[${requestId}] Resolving Vertex AI credential: ${credentialId}`) - - const resolved = await resolveOAuthAccountId(credentialId) - if (!resolved) { - throw new Error(`Vertex AI credential not found: ${credentialId}`) - } - - if (resolved.credentialType === 'service_account' && resolved.credentialId) { - const accessToken = await getServiceAccountToken(resolved.credentialId, [ - 'https://www.googleapis.com/auth/cloud-platform', - ]) - logger.info(`[${requestId}] Successfully resolved Vertex AI service account credential`) - return accessToken - } - - const credential = await db.query.account.findFirst({ - where: eq(account.id, resolved.accountId), - }) - - if (!credential) { - throw new Error(`Vertex AI credential not found: ${credentialId}`) - } - - const { accessToken } = await refreshTokenIfNeeded(requestId, credential, resolved.accountId) - - if (!accessToken) { - throw new Error('Failed to get Vertex AI access token') - } - - logger.info(`[${requestId}] Successfully resolved Vertex AI credential`) - return accessToken -} diff --git a/apps/sim/app/api/table/[tableId]/query/route.test.ts b/apps/sim/app/api/table/[tableId]/query/route.test.ts index bc758cfea01..cd372995ef8 100644 --- a/apps/sim/app/api/table/[tableId]/query/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/query/route.test.ts @@ -1,316 +1,159 @@ /** * @vitest-environment node - * - * v2 query route: predicate parsing, unconditional name→id translation - * (session auth included — the string grammar is name-keyed for every caller), - * cursor validation, and the response envelope. */ -import { createTableDefinition, hybridAuthMockFns } from '@sim/testing' + import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockCheckAccess, mockQueryRows, mockGate } = vi.hoisted(() => ({ - mockCheckAccess: vi.fn(), - mockQueryRows: vi.fn(), - mockGate: vi.fn(), -})) - -vi.mock('@/app/api/table/utils', async () => { - const { NextResponse } = await import('next/server') +const { mocks, MockTableV2FeatureDisabledError } = vi.hoisted(() => { + class MockTableV2FeatureDisabledError extends Error { + constructor() { + super('The v2 table query API is not enabled for this workspace') + } + } return { - checkAccess: mockCheckAccess, - accessError: (result: { status: number }) => - NextResponse.json({ error: 'Access denied' }, { status: result.status }), - tablesV2GateError: mockGate, + MockTableV2FeatureDisabledError, + mocks: { authenticate: vi.fn(), queryRows: vi.fn() }, } }) -vi.mock('@/lib/table', async () => { - // row-wire pulls the column-keys helpers through this barrel. - const columnKeys = await import('@/lib/table/column-keys') - return { ...columnKeys } -}) +vi.mock('@/lib/table/api', () => ({ + internalTableSessionOrExecutorAuth: { authenticate: mocks.authenticate }, +})) + +vi.mock('@/lib/table/api/row-route-policies', () => ({ + internalTableV2QueryErrorPolicy: { + project: (error: unknown) => + error instanceof MockTableV2FeatureDisabledError + ? { + status: 403, + body: { error: error.message, code: 'tables_v2_disabled' }, + } + : null, + }, +})) -vi.mock('@/lib/table/rows/service', () => ({ - queryRows: mockQueryRows, +vi.mock('@/lib/table/application/rows', () => ({ + TableV2FeatureDisabledError: MockTableV2FeatureDisabledError, + queryTableRows: { operation: { id: 'tables.rows.query' }, execute: mocks.queryRows }, })) -import { encodeCursor } from '@/lib/table/rows/cursor' +import { TableV2FeatureDisabledError } from '@/lib/table/application/rows' import { POST } from '@/app/api/table/[tableId]/query/route' -function authAs(authType: 'session' | 'internal_jwt') { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, +const TABLE = { + id: 'table-1', + workspaceId: 'workspace-1', + schema: { + columns: [ + { id: 'column-name', name: 'Name', type: 'string' as const }, + { id: 'column-age', name: 'Age', type: 'number' as const }, + ], + }, +} + +const ROW = { + id: 'row-1', + data: { 'column-name': 'Ada', 'column-age': 36 }, + position: 0, + orderKey: 'a0', + createdAt: new Date('2026-01-01T00:00:00.000Z'), + updatedAt: new Date('2026-01-01T00:00:00.000Z'), +} + +function sessionPrincipal() { + mocks.authenticate.mockResolvedValue({ + kind: 'session', userId: 'user-1', - authType, + sessionId: 'session-1', }) } -function callQuery(body: Record) { - const req = new NextRequest('http://localhost:3000/api/table/tbl_1/query', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), +function executorPrincipal() { + mocks.authenticate.mockResolvedValue({ + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'workspace-canonical', + delegationId: 'delegation-1', + audience: 'sim:tables', + issuedAt: new Date('2026-01-01'), + expiresAt: new Date('2026-01-02'), }) - return POST(req, { params: Promise.resolve({ tableId: 'tbl_1' }) }) } -const EMPTY_RESULT = { - rows: [], - rowCount: 0, - totalCount: 0, - limit: 0, - offset: 0, - nextCursor: null, +function callQuery(body: Record) { + return POST( + new NextRequest('http://localhost/api/table/table-1/query', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + }), + { params: Promise.resolve({ tableId: 'table-1' }) } + ) } -describe('POST /api/table/[tableId]/query', () => { +describe('POST /api/table/[tableId]/query application adapter', () => { beforeEach(() => { vi.clearAllMocks() - mockCheckAccess.mockResolvedValue({ - ok: true, - table: createTableDefinition({ - columns: [ - { id: 'col_aaa', name: 'name', type: 'string' }, - { id: 'col_bbb', name: 'wins', type: 'number' }, - ], - maxRows: 100, - createdAt: new Date('2024-01-01'), - updatedAt: new Date('2024-01-01'), - }), - }) - mockQueryRows.mockResolvedValue(EMPTY_RESULT) - mockGate.mockResolvedValue(null) - }) - - it('returns 404 when the tables-v2-api flag is off', async () => { - const { NextResponse } = await import('next/server') - authAs('session') - mockGate.mockResolvedValue(NextResponse.json({ error: 'Not found' }, { status: 404 })) - const res = await callQuery({ workspaceId: 'workspace-1' }) - expect(res.status).toBe(404) - expect(mockQueryRows).not.toHaveBeenCalled() - }) - - it('runs the flag gate only after the access check, so it cannot leak a cohort oracle', async () => { - authAs('session') - mockCheckAccess.mockResolvedValue({ ok: false, status: 403 }) - const res = await callQuery({ workspaceId: 'workspace-1' }) - expect(res.status).toBe(403) - expect(mockGate).not.toHaveBeenCalled() - }) - - it('translates predicate/sort column names to storage ids for SESSION auth too', async () => { - authAs('session') - const res = await callQuery({ - workspaceId: 'workspace-1', - predicate: { - all: [ - { field: 'name', op: 'eq', value: 'John' }, - { field: 'wins', op: 'gte', value: 10 }, - ], - }, - sort: [{ field: 'wins', direction: 'desc' }], - }) - - expect(res.status).toBe(200) - const options = mockQueryRows.mock.calls[0][1] - expect(options.predicate).toEqual({ - all: [ - { field: 'col_aaa', op: 'eq', value: 'John' }, - { field: 'col_bbb', op: 'gte', value: 10 }, - ], - }) - expect(options.sort).toEqual({ col_bbb: 'desc' }) - expect(options.withExecutions).toBe(false) - }) - - it('selects a stable column id and returns its current name to a workflow', async () => { - authAs('internal_jwt') - mockCheckAccess.mockResolvedValue({ - ok: true, - table: createTableDefinition({ - columns: [ - { id: 'col_aaa', name: 'renamed_name', type: 'string' }, - { id: 'col_bbb', name: 'wins', type: 'number' }, - ], - maxRows: 100, - createdAt: new Date('2024-01-01'), - updatedAt: new Date('2024-01-01'), - }), - }) - mockQueryRows.mockResolvedValue({ - ...EMPTY_RESULT, - rows: [ - { - id: 'row_1', - data: { col_aaa: 'Ana' }, - executions: {}, - position: 1, - orderKey: 'a0', - createdAt: new Date('2026-08-20T10:00:00.000Z'), - updatedAt: new Date('2026-08-20T10:00:00.000Z'), - }, - ], + sessionPrincipal() + mocks.queryRows.mockResolvedValue({ + table: TABLE, + rows: [ROW], rowCount: 1, totalCount: 1, - limit: 100, + limit: 10, + offset: 0, + nextCursor: null, }) - - const res = await callQuery({ workspaceId: 'workspace-1', columns: ['col_aaa'] }) - - expect(res.status).toBe(200) - // The service projects (so the byte budget measures the response); the route only resolves ids. - expect(mockQueryRows.mock.calls[0][1].columnIds).toEqual(new Set(['col_aaa'])) - expect((await res.json()).data.rows[0].data).toEqual({ renamed_name: 'Ana' }) }) - it('accepts an exact column name for direct callers', async () => { - authAs('internal_jwt') - mockQueryRows.mockResolvedValue({ - ...EMPTY_RESULT, - rows: [ - { - id: 'row_1', - data: { col_bbb: 12 }, - executions: {}, - position: 1, - orderKey: 'a0', - createdAt: new Date('2026-08-20T10:00:00.000Z'), - updatedAt: new Date('2026-08-20T10:00:00.000Z'), - }, - ], - rowCount: 1, - totalCount: 1, - limit: 100, - }) - - const res = await callQuery({ workspaceId: 'workspace-1', columns: ['wins'] }) - - expect(res.status).toBe(200) - expect(mockQueryRows.mock.calls[0][1].columnIds).toEqual(new Set(['col_bbb'])) - expect((await res.json()).data.rows[0].data).toEqual({ wins: 12 }) - }) - - it('asks for every column when the selection is omitted or empty', async () => { - authAs('internal_jwt') - - const omitted = await callQuery({ workspaceId: 'workspace-1' }) - const empty = await callQuery({ workspaceId: 'workspace-1', columns: [] }) - - expect(omitted.status).toBe(200) - expect(empty.status).toBe(200) - expect(mockQueryRows.mock.calls[0][1].columnIds).toBeUndefined() - expect(mockQueryRows.mock.calls[1][1].columnIds).toBeUndefined() - }) - - it('drops a column reference that no longer exists without exposing diagnostics', async () => { - authAs('internal_jwt') - const staleId = `col_${'0'.repeat(32)}` - - const res = await callQuery({ + it('passes typed query semantics and feature admission to the shared use case', async () => { + const response = await callQuery({ workspaceId: 'workspace-1', - columns: ['col_aaa', 'missing', staleId], + predicate: { field: 'Name', op: 'eq', value: 'Ada' }, + sort: [{ field: 'Age', direction: 'desc' }], + columns: ['Name'], + limit: 10, }) - expect(res.status).toBe(200) - expect(mockQueryRows.mock.calls[0][1].columnIds).toEqual(new Set(['col_aaa'])) - expect((await res.json()).data).not.toHaveProperty('ignoredColumns') - }) - - it('returns empty row data, not every column, when no requested column exists', async () => { - authAs('internal_jwt') - - const res = await callQuery({ workspaceId: 'workspace-1', columns: ['missing'] }) - - expect(res.status).toBe(200) - expect(mockQueryRows.mock.calls[0][1].columnIds).toEqual(new Set()) - expect((await res.json()).data).not.toHaveProperty('ignoredColumns') - }) - - it('does not expose ignored-column diagnostics for valid or omitted selections', async () => { - authAs('internal_jwt') - - const selected = await callQuery({ workspaceId: 'workspace-1', columns: ['col_aaa'] }) - const all = await callQuery({ workspaceId: 'workspace-1' }) - - expect((await selected.json()).data).not.toHaveProperty('ignoredColumns') - expect((await all.json()).data).not.toHaveProperty('ignoredColumns') - }) - - it('accepts a root condition and executes its canonical all group', async () => { - authAs('internal_jwt') - const res = await callQuery({ - workspaceId: 'workspace-1', - predicate: { field: 'name', op: 'eq', value: 'John' }, + expect(response.status).toBe(200) + expect(mocks.queryRows.mock.calls[0][0].input).toMatchObject({ + assertedWorkspaceId: 'workspace-1', + columns: ['Name'], + limit: 10, + allowExpandedLimit: true, + requireV2Feature: true, + includeTotal: true, }) - - expect(res.status).toBe(200) - expect(mockQueryRows.mock.calls[0][1].predicate).toEqual({ - all: [{ field: 'col_aaa', op: 'eq', value: 'John' }], + expect((await response.json()).data.rows[0].data).toEqual({ + 'column-name': 'Ada', + 'column-age': 36, }) }) - it('rejects a keyset cursor combined with a custom sort', async () => { - authAs('internal_jwt') - const cursor = encodeCursor({ - lastRow: { id: 'row_1', orderKey: 'a1' }, - keysetValid: true, - nextOffset: 1, - }) - const res = await callQuery({ - workspaceId: 'workspace-1', - sort: [{ field: 'wins', direction: 'desc' }], - cursor, - }) + it('uses canonical delegated workspace and returns name-keyed rows', async () => { + executorPrincipal() + const response = await callQuery({ workspaceId: 'workspace-forged', limit: 10 }) - expect(res.status).toBe(400) - const body = await res.json() - expect(body.error).toMatch(/not valid for a sorted query/) - expect(body.code).toBe('CURSOR_SORT_CONFLICT') - expect(mockQueryRows).not.toHaveBeenCalled() + expect(mocks.queryRows.mock.calls[0][0].input.assertedWorkspaceId).toBe('workspace-canonical') + expect((await response.json()).data.rows[0].data).toEqual({ Name: 'Ada', Age: 36 }) }) - it('returns 400 (not 500) for a cursor that decodes to a JSON primitive', async () => { - authAs('internal_jwt') - const res = await callQuery({ - workspaceId: 'workspace-1', - cursor: Buffer.from('42').toString('base64url'), - }) + it('projects the feature gate error with the compatibility code', async () => { + mocks.queryRows.mockRejectedValueOnce(new TableV2FeatureDisabledError()) - expect(res.status).toBe(400) - const body = await res.json() - expect(body.error).toBe('Invalid cursor') - expect(body.code).toBe('INVALID_CURSOR') - }) + const response = await callQuery({ workspaceId: 'workspace-1', limit: 10 }) - it('returns 400 for a predicate referencing an unknown column', async () => { - authAs('internal_jwt') - const res = await callQuery({ - workspaceId: 'workspace-1', - predicate: { all: [{ field: 'nope', op: 'eq', value: 1 }] }, + expect(response.status).toBe(403) + expect(await response.json()).toMatchObject({ + error: 'The v2 table query API is not enabled for this workspace', + code: 'tables_v2_disabled', }) - - expect(res.status).toBe(400) - expect((await res.json()).error).toMatch(/Unknown filter column/) }) - it('passes nextCursor through the response envelope and skips the count on later pages', async () => { - authAs('internal_jwt') - mockQueryRows.mockResolvedValue({ ...EMPTY_RESULT, nextCursor: 'tok' }) - const cursor = encodeCursor({ - lastRow: { id: 'row_1', orderKey: 'a1' }, - keysetValid: true, - nextOffset: 1, - }) - - const res = await callQuery({ workspaceId: 'workspace-1', cursor }) - - expect(res.status).toBe(200) - const body = await res.json() - expect(body.data.nextCursor).toBe('tok') - const options = mockQueryRows.mock.calls[0][1] - expect(options.includeTotal).toBe(false) - expect(options.after).toEqual({ orderKey: 'a1', id: 'row_1' }) + it('does not recompute the total count on cursor pages', async () => { + await callQuery({ workspaceId: 'workspace-1', limit: 10, cursor: 'cursor-1' }) + expect(mocks.queryRows.mock.calls[0][0].input.includeTotal).toBe(false) }) }) diff --git a/apps/sim/app/api/table/[tableId]/query/route.ts b/apps/sim/app/api/table/[tableId]/query/route.ts index cd526853377..0826de5b7f2 100644 --- a/apps/sim/app/api/table/[tableId]/query/route.ts +++ b/apps/sim/app/api/table/[tableId]/query/route.ts @@ -1,171 +1,53 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' import { rowQueryContract, TABLE_QUERY_MAX_BODY_BYTES } from '@/lib/api/contracts/tables' -import { parseRequest } from '@/lib/api/server' -import { isZodError, validationErrorResponse } from '@/lib/api/server/validation' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { Sort, TableSchema } from '@/lib/table' +import { defineInternalJsonRoute, internalRateLimits } from '@/lib/api/server/routes' +import { internalTableSessionOrExecutorAuth } from '@/lib/table/api' +import { internalTableV2QueryErrorPolicy } from '@/lib/table/api/row-route-policies' +import { tableOperations } from '@/lib/table/application/operations' +import { queryTableRows } from '@/lib/table/application/rows' import { - buildIdByName, - columnMatchesRef, - getColumnId, - sortSpecNamesToIds, -} from '@/lib/table/column-keys' -import { TableQueryValidationError } from '@/lib/table/errors' -import { validatePredicate, validateSortSpec } from '@/lib/table/query-builder/validate' -import { assertCursorQueryBinding, decodeCursor } from '@/lib/table/rows/cursor' -import { queryRows } from '@/lib/table/rows/service' -import { predicateToStorage } from '@/lib/table/select-values' -import { createTableRowsResponse } from '@/app/api/table/row-secret-provenance' -import { rowWireTranslators } from '@/app/api/table/row-wire' -import { accessError, checkAccess, tablesV2GateError } from '@/app/api/table/utils' - -const logger = createLogger('TableRowQueryAPI') - -interface RowQueryRouteParams { - params: Promise<{ tableId: string }> -} - -/** - * POST /api/table/[tableId]/query — v2 row query. Typed `predicate`/`sort` - * objects (validated server-side) + opaque cursor pagination (no offset on the - * wire). Shares the same engine as the legacy GET /rows route via `queryRows`. - */ -export const POST = withRouteHandler(async (request: NextRequest, context: RowQueryRouteParams) => { - const requestId = generateRequestId() - - try { - const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) - } - - const parsed = await parseRequest(rowQueryContract, request, context, { - maxBodyBytes: TABLE_QUERY_MAX_BODY_BYTES, - }) - if (!parsed.success) return parsed.response - const { params, body } = parsed.data - const { tableId } = params - - const accessResult = await checkAccess(tableId, authResult.userId, 'read') - if (!accessResult.ok) return accessError(accessResult, requestId, tableId) - const { table } = accessResult - - if (body.workspaceId !== table.workspaceId) { - logger.warn( - `[${requestId}] Workspace ID mismatch for table ${tableId}. Provided: ${body.workspaceId}, Actual: ${table.workspaceId}` - ) - return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) - } - - // After authz: the gate reads the workspace's org off the primary DB, and its - // 404 would otherwise distinguish "not in the rollout cohort" from "no access". - const gateError = await tablesV2GateError(authResult.userId, body.workspaceId) - if (gateError) return gateError - - const schema = table.schema as TableSchema - const wire = rowWireTranslators(authResult.authType, schema) - const cursor = body.cursor ? decodeCursor(body.cursor) : undefined - /** - * A reference that matches no column is dropped, not rejected: a workflow - * whose picked column was since deleted keeps running and simply gets the - * columns that still exist (the editor shows the orphaned id so it can be - * cleared). Skipped references are logged for server-side diagnostics. - */ - let selectedColumnIds: Set | undefined - const ignoredColumns: string[] = [] - if (body.columns?.length) { - selectedColumnIds = new Set() - for (const reference of body.columns) { - const column = schema.columns.find((candidate) => columnMatchesRef(candidate, reference)) - if (column) selectedColumnIds.add(getColumnId(column)) - else ignoredColumns.push(reference) - } - if (ignoredColumns.length > 0) { - logger.warn( - `[${requestId}] Ignoring output columns not on table ${tableId}: ${ignoredColumns.join(', ')}` - ) - } - } - - // Predicate/sort fields are column-NAME-keyed by construction (the caller - // authors names), so validate against the schema then translate names → - // storage ids unconditionally — unlike row data, this is not authType-dependent. - const idByName = buildIdByName(schema) - let predicate = body.predicate - if (predicate) { - validatePredicate(predicate, schema.columns) - predicate = predicateToStorage(predicate, schema) - } - let sortSpec = body.sort - if (sortSpec?.length) { - validateSortSpec(sortSpec, schema.columns) - sortSpec = sortSpecNamesToIds(sortSpec, idByName) - } - const sort: Sort | undefined = sortSpec?.length - ? Object.fromEntries(sortSpec.map((s) => [s.field, s.direction])) - : undefined - - // Cursor↔sort binding: keyset cursors are default-order only; an offset - // cursor must be replayed under the exact sort it was minted with. - if (cursor) assertCursorQueryBinding(cursor, { sort, predicate }) - - const result = await queryRows( - table, - { - predicate, - sort, - limit: body.limit, - after: cursor?.after, - offset: cursor?.offset, - // Only the first page (no inbound cursor) pays for the total count. - includeTotal: !body.cursor, - // Executions are grid UI state; the v2 surface returns row data only - // and the byte budget deliberately measures just `data`. - withExecutions: false, - // Projected inside the drain so the byte budget measures the response. - columnIds: selectedColumnIds, - }, - requestId - ) - - const responseBody = { - success: true, - data: { - rows: result.rows.map((r) => ({ - id: r.id, - data: wire.dataOut(r.data), - executions: r.executions, - position: r.position, - orderKey: r.orderKey ?? undefined, - createdAt: r.createdAt instanceof Date ? r.createdAt.toISOString() : String(r.createdAt), - updatedAt: r.updatedAt instanceof Date ? r.updatedAt.toISOString() : String(r.updatedAt), - })), - rowCount: result.rowCount, - totalCount: result.totalCount, - limit: result.limit, - nextCursor: result.nextCursor, - }, - } - return createTableRowsResponse({ + finalizeTableRowsProvenance, + negotiateTableRowsProvenance, +} from '@/app/api/table/row-secret-provenance' +import { presentQueryRowForPrincipal } from '@/app/api/table/row-wire' + +export const POST = defineInternalJsonRoute({ + contract: rowQueryContract, + operation: tableOperations.queryRows, + auth: internalTableSessionOrExecutorAuth, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal v2 table query behavior', + }), + errorPolicy: internalTableV2QueryErrorPolicy, + parseOptions: { maxBodyBytes: TABLE_QUERY_MAX_BODY_BYTES }, + mapInput: ({ params, body }, { principal, request }) => ({ + tableId: params.tableId, + assertedWorkspaceId: principal.kind === 'delegated' ? principal.workspaceId : body.workspaceId, + predicate: body.predicate, + sort: body.sort, + columns: body.columns, + limit: body.limit, + cursor: body.cursor, + includeTotal: !body.cursor, + includeRunState: false, + allowExpandedLimit: true, + requireV2Feature: true, + includePersistedSecretProvenance: negotiateTableRowsProvenance( request, - authType: authResult.authType, - userId: authResult.userId, - workspaceId: table.workspaceId, - body: responseBody, - rows: result.rows, - }) - } catch (error) { - if (isZodError(error)) return validationErrorResponse(error) - if (error instanceof TableQueryValidationError) { - return NextResponse.json( - { error: error.message, ...(error.code ? { code: error.code } : {}) }, - { status: 400 } - ) - } - logger.error(`[${requestId}] Error querying rows (v2):`, error) - return NextResponse.json({ error: 'Failed to query rows' }, { status: 500 }) - } + principal.kind === 'delegated' + ), + }), + useCase: queryTableRows, + present: (result, { principal }) => ({ + success: true as const, + data: { + rows: result.rows.map((row) => + presentQueryRowForPrincipal(row, result.table.schema, principal) + ), + rowCount: result.rowCount, + totalCount: result.totalCount, + limit: result.limit, + nextCursor: result.nextCursor, + }, + }), + finalizeResponse: ({ result }) => finalizeTableRowsProvenance(result.secretProvenance), }) diff --git a/apps/sim/app/api/table/[tableId]/route.test.ts b/apps/sim/app/api/table/[tableId]/route.test.ts index 366089ee9aa..eea9e53d5cf 100644 --- a/apps/sim/app/api/table/[tableId]/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/route.test.ts @@ -14,6 +14,8 @@ const { mockUpdateTableLocks, mockFindActiveFolder, mockGetLimits, + mockAuthenticate, + mockReadTable, } = vi.hoisted(() => ({ mockCheckAccess: vi.fn(), mockDeleteTable: vi.fn(), @@ -23,6 +25,18 @@ const { mockUpdateTableLocks: vi.fn(), mockFindActiveFolder: vi.fn(), mockGetLimits: vi.fn(), + mockAuthenticate: vi.fn(), + mockReadTable: vi.fn(), +})) + +vi.mock('@/lib/table/api', () => ({ + internalTableSessionOrExecutorAuth: { authenticate: mockAuthenticate }, + internalTableErrorPolicies: { + concealTableAuthorization: { project: () => null }, + }, +})) +vi.mock('@/lib/table/application/tables', () => ({ + readTableDetailsUseCase: { operation: { id: 'tables.read' }, execute: mockReadTable }, })) vi.mock('@/lib/table', () => ({ @@ -54,6 +68,7 @@ vi.mock('@/app/api/table/utils', () => ({ })) vi.mock('@/lib/table/wire', () => ({ normalizeColumn: (column: unknown) => column, + toWireTimestamp: (value: Date) => value.toISOString(), })) import { GET, PATCH } from '@/app/api/table/[tableId]/route' @@ -164,46 +179,46 @@ describe('PATCH /api/table/[tableId] folder moves', () => { }) }) -/** - * Pins which auth path this route hands a Bearer token to. - * - * `fetchTableSchema` in `@/tools/schema-enrichers` reaches this route with a legacy - * `type: 'internal'` token from the deprecated `buildAuthHeaders`. Only - * `checkSessionOrInternalAuth` accepts that token; the delegation policy the sibling - * table routes use rejects it outright. Migrating this route without moving that caller - * to `buildExecutorDelegationHeaders` in the same change breaks every table tool on an - * Agent block, so this fails first and names the caller. - * - * Scope: this pins the *route's* choice of verifier. That the legacy token is actually - * valid for that verifier — and rejected by the delegation one — is pinned separately in - * `@/lib/auth/internal.test.ts`. Both halves are needed; neither implies the other. - */ -describe('GET /api/table/[tableId] executor auth pairing', () => { +describe('GET /api/table/[tableId] application adapter', () => { beforeEach(() => { vi.clearAllMocks() - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', + mockAuthenticate.mockResolvedValue({ + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'workspace-canonical', + delegationId: 'delegation-1', + audience: 'sim:tables', + issuedAt: new Date('2026-01-01'), + expiresAt: new Date('2026-01-02'), + }) + mockReadTable.mockResolvedValue({ + table: { + ...TABLE, + description: null, + metadata: null, + rowCount: 0, + createdBy: 'user-1', + createdAt: new Date('2026-01-01T00:00:00.000Z'), + updatedAt: new Date('2026-01-01T00:00:00.000Z'), + }, + maxRows: 1000, + folderPath: '/', }) - mockCheckAccess.mockResolvedValue({ ok: true, table: TABLE }) - mockGetLimits.mockResolvedValue({ maxRowsPerTable: 1000 }) }) - it('routes a Bearer token to the legacy verifier fetchTableSchema mints for', async () => { - const request = new NextRequest('http://localhost:3000/api/table/tbl_1?workspaceId=workspace-1') - request.headers.set('authorization', 'Bearer legacy-internal-token') + it('uses the delegated principal workspace instead of the query assertion', async () => { + const request = new NextRequest( + 'http://localhost:3000/api/table/tbl_1?workspaceId=workspace-forged' + ) const response = await GET(request, routeContext) + expect(mockReadTable).toHaveBeenCalledOnce() expect(response.status).toBe(200) - expect(hybridAuthMockFns.mockCheckSessionOrInternalAuth).toHaveBeenCalledWith( - expect.objectContaining({ - headers: expect.objectContaining({ get: expect.any(Function) }), - }), - expect.anything() - ) - const [forwarded] = hybridAuthMockFns.mockCheckSessionOrInternalAuth.mock.calls[0] - expect(forwarded.headers.get('authorization')).toBe('Bearer legacy-internal-token') + expect(mockReadTable.mock.calls[0][0].input).toEqual({ + tableId: 'tbl_1', + workspaceId: 'workspace-canonical', + }) }) }) diff --git a/apps/sim/app/api/table/[tableId]/route.ts b/apps/sim/app/api/table/[tableId]/route.ts index aaf90a72f7e..2e0a6286181 100644 --- a/apps/sim/app/api/table/[tableId]/route.ts +++ b/apps/sim/app/api/table/[tableId]/route.ts @@ -1,14 +1,25 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { type NextRequest, NextResponse } from 'next/server' -import { getTableQuerySchema, updateTableContract } from '@/lib/api/contracts/tables' +import { + getTableContract, + getTableQuerySchema, + updateTableContract, +} from '@/lib/api/contracts/tables' +import { + defineInternalJsonRoute, + internalErrorResponse, + internalRateLimits, +} from '@/lib/api/server/routes' import { isZodError, parseRequest, validationErrorResponse } from '@/lib/api/server/validation' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { findActiveFolder } from '@/lib/folders/queries' -import { getTableById, TableConflictError, type TableSchema } from '@/lib/table' -import { getWorkspaceTableLimits } from '@/lib/table/billing' +import { getTableById, TableConflictError } from '@/lib/table' +import { internalTableErrorPolicies, internalTableSessionOrExecutorAuth } from '@/lib/table/api' +import { tableOperations } from '@/lib/table/application/operations' +import { readTableDetailsUseCase } from '@/lib/table/application/tables' import { signalTableSchemaChanged } from '@/lib/table/events' import { performDeleteTable, @@ -16,7 +27,7 @@ import { performRenameTable, performUpdateTableLocks, } from '@/lib/table/orchestration' -import { normalizeColumn } from '@/lib/table/wire' +import { normalizeColumn, toWireTimestamp } from '@/lib/table/wire' import { accessError, checkAccess, @@ -31,79 +42,50 @@ interface TableRouteParams { } /** GET /api/table/[tableId] - Retrieves a single table's details. */ -export const GET = withRouteHandler(async (request: NextRequest, { params }: TableRouteParams) => { - const requestId = generateRequestId() - const { tableId } = await params - - try { - const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized table access attempt`) - return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) - } - - const { searchParams } = new URL(request.url) - const validated = getTableQuerySchema.parse({ - workspaceId: searchParams.get('workspaceId'), - }) - - const result = await checkAccess(tableId, authResult.userId, 'read') - if (!result.ok) return accessError(result, requestId, tableId) - - const { table } = result - - if (table.workspaceId !== validated.workspaceId) { - return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) - } - - logger.info(`[${requestId}] Retrieved table ${tableId} for user ${authResult.userId}`) - - const schemaData = table.schema as TableSchema - - // Source the row cap from the workspace's live plan, not the value stored on - // the table at creation time (which goes stale when the plan changes). - const { maxRowsPerTable } = await getWorkspaceTableLimits(table.workspaceId) - - return NextResponse.json({ - success: true, - data: { - table: { - id: table.id, - name: table.name, - description: table.description, - schema: { - columns: schemaData.columns.map(normalizeColumn), - ...(schemaData.workflowGroups ? { workflowGroups: schemaData.workflowGroups } : {}), - }, - metadata: table.metadata ?? null, - rowCount: table.rowCount, - maxRows: maxRowsPerTable, - folderId: table.folderId ?? null, - locks: table.locks, - createdAt: - table.createdAt instanceof Date - ? table.createdAt.toISOString() - : String(table.createdAt), - updatedAt: - table.updatedAt instanceof Date - ? table.updatedAt.toISOString() - : String(table.updatedAt), - jobStatus: table.jobStatus ?? null, - jobId: table.jobId ?? null, - jobType: table.jobType ?? null, - jobError: table.jobError ?? null, - jobRowsProcessed: table.jobRowsProcessed ?? 0, +export const GET = defineInternalJsonRoute({ + contract: getTableContract, + operation: tableOperations.read, + auth: internalTableSessionOrExecutorAuth, + rateLimit: internalRateLimits.none({ + reason: 'Preserve existing internal table detail behavior', + }), + errorPolicy: { + ...internalTableErrorPolicies.concealTableAuthorization, + unhandled: () => internalErrorResponse(500, { error: 'Failed to get table' }), + }, + mapInput: ({ params, query }, { principal }) => ({ + tableId: params.tableId, + workspaceId: principal.kind === 'delegated' ? principal.workspaceId : query.workspaceId, + }), + useCase: readTableDetailsUseCase, + present: ({ table, maxRows }) => ({ + success: true as const, + data: { + table: { + id: table.id, + name: table.name, + description: table.description, + schema: { + columns: table.schema.columns.map(normalizeColumn), + ...(table.schema.workflowGroups ? { workflowGroups: table.schema.workflowGroups } : {}), }, + metadata: table.metadata ?? null, + rowCount: table.rowCount, + maxRows, + folderId: table.folderId ?? null, + locks: table.locks, + workspaceId: table.workspaceId, + createdBy: table.createdBy, + createdAt: toWireTimestamp(table.createdAt), + updatedAt: toWireTimestamp(table.updatedAt), + jobStatus: table.jobStatus ?? null, + jobId: table.jobId ?? null, + jobType: table.jobType ?? null, + jobError: table.jobError ?? null, + jobRowsProcessed: table.jobRowsProcessed ?? 0, }, - }) - } catch (error) { - if (isZodError(error)) { - return validationErrorResponse(error) - } - - logger.error(`[${requestId}] Error getting table:`, error) - return NextResponse.json({ error: 'Failed to get table' }, { status: 500 }) - } + }, + }), }) /** PATCH /api/table/[tableId] - Renames a table. */ diff --git a/apps/sim/app/api/table/[tableId]/rows/[rowId]/route.ts b/apps/sim/app/api/table/[tableId]/rows/[rowId]/route.ts index 02a5a45cfa4..7fa9f78e1cd 100644 --- a/apps/sim/app/api/table/[tableId]/rows/[rowId]/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/[rowId]/route.ts @@ -32,7 +32,7 @@ export const GET = defineInternalJsonRoute({ mapInput: ({ params, query }, { principal, request }) => ({ tableId: params.tableId, rowId: params.rowId, - assertedWorkspaceId: query.workspaceId, + assertedWorkspaceId: principal.kind === 'delegated' ? principal.workspaceId : query.workspaceId, includePersistedSecretProvenance: negotiateTableRowsProvenance( request, principal.kind !== 'session' @@ -56,7 +56,8 @@ export const PATCH = defineInternalJsonRoute({ return { tableId: params.tableId, rowId: params.rowId, - assertedWorkspaceId: body.workspaceId, + assertedWorkspaceId: + principal.kind === 'delegated' ? principal.workspaceId : body.workspaceId, data: body.data as RowData, dataKeying: rowKeyingForPrincipal(principal), strictWrite: false, @@ -87,10 +88,10 @@ export const DELETE = defineInternalJsonRoute({ auth: internalTableSessionOrExecutorAuth, rateLimit, errorPolicy: internalTableRowsErrorPolicy, - mapInput: ({ params, body }, { request }) => ({ + mapInput: ({ params, body }, { principal, request }) => ({ tableId: params.tableId, rowId: params.rowId, - assertedWorkspaceId: body.workspaceId, + assertedWorkspaceId: principal.kind === 'delegated' ? principal.workspaceId : body.workspaceId, actorClientId: readClientId(request), }), useCase: deleteTableRow, diff --git a/apps/sim/app/api/table/[tableId]/rows/route.test.ts b/apps/sim/app/api/table/[tableId]/rows/route.test.ts index b277f2eb83c..e57958b51cf 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.test.ts @@ -1,390 +1,237 @@ /** * @vitest-environment node */ -import { - createTableDefinition, - hybridAuthMockFns, - type TableDefinitionFactoryOptions, -} from '@sim/testing' + import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' -const { - mockCheckAccess, - mockInsertRow, - mockValidateRowData, - mockQueryRows, - mockUpdateRowsByFilter, - mockDeleteRowsByFilter, -} = vi.hoisted(() => ({ - mockCheckAccess: vi.fn(), - mockInsertRow: vi.fn(), - mockValidateRowData: vi.fn(), - mockQueryRows: vi.fn(), - mockUpdateRowsByFilter: vi.fn(), - mockDeleteRowsByFilter: vi.fn(), +const mocks = vi.hoisted(() => ({ + authenticate: vi.fn(), + createRows: vi.fn(), + queryRows: vi.fn(), + updateRows: vi.fn(), + batchUpdateRows: vi.fn(), + deleteRows: vi.fn(), })) -vi.mock('@/app/api/table/utils', async () => { - const { NextResponse } = await import('next/server') - return { - checkAccess: mockCheckAccess, - accessError: (result: { status: number }) => - NextResponse.json({ error: 'Access denied' }, { status: result.status }), - } -}) - -vi.mock('@/lib/table', async () => { - // Real column-keys translation functions; the row-wire helper under test - // imports them from this barrel. - const columnKeys = await import('@/lib/table/column-keys') - return { - ...columnKeys, - insertRow: mockInsertRow, - batchInsertRows: vi.fn(), - batchUpdateRows: vi.fn(), - deleteRowsByFilter: mockDeleteRowsByFilter, - deleteRowsByIds: vi.fn(), - updateRowsByFilter: mockUpdateRowsByFilter, - validateBatchRows: vi.fn(), - validateRowData: mockValidateRowData, - validateRowSize: vi.fn(() => ({ valid: true })), - } -}) +vi.mock('@/lib/table/api', () => ({ + internalTableSessionOrExecutorAuth: { authenticate: mocks.authenticate }, +})) -vi.mock('@/lib/table/rows/service', () => ({ - queryRows: mockQueryRows, +vi.mock('@/lib/table/api/row-route-policies', () => ({ + internalTableRowsErrorPolicy: { project: () => null }, })) -vi.mock('@/lib/table/sql', () => ({ - TableQueryValidationError: class TableQueryValidationError extends Error {}, +vi.mock('@/lib/table/application/rows', () => ({ + createTableRows: { operation: { id: 'tables.rows.create' }, execute: mocks.createRows }, + queryTableRows: { operation: { id: 'tables.rows.query' }, execute: mocks.queryRows }, + updateTableRows: { operation: { id: 'tables.rows.update_many' }, execute: mocks.updateRows }, + batchUpdateTableRows: { + operation: { id: 'tables.rows.update_many' }, + execute: mocks.batchUpdateRows, + }, + deleteTableRows: { operation: { id: 'tables.rows.delete_many' }, execute: mocks.deleteRows }, })) -import { DELETE, GET, POST, PUT } from '@/app/api/table/[tableId]/rows/route' +import { DELETE, GET, PATCH, POST, PUT } from '@/app/api/table/[tableId]/rows/route' + +const TABLE = { + id: 'table-1', + workspaceId: 'workspace-1', + schema: { + columns: [ + { id: 'column-name', name: 'Name', type: 'string' as const }, + { id: 'column-age', name: 'Age', type: 'number' as const }, + ], + }, +} -const TABLE_FIXTURE: TableDefinitionFactoryOptions = { - columns: [ - { id: 'col_aaa', name: 'Name', type: 'string' }, - { id: 'col_bbb', name: 'Age', type: 'number' }, - ], - maxRows: 100, - createdAt: new Date('2024-01-01'), - updatedAt: new Date('2024-01-01'), +const ROW = { + id: 'row-1', + data: { 'column-name': 'Ada', 'column-age': 36 }, + executions: {}, + position: 0, + orderKey: 'a0', + createdAt: new Date('2026-01-01T00:00:00.000Z'), + updatedAt: new Date('2026-01-01T00:00:00.000Z'), } -function authAs(authType: 'session' | 'internal_jwt') { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, +const routeContext = { params: Promise.resolve({ tableId: 'table-1' }) } + +function sessionPrincipal() { + mocks.authenticate.mockResolvedValue({ + kind: 'session', userId: 'user-1', - authType, + sessionId: 'session-1', }) } -function callPost(body: Record) { - const req = new NextRequest('http://localhost:3000/api/table/tbl_1/rows', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), +function executorPrincipal() { + mocks.authenticate.mockResolvedValue({ + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'workspace-canonical', + delegationId: 'delegation-1', + audience: 'sim:tables', + issuedAt: new Date('2026-01-01'), + expiresAt: new Date('2026-01-02'), }) - return POST(req, { params: Promise.resolve({ tableId: 'tbl_1' }) }) } -function callGet(query: Record) { - const params = new URLSearchParams(query) - const req = new NextRequest(`http://localhost:3000/api/table/tbl_1/rows?${params}`, { - method: 'GET', +function request(method: string, body?: unknown, query = '') { + return new NextRequest(`http://localhost/api/table/table-1/rows${query}`, { + method, + ...(body + ? { headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) } + : {}), }) - return GET(req, { params: Promise.resolve({ tableId: 'tbl_1' }) }) } -describe('POST /api/table/[tableId]/rows', () => { - beforeEach(() => { - vi.clearAllMocks() - mockCheckAccess.mockResolvedValue({ ok: true, table: createTableDefinition(TABLE_FIXTURE) }) - mockValidateRowData.mockResolvedValue({ valid: true }) - mockInsertRow.mockResolvedValue({ - id: 'row_1', - data: { col_aaa: 'Ada', col_bbb: 36 }, - position: 1, - orderKey: 'a0', - createdAt: new Date('2024-01-01'), - updatedAt: new Date('2024-01-01'), - }) - }) - - it('translates name-keyed data to column ids for internal-JWT (workflow tool) callers', async () => { - authAs('internal_jwt') - - const res = await callPost({ - workspaceId: 'workspace-1', - data: { Name: 'Ada', Age: 36 }, - }) - - expect(res.status).toBe(200) - expect(mockValidateRowData).toHaveBeenCalledWith( - expect.objectContaining({ rowData: { col_aaa: 'Ada', col_bbb: 36 } }) - ) - expect(mockInsertRow).toHaveBeenCalledWith( - expect.objectContaining({ data: { col_aaa: 'Ada', col_bbb: 36 } }), - expect.anything(), - expect.any(String) - ) - - const body = await res.json() - expect(body.data.row.data).toEqual({ Name: 'Ada', Age: 36 }) - }) - - it('passes id-keyed data through untouched for session (UI) callers', async () => { - authAs('session') - - const res = await callPost({ - workspaceId: 'workspace-1', - data: { col_aaa: 'Ada', col_bbb: 36 }, - }) - - expect(res.status).toBe(200) - expect(mockInsertRow).toHaveBeenCalledWith( - expect.objectContaining({ data: { col_aaa: 'Ada', col_bbb: 36 } }), - expect.anything(), - expect.any(String) - ) - - const body = await res.json() - expect(body.data.row.data).toEqual({ col_aaa: 'Ada', col_bbb: 36 }) - }) -}) - -describe('GET /api/table/[tableId]/rows', () => { +describe('/api/table/[tableId]/rows application adapter', () => { beforeEach(() => { vi.clearAllMocks() - mockCheckAccess.mockResolvedValue({ ok: true, table: createTableDefinition(TABLE_FIXTURE) }) - mockQueryRows.mockResolvedValue({ - rows: [ - { - id: 'row_1', - data: { col_aaa: 'Ada', col_bbb: 36 }, - position: 1, - orderKey: 'a0', - createdAt: new Date('2024-01-01'), - updatedAt: new Date('2024-01-01'), - }, - ], + sessionPrincipal() + mocks.createRows.mockResolvedValue({ kind: 'single', table: TABLE, row: ROW }) + mocks.queryRows.mockResolvedValue({ + table: TABLE, + rows: [ROW], rowCount: 1, totalCount: 1, - limit: 100, + limit: 10, offset: 0, + nextCursor: null, }) - }) - - it('translates name-keyed filter/sort and returns name-keyed rows for internal-JWT callers', async () => { - authAs('internal_jwt') - - const res = await callGet({ - workspaceId: 'workspace-1', - filter: JSON.stringify({ Name: { $eq: 'Ada' } }), - sort: JSON.stringify({ Age: 'desc' }), + mocks.updateRows.mockResolvedValue({ + table: TABLE, + affectedCount: 1, + affectedRowIds: ['row-1'], + }) + mocks.batchUpdateRows.mockResolvedValue({ + table: TABLE, + affectedCount: 1, + affectedRowIds: ['row-1'], + }) + mocks.deleteRows.mockResolvedValue({ + kind: 'filter', + table: TABLE, + affectedCount: 1, + affectedRowIds: ['row-1'], }) - - expect(res.status).toBe(200) - expect(mockQueryRows).toHaveBeenCalledWith( - expect.objectContaining({ id: 'tbl_1' }), - expect.objectContaining({ - filter: { col_aaa: { $eq: 'Ada' } }, - sort: { col_bbb: 'desc' }, - }), - expect.any(String) - ) - - const body = await res.json() - expect(body.data.rows[0].data).toEqual({ Name: 'Ada', Age: 36 }) - }) - - it('keeps counts but skips execution metadata for an omitted or expanded limit', async () => { - authAs('internal_jwt') - - const omitted = await callGet({ workspaceId: 'workspace-1' }) - expect(omitted.status).toBe(200) - expect(mockQueryRows.mock.calls[0][1]).toEqual( - expect.objectContaining({ limit: undefined, includeTotal: true, withExecutions: false }) - ) - - const expanded = await callGet({ workspaceId: 'workspace-1', limit: '1000000' }) - expect(expanded.status).toBe(200) - expect(mockQueryRows.mock.calls[1][1]).toEqual( - expect.objectContaining({ limit: 1000000, includeTotal: true, withExecutions: false }) - ) }) - it('retains metadata loading within the former query limit', async () => { - authAs('internal_jwt') - - const res = await callGet({ workspaceId: 'workspace-1', limit: '1000' }) - - expect(res.status).toBe(200) - expect(mockQueryRows.mock.calls[0][1]).toEqual( - expect.objectContaining({ limit: 1000, includeTotal: true, withExecutions: true }) + it('maps session inserts as id-keyed writes and preserves the row response', async () => { + const response = await POST( + request('POST', { + workspaceId: 'workspace-1', + data: { 'column-name': 'Ada' }, + }), + routeContext ) - }) - - it('passes id-keyed filter and rows through untouched for session callers', async () => { - authAs('session') - const res = await callGet({ - workspaceId: 'workspace-1', - filter: JSON.stringify({ col_aaa: { $eq: 'Ada' } }), + expect(response.status).toBe(200) + expect(mocks.createRows.mock.calls[0][0].input).toMatchObject({ + assertedWorkspaceId: 'workspace-1', + dataKeying: 'ids', + secretProvenanceEnvelope: { kind: 'none' }, }) - - expect(res.status).toBe(200) - expect(mockQueryRows).toHaveBeenCalledWith( - expect.objectContaining({ id: 'tbl_1' }), - expect.objectContaining({ filter: { col_aaa: { $eq: 'Ada' } } }), - expect.any(String) - ) - - const body = await res.json() - expect(body.data.rows[0].data).toEqual({ col_aaa: 'Ada', col_bbb: 36 }) - }) - - /** - * The grid now speaks the v2 grammar on this route: a predicate-shaped filter - * takes the NATIVE predicate path into queryRows (not a downgrade), and an - * ordered sort spec compiles to the record the engine's sort builder takes. - */ - it('routes a predicate filter + spec sort natively for session callers', async () => { - authAs('session') - - const res = await callGet({ - workspaceId: 'workspace-1', - filter: JSON.stringify({ all: [{ field: 'col_aaa', op: 'eq', value: 'Ada' }] }), - sort: JSON.stringify([{ field: 'col_bbb', direction: 'desc' }]), + expect((await response.json()).data.row.data).toEqual({ + 'column-name': 'Ada', + 'column-age': 36, }) - - expect(res.status).toBe(200) - const options = mockQueryRows.mock.calls[0][1] - expect(options.predicate).toEqual({ all: [{ field: 'col_aaa', op: 'eq', value: 'Ada' }] }) - expect(options.filter).toBeUndefined() - expect(options.sort).toEqual({ col_bbb: 'desc' }) }) - it('translates a name-keyed predicate for internal-JWT callers', async () => { - authAs('internal_jwt') + it('maps executor inserts as name-keyed and uses canonical delegated workspace', async () => { + executorPrincipal() + await POST( + request('POST', { + workspaceId: 'workspace-forged', + data: { Name: 'Ada' }, + }), + routeContext + ) - const res = await callGet({ - workspaceId: 'workspace-1', - filter: JSON.stringify({ all: [{ field: 'Name', op: 'eq', value: 'Ada' }] }), - sort: JSON.stringify([{ field: 'Age', direction: 'asc' }]), + expect(mocks.createRows.mock.calls[0][0].input).toMatchObject({ + assertedWorkspaceId: 'workspace-canonical', + dataKeying: 'names', }) - - expect(res.status).toBe(200) - const options = mockQueryRows.mock.calls[0][1] - expect(options.predicate).toEqual({ all: [{ field: 'col_aaa', op: 'eq', value: 'Ada' }] }) - expect(options.sort).toEqual({ col_bbb: 'asc' }) }) - /** - * Storage validation runs post-translation, mirroring bulk PUT/DELETE: a - * typo'd field must 400, not compile to a clause that matches nothing and - * read back as a plausible empty page. - */ - it('400s a predicate naming an unknown column instead of returning an empty page', async () => { - authAs('session') + it('preserves legacy query filters, sorts, counts, and expanded-limit policy', async () => { + const filter = encodeURIComponent(JSON.stringify({ 'column-name': { $eq: 'Ada' } })) + const sort = encodeURIComponent(JSON.stringify({ 'column-age': 'desc' })) + const response = await GET( + request( + 'GET', + undefined, + `?workspaceId=workspace-1&filter=${filter}&sort=${sort}&limit=10&offset=2` + ), + routeContext + ) - const res = await callGet({ - workspaceId: 'workspace-1', - filter: JSON.stringify({ all: [{ field: 'col_nope', op: 'eq', value: 'Ada' }] }), + expect(response.status).toBe(200) + expect(mocks.queryRows.mock.calls[0][0].input).toMatchObject({ + legacyFilter: { 'column-name': { $eq: 'Ada' } }, + legacySort: { 'column-age': 'desc' }, + legacyKeying: 'ids', + includeTotal: true, + allowExpandedLimit: true, + offset: 2, }) - - expect(res.status).toBe(400) - const body = await res.json() - expect(body.error).toMatch(/Unknown filter column "col_nope"/) - expect(mockQueryRows).not.toHaveBeenCalled() - }) -}) - -describe('PUT/DELETE /api/table/[tableId]/rows — predicate filters', () => { - beforeEach(() => { - vi.clearAllMocks() - mockCheckAccess.mockResolvedValue({ ok: true, table: createTableDefinition(TABLE_FIXTURE) }) - mockUpdateRowsByFilter.mockResolvedValue({ affectedCount: 1, affectedRowIds: ['row_1'] }) - mockDeleteRowsByFilter.mockResolvedValue({ affectedCount: 1, affectedRowIds: ['row_1'] }) }) - function callPut(body: Record) { - const req = new NextRequest('http://localhost:3000/api/table/tbl_1/rows', { - method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), - }) - return PUT(req, { params: Promise.resolve({ tableId: 'tbl_1' }) }) - } - - function callDelete(body: Record) { - const req = new NextRequest('http://localhost:3000/api/table/tbl_1/rows', { - method: 'DELETE', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), - }) - return DELETE(req, { params: Promise.resolve({ tableId: 'tbl_1' }) }) - } + it('hands unresolved provenance and caller keying to filter updates', async () => { + const response = await PUT( + request('PUT', { + workspaceId: 'workspace-1', + filter: { all: [{ field: 'column-name', op: 'eq', value: 'Ada' }] }, + data: { 'column-name': 'Grace' }, + }), + routeContext + ) - /** - * Keying follows the caller (PR #6067 review): the grid authors ID-keyed - * predicates and the session wire is identity, so ids pass through — and a - * NAME under session auth is just an unknown storage key, rejected like any - * other typo rather than half-translated. - */ - it('PUT passes an id-keyed predicate through untouched under SESSION auth', async () => { - authAs('session') - const res = await callPut({ - workspaceId: 'workspace-1', - filter: { all: [{ field: 'col_aaa', op: 'eq', value: 'Ada' }] }, - data: { col_aaa: 'Grace' }, + expect(response.status).toBe(200) + expect(mocks.updateRows.mock.calls[0][0].input).toMatchObject({ + filterKeying: 'ids', + dataKeying: 'ids', + secretProvenanceEnvelope: { kind: 'none' }, }) - - expect(res.status).toBe(200) - const args = mockUpdateRowsByFilter.mock.calls[0][1] - expect(args.filter).toEqual({ $and: [{ col_aaa: 'Ada' }] }) }) - it('PUT rejects an unknown storage key under SESSION auth with 400', async () => { - authAs('session') - const res = await callPut({ - workspaceId: 'workspace-1', - filter: { all: [{ field: 'Name', op: 'eq', value: 'Ada' }] }, - data: { col_aaa: 'Grace' }, - }) - expect(res.status).toBe(400) - expect(mockUpdateRowsByFilter).not.toHaveBeenCalled() - }) + it('routes filter deletes through the shared authorized use case', async () => { + const response = await DELETE( + request('DELETE', { + workspaceId: 'workspace-1', + filter: { all: [{ field: 'column-name', op: 'eq', value: 'Ada' }] }, + }), + routeContext + ) - it('PUT translates a name-keyed predicate for INTERNAL_JWT callers', async () => { - authAs('internal_jwt') - const res = await callPut({ - workspaceId: 'workspace-1', - filter: { all: [{ field: 'Name', op: 'eq', value: 'Ada' }] }, - data: { Name: 'Grace' }, + expect(response.status).toBe(200) + expect(mocks.deleteRows.mock.calls[0][0].input).toMatchObject({ + kind: 'filter', + filterKeying: 'ids', }) - - expect(res.status).toBe(200) - const args = mockUpdateRowsByFilter.mock.calls[0][1] - expect(args.filter).toEqual({ $and: [{ col_aaa: 'Ada' }] }) }) - it('DELETE accepts the predicate and rejects an unknown column with 400', async () => { - authAs('internal_jwt') - const ok = await callDelete({ - workspaceId: 'workspace-1', - filter: { all: [{ field: 'Age', op: 'gte', value: 30 }] }, - }) - expect(ok.status).toBe(200) - const args = mockDeleteRowsByFilter.mock.calls[0][1] - expect(args.filter).toEqual({ $and: [{ col_bbb: { $gte: 30 } }] }) + it('routes heterogeneous batch patches through one authorized application operation', async () => { + executorPrincipal() + const response = await PATCH( + request('PATCH', { + workspaceId: 'workspace-forged', + updates: [{ rowId: 'row-1', data: { Name: 'Grace' } }], + }), + routeContext + ) - const bad = await callDelete({ - workspaceId: 'workspace-1', - filter: { all: [{ field: 'Nope', op: 'eq', value: 1 }] }, + expect(response.status).toBe(200) + expect(mocks.batchUpdateRows.mock.calls[0][0].input).toMatchObject({ + tableId: 'table-1', + assertedWorkspaceId: 'workspace-canonical', + dataKeying: 'names', + strictWrite: false, + updates: [{ rowId: 'row-1', data: { Name: 'Grace' } }], + secretProvenanceEnvelope: { kind: 'none' }, }) - expect(bad.status).toBe(400) - expect((await bad.json()).error).toMatch(/Unknown filter column/) }) }) diff --git a/apps/sim/app/api/table/[tableId]/rows/route.ts b/apps/sim/app/api/table/[tableId]/rows/route.ts index 3b7d26a4bdf..9e4207bb70f 100644 --- a/apps/sim/app/api/table/[tableId]/rows/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/route.ts @@ -1,596 +1,213 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' import { readClientId } from '@/lib/api/client-id' import { - type BatchInsertTableRowsBodyInput, - batchUpdateTableRowsBodySchema, - deleteTableRowsBodySchema, + batchUpdateTableRowsContract, + deleteTableRowsContract, insertTableRowsContract, - tableRowsQuerySchema, - updateRowsByFilterBodySchema, + listTableRowsContract, + updateTableRowsByFilterContract, } from '@/lib/api/contracts/tables' -import { parseRequest } from '@/lib/api/server' -import { isZodError, parseJsonBody, validationErrorResponse } from '@/lib/api/server/validation' -import { type AuthTypeValue, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { Filter, RowData, Sort, SortSpec, TableRowsCursor, TableSchema } from '@/lib/table' import { - batchInsertRows, - batchUpdateRows, - deleteRowsByFilter, - deleteRowsByIds, - insertRow, - updateRowsByFilter, - validateBatchRows, - validateRowData, - validateRowSize, -} from '@/lib/table' -import { TABLE_LIMITS } from '@/lib/table/constants' -import { TableQueryValidationError } from '@/lib/table/errors' -import { signalTableRowsChanged, signalTableRowsChangedByActor } from '@/lib/table/events' -import { isTablePredicate, predicateToFilter } from '@/lib/table/query-builder/converters' + defineInternalJsonRoute, + internalErrorResponse, + internalRateLimits, +} from '@/lib/api/server/routes' +import type { Filter, RowData, Sort, SortSpec, TablePredicate } from '@/lib/table' +import { internalTableSessionOrExecutorAuth } from '@/lib/table/api' +import { internalTableRowsErrorPolicy } from '@/lib/table/api/row-route-policies' +import { tableOperations } from '@/lib/table/application/operations' import { - validatePredicateShape, - validateStoragePredicate, -} from '@/lib/table/query-builder/validate' -import { queryRows } from '@/lib/table/rows/service' -import type { TablePredicate } from '@/lib/table/types' + batchUpdateTableRows, + createTableRows, + deleteTableRows, + queryTableRows, + updateTableRows, +} from '@/lib/table/application/rows' +import { TABLE_LIMITS } from '@/lib/table/constants' +import { isTablePredicate } from '@/lib/table/query-builder/converters' import { - createTableRowsResponse, - createTableWriteProvenanceTargets, - resolveTableWriteSecretProvenance, + finalizeTableRowsProvenance, + negotiateTableRowsProvenance, + readTableRowProvenanceEnvelope, } from '@/app/api/table/row-secret-provenance' -import { type RowWireTranslators, rowWireTranslators } from '@/app/api/table/row-wire' -import { accessError, checkAccess, orchestrationErrorResponse } from '@/app/api/table/utils' - -const logger = createLogger('TableRowsAPI') - -/** Dual-grammar sort: an ordered spec (v2) or the legacy record, either keying. */ -function resolveWireSort( - sort: Sort | SortSpec | undefined, - wire: RowWireTranslators -): Sort | undefined { - if (!sort) return undefined - if (!Array.isArray(sort)) return wire.sortIn(sort) - const spec = wire.sortSpecIn(sort) - return spec.length > 0 ? Object.fromEntries(spec.map((s) => [s.field, s.direction])) : undefined -} - -/** - * Resolves a bulk-op filter to a storage-id-keyed legacy `Filter`. The v2 - * predicate tree is column-NAME-keyed by construction (the caller authors - * names), so it validates then translates names → ids unconditionally — unlike - * the legacy object form, whose keying follows the caller's wire dialect - * (`wire.filterIn`: ids from the UI, names from workflow tools). - */ -function resolveBulkFilter( - raw: TablePredicate | Filter, - schema: TableSchema, - wire: RowWireTranslators -): Filter { - if (isTablePredicate(raw)) { - // Shape first (keying-agnostic: hybrid nodes, leaf value rules), then let - // the wire translate — identity for the ID-keyed grid, names→ids for - // workflow tools — and validate the RESULT against storage keys. Post- - // translation, any unresolved field is a typo in the caller's own keying, - // and on a destructive path a typo must 400, not silently match nothing. - validatePredicateShape(raw) - const translated = wire.predicateIn(raw) - validateStoragePredicate(translated, schema.columns) - return predicateToFilter(translated) - } - return wire.filterIn(raw) -} - -interface TableRowsRouteParams { - params: Promise<{ tableId: string }> -} - -async function handleBatchInsert( - request: NextRequest, - requestId: string, - tableId: string, - validated: BatchInsertTableRowsBodyInput, - userId: string, - authType: AuthTypeValue | undefined -): Promise { - const accessResult = await checkAccess(tableId, userId, 'write') - if (!accessResult.ok) return accessError(accessResult, requestId, tableId) - - const { table } = accessResult - - if (validated.workspaceId !== table.workspaceId) { - logger.warn( - `[${requestId}] Workspace ID mismatch for table ${tableId}. Provided: ${validated.workspaceId}, Actual: ${table.workspaceId}` - ) - return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) - } - - const wire = rowWireTranslators(authType, table.schema as TableSchema) - const rows = (validated.rows as RowData[]).map((row) => wire.dataIn(row)) - const provenance = resolveTableWriteSecretProvenance({ - request, - payload: validated, - authType, - userId, - workspaceId: table.workspaceId, - targets: createTableWriteProvenanceTargets(validated.rows as RowData[], wire.dataIn), - rowKeys: validated.rows.map((_, index) => String(index)), - }) - if (!provenance.success) return provenance.response +import { presentQueryRowForPrincipal, rowKeyingForPrincipal } from '@/app/api/table/row-wire' - // Validate rows before calling service (service also validates, but route-level - // validation returns structured HTTP responses) - const validation = await validateBatchRows({ - rows, - schema: table.schema as TableSchema, - tableId, - }) - if (!validation.valid) return validation.response +const rateLimit = internalRateLimits.none({ + reason: 'Preserve existing internal table rows behavior', +}) - try { - const insertedRows = await batchInsertRows( - { - tableId, - rows, - workspaceId: validated.workspaceId, - userId, - orderKeys: validated.orderKeys, - secretProvenance: validated.rows.map( - (_, index) => provenance.provenanceByRowKey?.[String(index)] - ), - }, - table, - requestId - ) - signalTableRowsChanged(tableId) - - const responseBody = { - success: true, - data: { - rows: insertedRows.map((r) => ({ - id: r.id, - data: wire.dataOut(r.data), - position: r.position, - orderKey: r.orderKey ?? undefined, - createdAt: r.createdAt instanceof Date ? r.createdAt.toISOString() : r.createdAt, - updatedAt: r.updatedAt instanceof Date ? r.updatedAt.toISOString() : r.updatedAt, - })), - insertedCount: insertedRows.length, - message: `Successfully inserted ${insertedRows.length} rows`, - }, - } - return createTableRowsResponse({ - request, - authType, - userId, - workspaceId: table.workspaceId, - body: responseBody, - rows: insertedRows, - }) - } catch (error) { - const response = orchestrationErrorResponse(error) - if (response) return response - - logger.error(`[${requestId}] Error batch inserting rows:`, error) - return NextResponse.json({ error: 'Failed to insert rows' }, { status: 500 }) +function rowErrorPolicy(fallback: string) { + return { + ...internalTableRowsErrorPolicy, + unhandled: () => internalErrorResponse(500, { error: fallback }), } } -/** POST /api/table/[tableId]/rows - Inserts row(s). Supports single or batch insert. */ -export const POST = withRouteHandler( - async (request: NextRequest, context: TableRowsRouteParams) => { - const requestId = generateRequestId() - - try { - const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) - } - - const parsed = await parseRequest(insertTableRowsContract, request, context) - if (!parsed.success) return parsed.response - - const { tableId } = parsed.data.params - const body = parsed.data.body - - if ('rows' in body) { - return handleBatchInsert( - request, - requestId, - tableId, - body, - authResult.userId, - authResult.authType - ) - } - - const validated = body - - const accessResult = await checkAccess(tableId, authResult.userId, 'write') - if (!accessResult.ok) return accessError(accessResult, requestId, tableId) - - const { table } = accessResult - - if (validated.workspaceId !== table.workspaceId) { - logger.warn( - `[${requestId}] Workspace ID mismatch for table ${tableId}. Provided: ${validated.workspaceId}, Actual: ${table.workspaceId}` - ) - return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) - } - - const wire = rowWireTranslators(authResult.authType, table.schema as TableSchema) - const rowData = wire.dataIn(validated.data as RowData) - const provenance = resolveTableWriteSecretProvenance({ +export const POST = defineInternalJsonRoute({ + contract: insertTableRowsContract, + operation: tableOperations.createRows, + auth: internalTableSessionOrExecutorAuth, + rateLimit, + errorPolicy: rowErrorPolicy('Failed to insert row'), + mapInput: ({ params, body }, { principal, request }) => { + const shared = { + tableId: params.tableId, + assertedWorkspaceId: + principal.kind === 'delegated' ? principal.workspaceId : body.workspaceId, + strictWrite: false, + dataKeying: rowKeyingForPrincipal(principal), + secretProvenanceEnvelope: readTableRowProvenanceEnvelope(request, body), + includePersistedSecretProvenance: negotiateTableRowsProvenance( request, - payload: validated, - authType: authResult.authType, - userId: authResult.userId, - workspaceId: table.workspaceId, - targets: createTableWriteProvenanceTargets([validated.data as RowData], wire.dataIn), - rowKeys: ['0'], - }) - if (!provenance.success) return provenance.response - - // Validate at route level for structured HTTP error responses - const validation = await validateRowData({ - rowData, - schema: table.schema as TableSchema, - tableId, - }) - if (!validation.valid) return validation.response - - // Service handles atomic capacity check + insert in a transaction - const row = await insertRow( - { - tableId, - data: rowData, - workspaceId: validated.workspaceId, - userId: authResult.userId, - position: validated.position, - afterRowId: validated.afterRowId, - beforeRowId: validated.beforeRowId, - secretProvenance: provenance.provenanceByRowKey?.['0'], - }, - table, - requestId - ) - // Attributed unlike the batch path above: the acting tab's insert deliberately avoids - // invalidating the rows root to prevent flicker, which an unattributed echo would undo. - signalTableRowsChangedByActor(tableId, readClientId(request)) - - const responseBody = { - success: true, - data: { - row: { - id: row.id, - data: wire.dataOut(row.data), - position: row.position, - orderKey: row.orderKey ?? undefined, - createdAt: row.createdAt instanceof Date ? row.createdAt.toISOString() : row.createdAt, - updatedAt: row.updatedAt instanceof Date ? row.updatedAt.toISOString() : row.updatedAt, - }, - - message: 'Row inserted successfully', - }, - } - return createTableRowsResponse({ - request, - authType: authResult.authType, - userId: authResult.userId, - workspaceId: table.workspaceId, - body: responseBody, - rows: [row], - }) - } catch (error) { - if (isZodError(error)) { - return validationErrorResponse(error) - } - - const response = orchestrationErrorResponse(error) - if (response) return response - - logger.error(`[${requestId}] Error inserting row:`, error) - return NextResponse.json({ error: 'Failed to insert row' }, { status: 500 }) + principal.kind === 'delegated' + ), } - } -) - -/** GET /api/table/[tableId]/rows - Queries rows with filtering, sorting, and pagination. */ -export const GET = withRouteHandler( - async (request: NextRequest, { params }: TableRowsRouteParams) => { - const requestId = generateRequestId() - const { tableId } = await params - - try { - const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) - } - - const { searchParams } = new URL(request.url) - const workspaceId = searchParams.get('workspaceId') - const filterParam = searchParams.get('filter') - const sortParam = searchParams.get('sort') - const afterParam = searchParams.get('after') - const limit = searchParams.get('limit') - const offset = searchParams.get('offset') - const includeTotalParam = searchParams.get('includeTotal') - - let filter: Record | undefined - let sort: Sort | undefined - let after: TableRowsCursor | undefined - - try { - if (filterParam) { - filter = JSON.parse(filterParam) as Record + return 'rows' in body + ? { + ...shared, + kind: 'batch' as const, + rows: body.rows as RowData[], + orderKeys: body.orderKeys, } - if (sortParam) { - sort = JSON.parse(sortParam) as Sort + : { + ...shared, + kind: 'single' as const, + data: body.data as RowData, + position: body.position, + afterRowId: body.afterRowId, + beforeRowId: body.beforeRowId, + actorClientId: readClientId(request), } - if (afterParam) { - after = JSON.parse(afterParam) as TableRowsCursor + }, + useCase: createTableRows, + present: (result, { principal }) => + result.kind === 'single' + ? { + success: true as const, + data: { + row: presentQueryRowForPrincipal(result.row, result.table.schema, principal), + message: 'Row inserted successfully', + }, } - } catch { - return NextResponse.json({ error: 'Invalid filter, sort, or after JSON' }, { status: 400 }) - } - - const validated = tableRowsQuerySchema.parse({ - workspaceId, - filter, - sort, - after, - limit, - offset, - includeTotal: includeTotalParam, - }) - - const accessResult = await checkAccess(tableId, authResult.userId, 'read') - if (!accessResult.ok) return accessError(accessResult, requestId, tableId) - - const { table } = accessResult - - if (validated.workspaceId !== table.workspaceId) { - logger.warn( - `[${requestId}] Workspace ID mismatch for table ${tableId}. Provided: ${validated.workspaceId}, Actual: ${table.workspaceId}` - ) - return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) - } - - const wire = rowWireTranslators(authResult.authType, table.schema as TableSchema) - /** - * The newly expanded path can return up to the byte budget, so skip the - * per-row execution-sidecar load. Keep the count behavior unchanged so - * Query Rows continues to return totalCount for workflow callers. - */ - const isExpandedQuery = - validated.limit === undefined || validated.limit > TABLE_LIMITS.MAX_QUERY_LIMIT - const result = await queryRows( - table, - { - ...(validated.filter && isTablePredicate(validated.filter as Filter | TablePredicate) - ? { - predicate: (() => { - // Shape-check first: nothing upstream validates this branch, and - // an unchecked hybrid node would silently widen the result. - validatePredicateShape(validated.filter as TablePredicate) - const translated = wire.predicateIn(validated.filter as TablePredicate) - // Post-translation storage check, mirroring the bulk PUT/DELETE - // paths: a typo'd field must 400, not compile to a clause that - // matches nothing and read back as an empty page. - validateStoragePredicate(translated, (table.schema as TableSchema).columns) - return translated - })(), - } - : { filter: validated.filter ? wire.filterIn(validated.filter as Filter) : undefined }), - sort: resolveWireSort(validated.sort as Sort | SortSpec | undefined, wire), - limit: validated.limit, - offset: validated.offset, - after: validated.after, - includeTotal: validated.includeTotal, - withExecutions: !isExpandedQuery, - }, - requestId - ) - - const responseBody = { - success: true, - data: { - rows: result.rows.map((r) => ({ - id: r.id, - data: wire.dataOut(r.data), - executions: r.executions, - position: r.position, - orderKey: r.orderKey ?? undefined, - createdAt: - r.createdAt instanceof Date ? r.createdAt.toISOString() : String(r.createdAt), - updatedAt: - r.updatedAt instanceof Date ? r.updatedAt.toISOString() : String(r.updatedAt), - })), - rowCount: result.rowCount, - totalCount: result.totalCount, - limit: result.limit, - offset: result.offset, - nextCursor: result.nextCursor, + : { + success: true as const, + data: { + rows: result.rows.map((row) => + presentQueryRowForPrincipal(row, result.table.schema, principal) + ), + insertedCount: result.rows.length, + message: `Successfully inserted ${result.rows.length} rows`, + }, }, - } - return createTableRowsResponse({ + finalizeResponse: ({ result }) => finalizeTableRowsProvenance(result.secretProvenance), +}) + +export const GET = defineInternalJsonRoute({ + contract: listTableRowsContract, + operation: tableOperations.queryRows, + auth: internalTableSessionOrExecutorAuth, + rateLimit, + errorPolicy: rowErrorPolicy('Failed to query rows'), + mapInput: ({ params, query }, { principal, request }) => { + const filter = query.filter as Filter | TablePredicate | undefined + const sort = query.sort as Sort | SortSpec | undefined + return { + tableId: params.tableId, + assertedWorkspaceId: + principal.kind === 'delegated' ? principal.workspaceId : query.workspaceId, + ...(filter && isTablePredicate(filter) + ? { predicate: filter } + : { legacyFilter: filter as Filter | undefined }), + ...(Array.isArray(sort) + ? { sort: sort as SortSpec } + : { legacySort: sort as Sort | undefined }), + legacyKeying: rowKeyingForPrincipal(principal), + limit: query.limit, + offset: query.offset, + after: query.after, + includeTotal: query.includeTotal, + includeRunState: query.limit !== undefined && query.limit <= TABLE_LIMITS.MAX_QUERY_LIMIT, + allowExpandedLimit: true, + includePersistedSecretProvenance: negotiateTableRowsProvenance( request, - authType: authResult.authType, - userId: authResult.userId, - workspaceId: table.workspaceId, - body: responseBody, - rows: result.rows, - }) - } catch (error) { - if (isZodError(error)) { - return validationErrorResponse(error) - } - - if (error instanceof TableQueryValidationError) { - return NextResponse.json({ error: error.message }, { status: 400 }) - } - - logger.error(`[${requestId}] Error querying rows:`, error) - return NextResponse.json({ error: 'Failed to query rows' }, { status: 500 }) + principal.kind === 'delegated' + ), } - } -) - -/** PUT /api/table/[tableId]/rows - Updates rows matching filter criteria. */ -export const PUT = withRouteHandler( - async (request: NextRequest, { params }: TableRowsRouteParams) => { - const requestId = generateRequestId() - const { tableId } = await params - - try { - const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) - } - - // Bulk write bodies carry caller-supplied row data, so they can legitimately - // be large — but not unbounded. `parseJsonBody` applies the platform cap - // (413 past it) that a raw `request.json()` on this destructive surface skips. - const parsedBody = await parseJsonBody(request) - if (!parsedBody.success) return parsedBody.response - const body = parsedBody.data - - const validated = updateRowsByFilterBodySchema.parse(body) - - const accessResult = await checkAccess(tableId, authResult.userId, 'write') - if (!accessResult.ok) return accessError(accessResult, requestId, tableId) - - const { table } = accessResult - - if (validated.workspaceId !== table.workspaceId) { - logger.warn( - `[${requestId}] Workspace ID mismatch for table ${tableId}. Provided: ${validated.workspaceId}, Actual: ${table.workspaceId}` - ) - return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) - } - - const wire = rowWireTranslators(authResult.authType, table.schema as TableSchema) - const patchData = wire.dataIn(validated.data as RowData) - const provenance = resolveTableWriteSecretProvenance({ - request, - payload: validated, - authType: authResult.authType, - userId: authResult.userId, - workspaceId: table.workspaceId, - targets: createTableWriteProvenanceTargets([validated.data as RowData], wire.dataIn), - rowKeys: ['0'], - }) - if (!provenance.success) return provenance.response - - const sizeValidation = validateRowSize(patchData) - if (!sizeValidation.valid) { - return NextResponse.json( - { error: 'Invalid row data', details: sizeValidation.errors }, - { status: 400 } - ) - } - - const result = await updateRowsByFilter( - table, - { - filter: resolveBulkFilter( - validated.filter as TablePredicate | Filter, - table.schema as TableSchema, - wire - ), - data: patchData, - limit: validated.limit, - actorUserId: authResult.userId, - secretProvenance: provenance.provenanceByRowKey?.['0'], - }, - requestId - ) - - if (result.affectedCount === 0) { - return NextResponse.json( - { - success: true, - data: { - message: 'No rows matched the filter criteria', - updatedCount: 0, - }, - }, - { status: 200 } - ) - } - signalTableRowsChanged(tableId) - - return NextResponse.json({ - success: true, - data: { - message: 'Rows updated successfully', - updatedCount: result.affectedCount, - updatedRowIds: result.affectedRowIds, + }, + useCase: queryTableRows, + present: (result, { principal }) => ({ + success: true as const, + data: { + rows: result.rows.map((row) => + presentQueryRowForPrincipal(row, result.table.schema, principal) + ), + rowCount: result.rowCount, + totalCount: result.totalCount, + limit: result.limit, + offset: result.offset, + nextCursor: result.nextCursor, + }, + }), + finalizeResponse: ({ result }) => finalizeTableRowsProvenance(result.secretProvenance), +}) + +export const PUT = defineInternalJsonRoute({ + contract: updateTableRowsByFilterContract, + operation: tableOperations.updateRows, + auth: internalTableSessionOrExecutorAuth, + rateLimit, + errorPolicy: rowErrorPolicy('Failed to update rows'), + mapInput: ({ params, body }, { principal, request }) => ({ + tableId: params.tableId, + assertedWorkspaceId: principal.kind === 'delegated' ? principal.workspaceId : body.workspaceId, + filter: body.filter, + filterKeying: rowKeyingForPrincipal(principal), + data: body.data as RowData, + dataKeying: rowKeyingForPrincipal(principal), + strictWrite: false, + limit: body.limit, + secretProvenanceEnvelope: readTableRowProvenanceEnvelope(request, body), + }), + useCase: updateTableRows, + present: ({ affectedCount, affectedRowIds }) => ({ + success: true as const, + data: { + message: + affectedCount === 0 ? 'No rows matched the filter criteria' : 'Rows updated successfully', + updatedCount: affectedCount, + ...(affectedCount > 0 ? { updatedRowIds: affectedRowIds } : {}), + }, + }), +}) + +export const DELETE = defineInternalJsonRoute({ + contract: deleteTableRowsContract, + operation: tableOperations.deleteRows, + auth: internalTableSessionOrExecutorAuth, + rateLimit, + errorPolicy: rowErrorPolicy('Failed to delete rows'), + mapInput: ({ params, body }, { principal }) => + body.rowIds + ? { + kind: 'ids' as const, + tableId: params.tableId, + assertedWorkspaceId: + principal.kind === 'delegated' ? principal.workspaceId : body.workspaceId, + rowIds: body.rowIds, + } + : { + kind: 'filter' as const, + tableId: params.tableId, + assertedWorkspaceId: + principal.kind === 'delegated' ? principal.workspaceId : body.workspaceId, + filter: body.filter!, + filterKeying: rowKeyingForPrincipal(principal), + limit: body.limit, }, - }) - } catch (error) { - if (isZodError(error)) { - return validationErrorResponse(error) - } - - if (error instanceof TableQueryValidationError) { - return NextResponse.json({ error: error.message }, { status: 400 }) - } - - const response = orchestrationErrorResponse(error) - if (response) return response - - logger.error(`[${requestId}] Error updating rows by filter:`, error) - return NextResponse.json({ error: 'Failed to update rows' }, { status: 500 }) - } - } -) - -/** DELETE /api/table/[tableId]/rows - Deletes rows matching filter criteria or by IDs. */ -export const DELETE = withRouteHandler( - async (request: NextRequest, { params }: TableRowsRouteParams) => { - const requestId = generateRequestId() - const { tableId } = await params - - try { - const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) - } - - // Bulk write bodies carry caller-supplied row data, so they can legitimately - // be large — but not unbounded. `parseJsonBody` applies the platform cap - // (413 past it) that a raw `request.json()` on this destructive surface skips. - const parsedBody = await parseJsonBody(request) - if (!parsedBody.success) return parsedBody.response - const body = parsedBody.data - - const validated = deleteTableRowsBodySchema.parse(body) - - const accessResult = await checkAccess(tableId, authResult.userId, 'write') - if (!accessResult.ok) return accessError(accessResult, requestId, tableId) - - const { table } = accessResult - - if (validated.workspaceId !== table.workspaceId) { - logger.warn( - `[${requestId}] Workspace ID mismatch for table ${tableId}. Provided: ${validated.workspaceId}, Actual: ${table.workspaceId}` - ) - return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) - } - - if (validated.rowIds) { - const result = await deleteRowsByIds( - table, - { tableId, rowIds: validated.rowIds, workspaceId: validated.workspaceId }, - requestId - ) - if (result.deletedCount > 0) signalTableRowsChanged(tableId) - - return NextResponse.json({ - success: true, + useCase: deleteTableRows, + present: (result) => + result.kind === 'ids' + ? { + success: true as const, data: { message: result.deletedCount === 0 @@ -601,138 +218,44 @@ export const DELETE = withRouteHandler( requestedCount: result.requestedCount, ...(result.missingRowIds.length > 0 ? { missingRowIds: result.missingRowIds } : {}), }, - }) - } - - const wire = rowWireTranslators(authResult.authType, table.schema as TableSchema) - const result = await deleteRowsByFilter( - table, - { - filter: resolveBulkFilter( - validated.filter as TablePredicate | Filter, - table.schema as TableSchema, - wire - ), - limit: validated.limit, - }, - requestId - ) - if (result.affectedCount > 0) signalTableRowsChanged(tableId) - - return NextResponse.json({ - success: true, - data: { - message: - result.affectedCount === 0 - ? 'No rows matched the filter criteria' - : 'Rows deleted successfully', - deletedCount: result.affectedCount, - deletedRowIds: result.affectedRowIds, - }, - }) - } catch (error) { - if (isZodError(error)) { - return validationErrorResponse(error) - } - - if (error instanceof TableQueryValidationError) { - return NextResponse.json({ error: error.message }, { status: 400 }) - } - - const response = orchestrationErrorResponse(error) - if (response) return response - - logger.error(`[${requestId}] Error deleting rows:`, error) - return NextResponse.json({ error: 'Failed to delete rows' }, { status: 500 }) - } - } -) - -/** PATCH /api/table/[tableId]/rows - Batch updates rows by ID. */ -export const PATCH = withRouteHandler( - async (request: NextRequest, { params }: TableRowsRouteParams) => { - const requestId = generateRequestId() - const { tableId } = await params - - try { - const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) - } - - // Bulk write bodies carry caller-supplied row data, so they can legitimately - // be large — but not unbounded. `parseJsonBody` applies the platform cap - // (413 past it) that a raw `request.json()` on this destructive surface skips. - const parsedBody = await parseJsonBody(request) - if (!parsedBody.success) return parsedBody.response - const body = parsedBody.data - - const validated = batchUpdateTableRowsBodySchema.parse(body) - - const accessResult = await checkAccess(tableId, authResult.userId, 'write') - if (!accessResult.ok) return accessError(accessResult, requestId, tableId) - - const { table } = accessResult - - if (validated.workspaceId !== table.workspaceId) { - logger.warn( - `[${requestId}] Workspace ID mismatch for table ${tableId}. Provided: ${validated.workspaceId}, Actual: ${table.workspaceId}` - ) - return NextResponse.json({ error: 'Invalid workspace ID' }, { status: 400 }) - } - - const wire = rowWireTranslators(authResult.authType, table.schema as TableSchema) - const sourceRows = validated.updates.map((update) => update.data as RowData) - const provenance = resolveTableWriteSecretProvenance({ - request, - payload: validated, - authType: authResult.authType, - userId: authResult.userId, - workspaceId: table.workspaceId, - targets: createTableWriteProvenanceTargets(sourceRows, wire.dataIn), - rowKeys: validated.updates.map((_, index) => String(index)), - }) - if (!provenance.success) return provenance.response - - const result = await batchUpdateRows( - { - tableId, - updates: validated.updates.map((update) => ({ - rowId: update.rowId, - data: wire.dataIn(update.data as RowData), - })), - workspaceId: validated.workspaceId, - actorUserId: authResult.userId, - secretProvenanceByRowId: Object.fromEntries( - validated.updates.flatMap((update, index) => { - const rowProvenance = provenance.provenanceByRowKey?.[String(index)] - return rowProvenance ? [[update.rowId, rowProvenance]] : [] - }) - ), - }, - table, - requestId - ) - signalTableRowsChanged(tableId) - - return NextResponse.json({ - success: true, - data: { - message: 'Rows updated successfully', - updatedCount: result.affectedCount, - updatedRowIds: result.affectedRowIds, + } + : { + success: true as const, + data: { + message: + result.affectedCount === 0 + ? 'No rows matched the filter criteria' + : 'Rows deleted successfully', + deletedCount: result.affectedCount, + deletedRowIds: result.affectedRowIds, + }, }, - }) - } catch (error) { - if (isZodError(error)) { - return validationErrorResponse(error) - } - - const response = orchestrationErrorResponse(error) - if (response) return response - - logger.error(`[${requestId}] Error batch updating rows:`, error) - return NextResponse.json({ error: 'Failed to update rows' }, { status: 500 }) - } - } -) +}) + +export const PATCH = defineInternalJsonRoute({ + contract: batchUpdateTableRowsContract, + operation: tableOperations.updateRows, + auth: internalTableSessionOrExecutorAuth, + rateLimit, + errorPolicy: rowErrorPolicy('Failed to update rows'), + mapInput: ({ params, body }, { principal, request }) => ({ + tableId: params.tableId, + assertedWorkspaceId: principal.kind === 'delegated' ? principal.workspaceId : body.workspaceId, + strictWrite: false, + dataKeying: rowKeyingForPrincipal(principal), + updates: body.updates.map((update) => ({ + rowId: update.rowId, + data: update.data as RowData, + })), + secretProvenanceEnvelope: readTableRowProvenanceEnvelope(request, body), + }), + useCase: batchUpdateTableRows, + present: ({ affectedCount, affectedRowIds }) => ({ + success: true as const, + data: { + message: 'Rows updated successfully', + updatedCount: affectedCount, + updatedRowIds: affectedRowIds, + }, + }), +}) diff --git a/apps/sim/app/api/table/[tableId]/rows/upsert/route.test.ts b/apps/sim/app/api/table/[tableId]/rows/upsert/route.test.ts index 2d07bd5e0d0..071321e7549 100644 --- a/apps/sim/app/api/table/[tableId]/rows/upsert/route.test.ts +++ b/apps/sim/app/api/table/[tableId]/rows/upsert/route.test.ts @@ -125,12 +125,28 @@ describe('POST /api/table/[tableId]/rows/upsert', () => { mocks.authenticate.mockResolvedValue({ kind: 'delegated', serviceId: 'executor', - subjectUserId: 'user-1', workspaceId: WORKSPACE_ID, delegationId: 'delegation-1', audience: 'table', issuedAt: new Date('2026-01-01'), - expiresAt: new Date('2026-01-02'), + expiresAt: new Date('2099-01-02'), + delegationContext: { + kind: 'workflow_execution', + workflowId: 'workflow-1', + currentWorkflow: { + workflowId: 'workflow-1', + mode: 'deployment', + deploymentVersionId: 'deployment-1', + }, + principal: { + kind: 'system', + serviceId: 'webhook', + workspaceId: WORKSPACE_ID, + workflowId: 'workflow-1', + webhookId: 'webhook-1', + provider: 'generic', + }, + }, }) await POST(request({ ...BODY, data: { Name: 'Ada' }, conflictTarget: 'Name' }), routeContext()) diff --git a/apps/sim/app/api/table/[tableId]/rows/upsert/route.ts b/apps/sim/app/api/table/[tableId]/rows/upsert/route.ts index d34559cb66f..b9939bb0d6a 100644 --- a/apps/sim/app/api/table/[tableId]/rows/upsert/route.ts +++ b/apps/sim/app/api/table/[tableId]/rows/upsert/route.ts @@ -25,7 +25,7 @@ export const POST = defineInternalJsonRoute({ errorPolicy: internalTableRowsErrorPolicy, mapInput: ({ params, body }, { principal, request }) => ({ tableId: params.tableId, - assertedWorkspaceId: body.workspaceId, + assertedWorkspaceId: principal.kind === 'delegated' ? principal.workspaceId : body.workspaceId, data: body.data as RowData, dataKeying: rowKeyingForPrincipal(principal), strictWrite: false, diff --git a/apps/sim/app/api/table/route.test.ts b/apps/sim/app/api/table/route.test.ts index 0932258551d..ce7c3e045bb 100644 --- a/apps/sim/app/api/table/route.test.ts +++ b/apps/sim/app/api/table/route.test.ts @@ -1,154 +1,185 @@ /** * @vitest-environment node */ -import { hybridAuthMockFns, permissionsMock, permissionsMockFns } from '@sim/testing' + import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockCreateTable, mockGetLimits, mockListTables, mockFindActiveFolder } = vi.hoisted(() => ({ - mockCreateTable: vi.fn(), - mockGetLimits: vi.fn(), - mockListTables: vi.fn(), - mockFindActiveFolder: vi.fn(), +const mocks = vi.hoisted(() => ({ + authenticate: vi.fn(), + createTable: vi.fn(), + listTables: vi.fn(), + capture: vi.fn(), })) -vi.mock('@/lib/table', () => ({ - createTable: mockCreateTable, - getWorkspaceTableLimits: mockGetLimits, - listTables: mockListTables, +vi.mock('@/lib/table/api', () => ({ + internalTableSessionOrExecutorAuth: { authenticate: mocks.authenticate }, })) -vi.mock('@/lib/folders/queries', () => ({ findActiveFolder: mockFindActiveFolder })) -vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock) -vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: vi.fn() })) -vi.mock('@sim/audit', () => ({ - AuditAction: { TABLE_CREATED: 'table.created' }, - AuditResourceType: { TABLE: 'table' }, - recordAudit: vi.fn(), + +vi.mock('@/lib/table/application/tables', () => ({ + createTableUseCase: { operation: { id: 'tables.create' }, execute: mocks.createTable }, + listTableDefinitionsUseCase: { operation: { id: 'tables.list' }, execute: mocks.listTables }, })) +vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: mocks.capture })) + import { GET, POST } from '@/app/api/table/route' -const CREATED_TABLE = { - id: 'tbl_1', +const TABLE = { + id: 'table-1', name: 'people', description: null, - schema: { columns: [{ name: 'name', type: 'string' }] }, + schema: { columns: [{ id: 'column-1', name: 'name', type: 'string' as const }] }, rowCount: 0, maxRows: 10_000, - folderId: null as string | null, + workspaceId: 'workspace-1', + folderId: null, + createdBy: 'user-1', locks: { schemaLocked: false, insertLocked: false, updateLocked: false, deleteLocked: false, }, + archivedAt: null, createdAt: new Date('2026-01-01T00:00:00.000Z'), updatedAt: new Date('2026-01-01T00:00:00.000Z'), } -function postRequest(body: unknown): NextRequest { - return new NextRequest('http://localhost:3000/api/table', { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify(body), +function sessionPrincipal() { + mocks.authenticate.mockResolvedValue({ + kind: 'session', + userId: 'user-1', + sessionId: 'session-1', }) } -const createBody = { - workspaceId: 'workspace-1', - name: 'people', - schema: { columns: [{ name: 'name', type: 'string' }] }, +function executorPrincipal() { + mocks.authenticate.mockResolvedValue({ + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'workspace-canonical', + delegationId: 'delegation-1', + audience: 'sim:tables', + issuedAt: new Date('2026-01-01'), + expiresAt: new Date('2026-01-02'), + }) } -describe('POST /api/table folder assignment', () => { - beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'session', - }) - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('write') - mockGetLimits.mockResolvedValue({ maxRowsPerTable: 1_000_000, maxTables: 50 }) - mockCreateTable.mockResolvedValue(CREATED_TABLE) - mockFindActiveFolder.mockResolvedValue({ id: 'folder-1' }) +function actorlessExecutorPrincipal() { + mocks.authenticate.mockResolvedValue({ + kind: 'delegated', + serviceId: 'executor', + workspaceId: 'workspace-canonical', + delegationId: 'delegation-1', + audience: 'sim:tables', + issuedAt: new Date('2026-01-01'), + expiresAt: new Date('2026-01-02'), + delegationContext: { + kind: 'workflow_execution', + workflowId: 'parent-workflow', + principal: { + kind: 'system', + serviceId: 'internal', + workspaceId: 'workspace-canonical', + workflowId: 'parent-workflow', + }, + }, }) +} - it('creates the table at the workspace root when no folder is given', async () => { - const response = await POST(postRequest(createBody)) +function post(body: unknown) { + return POST( + new NextRequest('http://localhost/api/table', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + }), + {} + ) +} - expect(response.status).toBe(200) - expect(mockFindActiveFolder).not.toHaveBeenCalled() - expect(mockCreateTable).toHaveBeenCalledWith( - expect.objectContaining({ folderId: null }), - expect.any(String) - ) +describe('/api/table application adapter', () => { + beforeEach(() => { + vi.clearAllMocks() + sessionPrincipal() + mocks.createTable.mockResolvedValue({ table: TABLE, folderPath: '/' }) + mocks.listTables.mockResolvedValue({ + tables: [TABLE], + }) }) - it('creates the table inside the requested folder', async () => { - mockCreateTable.mockResolvedValue({ ...CREATED_TABLE, folderId: 'folder-1' }) - - const response = await POST(postRequest({ ...createBody, folderId: 'folder-1' })) - const json = await response.json() + it('passes the session workspace and folder id into the shared create use case', async () => { + const response = await post({ + workspaceId: 'workspace-1', + name: 'people', + folderId: 'folder-1', + schema: { columns: [{ name: 'name', type: 'string' }] }, + }) expect(response.status).toBe(200) - expect(mockCreateTable).toHaveBeenCalledWith( - expect.objectContaining({ folderId: 'folder-1' }), - expect.any(String) - ) - expect(json.data.table.folderId).toBe('folder-1') + expect(mocks.createTable.mock.calls[0][0].input).toMatchObject({ + workspaceId: 'workspace-1', + folderId: 'folder-1', + }) + expect((await response.json()).data.table.folderId).toBeNull() }) - it('scopes the folder lookup to the workspace and the table folder tree', async () => { - await POST(postRequest({ ...createBody, folderId: 'folder-1' })) + it('uses canonical delegated workspace instead of the body assertion', async () => { + executorPrincipal() + await post({ + workspaceId: 'workspace-forged', + name: 'people', + schema: { columns: [{ name: 'name', type: 'string' }] }, + }) - expect(mockFindActiveFolder).toHaveBeenCalledWith('folder-1', 'workspace-1', 'table') + expect(mocks.createTable.mock.calls[0][0].input.workspaceId).toBe('workspace-canonical') }) - it('rejects a folder id that does not resolve in this workspace or tree', async () => { - mockFindActiveFolder.mockResolvedValue(null) + it('creates for an actorless executor without attributing user analytics', async () => { + actorlessExecutorPrincipal() - const response = await POST(postRequest({ ...createBody, folderId: 'kb-folder' })) + const response = await post({ + workspaceId: 'workspace-forged', + name: 'people', + schema: { columns: [{ name: 'name', type: 'string' }] }, + }) - expect(response.status).toBe(404) - expect(mockCreateTable).not.toHaveBeenCalled() + expect(response.status).toBe(200) + expect(mocks.createTable.mock.calls[0][0]).toMatchObject({ + principal: { + kind: 'delegated', + serviceId: 'executor', + workspaceId: 'workspace-canonical', + }, + input: { workspaceId: 'workspace-canonical' }, + }) + expect(mocks.capture).not.toHaveBeenCalled() }) - it('rejects an empty folder id before touching the database', async () => { - const response = await POST(postRequest({ ...createBody, folderId: '' })) + it('validates the contract before executing the create use case', async () => { + const response = await post({ + workspaceId: 'workspace-1', + name: 'people', + folderId: '', + schema: { columns: [{ name: 'name', type: 'string' }] }, + }) expect(response.status).toBe(400) - expect(mockFindActiveFolder).not.toHaveBeenCalled() - expect(mockCreateTable).not.toHaveBeenCalled() - }) -}) - -describe('GET /api/table folder placement', () => { - beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'session', - }) - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('read') + expect(mocks.createTable).not.toHaveBeenCalled() }) - it('emits each table folderId so the list can group rows by folder', async () => { - mockListTables.mockResolvedValue([ - { ...CREATED_TABLE, id: 'tbl_1', folderId: 'folder-1', workspaceId: 'ws', createdBy: 'u' }, - { ...CREATED_TABLE, id: 'tbl_2', folderId: null, workspaceId: 'ws', createdBy: 'u' }, - ]) - + it('lists through the shared use case and preserves the table list projection', async () => { const response = await GET( - new NextRequest('http://localhost:3000/api/table?workspaceId=workspace-1') + new NextRequest('http://localhost/api/table?workspaceId=workspace-1'), + {} ) - const json = await response.json() expect(response.status).toBe(200) - expect(json.data.tables.map((t: { folderId: string | null }) => t.folderId)).toEqual([ - 'folder-1', - null, - ]) + expect(mocks.listTables.mock.calls[0][0].input).toMatchObject({ + workspaceId: 'workspace-1', + }) + expect((await response.json()).data).toMatchObject({ totalCount: 1 }) }) }) diff --git a/apps/sim/app/api/table/route.ts b/apps/sim/app/api/table/route.ts index be28a064fd1..b7ba0880be1 100644 --- a/apps/sim/app/api/table/route.ts +++ b/apps/sim/app/api/table/route.ts @@ -1,211 +1,101 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { createTableContract, listTablesQuerySchema } from '@/lib/api/contracts/tables' -import { isZodError, parseRequest, validationErrorResponse } from '@/lib/api/server/validation' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { findActiveFolder } from '@/lib/folders/queries' -import { captureServerEvent } from '@/lib/posthog/server' +import { resolvePrincipalSubject } from '@sim/auth/principal' +import { createTableContract, listTablesContract } from '@/lib/api/contracts/tables' import { - createTable, - getWorkspaceTableLimits, - listTables, - type TableSchema, - type TableScope, -} from '@/lib/table' + defineInternalJsonRoute, + internalErrorResponse, + internalOrchestrationErrorPolicy, + internalRateLimits, +} from '@/lib/api/server/routes' +import { captureServerEvent } from '@/lib/posthog/server' +import { internalTableSessionOrExecutorAuth } from '@/lib/table/api' +import { tableOperations } from '@/lib/table/application/operations' +import { createTableUseCase, listTableDefinitionsUseCase } from '@/lib/table/application/tables' import { normalizeColumn, toTableListItem, toWireTimestamp } from '@/lib/table/wire' -import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' -import { orchestrationErrorResponse } from '@/app/api/table/utils' -const logger = createLogger('TableAPI') +const rateLimit = internalRateLimits.none({ + reason: 'Preserve existing internal table list and create behavior', +}) -interface WorkspaceAccessResult { - hasAccess: boolean - canWrite: boolean +const createErrorPolicy = { + ...internalOrchestrationErrorPolicy, + unhandled: () => internalErrorResponse(500, { error: 'Failed to create table' }), } -async function checkWorkspaceAccess( - workspaceId: string, - userId: string -): Promise { - const permission = await getUserEntityPermissions(userId, 'workspace', workspaceId) - - if (permission === null) { - return { hasAccess: false, canWrite: false } - } - - const canWrite = permission === 'admin' || permission === 'write' - return { hasAccess: true, canWrite } +const listErrorPolicy = { + ...internalOrchestrationErrorPolicy, + unhandled: () => internalErrorResponse(500, { error: 'Failed to list tables' }), } -/** POST /api/table - Creates a new user-defined table. */ -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) - } - - const parsed = await parseRequest( - createTableContract, - request, - {}, - { - validationErrorResponse: (error) => validationErrorResponse(error), - } - ) - if (!parsed.success) return parsed.response - - const params = parsed.data.body - - const { hasAccess, canWrite } = await checkWorkspaceAccess( - params.workspaceId, - authResult.userId - ) - - if (!hasAccess || !canWrite) { - return NextResponse.json({ error: 'Access denied' }, { status: 403 }) - } - - // Scoped to `resourceType: 'table'` so a folder id belonging to another - // resource's tree can't file a table where its own surface will never list it. - if ( - params.folderId && - !(await findActiveFolder(params.folderId, params.workspaceId, 'table')) - ) { - return NextResponse.json({ error: 'Folder not found in this workspace' }, { status: 404 }) - } - - const planLimits = await getWorkspaceTableLimits(params.workspaceId) - - const normalizedSchema: TableSchema = { - columns: params.schema.columns.map(normalizeColumn), - } - - const table = await createTable( - { - name: params.name, - description: params.description, - schema: normalizedSchema, - workspaceId: params.workspaceId, - folderId: params.folderId ?? null, - userId: authResult.userId, - maxTables: planLimits.maxTables, - initialRowCount: params.initialRowCount, - }, - requestId - ) - +export const POST = defineInternalJsonRoute({ + contract: createTableContract, + operation: tableOperations.create, + auth: internalTableSessionOrExecutorAuth, + rateLimit, + errorPolicy: createErrorPolicy, + mapInput: ({ body }, { principal }) => ({ + workspaceId: principal.kind === 'delegated' ? principal.workspaceId : body.workspaceId, + name: body.name, + description: body.description, + schema: { columns: body.schema.columns.map(normalizeColumn) }, + folderId: body.folderId, + initialRowCount: body.initialRowCount, + }), + useCase: createTableUseCase, + onSuccess: ({ principal, result }) => { + const subject = resolvePrincipalSubject(principal) + if (subject?.kind !== 'sim_user') return captureServerEvent( - authResult.userId, + subject.userId, 'table_created', { - table_id: table.id, - workspace_id: params.workspaceId, - column_count: params.schema.columns.length, + table_id: result.table.id, + workspace_id: result.table.workspaceId, + column_count: result.table.schema.columns.length, }, { - groups: { workspace: params.workspaceId }, + groups: { workspace: result.table.workspaceId }, setOnce: { first_table_created_at: new Date().toISOString() }, } ) - - recordAudit({ - workspaceId: params.workspaceId, - actorId: authResult.userId, - actorName: authResult.userName ?? undefined, - actorEmail: authResult.userEmail ?? undefined, - action: AuditAction.TABLE_CREATED, - resourceType: AuditResourceType.TABLE, - resourceId: table.id, - resourceName: table.name, - description: `Created table "${table.name}"`, - request, - }) - - return NextResponse.json({ - success: true, - data: { - table: { - id: table.id, - name: table.name, - description: table.description, - schema: { - columns: (table.schema as TableSchema).columns.map(normalizeColumn), - }, - rowCount: table.rowCount, - maxRows: table.maxRows, - folderId: table.folderId ?? null, - locks: table.locks, - createdAt: toWireTimestamp(table.createdAt), - updatedAt: toWireTimestamp(table.updatedAt), - }, - message: 'Table created successfully', + }, + present: ({ table }) => ({ + success: true as const, + data: { + table: { + id: table.id, + name: table.name, + description: table.description, + schema: { columns: table.schema.columns.map(normalizeColumn) }, + rowCount: table.rowCount, + maxRows: table.maxRows, + folderId: table.folderId ?? null, + locks: table.locks, + workspaceId: table.workspaceId, + createdBy: table.createdBy, + createdAt: toWireTimestamp(table.createdAt), + updatedAt: toWireTimestamp(table.updatedAt), }, - }) - } catch (error) { - const classified = orchestrationErrorResponse(error) - if (classified) return classified - - logger.error(`[${requestId}] Error creating table:`, error) - return NextResponse.json({ error: 'Failed to create table' }, { status: 500 }) - } + message: 'Table created successfully', + }, + }), }) -/** GET /api/table - Lists all tables in a workspace. */ -export const GET = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json({ error: 'Authentication required' }, { status: 401 }) - } - - const { searchParams } = new URL(request.url) - const workspaceId = searchParams.get('workspaceId') - const scope = searchParams.get('scope') - - const validation = listTablesQuerySchema.safeParse({ - workspaceId, - scope: scope ?? undefined, - }) - if (!validation.success) { - return NextResponse.json( - { error: 'Validation error', details: validation.error.issues }, - { status: 400 } - ) - } - - const params = validation.data - - const { hasAccess } = await checkWorkspaceAccess(params.workspaceId, authResult.userId) - - if (!hasAccess) { - return NextResponse.json({ error: 'Access denied' }, { status: 403 }) - } - - const tables = await listTables(params.workspaceId, { scope: params.scope as TableScope }) - - logger.info(`[${requestId}] Listed ${tables.length} tables in workspace ${params.workspaceId}`) - - return NextResponse.json({ - success: true, - data: { - tables: tables.map(toTableListItem), - totalCount: tables.length, - }, - }) - } catch (error) { - if (isZodError(error)) { - return validationErrorResponse(error) - } - - logger.error(`[${requestId}] Error listing tables:`, error) - return NextResponse.json({ error: 'Failed to list tables' }, { status: 500 }) - } +export const GET = defineInternalJsonRoute({ + contract: listTablesContract, + operation: tableOperations.list, + auth: internalTableSessionOrExecutorAuth, + rateLimit, + errorPolicy: listErrorPolicy, + mapInput: ({ query }, { principal }) => ({ + workspaceId: principal.kind === 'delegated' ? principal.workspaceId : query.workspaceId, + scope: query.scope, + }), + useCase: listTableDefinitionsUseCase, + present: ({ tables }) => ({ + success: true as const, + data: { + tables: tables.map(toTableListItem), + totalCount: tables.length, + }, + }), }) diff --git a/apps/sim/app/api/table/row-wire.ts b/apps/sim/app/api/table/row-wire.ts index b20121a621e..880f5f5695e 100644 --- a/apps/sim/app/api/table/row-wire.ts +++ b/apps/sim/app/api/table/row-wire.ts @@ -121,6 +121,24 @@ export function presentRowForPrincipal( } } +export function presentQueryRowForPrincipal( + row: TableRow, + schema: TableSchema, + principal: TableRowRoutePrincipal +) { + const dataOut = + rowKeyingForPrincipal(principal) === 'names' ? namedRowMapper(schema.columns) : identity + return { + id: row.id, + data: dataOut(row.data), + executions: row.executions, + position: row.position, + orderKey: row.orderKey ?? undefined, + createdAt: toWireTimestamp(row.createdAt), + updatedAt: toWireTimestamp(row.updatedAt), + } +} + function identity(value: T): T { return value } diff --git a/apps/sim/app/api/table/table-tool-auth.test.ts b/apps/sim/app/api/table/table-tool-auth.test.ts index 9f2a6a3bede..6766d18ed60 100644 --- a/apps/sim/app/api/table/table-tool-auth.test.ts +++ b/apps/sim/app/api/table/table-tool-auth.test.ts @@ -1,17 +1,11 @@ /** * @vitest-environment node * - * The executor reaches the internal table row routes through the Table block's - * tools, and those routes now authenticate with the delegation policy rather - * than the legacy internal token. Two things have to line up for that to work, - * and neither is visible to a route test that mocks the auth policy: + * Table block tools execute through the in-process operation boundary. Two + * things have to line up for that to work: * - * 1. the tool must ask the executor to mint a delegation token, and + * 1. the tool must declare an in-process operation, and * 2. the operation's policy must admit the `executor` delegated service. - * - * Both are pinned here because getting either wrong fails every workflow call - * to these endpoints — the first as a 401, the second as a 403 — while every - * route-level test keeps passing. */ import { describe, expect, it } from 'vitest' import { tableOperations } from '@/lib/table/application/operations' @@ -29,10 +23,9 @@ const EXECUTOR_ROW_TOOLS = [ ] as const describe('executor access to the migrated table row routes', () => { - it.each(EXECUTOR_ROW_TOOLS)('%s asks the executor for a delegation token', (_name, tool) => { - // Without this the executor mints a legacy internal token, which the - // delegation policy rejects outright. - expect(tool.request.internalAuth).toBe('executor_delegation') + it.each(EXECUTOR_ROW_TOOLS)('%s declares an in-process operation', (_name, tool) => { + expect(tool.request).toBeUndefined() + expect(tool.operation.input).toBeTypeOf('function') }) it.each(EXECUTOR_ROW_TOOLS)( diff --git a/apps/sim/app/api/tools/a2a/cancel-task/route.ts b/apps/sim/app/api/tools/a2a/cancel-task/route.ts deleted file mode 100644 index c65f6381f09..00000000000 --- a/apps/sim/app/api/tools/a2a/cancel-task/route.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { createA2AClient, taskOutput } from '@/lib/a2a/client' -import { a2aCancelTaskContract } from '@/lib/api/contracts/tools/a2a' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { enforceUserOrIpRateLimit } from '@/lib/core/rate-limiter' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' -export const maxDuration = 60 - -const logger = createLogger('A2ACancelTaskAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success) { - return NextResponse.json( - { success: false, error: auth.error || 'Authentication required' }, - { status: 401 } - ) - } - - const rateLimited = await enforceUserOrIpRateLimit('a2a-cancel-task', auth.userId, request) - if (rateLimited) return rateLimited - - const parsed = await parseRequest( - a2aCancelTaskContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Invalid request data') }, - { status: 400 } - ), - } - ) - if (!parsed.success) return parsed.response - const body = parsed.data.body - - try { - const client = await createA2AClient(body.agentUrl, body.apiKey, { signal: request.signal }) - const task = await client.cancelTask({ tenant: '', id: body.taskId, metadata: undefined }) - const out = taskOutput(task) - - logger.info(`[${requestId}] Cancel requested for A2A task ${task.id}`) - return NextResponse.json({ - success: true, - output: { taskId: out.taskId, state: out.state, canceled: out.state === 'canceled' }, - }) - } catch (error) { - logger.error(`[${requestId}] A2A cancel-task failed`, { error: getErrorMessage(error) }) - return NextResponse.json({ success: false, error: getErrorMessage(error) }, { status: 502 }) - } -}) diff --git a/apps/sim/app/api/tools/a2a/get-agent-card/route.ts b/apps/sim/app/api/tools/a2a/get-agent-card/route.ts deleted file mode 100644 index 52a8c055446..00000000000 --- a/apps/sim/app/api/tools/a2a/get-agent-card/route.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { agentCardOutput, createA2AClient } from '@/lib/a2a/client' -import { a2aGetAgentCardContract } from '@/lib/api/contracts/tools/a2a' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { enforceUserOrIpRateLimit } from '@/lib/core/rate-limiter' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' -export const maxDuration = 60 - -const logger = createLogger('A2AGetAgentCardAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success) { - return NextResponse.json( - { success: false, error: auth.error || 'Authentication required' }, - { status: 401 } - ) - } - - const rateLimited = await enforceUserOrIpRateLimit('a2a-get-agent-card', auth.userId, request) - if (rateLimited) return rateLimited - - const parsed = await parseRequest( - a2aGetAgentCardContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Invalid request data') }, - { status: 400 } - ), - } - ) - if (!parsed.success) return parsed.response - const body = parsed.data.body - - try { - const client = await createA2AClient(body.agentUrl, body.apiKey, { signal: request.signal }) - const card = await client.getAgentCard() - - logger.info(`[${requestId}] Fetched agent card for ${card.name}`) - return NextResponse.json({ success: true, output: agentCardOutput(card, body.agentUrl) }) - } catch (error) { - logger.error(`[${requestId}] A2A get-agent-card failed`, { error: getErrorMessage(error) }) - return NextResponse.json({ success: false, error: getErrorMessage(error) }, { status: 502 }) - } -}) diff --git a/apps/sim/app/api/tools/a2a/get-task/route.ts b/apps/sim/app/api/tools/a2a/get-task/route.ts deleted file mode 100644 index 2e955c8834f..00000000000 --- a/apps/sim/app/api/tools/a2a/get-task/route.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { createA2AClient, taskOutput } from '@/lib/a2a/client' -import { a2aGetTaskContract } from '@/lib/api/contracts/tools/a2a' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { enforceUserOrIpRateLimit } from '@/lib/core/rate-limiter' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' -export const maxDuration = 60 - -const logger = createLogger('A2AGetTaskAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success) { - return NextResponse.json( - { success: false, error: auth.error || 'Authentication required' }, - { status: 401 } - ) - } - - const rateLimited = await enforceUserOrIpRateLimit('a2a-get-task', auth.userId, request) - if (rateLimited) return rateLimited - - const parsed = await parseRequest( - a2aGetTaskContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Invalid request data') }, - { status: 400 } - ), - } - ) - if (!parsed.success) return parsed.response - const body = parsed.data.body - - try { - const client = await createA2AClient(body.agentUrl, body.apiKey, { signal: request.signal }) - const task = await client.getTask({ - tenant: '', - id: body.taskId, - historyLength: body.historyLength, - }) - - logger.info(`[${requestId}] Retrieved A2A task ${task.id}`) - return NextResponse.json({ success: true, output: taskOutput(task) }) - } catch (error) { - logger.error(`[${requestId}] A2A get-task failed`, { error: getErrorMessage(error) }) - return NextResponse.json({ success: false, error: getErrorMessage(error) }, { status: 502 }) - } -}) diff --git a/apps/sim/app/api/tools/a2a/send-message/route.ts b/apps/sim/app/api/tools/a2a/send-message/route.ts deleted file mode 100644 index 1be1ae0f1b5..00000000000 --- a/apps/sim/app/api/tools/a2a/send-message/route.ts +++ /dev/null @@ -1,170 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { - type A2AFileInput, - buildUserMessage, - createA2AClient, - isTaskResult, - messageOutput, - taskErrored, - taskOutput, -} from '@/lib/a2a/client' -import { a2aSendMessageContract } from '@/lib/api/contracts/tools/a2a' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { AuthType, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { enforceUserOrIpRateLimit } from '@/lib/core/rate-limiter' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' -import { - isModelSafeWorkspaceFileKey, - MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, -} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' -import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' - -export const dynamic = 'force-dynamic' - -/** Blocking sends wait until the agent reaches a terminal/interrupted state. */ -export const maxDuration = 300 - -/** Per-file cap on attachments resolved from storage. */ -const A2A_MAX_FILE_BYTES = 10 * 1024 * 1024 - -const logger = createLogger('A2ASendMessageAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success) { - return NextResponse.json( - { success: false, error: auth.error || 'Authentication required' }, - { status: 401 } - ) - } - - const rateLimited = await enforceUserOrIpRateLimit('a2a-send-message', auth.userId, request) - if (rateLimited) return rateLimited - - const parsed = await parseRequest( - a2aSendMessageContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Invalid request data') }, - { status: 400 } - ), - } - ) - if (!parsed.success) return parsed.response - const body = parsed.data.body - const modelInputProvenance = validateOpaqueModelInputProvenance({ - headers: request.headers, - payload: body, - isInternalRequest: auth.authType === AuthType.INTERNAL_JWT, - }) - if (!modelInputProvenance.success) { - return NextResponse.json( - { success: false, error: modelInputProvenance.error }, - { status: modelInputProvenance.status } - ) - } - - let data: unknown - if (body.data !== undefined) { - if (typeof body.data === 'string') { - try { - data = JSON.parse(body.data) - } catch { - return NextResponse.json( - { success: false, error: 'Data must be valid JSON' }, - { status: 400 } - ) - } - } else { - data = body.data - } - } - - try { - let files: A2AFileInput[] | undefined - if (body.files?.length) { - if (!auth.userId) { - return NextResponse.json( - { success: false, error: 'Authentication required to attach files' }, - { status: 401 } - ) - } - const userFiles = processFilesToUserFiles(body.files, requestId, logger) - for (const userFile of userFiles) { - const denied = await assertToolFileAccess(userFile.key, auth.userId, requestId, logger) - if (denied) return denied - if (!(await isModelSafeWorkspaceFileKey(userFile.key))) { - return NextResponse.json( - { - success: false, - error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, - }, - { status: 400 } - ) - } - } - files = await Promise.all( - userFiles.map(async (userFile) => { - const { buffer, contentType } = await downloadServableFileFromStorage( - userFile, - requestId, - logger, - { maxBytes: A2A_MAX_FILE_BYTES } - ) - return { - bytes: buffer, - name: userFile.name, - mediaType: contentType || userFile.type || 'application/octet-stream', - } - }) - ) - } - - const client = await createA2AClient(body.agentUrl, body.apiKey, { signal: request.signal }) - const message = buildUserMessage({ - text: body.message, - data, - files, - taskId: body.taskId, - contextId: body.contextId, - }) - - const result = await client.sendMessage({ - tenant: '', - message, - configuration: undefined, - metadata: undefined, - }) - - if (!isTaskResult(result)) { - logger.info(`[${requestId}] A2A send returned a direct message`) - return NextResponse.json({ success: true, output: messageOutput(result) }) - } - - const output = taskOutput(result) - const errored = taskErrored(result) - logger.info(`[${requestId}] A2A send produced task ${result.id} (${output.state})`) - return NextResponse.json({ - success: !errored, - ...(errored ? { error: output.content || `Agent task ${output.state}` } : {}), - output, - }) - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - logger.error(`[${requestId}] A2A send failed`, { error: getErrorMessage(error) }) - return NextResponse.json({ success: false, error: getErrorMessage(error) }, { status: 502 }) - } -}) diff --git a/apps/sim/app/api/tools/agiloft/async_status/route.ts b/apps/sim/app/api/tools/agiloft/async_status/route.ts deleted file mode 100644 index f034d4a0bf6..00000000000 --- a/apps/sim/app/api/tools/agiloft/async_status/route.ts +++ /dev/null @@ -1,96 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { agiloftAsyncStatusContract } from '@/lib/api/contracts/tools/agiloft' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { AgiloftAsyncStatusResponse } from '@/tools/agiloft/types' -import { AGILOFT_ASYNC_STATUS, buildAsyncStatusUrl } from '@/tools/agiloft/utils' -import { executeEwRequest } from '@/tools/agiloft/utils.server' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AgiloftAsyncStatusAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Agiloft async_status attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - agiloftAsyncStatusContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const result = await executeEwRequest( - params, - (base) => ({ url: buildAsyncStatusUrl(base, params), method: 'GET' }), - async (response) => { - /** - * EWAsyncStatus communicates entirely through the status code and - * returns an empty body, so the code is the result rather than an - * error signal — 501 means the async operation failed, not that the - * status check did. - */ - const known = AGILOFT_ASYNC_STATUS[response.status] - - if (!known) { - const body = await response.text() - return { - success: false, - output: { - callbackId: params.callbackId.trim(), - statusCode: response.status, - status: 'unrecognized', - complete: false, - }, - error: `Agiloft returned an unrecognized async status ${response.status}: ${body.trim() || '(empty response)'}`, - } - } - - return { - success: true, - output: { - callbackId: params.callbackId.trim(), - statusCode: response.status, - status: known.status, - complete: known.complete, - }, - } - } - ) - - return NextResponse.json(result) - } catch (error) { - logger.error(`[${requestId}] Error checking Agiloft async status:`, error) - - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/agiloft/attach/route.test.ts b/apps/sim/app/api/tools/agiloft/attach/route.test.ts deleted file mode 100644 index a5db37f2079..00000000000 --- a/apps/sim/app/api/tools/agiloft/attach/route.test.ts +++ /dev/null @@ -1,146 +0,0 @@ -/** - * @vitest-environment node - */ -import { - createMockRequest, - hybridAuthMockFns, - inputValidationMock, - inputValidationMockFns, -} from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -/** Obvious non-secret so credential scanners do not flag these fixtures. */ -const PLACEHOLDER_PASSWORD = 'not-a-real-password' - -const { mockProcessFilesToUserFiles, mockDownloadFileFromStorage, mockAssertToolFileAccess } = - vi.hoisted(() => ({ - mockProcessFilesToUserFiles: vi.fn(), - mockDownloadFileFromStorage: vi.fn(), - mockAssertToolFileAccess: vi.fn(), - })) - -vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) -vi.mock('@/lib/uploads/utils/file-utils', () => ({ - processFilesToUserFiles: mockProcessFilesToUserFiles, -})) -vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ - downloadServableFileFromStorage: mockDownloadFileFromStorage, -})) -vi.mock('@/app/api/files/authorization', () => ({ - assertToolFileAccess: mockAssertToolFileAccess, -})) - -import { POST } from '@/app/api/tools/agiloft/attach/route' - -const PINNED_IP = '93.184.216.34' - -const baseBody = { - instanceUrl: 'https://example.agiloft.com', - knowledgeBase: 'demo', - login: 'admin', - password: PLACEHOLDER_PASSWORD, - table: 'contracts', - recordId: '42', - fieldName: 'attachments', - file: { key: 's3://bucket/file.txt', name: 'file.txt', size: 5, type: 'text/plain' }, - fileName: 'file.txt', -} - -function mockSecureFetchResponse(body: { - ok?: boolean - status?: number - json?: unknown - text?: string -}) { - return { - ok: body.ok ?? true, - status: body.status ?? 200, - statusText: '', - headers: new Headers(), - body: null, - text: async () => body.text ?? JSON.stringify(body.json ?? {}), - json: async () => body.json ?? {}, - arrayBuffer: async () => new ArrayBuffer(0), - } -} - -beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ - isValid: true, - resolvedIP: PINNED_IP, - originalHostname: 'example.agiloft.com', - }) - mockProcessFilesToUserFiles.mockReturnValue([ - { key: 's3://bucket/file.txt', name: 'file.txt', size: 5, type: 'text/plain' }, - ]) - mockAssertToolFileAccess.mockResolvedValue(null) - mockDownloadFileFromStorage.mockResolvedValue({ - buffer: Buffer.from('hello'), - contentType: 'application/octet-stream', - }) -}) - -describe('POST /api/tools/agiloft/attach', () => { - it('rejects unauthenticated requests', async () => { - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValueOnce({ - success: false, - error: 'unauthorized', - }) - - const response = await POST(createMockRequest('POST', baseBody)) - expect(response.status).toBe(401) - expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() - }) - - it('blocks SSRF when the instance URL fails DNS validation', async () => { - inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValueOnce({ - isValid: false, - error: 'instanceUrl resolves to a blocked IP address', - }) - - const response = await POST( - createMockRequest('POST', { ...baseBody, instanceUrl: 'https://attacker.example.com' }) - ) - - expect(response.status).toBe(400) - expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() - }) - - it('attaches with inline credentials on the pinned IP, with no login round trip', async () => { - /** Documented response: EWREST_.length='1'; */ - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( - mockSecureFetchResponse({ text: "EWREST_attachments.length='1';" }) - ) - - const response = await POST(createMockRequest('POST', baseBody)) - expect(response.status).toBe(200) - const data = (await response.json()) as { - success: true - output: { totalAttachments: number; fileName: string } - } - expect(data.output.totalAttachments).toBe(1) - expect(data.output.fileName).toBe('file.txt') - - const calls = inputValidationMockFns.mockSecureFetchWithPinnedIP.mock.calls - expect(calls).toHaveLength(1) - expect(calls[0][1]).toBe(PINNED_IP) - - expect(calls[0][0]).toContain('https://example.agiloft.com/ewws/EWAttach') - expect(calls[0][0]).toContain('&$login=admin') - expect(calls[0][2]).toMatchObject({ - method: 'PUT', - headers: { 'Content-Type': 'application/octet-stream' }, - }) - // A bearer token on this surface is rejected; it must not be sent. - expect(calls[0][2].headers.Authorization).toBeUndefined() - - // DNS only resolved once. - expect(inputValidationMockFns.mockValidateUrlWithDNS).toHaveBeenCalledTimes(1) - }) -}) diff --git a/apps/sim/app/api/tools/agiloft/attach/route.ts b/apps/sim/app/api/tools/agiloft/attach/route.ts deleted file mode 100644 index a9f0ce22ac6..00000000000 --- a/apps/sim/app/api/tools/agiloft/attach/route.ts +++ /dev/null @@ -1,187 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { agiloftAttachContract } from '@/lib/api/contracts/tools/agiloft' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { secureFetchWithPinnedIP } from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' -import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { parseEwRest } from '@/tools/agiloft/ewrest' -import { buildAttachFileUrl, describeAgiloftError } from '@/tools/agiloft/utils' -import { resolveAgiloftInstance } from '@/tools/agiloft/utils.server' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AgiloftAttachAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Agiloft attach attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - agiloftAttachContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - if (!data.file) { - return NextResponse.json({ success: false, error: 'File is required' }, { status: 400 }) - } - - const userFiles = processFilesToUserFiles([data.file as RawFileInput], requestId, logger) - - if (userFiles.length === 0) { - return NextResponse.json({ success: false, error: 'Invalid file input' }, { status: 400 }) - } - - const userFile = userFiles[0] - logger.info( - `[${requestId}] Downloading file for Agiloft attach: ${userFile.name} (${userFile.size} bytes)` - ) - - const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) - if (denied) return denied - - let fileBuffer: Buffer - try { - const servable = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - }) - fileBuffer = servable.buffer - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - logger.error(`[${requestId}] Failed to download file from storage:`, error) - return NextResponse.json( - { success: false, error: toError(error).message }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } - - const resolvedFileName = data.fileName || userFile.name || 'attachment' - - let resolvedIP: string - try { - resolvedIP = await resolveAgiloftInstance(data.instanceUrl) - } catch (error) { - logger.warn(`[${requestId}] SSRF attempt blocked for Agiloft instance URL`, { - instanceUrl: data.instanceUrl, - }) - return NextResponse.json({ success: false, error: toError(error).message }, { status: 400 }) - } - - const base = data.instanceUrl.replace(/\/$/, '') - - { - /** - * EWAttach lives on the legacy surface, which authenticates from the - * inline credentials in the URL and rejects a bearer token — so there is - * no login/logout pair here. - */ - const url = buildAttachFileUrl(base, data, resolvedFileName) - - logger.info(`[${requestId}] Uploading file to Agiloft: ${resolvedFileName}`) - - const agiloftResponse = await secureFetchWithPinnedIP(url, resolvedIP, { - method: 'PUT', - headers: { - 'Content-Type': 'application/octet-stream', - }, - body: new Uint8Array(fileBuffer), - }) - - if (!agiloftResponse.ok) { - const errorText = await agiloftResponse.text() - logger.error( - `[${requestId}] Agiloft attach error: ${agiloftResponse.status} - ${errorText}` - ) - return NextResponse.json( - { - success: false, - error: `Agiloft error ${agiloftResponse.status}: ${describeAgiloftError(errorText)}`, - }, - { status: agiloftResponse.status } - ) - } - - /** - * EWAttach reports the new file count against the field it wrote to, as - * EWREST_.length='1'; — the key is field-specific, so the - * single returned assignment is read rather than a fixed key. - */ - const responseText = await agiloftResponse.text() - const assignments = parseEwRest(responseText) - const countRaw = - assignments.get(`${data.fieldName.trim()}.length`) ?? [...assignments.values()][0] - const totalAttachments = Number(countRaw) - - /** - * A 200 with no parsable count is not a confirmed write, so it fails - * closed rather than reporting success with zero attachments. - */ - if (!Number.isFinite(totalAttachments)) { - logger.error(`[${requestId}] Agiloft attach returned an unrecognised body`, { - body: responseText.slice(0, 200), - }) - return NextResponse.json( - { - success: false, - error: `Agiloft did not confirm the attachment: ${describeAgiloftError(responseText) || '(empty response)'}`, - }, - { status: 502 } - ) - } - - logger.info( - `[${requestId}] File attached successfully. Total attachments: ${totalAttachments}` - ) - - return NextResponse.json({ - success: true, - output: { - recordId: data.recordId.trim(), - fieldName: data.fieldName.trim(), - fileName: resolvedFileName, - totalAttachments, - }, - }) - } - } catch (error) { - logger.error(`[${requestId}] Error attaching file to Agiloft:`, error) - - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/agiloft/attachment_info/route.ts b/apps/sim/app/api/tools/agiloft/attachment_info/route.ts deleted file mode 100644 index 3404758ad1b..00000000000 --- a/apps/sim/app/api/tools/agiloft/attachment_info/route.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { agiloftAttachmentInfoContract } from '@/lib/api/contracts/tools/agiloft' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { AgiloftAttachmentInfoResponse } from '@/tools/agiloft/types' -import { buildAttachmentInfoUrl, describeAgiloftError } from '@/tools/agiloft/utils' -import { executeEwRequest } from '@/tools/agiloft/utils.server' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AgiloftAttachmentInfoAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn( - `[${requestId}] Unauthorized Agiloft attachment_info attempt: ${authResult.error}` - ) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - agiloftAttachmentInfoContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const result = await executeEwRequest( - params, - (base) => ({ - url: buildAttachmentInfoUrl(base, params), - method: 'GET', - }), - async (response) => { - if (!response.ok) { - const errorText = await response.text() - return { - success: false, - output: { attachments: [], totalCount: 0 }, - error: `Agiloft error ${response.status}: ${describeAgiloftError(errorText)}`, - } - } - - const data = (await response.json()) as Record - const result = (data.result ?? data) as Record - - const attachments: Array<{ position: number; name: string; size: number }> = [] - - if (Array.isArray(result)) { - for (let i = 0; i < result.length; i++) { - const item = result[i] as Record - attachments.push({ - position: (item.filePosition as number) ?? (item.position as number) ?? i, - name: - (item.fileName as string) ?? - (item.name as string) ?? - (item.filename as string) ?? - '', - size: (item.size as number) ?? (item.fileSize as number) ?? 0, - }) - } - } - - return { - success: data.success !== false, - output: { - attachments, - totalCount: attachments.length, - }, - } - } - ) - - return NextResponse.json(result) - } catch (error) { - logger.error(`[${requestId}] Error getting Agiloft attachment info:`, error) - - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/agiloft/create_record/route.test.ts b/apps/sim/app/api/tools/agiloft/create_record/route.test.ts deleted file mode 100644 index 4e8f8e98718..00000000000 --- a/apps/sim/app/api/tools/agiloft/create_record/route.test.ts +++ /dev/null @@ -1,631 +0,0 @@ -/** - * @vitest-environment node - */ -import { - createMockRequest, - hybridAuthMockFns, - inputValidationMock, - inputValidationMockFns, -} from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) - -import { POST } from '@/app/api/tools/agiloft/create_record/route' -import { POST as READ } from '@/app/api/tools/agiloft/read_record/route' -import { POST as SEARCH } from '@/app/api/tools/agiloft/search_records/route' - -/** Obvious non-secret so credential scanners do not flag these fixtures. */ -const PLACEHOLDER_PASSWORD = 'not-a-real-password' - -const PINNED_IP = '93.184.216.34' - -const baseBody = { - instanceUrl: 'https://example.agiloft.com', - knowledgeBase: 'Russell Investments', - login: 'svc.user', - password: PLACEHOLDER_PASSWORD, - table: 'contract', -} - -function res(body: { ok?: boolean; status?: number; json?: unknown; text?: string }) { - const text = body.text ?? JSON.stringify(body.json ?? {}) - return { - ok: body.ok ?? true, - status: body.status ?? 200, - statusText: '', - headers: new Headers(), - body: null, - text: async () => text, - json: async () => JSON.parse(text), - arrayBuffer: async () => new ArrayBuffer(0), - } -} - -/** Login envelope Agiloft returns — note the trailing space on the scheme. */ -const LOGIN_OK = res({ - json: { access_token: 'tok-123', authentication_scheme: 'Bearer ' }, -}) - -beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ - isValid: true, - resolvedIP: PINNED_IP, - originalHostname: 'example.agiloft.com', - }) -}) - -function arrange(operationResponse: ReturnType) { - inputValidationMockFns.mockSecureFetchWithPinnedIP - .mockResolvedValueOnce(LOGIN_OK) - .mockResolvedValueOnce(operationResponse) - .mockResolvedValueOnce(res({})) -} - -describe('EWLogin', () => { - it('sends $table and $lang alongside $KB in a form body, which the live server requires', async () => { - arrange(res({ json: { success: true, result: { id: 6342 } } })) - - await POST(createMockRequest('POST', { ...baseBody, data: '{"a":"b"}' })) - - const [url, ip, init] = inputValidationMockFns.mockSecureFetchWithPinnedIP.mock.calls[0] - expect(url).toBe('https://example.agiloft.com/ewws/EWLogin') - expect(ip).toBe(PINNED_IP) - expect(init.method).toBe('POST') - expect(init.headers['Content-Type']).toBe('application/x-www-form-urlencoded') - - const sent = new URLSearchParams(init.body as string) - expect(sent.get('$KB')).toBe('Russell Investments') - expect(sent.get('$table')).toBe('contract') - expect(sent.get('$lang')).toBe('en') - expect(sent.get('$login')).toBe('svc.user') - expect(sent.get('$password')).toBe(PLACEHOLDER_PASSWORD) - // Credentials must not leak into the URL. - expect(url).not.toContain(PLACEHOLDER_PASSWORD) - }) - - it('trims the trailing space Agiloft puts on authentication_scheme', async () => { - arrange(res({ json: { success: true, result: { id: 6342 } } })) - - await POST(createMockRequest('POST', { ...baseBody, data: '{"a":"b"}' })) - - const [, , init] = inputValidationMockFns.mockSecureFetchWithPinnedIP.mock.calls[1] - expect(init.headers.Authorization).toBe('Bearer tok-123') - }) - - it('surfaces the live "One has to specify" refusal instead of a generic failure', async () => { - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( - res({ - ok: false, - status: 400, - text: 'EWWrongDataException has occurred: One has to specify $table, $KB, $lang parameters', - }) - ) - - const response = await POST(createMockRequest('POST', { ...baseBody, data: '{"a":"b"}' })) - const data = (await response.json()) as { success: boolean; error?: string } - - expect(data.success).toBe(false) - expect(data.error).toContain('One has to specify $table, $KB, $lang') - }) -}) - -describe('alrest envelope handling', () => { - it('targets /ewws/alrest/{KB} for record creation', async () => { - arrange(res({ json: { success: true, result: { id: 6342, contract_title1: 'X' } } })) - - const response = await POST( - createMockRequest('POST', { ...baseBody, data: '{"contract_title1":"X"}' }) - ) - const data = (await response.json()) as { success: boolean; output: { id: string | null } } - - const [url] = inputValidationMockFns.mockSecureFetchWithPinnedIP.mock.calls[1] - expect(url).toBe( - 'https://example.agiloft.com/ewws/alrest/Russell%20Investments/contract?lang=en' - ) - expect(data.output.id).toBe('6342') - }) - - it('treats HTTP 200 with success:false as a failure, not a successful create', async () => { - arrange( - res({ - json: { success: false, errors: [{ message: 'Field contract_title1 is required' }] }, - }) - ) - - const response = await POST(createMockRequest('POST', { ...baseBody, data: '{"a":"b"}' })) - const data = (await response.json()) as { success: boolean; error?: string } - - expect(data.success).toBe(false) - expect(data.error).toContain('Field contract_title1 is required') - }) -}) - -describe('field projection', () => { - it('reads through search when fields are named, so a 184KB record is not pulled whole', async () => { - arrange(res({ json: { success: true, result: [{ id: 6342, contract_title1: 'X' }] } })) - - await READ( - createMockRequest('POST', { - ...baseBody, - recordId: '6342', - fields: 'contract_title1, company_name', - }) - ) - - const [url, , init] = inputValidationMockFns.mockSecureFetchWithPinnedIP.mock.calls[1] - expect(url).toContain('/contract/search?lang=en') - expect(JSON.parse(init.body as string)).toEqual({ - field: ['id', 'contract_title1', 'company_name'], - query: 'id=6342', - }) - }) - - it('fetches the record directly when no projection was asked for', async () => { - arrange(res({ json: { success: true, result: { id: 6342 } } })) - - await READ(createMockRequest('POST', { ...baseBody, recordId: '6342' })) - - const [url, , init] = inputValidationMockFns.mockSecureFetchWithPinnedIP.mock.calls[1] - expect(url).toContain('/contract/6342?lang=en') - expect(init.method).toBe('GET') - }) -}) - -describe('optional inputs arriving as null', () => { - it('accepts a null page instead of rejecting the call before it is made', async () => { - arrange(res({ json: { success: true, result: [] } })) - - const response = await SEARCH( - createMockRequest('POST', { - ...baseBody, - query: "status='Active'", - page: null, - limit: null, - fields: null, - search: null, - }) - ) - const data = (await response.json()) as { success: boolean; error?: string } - - expect(data.success).toBe(true) - expect(data.error).toBeUndefined() - }) - - it('omits unset optional fields from the search body rather than sending null', async () => { - arrange(res({ json: { success: true, result: [] } })) - - await SEARCH(createMockRequest('POST', { ...baseBody, query: "status='Active'", page: null })) - - const [, , init] = inputValidationMockFns.mockSecureFetchWithPinnedIP.mock.calls[1] - expect(JSON.parse(init.body as string)).toEqual({ query: "status='Active'" }) - }) -}) - -describe('search result ceiling', () => { - it('caps the returned records, since alrest honouring limit is unverified', async () => { - arrange( - res({ - json: { - success: true, - result: Array.from({ length: 250 }, (_, i) => ({ id: i })), - }, - }) - ) - - const response = await SEARCH( - createMockRequest('POST', { ...baseBody, query: "status='Active'" }) - ) - const data = (await response.json()) as { - output: { records: unknown[]; totalCount: number } - } - - expect(data.output.records).toHaveLength(200) - expect(data.output.totalCount).toBe(200) - }) -}) - -describe('review round 1 fixes', () => { - it('fails a create that comes back without a record ID', async () => { - arrange(res({ json: { success: true, result: {} } })) - - const response = await POST(createMockRequest('POST', { ...baseBody, data: '{"a":"b"}' })) - const data = (await response.json()) as { success: boolean; error?: string } - - expect(data.success).toBe(false) - expect(data.error).toContain('did not return an ID') - }) - - it('refuses a non-numeric record ID on a projected read rather than interpolating it', async () => { - const response = await READ( - createMockRequest('POST', { - ...baseBody, - recordId: "1' || priority='High", - fields: 'contract_title1', - }) - ) - const data = (await response.json()) as { success: boolean; error?: string } - - expect(data.success).toBe(false) - expect(data.error).toContain('must be numeric') - expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() - }) - - it('does not return an unrelated record when the search matches something else', async () => { - arrange(res({ json: { success: true, result: [{ id: 99, contract_title1: 'Other' }] } })) - - const response = await READ( - createMockRequest('POST', { ...baseBody, recordId: '6342', fields: 'contract_title1' }) - ) - const data = (await response.json()) as { success: boolean; error?: string } - - expect(data.success).toBe(false) - expect(data.error).toContain('no record for ID 6342') - }) -}) - -describe('documented EWREST response keys', () => { - it('reads the choice line ID from EWREST_choiceLineId', async () => { - const { POST: CHOICE } = await import('@/app/api/tools/agiloft/get_choice_line_id/route') - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( - res({ text: "EWREST_choiceLineId = '1';" }) - ) - - const response = await CHOICE( - createMockRequest('POST', { ...baseBody, fieldName: 'priority', value: 'High' }) - ) - const data = (await response.json()) as { success: boolean; output: { choiceLineId: number } } - - expect(data.success).toBe(true) - expect(data.output.choiceLineId).toBe(1) - }) -}) - -describe('EWTable', () => { - it('is KB-scoped: no $table, mandatory .json, and no inline credentials', async () => { - const { POST: LIST } = await import('@/app/api/tools/agiloft/list_tables/route') - arrange(res({ json: { success: true, result: { tables: [] } } })) - - await LIST( - createMockRequest('POST', { - instanceUrl: baseBody.instanceUrl, - knowledgeBase: baseBody.knowledgeBase, - login: baseBody.login, - password: baseBody.password, - includeLinkedInfo: true, - }) - ) - - const [url, , init] = inputValidationMockFns.mockSecureFetchWithPinnedIP.mock.calls[1] - expect(url).toContain('/ewws/EWTable/.json?') - expect(url).toContain('&includelinkedinfo=true') - expect(url).not.toContain('$table=') - expect(url).not.toContain('$password') - expect(init.headers.Authorization).toBe('Bearer tok-123') - }) - - it('narrows to one table with the plain table parameter, not $table', async () => { - const { POST: LIST } = await import('@/app/api/tools/agiloft/list_tables/route') - arrange(res({ json: { success: true, result: { tables: [] } } })) - - await LIST(createMockRequest('POST', { ...baseBody, table: 'contacts' })) - - const [url] = inputValidationMockFns.mockSecureFetchWithPinnedIP.mock.calls[1] - expect(url).toContain('&table=contacts') - expect(url).not.toContain('$table=') - }) - - it('flattens tables and fields into the documented shape', async () => { - const { POST: LIST } = await import('@/app/api/tools/agiloft/list_tables/route') - arrange( - res({ - json: { - success: true, - result: { - tables: [ - { - label: 'WMI Sample', - logicalName: 'wmi_sample', - fields: [ - { - columnLabel: 'ID', - columnName: 'id', - columnType: 'BIGINT', - columnTypeDomain: 'swautoincrementfield', - }, - { - columnLabel: 'Updated By', - columnName: '_1794_full_name', - columnType: 'VARCHAR', - columnTypeDomain: 'swshorttextfield', - isLinked: true, - linkedInfo: [ - { - linkedTable: 'contacts', - linkedColumn: 'full_name', - linkedDao: '_dao3_link0', - }, - ], - textFieldType: 'text/plain', - }, - ], - }, - ], - }, - }, - }) - ) - - const response = await LIST(createMockRequest('POST', baseBody)) - const data = (await response.json()) as { - output: { tables: Array<{ logicalName: string; fields: unknown[] }>; totalCount: number } - } - - expect(data.output.totalCount).toBe(1) - expect(data.output.tables[0].logicalName).toBe('wmi_sample') - expect(data.output.tables[0].fields).toEqual([ - { - columnName: 'id', - columnLabel: 'ID', - columnType: 'BIGINT', - columnTypeDomain: 'swautoincrementfield', - required: false, - isLinked: false, - linkedInfo: [], - textFieldType: null, - }, - { - columnName: '_1794_full_name', - columnLabel: 'Updated By', - columnType: 'VARCHAR', - columnTypeDomain: 'swshorttextfield', - required: false, - isLinked: true, - // includeLinkedInfo must actually surface the source table/column. - linkedInfo: [{ linkedTable: 'contacts', linkedColumn: 'full_name' }], - textFieldType: 'text/plain', - }, - ]) - }) -}) - -describe('EWUpsert', () => { - const upsertBody = { ...baseBody, match: 'ext_id', data: '{"first_name":"John"}' } - - it('sends every parameter in the form body, keeping credentials out of the URL', async () => { - const { POST: UPSERT } = await import('@/app/api/tools/agiloft/upsert_record/route') - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( - res({ status: 201, text: "EWREST_id='353';" }) - ) - - await UPSERT(createMockRequest('POST', upsertBody)) - - const calls = inputValidationMockFns.mockSecureFetchWithPinnedIP.mock.calls - // Inline-credential auth: a single call, no login/logout pair. - expect(calls).toHaveLength(1) - expect(calls[0][0]).toBe('https://example.agiloft.com/ewws/EWUpsert') - expect(calls[0][0]).not.toContain('?') - - const sent = new URLSearchParams(calls[0][2].body as string) - expect(sent.get('$match')).toBe('ext_id') - expect(sent.get('$table')).toBe('contract') - expect(sent.get('$password')).toBe(PLACEHOLDER_PASSWORD) - expect(sent.get('first_name')).toBe('John') - }) - - it('reports 201 as a create and 200 as an update', async () => { - const { POST: UPSERT } = await import('@/app/api/tools/agiloft/upsert_record/route') - - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( - res({ status: 201, text: "EWREST_id='353';" }) - ) - let data = (await (await UPSERT(createMockRequest('POST', upsertBody))).json()) as { - output: { id: string; created: boolean } - } - expect(data.output).toEqual({ id: '353', created: true, callbackId: null }) - - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( - res({ status: 200, text: "EWREST_id='353';" }) - ) - data = (await (await UPSERT(createMockRequest('POST', upsertBody))).json()) as { - output: { id: string; created: boolean } - } - expect(data.output).toEqual({ id: '353', created: false, callbackId: null }) - }) - - it('surfaces a 409 as an ambiguous match rather than a generic failure', async () => { - const { POST: UPSERT } = await import('@/app/api/tools/agiloft/upsert_record/route') - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( - res({ ok: false, status: 409, text: 'Multiple matching records found' }) - ) - - const response = await UPSERT(createMockRequest('POST', upsertBody)) - const data = (await response.json()) as { success: boolean; error?: string } - - expect(data.success).toBe(false) - expect(data.error).toContain('more than one record matching "ext_id"') - }) -}) - -describe('EWAsyncStatus', () => { - const statusBody = { ...baseBody, callbackId: '10100_1' } - - it('maps each documented status code to its meaning', async () => { - const { POST: STATUS } = await import('@/app/api/tools/agiloft/async_status/route') - - const cases: Array<[number, string, boolean]> = [ - [200, 'completed', true], - [201, 'queued', false], - [202, 'in_progress', false], - [501, 'failed', true], - [523, 'unknown_callback', true], - ] - - for (const [status, expected, complete] of cases) { - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( - res({ ok: status < 400, status, text: '' }) - ) - - const response = await STATUS(createMockRequest('POST', statusBody)) - const data = (await response.json()) as { - success: boolean - output: { status: string; complete: boolean; statusCode: number } - } - - // 501 is a failed *operation*, not a failed status check. - expect(data.success).toBe(true) - expect(data.output).toMatchObject({ status: expected, complete, statusCode: status }) - } - }) - - it('authenticates inline and sends the documented callback_id param', async () => { - const { POST: STATUS } = await import('@/app/api/tools/agiloft/async_status/route') - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( - res({ status: 200, text: '' }) - ) - - await STATUS(createMockRequest('POST', statusBody)) - - const calls = inputValidationMockFns.mockSecureFetchWithPinnedIP.mock.calls - expect(calls).toHaveLength(1) - expect(calls[0][0]).toContain('/ewws/EWAsyncStatus?') - expect(calls[0][0]).toContain('&callback_id=10100_1') - }) -}) - -describe('EWActionButton single-line response', () => { - it('parses both assignments when Agiloft returns them on one line', async () => { - const { POST: ACTION } = await import('@/app/api/tools/agiloft/run_action_button/route') - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( - res({ text: "EWREST_id='82'; EWREST_EWCALLBACK_ID='10100_1';" }) - ) - - const response = await ACTION( - createMockRequest('POST', { ...baseBody, recordId: '82', actionButtonField: 'ab_field' }) - ) - const data = (await response.json()) as { - success: boolean - output: { recordId: string; callbackId: string | null } - } - - expect(data.output.recordId).toBe('82') - expect(data.output.callbackId).toBe('10100_1') - }) -}) - -describe('review round 2 fixes', () => { - const upsertBody = { ...baseBody, match: 'ext_id', data: '{"a":"b"}' } - - it('returns a callback ID for a queued upsert so Async Status can poll it', async () => { - const { POST: UPSERT } = await import('@/app/api/tools/agiloft/upsert_record/route') - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( - res({ status: 202, text: "EWREST_EWCALLBACK_ID='10100_7';" }) - ) - - const response = await UPSERT(createMockRequest('POST', upsertBody)) - const data = (await response.json()) as { - success: boolean - output: { id: string | null; callbackId: string | null } - } - - expect(data.success).toBe(true) - expect(data.output.callbackId).toBe('10100_7') - }) - - it('matches a projected read on the canonical numeric ID, not the string', async () => { - arrange(res({ json: { success: true, result: [{ id: 123, contract_title1: 'X' }] } })) - - // Agiloft echoes 123 for a request made as 00123. - const response = await READ( - createMockRequest('POST', { ...baseBody, recordId: '00123', fields: 'contract_title1' }) - ) - const data = (await response.json()) as { success: boolean; output: { id: string | null } } - - expect(data.success).toBe(true) - expect(data.output.id).toBe('123') - }) -}) - -describe('review round 3 fixes', () => { - it('reports an Agiloft refusal as a non-retryable failure, not a 500', async () => { - arrange( - res({ json: { success: false, errors: [{ message: 'Field contract_title1 is required' }] } }) - ) - - const response = await POST(createMockRequest('POST', { ...baseBody, data: '{"a":"b"}' })) - - /** - * The tool runner retries 500s, and retrying a refused create can duplicate - * a record — a refusal must come back as a settled failure. - */ - expect(response.status).toBe(200) - const data = (await response.json()) as { success: boolean; error?: string } - expect(data.success).toBe(false) - expect(data.error).toContain('Field contract_title1 is required') - }) - - it('tells the user what to do when EWTable login needs a table', async () => { - const { POST: LIST } = await import('@/app/api/tools/agiloft/list_tables/route') - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( - res({ - ok: false, - status: 400, - text: 'EWWrongDataException has occurred: One has to specify $table, $KB, $lang parameters', - }) - ) - - const response = await LIST( - createMockRequest('POST', { - instanceUrl: baseBody.instanceUrl, - knowledgeBase: baseBody.knowledgeBase, - login: baseBody.login, - password: baseBody.password, - }) - ) - const data = (await response.json()) as { success: boolean; error?: string } - - expect(data.success).toBe(false) - expect(data.error).toContain('requires a table name to authenticate') - }) - - it('encodes a multi-value upsert field as repeated pairs, not a joined string', async () => { - const { POST: UPSERT } = await import('@/app/api/tools/agiloft/upsert_record/route') - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( - res({ status: 200, text: "EWREST_id='353';" }) - ) - - await UPSERT( - createMockRequest('POST', { - ...baseBody, - match: 'ext_id', - data: '{"contactMethod":["phone","email"]}', - }) - ) - - const body = inputValidationMockFns.mockSecureFetchWithPinnedIP.mock.calls[0][2].body as string - expect(new URLSearchParams(body).getAll('contactMethod')).toEqual(['phone', 'email']) - }) - - it('refuses an object field value rather than writing [object Object]', async () => { - const { POST: UPSERT } = await import('@/app/api/tools/agiloft/upsert_record/route') - - const response = await UPSERT( - createMockRequest('POST', { - ...baseBody, - match: 'ext_id', - data: '{"nested":{"a":1}}', - }) - ) - const data = (await response.json()) as { success: boolean; error?: string } - - expect(data.success).toBe(false) - expect(data.error).toContain('has no encoding for') - expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/app/api/tools/agiloft/create_record/route.ts b/apps/sim/app/api/tools/agiloft/create_record/route.ts deleted file mode 100644 index e6f183c0699..00000000000 --- a/apps/sim/app/api/tools/agiloft/create_record/route.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { agiloftCreateRecordContract } from '@/lib/api/contracts/tools/agiloft' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { AgiloftRecordResponse } from '@/tools/agiloft/types' -import { alrestRecordCollectionUrl } from '@/tools/agiloft/utils' -import { - executeAlrestRequest, - isAgiloftRefusal, - readAlrestJson, -} from '@/tools/agiloft/utils.server' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AgiloftCreateRecordAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Agiloft create_record attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - agiloftCreateRecordContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - let fieldValues: Record - try { - const parsedData = JSON.parse(params.data) - if (typeof parsedData !== 'object' || parsedData === null || Array.isArray(parsedData)) { - throw new Error('not an object') - } - fieldValues = parsedData as Record - } catch { - return NextResponse.json({ - success: false, - output: { id: null, fields: {} }, - error: 'The data parameter must be a JSON object of field names to values', - }) - } - - const result = await executeAlrestRequest( - params, - (base) => ({ - url: alrestRecordCollectionUrl(base, params.table), - method: 'POST', - headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, - body: JSON.stringify(fieldValues), - }), - async (response) => { - const record = await readAlrestJson>(response) - const id = record?.id - - /** - * A create that reports no ID did not create anything usable — callers - * chain on this ID, so surface it as a failure rather than handing back - * a successful-looking null. - */ - if (id == null) { - return { - success: false, - output: { id: null, fields: record ?? {} }, - error: 'Agiloft did not return an ID for the created record', - } - } - - return { - success: true, - output: { id: String(id), fields: record ?? {} }, - } - } - ) - - return NextResponse.json(result) - } catch (error) { - /** - * A refusal Agiloft already decided on is a final answer, not a transient - * fault — returning 500 would make the tool runner retry it. - */ - if (isAgiloftRefusal(error)) { - logger.warn(`[${requestId}] Agiloft refused the request`, { error: error.message }) - return NextResponse.json({ - success: false, - output: { id: null, fields: {} }, - error: error.message, - }) - } - - logger.error(`[${requestId}] Error creating Agiloft record:`, error) - - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/agiloft/delete_record/route.ts b/apps/sim/app/api/tools/agiloft/delete_record/route.ts deleted file mode 100644 index e6381ce0137..00000000000 --- a/apps/sim/app/api/tools/agiloft/delete_record/route.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { agiloftDeleteRecordContract } from '@/lib/api/contracts/tools/agiloft' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { AgiloftDeleteResponse } from '@/tools/agiloft/types' -import { alrestDeleteRecordUrl } from '@/tools/agiloft/utils' -import { - executeAlrestRequest, - isAgiloftRefusal, - readAlrestJson, -} from '@/tools/agiloft/utils.server' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AgiloftDeleteRecordAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Agiloft delete_record attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - agiloftDeleteRecordContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const result = await executeAlrestRequest( - params, - (base) => ({ - url: alrestDeleteRecordUrl( - base, - params.table, - params.recordId, - params.deleteRule, - params.substituteIds - ), - method: 'DELETE', - headers: { Accept: 'application/json' }, - }), - async (response) => { - await readAlrestJson(response) - return { - success: true, - output: { id: params.recordId.trim(), deleted: true }, - } - } - ) - - return NextResponse.json(result) - } catch (error) { - /** - * A refusal Agiloft already decided on is a final answer, not a transient - * fault — returning 500 would make the tool runner retry it. - */ - if (isAgiloftRefusal(error)) { - logger.warn(`[${requestId}] Agiloft refused the request`, { error: error.message }) - return NextResponse.json({ - success: false, - output: { id: '', deleted: false }, - error: error.message, - }) - } - - logger.error(`[${requestId}] Error deleting Agiloft record:`, error) - - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/agiloft/get_choice_line_id/route.ts b/apps/sim/app/api/tools/agiloft/get_choice_line_id/route.ts deleted file mode 100644 index 3324c6970ef..00000000000 --- a/apps/sim/app/api/tools/agiloft/get_choice_line_id/route.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { agiloftGetChoiceLineIdContract } from '@/lib/api/contracts/tools/agiloft' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { parseEwRest } from '@/tools/agiloft/ewrest' -import type { AgiloftGetChoiceLineIdResponse } from '@/tools/agiloft/types' -import { buildGetChoiceLineIdUrl, describeAgiloftError } from '@/tools/agiloft/utils' -import { executeEwRequest } from '@/tools/agiloft/utils.server' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AgiloftGetChoiceLineIdAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn( - `[${requestId}] Unauthorized Agiloft get_choice_line_id attempt: ${authResult.error}` - ) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - agiloftGetChoiceLineIdContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const result = await executeEwRequest( - params, - (base) => ({ - url: buildGetChoiceLineIdUrl(base, params), - method: 'GET', - }), - async (response) => { - const body = await response.text() - - if (!response.ok) { - return { - success: false, - output: { choiceLineId: null }, - error: `Agiloft error ${response.status}: ${describeAgiloftError(body)}`, - } - } - - /** Documented response: EWREST_choiceLineId = '1'; */ - const raw = parseEwRest(body).get('choiceLineId') - const parsedId = Number(raw) - const choiceLineId = - raw !== undefined && raw.trim() !== '' && Number.isFinite(parsedId) ? parsedId : null - - if (raw === undefined) { - return { - success: false, - output: { choiceLineId: null }, - error: `Agiloft did not return a choice line ID for "${params.value}" in field "${params.fieldName}": ${body.trim() || '(empty response)'}`, - } - } - - if (choiceLineId === null) { - return { - success: false, - output: { choiceLineId: null }, - error: `Agiloft returned a non-numeric choice line ID for "${params.value}" in field "${params.fieldName}": "${raw}"`, - } - } - - return { success: true, output: { choiceLineId } } - } - ) - - return NextResponse.json(result) - } catch (error) { - logger.error(`[${requestId}] Error getting Agiloft choice line ID:`, error) - - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/agiloft/list_tables/route.ts b/apps/sim/app/api/tools/agiloft/list_tables/route.ts deleted file mode 100644 index 42187655c0c..00000000000 --- a/apps/sim/app/api/tools/agiloft/list_tables/route.ts +++ /dev/null @@ -1,152 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { agiloftListTablesContract } from '@/lib/api/contracts/tools/agiloft' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { - AgiloftListTablesParams, - AgiloftListTablesResponse, - AgiloftTableField, -} from '@/tools/agiloft/types' -import { buildListTablesUrl } from '@/tools/agiloft/utils' -import { - executeAgiloftRequest, - isAgiloftRefusal, - readAlrestJson, -} from '@/tools/agiloft/utils.server' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AgiloftListTablesAPI') - -/** Shape of the `result` object EWTable returns. */ -interface EwTableResult { - tables?: Array<{ - label?: string - logicalName?: string - fields?: Array<{ - columnName?: string - columnLabel?: string - columnType?: string - columnTypeDomain?: string - required?: boolean - isLinked?: boolean - linkedInfo?: Array<{ linkedTable?: string; linkedColumn?: string }> - textFieldType?: string - }> - }> -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - let params: AgiloftListTablesParams | undefined - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Agiloft list_tables attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - agiloftListTablesContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const listParams = parsed.data.body - params = listParams - - /** EWTable must run under EWLogin or OAuth authorization. */ - const result = await executeAgiloftRequest( - listParams, - (base) => ({ - url: buildListTablesUrl(base, listParams), - method: 'GET', - headers: { Accept: 'application/json' }, - }), - async (response) => { - const payload = await readAlrestJson(response) - - const tables = (payload?.tables ?? []).map((table) => ({ - label: table.label ?? '', - logicalName: table.logicalName ?? '', - fields: (table.fields ?? []).map( - (field): AgiloftTableField => ({ - columnName: field.columnName ?? '', - columnLabel: field.columnLabel ?? '', - columnType: field.columnType ?? '', - columnTypeDomain: field.columnTypeDomain ?? '', - required: field.required === true, - isLinked: field.isLinked === true, - /** - * Only present when includeLinkedInfo was requested; dropping it - * would make that option have no observable effect. - */ - linkedInfo: (field.linkedInfo ?? []).map((link) => ({ - linkedTable: link.linkedTable ?? '', - linkedColumn: link.linkedColumn ?? '', - })), - textFieldType: field.textFieldType ?? null, - }) - ), - })) - - return { success: true, output: { tables, totalCount: tables.length } } - } - ) - - return NextResponse.json(result) - } catch (error) { - /** - * A refusal Agiloft already decided on is a final answer, not a transient - * fault — returning 500 would make the tool runner retry it. - */ - /** - * EWTable is knowledge-base scoped, but some instances reject EWLogin - * without a $table. When that happens there is nothing to fall back to, so - * say what the caller can actually do about it. - */ - if (!params?.table && /\$table/.test(toError(error).message)) { - return NextResponse.json({ - success: false, - output: { tables: [], totalCount: 0 }, - error: - 'This Agiloft instance requires a table name to authenticate. Put any known table in the Table field — it also narrows the result to that table.', - }) - } - - if (isAgiloftRefusal(error)) { - logger.warn(`[${requestId}] Agiloft refused the request`, { error: error.message }) - return NextResponse.json({ - success: false, - output: { tables: [], totalCount: 0 }, - error: error.message, - }) - } - - logger.error(`[${requestId}] Error listing Agiloft tables:`, error) - - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/agiloft/lock_record/route.ts b/apps/sim/app/api/tools/agiloft/lock_record/route.ts deleted file mode 100644 index 96ebe9f587b..00000000000 --- a/apps/sim/app/api/tools/agiloft/lock_record/route.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { agiloftLockRecordContract } from '@/lib/api/contracts/tools/agiloft' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { AgiloftLockResponse } from '@/tools/agiloft/types' -import { buildLockRecordUrl, describeAgiloftError, getLockHttpMethod } from '@/tools/agiloft/utils' -import { executeEwRequest } from '@/tools/agiloft/utils.server' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AgiloftLockRecordAPI') - -/** Lock output for a call that never returned a lock state. */ -function emptyLock(recordId: string) { - return { - id: recordId.trim(), - tableId: null, - lockStatus: '', - lockedBy: null, - lockExpiresInMinutes: null, - } -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Agiloft lock_record attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - agiloftLockRecordContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const result = await executeEwRequest( - params, - (base) => ({ - url: buildLockRecordUrl(base, params), - method: getLockHttpMethod(params.lockAction), - }), - async (response) => { - if (!response.ok) { - const errorText = await response.text() - return { - success: false, - output: emptyLock(params.recordId), - error: `Agiloft error ${response.status}: ${describeAgiloftError(errorText)}`, - } - } - - const data = (await response.json()) as Record - - /** - * EWLock answers failures with `{error, error_description}` rather than - * a lock body, and can do so on a 200 — so the absence of a - * `lock_status` is the reliable signal, not the status code. - */ - if (typeof data.lock_status !== 'string') { - const code = typeof data.error === 'string' ? data.error : 'UNKNOWN' - const detail = - typeof data.error_description === 'string' - ? data.error_description - : JSON.stringify(data) - return { - success: false, - output: emptyLock(params.recordId), - error: `Agiloft lock error (${code}): ${detail}`, - } - } - - return { - success: true, - output: { - id: String(data.id ?? params.recordId.trim()), - tableId: typeof data.table_id === 'number' ? data.table_id : null, - lockStatus: data.lock_status, - lockedBy: typeof data.locked_by === 'string' ? data.locked_by : null, - lockExpiresInMinutes: - typeof data.lock_expires_in_minutes === 'number' - ? data.lock_expires_in_minutes - : null, - }, - } - } - ) - - return NextResponse.json(result) - } catch (error) { - logger.error(`[${requestId}] Error locking Agiloft record:`, error) - - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/agiloft/nlp_search/route.ts b/apps/sim/app/api/tools/agiloft/nlp_search/route.ts deleted file mode 100644 index 4b6c2e50a01..00000000000 --- a/apps/sim/app/api/tools/agiloft/nlp_search/route.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { filterUndefined } from '@sim/utils/object' -import { type NextRequest, NextResponse } from 'next/server' -import { agiloftNlpSearchContract } from '@/lib/api/contracts/tools/agiloft' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { AgiloftNlpSearchResponse } from '@/tools/agiloft/types' -import { - AGILOFT_LANG, - AGILOFT_MAX_SEARCH_RECORDS, - buildNlpSearchUrl, - parseFieldList, -} from '@/tools/agiloft/utils' -import { executeEwRequest, readAlrestJson } from '@/tools/agiloft/utils.server' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AgiloftNlpSearchAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Agiloft nlp_search attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - agiloftNlpSearchContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const result = await executeEwRequest( - params, - (base) => ({ - url: buildNlpSearchUrl(base), - method: 'POST', - headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, - /** - * EWNLPSearch accepts application/json, so credentials travel in the - * body rather than the query string. - */ - body: JSON.stringify( - filterUndefined({ - $KB: params.knowledgeBase, - $login: params.login, - $password: params.password, - $lang: AGILOFT_LANG, - field: parseFieldList(params.fields), - nlp_query: params.nlpQuery.trim(), - page: params.page ? Number(params.page) : undefined, - limit: params.limit ? Number(params.limit) : undefined, - }) - ), - }), - async (response) => { - const returned = (await readAlrestJson[]>(response)) ?? [] - const records = returned.slice(0, AGILOFT_MAX_SEARCH_RECORDS) - - if (returned.length > records.length) { - logger.warn( - `[${requestId}] Agiloft NLP search returned ${returned.length} records; truncated to ${AGILOFT_MAX_SEARCH_RECORDS}` - ) - } - - return { - success: true, - output: { - records, - totalCount: records.length, - truncated: returned.length > records.length, - }, - } - } - ) - - return NextResponse.json(result) - } catch (error) { - logger.error(`[${requestId}] Error running Agiloft NLP search:`, error) - - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/agiloft/read_record/route.ts b/apps/sim/app/api/tools/agiloft/read_record/route.ts deleted file mode 100644 index 53c1479024d..00000000000 --- a/apps/sim/app/api/tools/agiloft/read_record/route.ts +++ /dev/null @@ -1,159 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { agiloftReadRecordContract } from '@/lib/api/contracts/tools/agiloft' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { AgiloftRecordResponse } from '@/tools/agiloft/types' -import { alrestRecordUrl, alrestSearchUrl, parseFieldList } from '@/tools/agiloft/utils' -import { - type AgiloftRequestConfig, - executeAlrestRequest, - isAgiloftRefusal, - readAlrestJson, -} from '@/tools/agiloft/utils.server' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AgiloftReadRecordAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Agiloft read_record attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - agiloftReadRecordContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - /** - * A full record is large — a contract runs to roughly 184 KB — so when the - * caller names the fields they want, the read goes through the search - * endpoint, which is the only route that accepts a `field` projection. - * Without a field list there is nothing to project and the plain record - * fetch is cheaper. - */ - const requestedFields = parseFieldList(params.fields) - const recordId = params.recordId.trim() - - /** - * The projected read puts the ID inside a search predicate, so anything - * other than a plain number could change which records the query selects. - * Agiloft record IDs are integers, so reject everything else rather than - * trying to escape it. - */ - if (requestedFields && !/^\d+$/.test(recordId)) { - return NextResponse.json({ - success: false, - output: { id: null, fields: {} }, - error: `Record ID must be numeric to read specific fields, got "${recordId}"`, - }) - } - - const result = await executeAlrestRequest( - params, - (base): AgiloftRequestConfig => { - if (!requestedFields) { - return { - url: alrestRecordUrl(base, params.table, params.recordId), - method: 'GET', - headers: { Accept: 'application/json' }, - } - } - - return { - url: alrestSearchUrl(base, params.table), - method: 'POST', - headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, - body: JSON.stringify({ - field: requestedFields.includes('id') ? requestedFields : ['id', ...requestedFields], - query: `id=${recordId}`, - }), - } - }, - async (response) => { - const payload = await readAlrestJson | Record[]>( - response - ) - - /** - * Match on ID rather than taking the first row: a search answers with a - * result set, and returning an unrelated record as a successful read - * would be worse than failing. - */ - /** - * Compare numerically: Agiloft echoes the canonical ID, so a request - * for `00123` comes back as `123` and a string compare would discard - * the very record that was asked for. - */ - const record = Array.isArray(payload) - ? payload.find((row) => { - const rowId = String(row?.id ?? '') - return /^\d+$/.test(rowId) && BigInt(rowId) === BigInt(recordId) - }) - : payload - - if (!record) { - return { - success: false, - output: { id: null, fields: {} }, - error: `Agiloft returned no record for ID ${recordId}`, - } - } - - const id = record.id - return { - success: true, - output: { id: id == null ? null : String(id), fields: record }, - } - } - ) - - return NextResponse.json(result) - } catch (error) { - /** - * A refusal Agiloft already decided on is a final answer, not a transient - * fault — returning 500 would make the tool runner retry it. - */ - if (isAgiloftRefusal(error)) { - logger.warn(`[${requestId}] Agiloft refused the request`, { error: error.message }) - return NextResponse.json({ - success: false, - output: { id: null, fields: {} }, - error: error.message, - }) - } - - logger.error(`[${requestId}] Error reading Agiloft record:`, error) - - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/agiloft/remove_attachment/route.test.ts b/apps/sim/app/api/tools/agiloft/remove_attachment/route.test.ts deleted file mode 100644 index 027f4dfadd2..00000000000 --- a/apps/sim/app/api/tools/agiloft/remove_attachment/route.test.ts +++ /dev/null @@ -1,84 +0,0 @@ -/** - * @vitest-environment node - */ -import { - createMockRequest, - hybridAuthMockFns, - inputValidationMock, - inputValidationMockFns, -} from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) - -import { POST } from '@/app/api/tools/agiloft/remove_attachment/route' - -/** Obvious non-secret so credential scanners do not flag these fixtures. */ -const PLACEHOLDER_PASSWORD = 'not-a-real-password' - -const PINNED_IP = '93.184.216.34' - -const baseBody = { - instanceUrl: 'https://example.agiloft.com', - knowledgeBase: 'demo', - login: 'admin', - password: PLACEHOLDER_PASSWORD, - table: 'contracts', - recordId: '42', - fieldName: 'attachments', - position: '0', -} - -function mockSecureFetchResponse(body: { ok?: boolean; json?: unknown; text?: string }) { - return { - ok: body.ok ?? true, - status: 200, - statusText: '', - headers: new Headers(), - body: null, - text: async () => body.text ?? JSON.stringify(body.json ?? {}), - json: async () => body.json ?? {}, - arrayBuffer: async () => new ArrayBuffer(0), - } -} - -beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ - isValid: true, - resolvedIP: PINNED_IP, - originalHostname: 'example.agiloft.com', - }) -}) - -describe('POST /api/tools/agiloft/remove_attachment', () => { - it('calls EWRemoveAttachment with GET, the only verb it accepts besides POST', async () => { - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( - mockSecureFetchResponse({ text: "EWREST_attachments.length='2';" }) - ) - - const response = await POST(createMockRequest('POST', baseBody)) - expect(response.status).toBe(200) - - const data = (await response.json()) as { - success: boolean - output: { remainingAttachments: number } - } - expect(data.output.remainingAttachments).toBe(2) - - /** - * One call, not three: the EW* surface rejects the bearer token, so it - * authenticates from inline credentials and needs no login/logout pair. - */ - const calls = inputValidationMockFns.mockSecureFetchWithPinnedIP.mock.calls - expect(calls).toHaveLength(1) - expect(calls[0][0]).toContain('/ewws/EWRemoveAttachment') - expect(calls[0][0]).toContain('&$login=admin') - expect(calls[0][2]).toMatchObject({ method: 'GET' }) - }) -}) diff --git a/apps/sim/app/api/tools/agiloft/remove_attachment/route.ts b/apps/sim/app/api/tools/agiloft/remove_attachment/route.ts deleted file mode 100644 index d9abd48a518..00000000000 --- a/apps/sim/app/api/tools/agiloft/remove_attachment/route.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { agiloftRemoveAttachmentContract } from '@/lib/api/contracts/tools/agiloft' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { parseEwRest } from '@/tools/agiloft/ewrest' -import type { AgiloftRemoveAttachmentResponse } from '@/tools/agiloft/types' -import { buildRemoveAttachmentUrl, describeAgiloftError } from '@/tools/agiloft/utils' -import { executeEwRequest } from '@/tools/agiloft/utils.server' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AgiloftRemoveAttachmentAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn( - `[${requestId}] Unauthorized Agiloft remove_attachment attempt: ${authResult.error}` - ) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - agiloftRemoveAttachmentContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const result = await executeEwRequest( - params, - (base) => ({ - url: buildRemoveAttachmentUrl(base, params), - /** EWRemoveAttachment is a GET/POST operation; it does not accept DELETE. */ - method: 'GET', - }), - async (response) => { - const text = await response.text() - - if (!response.ok) { - return { - success: false, - output: { - recordId: params.recordId?.trim() ?? '', - fieldName: params.fieldName?.trim() ?? '', - remainingAttachments: 0, - }, - error: `Agiloft error ${response.status}: ${describeAgiloftError(text)}`, - } - } - - /** - * The URL carries no `.json` decorator, so the body is the EWREST - * assignment form — `EWREST_.length='0';` — exactly as - * EWAttach returns. Parsing it as JSON silently produced 0 on every - * call regardless of what Agiloft reported. - */ - const fieldName = params.fieldName.trim() - const assignments = parseEwRest(text) - const countRaw = assignments.get(`${fieldName}.length`) ?? [...assignments.values()][0] - const remainingAttachments = Number(countRaw) - - if (!Number.isFinite(remainingAttachments)) { - return { - success: false, - output: { recordId: params.recordId.trim(), fieldName, remainingAttachments: 0 }, - error: `Agiloft did not report the remaining attachment count: ${text.trim() || '(empty response)'}`, - } - } - - return { - success: true, - output: { - recordId: params.recordId.trim(), - fieldName, - remainingAttachments, - }, - } - } - ) - - return NextResponse.json(result) - } catch (error) { - logger.error(`[${requestId}] Error removing Agiloft attachment:`, error) - - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/agiloft/retrieve/route.test.ts b/apps/sim/app/api/tools/agiloft/retrieve/route.test.ts deleted file mode 100644 index c738a1e952b..00000000000 --- a/apps/sim/app/api/tools/agiloft/retrieve/route.test.ts +++ /dev/null @@ -1,202 +0,0 @@ -/** - * @vitest-environment node - */ -import { - createMockRequest, - hybridAuthMockFns, - inputValidationMock, - inputValidationMockFns, -} from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) - -import { POST } from '@/app/api/tools/agiloft/retrieve/route' - -/** Obvious non-secret so credential scanners do not flag these fixtures. */ -const PLACEHOLDER_PASSWORD = 'not-a-real-password' - -const PINNED_IP = '93.184.216.34' - -const baseBody = { - instanceUrl: 'https://example.agiloft.com', - knowledgeBase: 'demo', - login: 'admin', - password: PLACEHOLDER_PASSWORD, - table: 'contracts', - recordId: '42', - fieldName: 'attachments', - position: '0', -} - -function mockSecureFetchResponse(body: { - ok?: boolean - status?: number - json?: unknown - text?: string - arrayBuffer?: ArrayBuffer - headers?: Headers -}) { - return { - ok: body.ok ?? true, - status: body.status ?? 200, - statusText: '', - headers: body.headers ?? new Headers(), - body: null, - text: async () => body.text ?? JSON.stringify(body.json ?? {}), - json: async () => body.json ?? {}, - arrayBuffer: async () => body.arrayBuffer ?? new ArrayBuffer(0), - } -} - -beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ - isValid: true, - resolvedIP: PINNED_IP, - originalHostname: 'example.agiloft.com', - }) -}) - -describe('POST /api/tools/agiloft/retrieve', () => { - it('rejects unauthenticated requests', async () => { - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValueOnce({ - success: false, - error: 'unauthorized', - }) - - const response = await POST(createMockRequest('POST', baseBody)) - expect(response.status).toBe(401) - expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() - }) - - it('blocks SSRF when the instance URL fails DNS validation', async () => { - inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValueOnce({ - isValid: false, - error: 'instanceUrl resolves to a blocked IP address', - }) - - const response = await POST( - createMockRequest('POST', { ...baseBody, instanceUrl: 'https://attacker.example.com' }) - ) - - expect(response.status).toBe(400) - const data = (await response.json()) as { success: false; error: string } - expect(data.success).toBe(false) - expect(data.error).toContain('blocked IP') - expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() - }) - - it('retrieves on the pinned IP in a single call (TOCTOU fix)', async () => { - const fileBytes = Buffer.from('hello-attachment', 'utf-8') - - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( - mockSecureFetchResponse({ - arrayBuffer: fileBytes.buffer.slice( - fileBytes.byteOffset, - fileBytes.byteOffset + fileBytes.byteLength - ) as ArrayBuffer, - headers: new Headers({ - 'content-type': 'text/plain', - 'content-disposition': 'attachment; filename="report.txt"', - }), - }) - ) - - const response = await POST(createMockRequest('POST', baseBody)) - expect(response.status).toBe(200) - const data = (await response.json()) as { - success: true - output: { file: { name: string; mimeType: string; data: string; size: number } } - } - - expect(data.output.file.name).toBe('report.txt') - expect(data.output.file.mimeType).toBe('text/plain') - expect(data.output.file.size).toBe(fileBytes.length) - expect(Buffer.from(data.output.file.data, 'base64').toString('utf-8')).toBe('hello-attachment') - - const calls = inputValidationMockFns.mockSecureFetchWithPinnedIP.mock.calls - /** - * EWRetrieve authenticates from inline credentials, so there is no - * login/logout pair — one call, on the pre-resolved IP. - */ - expect(calls).toHaveLength(1) - expect(calls[0][1]).toBe(PINNED_IP) - - // Original hostname is preserved in the URL (so TLS SNI works). - expect(calls[0][0]).toContain('https://example.agiloft.com/ewws/EWRetrieve') - expect(calls[0][0]).toContain('&filePosition=') - expect(calls[0][2]).toMatchObject({ method: 'GET' }) - expect(calls[0][2].headers?.Authorization).toBeUndefined() - // Attachment downloads are byte-capped rather than inheriting the global default. - expect(calls[0][2].maxResponseBytes).toBe(25 * 1024 * 1024) - - // DNS only resolved once — no second lookup that could rebind. - expect(inputValidationMockFns.mockValidateUrlWithDNS).toHaveBeenCalledTimes(1) - }) - - it('resolves the real type when Agiloft labels an attachment octet-stream', async () => { - const fileBytes = Buffer.from('PKdocx-bytes', 'utf-8') - - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( - mockSecureFetchResponse({ - arrayBuffer: fileBytes.buffer.slice( - fileBytes.byteOffset, - fileBytes.byteOffset + fileBytes.byteLength - ) as ArrayBuffer, - headers: new Headers({ - 'content-type': 'application/octet-stream', - 'content-disposition': 'attachment; filename="Master Agreement.docx"', - }), - }) - ) - - const response = await POST(createMockRequest('POST', baseBody)) - const data = (await response.json()) as { output: { file: { mimeType: string } } } - - /** - * Agiloft labels most attachments octet-stream whatever they are. The - * filename disambiguates what the leading bytes cannot: a ZIP header is - * equally a .docx, .xlsx or a plain archive. - */ - expect(data.output.file.mimeType).toBe( - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' - ) - }) - - it('propagates upstream errors', async () => { - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( - mockSecureFetchResponse({ ok: false, status: 404, text: 'Record not found' }) - ) - - const response = await POST(createMockRequest('POST', baseBody)) - expect(response.status).toBe(404) - const data = (await response.json()) as { success: false; error: string } - expect(data.error).toContain('Record not found') - }) - - it('rejects an EWREST error document returned in place of file bytes', async () => { - const errorBody = Buffer.from( - "EWREST_error='Search in table contracts returns no records for key 1';" - ) - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( - mockSecureFetchResponse({ - arrayBuffer: errorBody.buffer.slice( - errorBody.byteOffset, - errorBody.byteOffset + errorBody.byteLength - ) as ArrayBuffer, - headers: new Headers({ 'content-type': 'text/html' }), - }) - ) - - const response = await POST(createMockRequest('POST', baseBody)) - expect(response.status).toBe(502) - const data = (await response.json()) as { success: false; error: string } - expect(data.error).toContain('no records for key') - }) -}) diff --git a/apps/sim/app/api/tools/agiloft/retrieve/route.ts b/apps/sim/app/api/tools/agiloft/retrieve/route.ts deleted file mode 100644 index f839a0a23fb..00000000000 --- a/apps/sim/app/api/tools/agiloft/retrieve/route.ts +++ /dev/null @@ -1,164 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { agiloftRetrieveContract } from '@/lib/api/contracts/tools/agiloft' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { secureFetchWithPinnedIP } from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { resolveEffectiveMimeType } from '@/lib/uploads/utils/file-utils' -import { isEwRestBody } from '@/tools/agiloft/ewrest' -import { - AGILOFT_MAX_ATTACHMENT_BYTES, - buildRetrieveAttachmentUrl, - describeAgiloftError, -} from '@/tools/agiloft/utils' -import { resolveAgiloftInstance } from '@/tools/agiloft/utils.server' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AgiloftRetrieveAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Agiloft retrieve attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - agiloftRetrieveContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - let resolvedIP: string - try { - resolvedIP = await resolveAgiloftInstance(data.instanceUrl) - } catch (error) { - logger.warn(`[${requestId}] SSRF attempt blocked for Agiloft instance URL`, { - instanceUrl: data.instanceUrl, - }) - return NextResponse.json({ success: false, error: toError(error).message }, { status: 400 }) - } - - const base = data.instanceUrl.replace(/\/$/, '') - - { - /** - * EWRetrieve is the documented endpoint for this: GET on the legacy - * surface, authenticating from inline credentials, answering with the - * raw file bytes. It needs no login/logout pair, which also avoids two - * extra round trips against Agiloft's one-second per-call delay. - */ - const url = buildRetrieveAttachmentUrl(base, data) - - logger.info(`[${requestId}] Downloading attachment from Agiloft`, { - recordId: data.recordId, - fieldName: data.fieldName, - position: data.position, - }) - - const agiloftResponse = await secureFetchWithPinnedIP(url, resolvedIP, { - method: 'GET', - maxResponseBytes: AGILOFT_MAX_ATTACHMENT_BYTES, - }) - - if (!agiloftResponse.ok) { - const errorText = await agiloftResponse.text() - logger.error( - `[${requestId}] Agiloft retrieve error: ${agiloftResponse.status} - ${errorText}` - ) - return NextResponse.json( - { - success: false, - error: `Agiloft error ${agiloftResponse.status}: ${describeAgiloftError(errorText)}`, - }, - { status: agiloftResponse.status } - ) - } - - const contentType = agiloftResponse.headers.get('content-type') || 'application/octet-stream' - const contentDisposition = agiloftResponse.headers.get('content-disposition') - let fileName = 'attachment' - - if (contentDisposition) { - const match = contentDisposition.match(/filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/) - if (match?.[1]) { - fileName = match[1].replace(/['"]/g, '') - } - } - - const arrayBuffer = await agiloftResponse.arrayBuffer() - const fileBuffer = Buffer.from(arrayBuffer) - - /** - * A refusal comes back as an EWREST assignment or a plain-text error - * rather than file bytes, so a body that parses as one is an error - * document, not an attachment. - */ - if (isEwRestBody(fileBuffer.subarray(0, 512).toString('utf8'))) { - const envelope = fileBuffer.toString('utf8').slice(0, 300) - logger.error(`[${requestId}] Agiloft refused the attachment retrieve`, { envelope }) - return NextResponse.json( - { success: false, error: `Agiloft error: ${envelope}` }, - { status: 502 } - ) - } - - /** - * Agiloft labels most attachments application/octet-stream whatever they - * are, so downstream consumers keyed on the header mis-handle them. The - * filename Agiloft sends in Content-Disposition carries the real type. - */ - const mimeType = resolveEffectiveMimeType(contentType, fileName) - - logger.info(`[${requestId}] Attachment downloaded successfully`, { - name: fileName, - size: fileBuffer.length, - mimeType, - }) - - const base64Data = fileBuffer.toString('base64') - - return NextResponse.json({ - success: true, - output: { - file: { - name: fileName, - mimeType, - data: base64Data, - size: fileBuffer.length, - }, - }, - }) - } - } catch (error) { - logger.error(`[${requestId}] Error retrieving Agiloft attachment:`, error) - - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/agiloft/run_action_button/route.ts b/apps/sim/app/api/tools/agiloft/run_action_button/route.ts deleted file mode 100644 index f27fbe0685b..00000000000 --- a/apps/sim/app/api/tools/agiloft/run_action_button/route.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { agiloftRunActionButtonContract } from '@/lib/api/contracts/tools/agiloft' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { parseEwRest } from '@/tools/agiloft/ewrest' -import type { AgiloftRunActionButtonResponse } from '@/tools/agiloft/types' -import { buildRunActionButtonUrl, describeAgiloftError } from '@/tools/agiloft/utils' -import { executeEwRequest } from '@/tools/agiloft/utils.server' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AgiloftRunActionButtonAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn( - `[${requestId}] Unauthorized Agiloft run_action_button attempt: ${authResult.error}` - ) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - agiloftRunActionButtonContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const result = await executeEwRequest( - params, - (base) => ({ - url: buildRunActionButtonUrl(base, params), - /** EWActionButton is documented as POST-only. */ - method: 'POST', - headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, - }), - async (response) => { - const body = await response.text() - const recordId = params.recordId.trim() - - if (!response.ok) { - return { - success: false, - output: { recordId, callbackId: null }, - error: `Agiloft error ${response.status}: ${describeAgiloftError(body)}`, - } - } - - /** Documented response: EWREST_id='82'; EWREST_EWCALLBACK_ID='10100_1'; */ - const values = parseEwRest(body) - if (values.size === 0) { - return { - success: false, - output: { recordId, callbackId: null }, - error: `Agiloft did not acknowledge the action button: ${body.trim() || '(empty response)'}`, - } - } - - return { - success: true, - output: { - recordId: values.get('id') ?? recordId, - callbackId: values.get('EWCALLBACK_ID') ?? null, - }, - } - } - ) - - return NextResponse.json(result) - } catch (error) { - logger.error(`[${requestId}] Error running Agiloft action button:`, error) - - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/agiloft/saved_search/route.ts b/apps/sim/app/api/tools/agiloft/saved_search/route.ts deleted file mode 100644 index 008156693b8..00000000000 --- a/apps/sim/app/api/tools/agiloft/saved_search/route.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { agiloftSavedSearchContract } from '@/lib/api/contracts/tools/agiloft' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { AgiloftSavedSearchResponse } from '@/tools/agiloft/types' -import { buildSavedSearchUrl } from '@/tools/agiloft/utils' -import { - executeAgiloftRequest, - isAgiloftRefusal, - readAlrestJson, -} from '@/tools/agiloft/utils.server' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AgiloftSavedSearchAPI') - -interface SavedSearchRow { - name?: string - label?: string - id?: number - description?: string -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Agiloft saved_search attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - agiloftSavedSearchContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - /** - * EWSavedSearch must run under EWLogin or OAuth authorization, so this goes - * through the token-bearing executor rather than the inline-credential one - * the other legacy operations use. - */ - const result = await executeAgiloftRequest( - params, - (base) => ({ - url: buildSavedSearchUrl(base, params), - method: 'GET', - headers: { Accept: 'application/json' }, - }), - async (response) => { - const rows = (await readAlrestJson(response)) ?? [] - - const searches = rows.map((row) => ({ - name: row.name ?? '', - label: row.label ?? row.name ?? '', - id: row.id ?? null, - description: row.description ?? null, - })) - - return { success: true, output: { searches, totalCount: searches.length } } - } - ) - - return NextResponse.json(result) - } catch (error) { - /** - * A refusal Agiloft already decided on is a final answer, not a transient - * fault — returning 500 would make the tool runner retry it. - */ - if (isAgiloftRefusal(error)) { - logger.warn(`[${requestId}] Agiloft refused the request`, { error: error.message }) - return NextResponse.json({ - success: false, - output: { searches: [], totalCount: 0 }, - error: error.message, - }) - } - - logger.error(`[${requestId}] Error listing Agiloft saved searches:`, error) - - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/agiloft/search_records/route.ts b/apps/sim/app/api/tools/agiloft/search_records/route.ts deleted file mode 100644 index 2e7396deee6..00000000000 --- a/apps/sim/app/api/tools/agiloft/search_records/route.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { filterUndefined } from '@sim/utils/object' -import { type NextRequest, NextResponse } from 'next/server' -import { agiloftSearchRecordsContract } from '@/lib/api/contracts/tools/agiloft' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { AgiloftSearchResponse } from '@/tools/agiloft/types' -import { AGILOFT_MAX_SEARCH_RECORDS, alrestSearchUrl, parseFieldList } from '@/tools/agiloft/utils' -import { - executeAlrestRequest, - isAgiloftRefusal, - readAlrestJson, -} from '@/tools/agiloft/utils.server' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AgiloftSearchRecordsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Agiloft search_records attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - agiloftSearchRecordsContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const page = params.page ? Number(params.page) : 0 - const limit = params.limit ? Number(params.limit) : 0 - - const result = await executeAlrestRequest( - params, - (base) => ({ - url: alrestSearchUrl(base, params.table), - method: 'POST', - headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, - body: JSON.stringify( - filterUndefined({ - search: params.search?.trim() || undefined, - query: params.query?.trim() || undefined, - field: parseFieldList(params.fields), - page: params.page ? page : undefined, - limit: params.limit ? limit : undefined, - }) - ), - }), - async (response) => { - const returned = (await readAlrestJson[]>(response)) ?? [] - const records = returned.slice(0, AGILOFT_MAX_SEARCH_RECORDS) - - if (returned.length > records.length) { - logger.warn( - `[${requestId}] Agiloft search returned ${returned.length} records; truncated to ${AGILOFT_MAX_SEARCH_RECORDS}`, - { table: params.table } - ) - } - - return { - success: true, - output: { - records, - totalCount: records.length, - page, - limit, - truncated: returned.length > records.length, - }, - } - } - ) - - return NextResponse.json(result) - } catch (error) { - /** - * A refusal Agiloft already decided on is a final answer, not a transient - * fault — returning 500 would make the tool runner retry it. - */ - if (isAgiloftRefusal(error)) { - logger.warn(`[${requestId}] Agiloft refused the request`, { error: error.message }) - return NextResponse.json({ - success: false, - output: { records: [], totalCount: 0, page: 0, limit: 0, truncated: false }, - error: error.message, - }) - } - - logger.error(`[${requestId}] Error searching Agiloft records:`, error) - - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/agiloft/select_records/route.ts b/apps/sim/app/api/tools/agiloft/select_records/route.ts deleted file mode 100644 index dd7be15039e..00000000000 --- a/apps/sim/app/api/tools/agiloft/select_records/route.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { agiloftSelectRecordsContract } from '@/lib/api/contracts/tools/agiloft' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { parseEwRest, toRecordIds } from '@/tools/agiloft/ewrest' -import type { AgiloftSelectResponse } from '@/tools/agiloft/types' -import { - AGILOFT_MAX_SELECT_IDS, - buildSelectRecordsUrl, - describeAgiloftError, - ewCredentialBody, -} from '@/tools/agiloft/utils' -import { executeEwRequest } from '@/tools/agiloft/utils.server' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AgiloftSelectRecordsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Agiloft select_records attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - agiloftSelectRecordsContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const result = await executeEwRequest( - params, - (base) => ({ - url: buildSelectRecordsUrl(base, params), - /** - * POST with the credentials in the body: EWSelect is one of the five - * operations that support it, and it keeps the password out of the URL - * and out of the server's access logs. - */ - method: 'POST', - headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, - body: ewCredentialBody(params), - }), - async (response) => { - const body = await response.text() - - if (!response.ok) { - return { - success: false, - output: { recordIds: [], totalCount: 0, truncated: false }, - error: `Agiloft error ${response.status}: ${describeAgiloftError(body)}`, - } - } - - /** - * EWSelect answers with EWREST_id_length followed by one EWREST_id_ - * assignment per match, and zero matches still yields the length line. - * A body with no assignments at all is therefore never a legitimate - * empty result — it is a refusal Agiloft returned with HTTP 200, most - * often invalid WHERE-clause SQL. - */ - const values = parseEwRest(body) - if (values.size === 0) { - return { - success: false, - output: { recordIds: [], totalCount: 0, truncated: false }, - error: `Agiloft did not return a result set: ${body.trim() || '(empty response)'}`, - } - } - - const { recordIds } = toRecordIds(values) - const capped = recordIds.slice(0, AGILOFT_MAX_SELECT_IDS) - - if (recordIds.length > capped.length) { - logger.warn( - `[${requestId}] Agiloft select returned ${recordIds.length} IDs; truncated to ${AGILOFT_MAX_SELECT_IDS}`, - { table: params.table } - ) - } - - return { - success: true, - output: { - recordIds: capped, - totalCount: capped.length, - truncated: recordIds.length > capped.length, - }, - } - } - ) - - return NextResponse.json(result) - } catch (error) { - logger.error(`[${requestId}] Error selecting Agiloft records:`, error) - - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/agiloft/update_record/route.ts b/apps/sim/app/api/tools/agiloft/update_record/route.ts deleted file mode 100644 index 2ff0102200a..00000000000 --- a/apps/sim/app/api/tools/agiloft/update_record/route.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { agiloftUpdateRecordContract } from '@/lib/api/contracts/tools/agiloft' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { AgiloftRecordResponse } from '@/tools/agiloft/types' -import { alrestRecordUrl } from '@/tools/agiloft/utils' -import { - executeAlrestRequest, - isAgiloftRefusal, - readAlrestJson, -} from '@/tools/agiloft/utils.server' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AgiloftUpdateRecordAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Agiloft update_record attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - agiloftUpdateRecordContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - let fieldValues: Record - try { - const parsedData = JSON.parse(params.data) - if (typeof parsedData !== 'object' || parsedData === null || Array.isArray(parsedData)) { - throw new Error('not an object') - } - fieldValues = parsedData as Record - } catch { - return NextResponse.json({ - success: false, - output: { id: null, fields: {} }, - error: 'The data parameter must be a JSON object of field names to values', - }) - } - - const result = await executeAlrestRequest( - params, - (base) => ({ - url: alrestRecordUrl(base, params.table, params.recordId), - method: 'PUT', - headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, - body: JSON.stringify(fieldValues), - }), - async (response) => { - const record = await readAlrestJson>(response) - const id = record?.id ?? params.recordId.trim() - - return { - success: true, - output: { id: String(id), fields: record ?? {} }, - } - } - ) - - return NextResponse.json(result) - } catch (error) { - /** - * A refusal Agiloft already decided on is a final answer, not a transient - * fault — returning 500 would make the tool runner retry it. - */ - if (isAgiloftRefusal(error)) { - logger.warn(`[${requestId}] Agiloft refused the request`, { error: error.message }) - return NextResponse.json({ - success: false, - output: { id: null, fields: {} }, - error: error.message, - }) - } - - logger.error(`[${requestId}] Error updating Agiloft record:`, error) - - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/agiloft/upsert_record/route.ts b/apps/sim/app/api/tools/agiloft/upsert_record/route.ts deleted file mode 100644 index 891b9c91c84..00000000000 --- a/apps/sim/app/api/tools/agiloft/upsert_record/route.ts +++ /dev/null @@ -1,149 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { agiloftUpsertRecordContract } from '@/lib/api/contracts/tools/agiloft' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { parseEwRest } from '@/tools/agiloft/ewrest' -import type { AgiloftUpsertRecordResponse } from '@/tools/agiloft/types' -import { - buildUpsertRecordBody, - buildUpsertRecordUrl, - describeAgiloftError, -} from '@/tools/agiloft/utils' -import { executeEwRequest } from '@/tools/agiloft/utils.server' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AgiloftUpsertRecordAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Agiloft upsert_record attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - agiloftUpsertRecordContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - let fieldValues: Record - try { - const parsedData = JSON.parse(params.data) - if (typeof parsedData !== 'object' || parsedData === null || Array.isArray(parsedData)) { - throw new Error('not an object') - } - fieldValues = parsedData as Record - } catch { - return NextResponse.json({ - success: false, - output: { id: null, created: false, callbackId: null }, - error: 'The data parameter must be a JSON object of field names to values', - }) - } - - let body: string - try { - body = buildUpsertRecordBody(params, fieldValues) - } catch (error) { - return NextResponse.json({ - success: false, - output: { id: null, created: false, callbackId: null }, - error: toError(error).message, - }) - } - - const result = await executeEwRequest( - params, - (base) => ({ - url: buildUpsertRecordUrl(base), - method: 'POST', - headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, - body, - }), - async (response) => { - const body = await response.text() - - /** 409 means the match criteria selected more than one record. */ - if (response.status === 409) { - return { - success: false, - output: { id: null, created: false, callbackId: null }, - error: `Agiloft found more than one record matching "${params.match}", so it did not write: ${body.trim()}`, - } - } - - if (!response.ok) { - return { - success: false, - output: { id: null, created: false, callbackId: null }, - error: `Agiloft error ${response.status}: ${describeAgiloftError(body)}`, - } - } - - /** - * 202 is the documented acknowledgement for an asynchronous upsert. It - * carries no record ID, so it must not be read as a failed write. - */ - if (response.status === 202) { - const callbackId = parseEwRest(body).get('EWCALLBACK_ID') ?? null - - if (!callbackId) { - logger.warn( - `[${requestId}] Agiloft queued the upsert without a callback ID; the result cannot be polled`, - { body: body.slice(0, 200) } - ) - } - - return { success: true, output: { id: null, created: false, callbackId } } - } - - /** Documented response: EWREST_id='353'; with 201 on create, 200 on update. */ - const id = parseEwRest(body).get('id') - - if (id === undefined) { - return { - success: false, - output: { id: null, created: false, callbackId: null }, - error: `Agiloft did not return a record ID: ${body.trim() || '(empty response)'}`, - } - } - - return { success: true, output: { id, created: response.status === 201, callbackId: null } } - } - ) - - return NextResponse.json(result) - } catch (error) { - logger.error(`[${requestId}] Error upserting Agiloft record:`, error) - - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/create-application/route.ts b/apps/sim/app/api/tools/appconfig/create-application/route.ts deleted file mode 100644 index f539bb43897..00000000000 --- a/apps/sim/app/api/tools/appconfig/create-application/route.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigCreateApplicationContract } from '@/lib/api/contracts/tools/aws/appconfig-create-application' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, createApplication } from '../utils' - -const logger = createLogger('AppConfigCreateApplicationAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsAppConfigCreateApplicationContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Creating AppConfig application ${params.name}`) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await createApplication(client, params.name, params.description) - logger.info(`[${requestId}] Created application ${result.id}`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to create application:`, error) - return NextResponse.json( - { error: `Failed to create application: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/create-configuration-profile/route.ts b/apps/sim/app/api/tools/appconfig/create-configuration-profile/route.ts deleted file mode 100644 index cf61b1f79fa..00000000000 --- a/apps/sim/app/api/tools/appconfig/create-configuration-profile/route.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigCreateConfigurationProfileContract } from '@/lib/api/contracts/tools/aws/appconfig-create-configuration-profile' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, createConfigurationProfile } from '../utils' - -const logger = createLogger('AppConfigCreateConfigurationProfileAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsAppConfigCreateConfigurationProfileContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Creating AppConfig configuration profile ${params.name}`) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await createConfigurationProfile( - client, - params.applicationId, - params.name, - params.locationUri, - params.description, - params.retrievalRoleArn, - params.type - ) - logger.info(`[${requestId}] Created configuration profile ${result.id}`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to create configuration profile:`, error) - return NextResponse.json( - { error: `Failed to create configuration profile: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/create-environment/route.ts b/apps/sim/app/api/tools/appconfig/create-environment/route.ts deleted file mode 100644 index dff11740c9b..00000000000 --- a/apps/sim/app/api/tools/appconfig/create-environment/route.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigCreateEnvironmentContract } from '@/lib/api/contracts/tools/aws/appconfig-create-environment' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, createEnvironment } from '../utils' - -const logger = createLogger('AppConfigCreateEnvironmentAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsAppConfigCreateEnvironmentContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Creating AppConfig environment ${params.name}`) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await createEnvironment( - client, - params.applicationId, - params.name, - params.description - ) - logger.info(`[${requestId}] Created environment ${result.id}`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to create environment:`, error) - return NextResponse.json( - { error: `Failed to create environment: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/create-hosted-configuration-version/route.ts b/apps/sim/app/api/tools/appconfig/create-hosted-configuration-version/route.ts deleted file mode 100644 index dbce4d4966f..00000000000 --- a/apps/sim/app/api/tools/appconfig/create-hosted-configuration-version/route.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigCreateHostedConfigurationVersionContract } from '@/lib/api/contracts/tools/aws/appconfig-create-hosted-configuration-version' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, createHostedConfigurationVersion } from '../utils' - -const logger = createLogger('AppConfigCreateHostedConfigurationVersionAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest( - awsAppConfigCreateHostedConfigurationVersionContract, - request, - { errorFormat: 'details', logger } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Creating hosted configuration version for profile ${params.configurationProfileId}` - ) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await createHostedConfigurationVersion( - client, - params.applicationId, - params.configurationProfileId, - params.content, - params.contentType, - params.description, - params.latestVersionNumber, - params.versionLabel - ) - logger.info(`[${requestId}] Created hosted configuration version ${result.versionNumber}`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to create hosted configuration version:`, error) - return NextResponse.json( - { error: `Failed to create hosted configuration version: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/delete-application/route.ts b/apps/sim/app/api/tools/appconfig/delete-application/route.ts deleted file mode 100644 index 3e7c8929d49..00000000000 --- a/apps/sim/app/api/tools/appconfig/delete-application/route.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigDeleteApplicationContract } from '@/lib/api/contracts/tools/aws/appconfig-delete-application' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, deleteApplication } from '../utils' - -const logger = createLogger('AppConfigDeleteApplicationAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsAppConfigDeleteApplicationContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Deleting AppConfig application ${params.applicationId}`) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await deleteApplication(client, params.applicationId) - logger.info(`[${requestId}] Deleted application`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to delete application:`, error) - return NextResponse.json( - { error: `Failed to delete application: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/delete-configuration-profile/route.ts b/apps/sim/app/api/tools/appconfig/delete-configuration-profile/route.ts deleted file mode 100644 index 202ca4886cd..00000000000 --- a/apps/sim/app/api/tools/appconfig/delete-configuration-profile/route.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigDeleteConfigurationProfileContract } from '@/lib/api/contracts/tools/aws/appconfig-delete-configuration-profile' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, deleteConfigurationProfile } from '../utils' - -const logger = createLogger('AppConfigDeleteConfigurationProfileAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsAppConfigDeleteConfigurationProfileContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Deleting AppConfig configuration profile ${params.configurationProfileId}` - ) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await deleteConfigurationProfile( - client, - params.applicationId, - params.configurationProfileId - ) - logger.info(`[${requestId}] Deleted configuration profile`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to delete configuration profile:`, error) - return NextResponse.json( - { error: `Failed to delete configuration profile: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/delete-environment/route.ts b/apps/sim/app/api/tools/appconfig/delete-environment/route.ts deleted file mode 100644 index 5cbefc817ef..00000000000 --- a/apps/sim/app/api/tools/appconfig/delete-environment/route.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigDeleteEnvironmentContract } from '@/lib/api/contracts/tools/aws/appconfig-delete-environment' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, deleteEnvironment } from '../utils' - -const logger = createLogger('AppConfigDeleteEnvironmentAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsAppConfigDeleteEnvironmentContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Deleting AppConfig environment ${params.environmentId}`) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await deleteEnvironment(client, params.applicationId, params.environmentId) - logger.info(`[${requestId}] Deleted environment`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to delete environment:`, error) - return NextResponse.json( - { error: `Failed to delete environment: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/delete-hosted-configuration-version/route.ts b/apps/sim/app/api/tools/appconfig/delete-hosted-configuration-version/route.ts deleted file mode 100644 index 94f9d34b4ba..00000000000 --- a/apps/sim/app/api/tools/appconfig/delete-hosted-configuration-version/route.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigDeleteHostedConfigurationVersionContract } from '@/lib/api/contracts/tools/aws/appconfig-delete-hosted-configuration-version' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, deleteHostedConfigurationVersion } from '../utils' - -const logger = createLogger('AppConfigDeleteHostedConfigurationVersionAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest( - awsAppConfigDeleteHostedConfigurationVersionContract, - request, - { errorFormat: 'details', logger } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Deleting hosted configuration version ${params.versionNumber} for profile ${params.configurationProfileId}` - ) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await deleteHostedConfigurationVersion( - client, - params.applicationId, - params.configurationProfileId, - params.versionNumber - ) - logger.info(`[${requestId}] Deleted hosted configuration version`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to delete hosted configuration version:`, error) - return NextResponse.json( - { error: `Failed to delete hosted configuration version: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/get-application/route.ts b/apps/sim/app/api/tools/appconfig/get-application/route.ts deleted file mode 100644 index ac1c0c86294..00000000000 --- a/apps/sim/app/api/tools/appconfig/get-application/route.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigGetApplicationContract } from '@/lib/api/contracts/tools/aws/appconfig-get-application' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, getApplication } from '../utils' - -const logger = createLogger('AppConfigGetApplicationAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsAppConfigGetApplicationContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Getting AppConfig application ${params.applicationId}`) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await getApplication(client, params.applicationId) - logger.info(`[${requestId}] Retrieved application`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to get application:`, error) - return NextResponse.json( - { error: `Failed to get application: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/get-configuration-profile/route.ts b/apps/sim/app/api/tools/appconfig/get-configuration-profile/route.ts deleted file mode 100644 index 61d26098a98..00000000000 --- a/apps/sim/app/api/tools/appconfig/get-configuration-profile/route.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigGetConfigurationProfileContract } from '@/lib/api/contracts/tools/aws/appconfig-get-configuration-profile' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, getConfigurationProfile } from '../utils' - -const logger = createLogger('AppConfigGetConfigurationProfileAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsAppConfigGetConfigurationProfileContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Getting AppConfig configuration profile ${params.configurationProfileId}` - ) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await getConfigurationProfile( - client, - params.applicationId, - params.configurationProfileId - ) - logger.info(`[${requestId}] Retrieved configuration profile`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to get configuration profile:`, error) - return NextResponse.json( - { error: `Failed to get configuration profile: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/get-configuration/route.ts b/apps/sim/app/api/tools/appconfig/get-configuration/route.ts deleted file mode 100644 index 32387a4505f..00000000000 --- a/apps/sim/app/api/tools/appconfig/get-configuration/route.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigGetConfigurationContract } from '@/lib/api/contracts/tools/aws/appconfig-get-configuration' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigDataClient, getConfiguration } from '../utils' - -const logger = createLogger('AppConfigGetConfigurationAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsAppConfigGetConfigurationContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Retrieving AppConfig configuration for ${params.applicationId}/${params.environmentId}/${params.configurationProfileId}` - ) - - const client = createAppConfigDataClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await getConfiguration( - client, - params.applicationId, - params.environmentId, - params.configurationProfileId - ) - logger.info(`[${requestId}] Retrieved configuration`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to retrieve configuration:`, error) - return NextResponse.json( - { error: `Failed to retrieve configuration: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/get-deployment/route.ts b/apps/sim/app/api/tools/appconfig/get-deployment/route.ts deleted file mode 100644 index 4e4313dfb24..00000000000 --- a/apps/sim/app/api/tools/appconfig/get-deployment/route.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigGetDeploymentContract } from '@/lib/api/contracts/tools/aws/appconfig-get-deployment' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, getDeployment } from '../utils' - -const logger = createLogger('AppConfigGetDeploymentAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsAppConfigGetDeploymentContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Getting AppConfig deployment ${params.deploymentNumber} in env ${params.environmentId}` - ) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await getDeployment( - client, - params.applicationId, - params.environmentId, - params.deploymentNumber - ) - logger.info(`[${requestId}] Retrieved deployment`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to get deployment:`, error) - return NextResponse.json( - { error: `Failed to get deployment: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/get-environment/route.ts b/apps/sim/app/api/tools/appconfig/get-environment/route.ts deleted file mode 100644 index 37a9918f801..00000000000 --- a/apps/sim/app/api/tools/appconfig/get-environment/route.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigGetEnvironmentContract } from '@/lib/api/contracts/tools/aws/appconfig-get-environment' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, getEnvironment } from '../utils' - -const logger = createLogger('AppConfigGetEnvironmentAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsAppConfigGetEnvironmentContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Getting AppConfig environment ${params.environmentId}`) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await getEnvironment(client, params.applicationId, params.environmentId) - logger.info(`[${requestId}] Retrieved environment`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to get environment:`, error) - return NextResponse.json( - { error: `Failed to get environment: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/get-hosted-configuration-version/route.ts b/apps/sim/app/api/tools/appconfig/get-hosted-configuration-version/route.ts deleted file mode 100644 index 74011556eeb..00000000000 --- a/apps/sim/app/api/tools/appconfig/get-hosted-configuration-version/route.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigGetHostedConfigurationVersionContract } from '@/lib/api/contracts/tools/aws/appconfig-get-hosted-configuration-version' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, getHostedConfigurationVersion } from '../utils' - -const logger = createLogger('AppConfigGetHostedConfigurationVersionAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest( - awsAppConfigGetHostedConfigurationVersionContract, - request, - { errorFormat: 'details', logger } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Getting hosted configuration version ${params.versionNumber} for profile ${params.configurationProfileId}` - ) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await getHostedConfigurationVersion( - client, - params.applicationId, - params.configurationProfileId, - params.versionNumber - ) - logger.info(`[${requestId}] Retrieved hosted configuration version`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to get hosted configuration version:`, error) - return NextResponse.json( - { error: `Failed to get hosted configuration version: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/list-applications/route.ts b/apps/sim/app/api/tools/appconfig/list-applications/route.ts deleted file mode 100644 index 5cd42681bc4..00000000000 --- a/apps/sim/app/api/tools/appconfig/list-applications/route.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigListApplicationsContract } from '@/lib/api/contracts/tools/aws/appconfig-list-applications' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, listApplications } from '../utils' - -const logger = createLogger('AppConfigListApplicationsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsAppConfigListApplicationsContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Listing AppConfig applications`) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await listApplications(client, params.maxResults, params.nextToken) - logger.info(`[${requestId}] Listed ${result.count} applications`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to list applications:`, error) - return NextResponse.json( - { error: `Failed to list applications: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/list-configuration-profiles/route.ts b/apps/sim/app/api/tools/appconfig/list-configuration-profiles/route.ts deleted file mode 100644 index d46d6f147b9..00000000000 --- a/apps/sim/app/api/tools/appconfig/list-configuration-profiles/route.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigListConfigurationProfilesContract } from '@/lib/api/contracts/tools/aws/appconfig-list-configuration-profiles' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, listConfigurationProfiles } from '../utils' - -const logger = createLogger('AppConfigListConfigurationProfilesAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsAppConfigListConfigurationProfilesContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Listing AppConfig configuration profiles for ${params.applicationId}` - ) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await listConfigurationProfiles( - client, - params.applicationId, - params.maxResults, - params.nextToken - ) - logger.info(`[${requestId}] Listed ${result.count} configuration profiles`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to list configuration profiles:`, error) - return NextResponse.json( - { error: `Failed to list configuration profiles: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/list-deployment-strategies/route.ts b/apps/sim/app/api/tools/appconfig/list-deployment-strategies/route.ts deleted file mode 100644 index 6aa05d4b31a..00000000000 --- a/apps/sim/app/api/tools/appconfig/list-deployment-strategies/route.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigListDeploymentStrategiesContract } from '@/lib/api/contracts/tools/aws/appconfig-list-deployment-strategies' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, listDeploymentStrategies } from '../utils' - -const logger = createLogger('AppConfigListDeploymentStrategiesAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsAppConfigListDeploymentStrategiesContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Listing AppConfig deployment strategies`) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await listDeploymentStrategies(client, params.maxResults, params.nextToken) - logger.info(`[${requestId}] Listed ${result.count} deployment strategies`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to list deployment strategies:`, error) - return NextResponse.json( - { error: `Failed to list deployment strategies: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/list-deployments/route.ts b/apps/sim/app/api/tools/appconfig/list-deployments/route.ts deleted file mode 100644 index 0eb3eac0a9a..00000000000 --- a/apps/sim/app/api/tools/appconfig/list-deployments/route.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigListDeploymentsContract } from '@/lib/api/contracts/tools/aws/appconfig-list-deployments' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, listDeployments } from '../utils' - -const logger = createLogger('AppConfigListDeploymentsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsAppConfigListDeploymentsContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Listing AppConfig deployments in env ${params.environmentId}`) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await listDeployments( - client, - params.applicationId, - params.environmentId, - params.maxResults, - params.nextToken - ) - logger.info(`[${requestId}] Listed ${result.count} deployments`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to list deployments:`, error) - return NextResponse.json( - { error: `Failed to list deployments: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/list-environments/route.ts b/apps/sim/app/api/tools/appconfig/list-environments/route.ts deleted file mode 100644 index 8daae842e47..00000000000 --- a/apps/sim/app/api/tools/appconfig/list-environments/route.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigListEnvironmentsContract } from '@/lib/api/contracts/tools/aws/appconfig-list-environments' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, listEnvironments } from '../utils' - -const logger = createLogger('AppConfigListEnvironmentsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsAppConfigListEnvironmentsContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Listing AppConfig environments for ${params.applicationId}`) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await listEnvironments( - client, - params.applicationId, - params.maxResults, - params.nextToken - ) - logger.info(`[${requestId}] Listed ${result.count} environments`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to list environments:`, error) - return NextResponse.json( - { error: `Failed to list environments: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/list-hosted-configuration-versions/route.ts b/apps/sim/app/api/tools/appconfig/list-hosted-configuration-versions/route.ts deleted file mode 100644 index 9db3d331902..00000000000 --- a/apps/sim/app/api/tools/appconfig/list-hosted-configuration-versions/route.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigListHostedConfigurationVersionsContract } from '@/lib/api/contracts/tools/aws/appconfig-list-hosted-configuration-versions' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, listHostedConfigurationVersions } from '../utils' - -const logger = createLogger('AppConfigListHostedConfigurationVersionsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest( - awsAppConfigListHostedConfigurationVersionsContract, - request, - { errorFormat: 'details', logger } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Listing AppConfig hosted configuration versions for profile ${params.configurationProfileId}` - ) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await listHostedConfigurationVersions( - client, - params.applicationId, - params.configurationProfileId, - params.maxResults, - params.nextToken - ) - logger.info(`[${requestId}] Listed ${result.count} hosted configuration versions`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to list hosted configuration versions:`, error) - return NextResponse.json( - { error: `Failed to list hosted configuration versions: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/start-deployment/route.ts b/apps/sim/app/api/tools/appconfig/start-deployment/route.ts deleted file mode 100644 index 743f1ae330a..00000000000 --- a/apps/sim/app/api/tools/appconfig/start-deployment/route.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigStartDeploymentContract } from '@/lib/api/contracts/tools/aws/appconfig-start-deployment' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, startDeployment } from '../utils' - -const logger = createLogger('AppConfigStartDeploymentAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsAppConfigStartDeploymentContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Starting AppConfig deployment in env ${params.environmentId}`) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await startDeployment( - client, - params.applicationId, - params.environmentId, - params.deploymentStrategyId, - params.configurationProfileId, - params.configurationVersion, - params.description - ) - logger.info(`[${requestId}] Started deployment ${result.deploymentNumber}`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to start deployment:`, error) - return NextResponse.json( - { error: `Failed to start deployment: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/stop-deployment/route.ts b/apps/sim/app/api/tools/appconfig/stop-deployment/route.ts deleted file mode 100644 index dcbdf7db88a..00000000000 --- a/apps/sim/app/api/tools/appconfig/stop-deployment/route.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigStopDeploymentContract } from '@/lib/api/contracts/tools/aws/appconfig-stop-deployment' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, stopDeployment } from '../utils' - -const logger = createLogger('AppConfigStopDeploymentAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsAppConfigStopDeploymentContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Stopping AppConfig deployment ${params.deploymentNumber} in env ${params.environmentId}` - ) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await stopDeployment( - client, - params.applicationId, - params.environmentId, - params.deploymentNumber - ) - logger.info(`[${requestId}] Stopped deployment ${result.deploymentNumber}`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to stop deployment:`, error) - return NextResponse.json( - { error: `Failed to stop deployment: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/update-application/route.ts b/apps/sim/app/api/tools/appconfig/update-application/route.ts deleted file mode 100644 index d2c2ac58260..00000000000 --- a/apps/sim/app/api/tools/appconfig/update-application/route.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigUpdateApplicationContract } from '@/lib/api/contracts/tools/aws/appconfig-update-application' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, updateApplication } from '../utils' - -const logger = createLogger('AppConfigUpdateApplicationAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsAppConfigUpdateApplicationContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Updating AppConfig application ${params.applicationId}`) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await updateApplication( - client, - params.applicationId, - params.name, - params.description - ) - logger.info(`[${requestId}] Updated application`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to update application:`, error) - return NextResponse.json( - { error: `Failed to update application: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/update-configuration-profile/route.ts b/apps/sim/app/api/tools/appconfig/update-configuration-profile/route.ts deleted file mode 100644 index 002bacdf665..00000000000 --- a/apps/sim/app/api/tools/appconfig/update-configuration-profile/route.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigUpdateConfigurationProfileContract } from '@/lib/api/contracts/tools/aws/appconfig-update-configuration-profile' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, updateConfigurationProfile } from '../utils' - -const logger = createLogger('AppConfigUpdateConfigurationProfileAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsAppConfigUpdateConfigurationProfileContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Updating AppConfig configuration profile ${params.configurationProfileId}` - ) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await updateConfigurationProfile( - client, - params.applicationId, - params.configurationProfileId, - params.name, - params.description, - params.retrievalRoleArn - ) - logger.info(`[${requestId}] Updated configuration profile`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to update configuration profile:`, error) - return NextResponse.json( - { error: `Failed to update configuration profile: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/update-environment/route.ts b/apps/sim/app/api/tools/appconfig/update-environment/route.ts deleted file mode 100644 index b1f611b8f2f..00000000000 --- a/apps/sim/app/api/tools/appconfig/update-environment/route.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAppConfigUpdateEnvironmentContract } from '@/lib/api/contracts/tools/aws/appconfig-update-environment' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAppConfigClient, updateEnvironment } from '../utils' - -const logger = createLogger('AppConfigUpdateEnvironmentAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsAppConfigUpdateEnvironmentContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Updating AppConfig environment ${params.environmentId}`) - - const client = createAppConfigClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await updateEnvironment( - client, - params.applicationId, - params.environmentId, - params.name, - params.description - ) - logger.info(`[${requestId}] Updated environment`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to update environment:`, error) - return NextResponse.json( - { error: `Failed to update environment: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/appconfig/utils.ts b/apps/sim/app/api/tools/appconfig/utils.ts deleted file mode 100644 index 038109be133..00000000000 --- a/apps/sim/app/api/tools/appconfig/utils.ts +++ /dev/null @@ -1,681 +0,0 @@ -import { - AppConfigClient, - CreateApplicationCommand, - CreateConfigurationProfileCommand, - CreateEnvironmentCommand, - CreateHostedConfigurationVersionCommand, - DeleteApplicationCommand, - DeleteConfigurationProfileCommand, - DeleteEnvironmentCommand, - DeleteHostedConfigurationVersionCommand, - GetApplicationCommand, - GetConfigurationProfileCommand, - GetDeploymentCommand, - GetEnvironmentCommand, - GetHostedConfigurationVersionCommand, - ListApplicationsCommand, - ListConfigurationProfilesCommand, - ListDeploymentStrategiesCommand, - ListDeploymentsCommand, - ListEnvironmentsCommand, - ListHostedConfigurationVersionsCommand, - StartDeploymentCommand, - StopDeploymentCommand, - UpdateApplicationCommand, - UpdateConfigurationProfileCommand, - UpdateEnvironmentCommand, -} from '@aws-sdk/client-appconfig' -import { - AppConfigDataClient, - GetLatestConfigurationCommand, - StartConfigurationSessionCommand, -} from '@aws-sdk/client-appconfigdata' -import type { AppConfigConnectionConfig } from '@/tools/appconfig/types' - -export function createAppConfigClient(config: AppConfigConnectionConfig): AppConfigClient { - return new AppConfigClient({ - region: config.region, - credentials: { - accessKeyId: config.accessKeyId, - secretAccessKey: config.secretAccessKey, - }, - }) -} - -export function createAppConfigDataClient(config: AppConfigConnectionConfig): AppConfigDataClient { - return new AppConfigDataClient({ - region: config.region, - credentials: { - accessKeyId: config.accessKeyId, - secretAccessKey: config.secretAccessKey, - }, - }) -} - -const textDecoder = new TextDecoder() - -function decodeContent(content?: Uint8Array): string { - if (!content || content.length === 0) return '' - return textDecoder.decode(content) -} - -export async function listApplications( - client: AppConfigClient, - maxResults?: number | null, - nextToken?: string | null -) { - const response = await client.send( - new ListApplicationsCommand({ - ...(maxResults ? { MaxResults: maxResults } : {}), - ...(nextToken ? { NextToken: nextToken } : {}), - }) - ) - - const applications = (response.Items ?? []).map((item) => ({ - id: item.Id ?? '', - name: item.Name ?? '', - description: item.Description ?? null, - })) - - return { - applications, - nextToken: response.NextToken ?? null, - count: applications.length, - } -} - -export async function createApplication( - client: AppConfigClient, - name: string, - description?: string | null -) { - const response = await client.send( - new CreateApplicationCommand({ - Name: name, - ...(description ? { Description: description } : {}), - }) - ) - - return { - message: `Application "${response.Name ?? name}" created`, - id: response.Id ?? '', - name: response.Name ?? '', - description: response.Description ?? null, - } -} - -export async function listEnvironments( - client: AppConfigClient, - applicationId: string, - maxResults?: number | null, - nextToken?: string | null -) { - const response = await client.send( - new ListEnvironmentsCommand({ - ApplicationId: applicationId, - ...(maxResults ? { MaxResults: maxResults } : {}), - ...(nextToken ? { NextToken: nextToken } : {}), - }) - ) - - const environments = (response.Items ?? []).map((item) => ({ - applicationId: item.ApplicationId ?? '', - id: item.Id ?? '', - name: item.Name ?? '', - description: item.Description ?? null, - state: item.State ?? null, - })) - - return { - environments, - nextToken: response.NextToken ?? null, - count: environments.length, - } -} - -export async function createEnvironment( - client: AppConfigClient, - applicationId: string, - name: string, - description?: string | null -) { - const response = await client.send( - new CreateEnvironmentCommand({ - ApplicationId: applicationId, - Name: name, - ...(description ? { Description: description } : {}), - }) - ) - - return { - message: `Environment "${response.Name ?? name}" created`, - applicationId: response.ApplicationId ?? applicationId, - id: response.Id ?? '', - name: response.Name ?? '', - state: response.State ?? null, - } -} - -export async function listConfigurationProfiles( - client: AppConfigClient, - applicationId: string, - maxResults?: number | null, - nextToken?: string | null -) { - const response = await client.send( - new ListConfigurationProfilesCommand({ - ApplicationId: applicationId, - ...(maxResults ? { MaxResults: maxResults } : {}), - ...(nextToken ? { NextToken: nextToken } : {}), - }) - ) - - const configurationProfiles = (response.Items ?? []).map((item) => ({ - applicationId: item.ApplicationId ?? '', - id: item.Id ?? '', - name: item.Name ?? '', - description: null, - locationUri: item.LocationUri ?? null, - retrievalRoleArn: null, - type: item.Type ?? null, - validatorTypes: item.ValidatorTypes ?? [], - })) - - return { - configurationProfiles, - nextToken: response.NextToken ?? null, - count: configurationProfiles.length, - } -} - -export async function createConfigurationProfile( - client: AppConfigClient, - applicationId: string, - name: string, - locationUri: string, - description?: string | null, - retrievalRoleArn?: string | null, - type?: string | null -) { - const response = await client.send( - new CreateConfigurationProfileCommand({ - ApplicationId: applicationId, - Name: name, - LocationUri: locationUri, - ...(description ? { Description: description } : {}), - ...(retrievalRoleArn ? { RetrievalRoleArn: retrievalRoleArn } : {}), - ...(type ? { Type: type } : {}), - }) - ) - - return { - message: `Configuration profile "${response.Name ?? name}" created`, - applicationId: response.ApplicationId ?? applicationId, - id: response.Id ?? '', - name: response.Name ?? '', - locationUri: response.LocationUri ?? null, - type: response.Type ?? null, - } -} - -export async function createHostedConfigurationVersion( - client: AppConfigClient, - applicationId: string, - configurationProfileId: string, - content: string, - contentType: string, - description?: string | null, - latestVersionNumber?: number | null, - versionLabel?: string | null -) { - const response = await client.send( - new CreateHostedConfigurationVersionCommand({ - ApplicationId: applicationId, - ConfigurationProfileId: configurationProfileId, - Content: new TextEncoder().encode(content), - ContentType: contentType, - ...(description ? { Description: description } : {}), - ...(latestVersionNumber != null ? { LatestVersionNumber: latestVersionNumber } : {}), - ...(versionLabel ? { VersionLabel: versionLabel } : {}), - }) - ) - - return { - message: `Hosted configuration version ${response.VersionNumber ?? ''} created`, - applicationId: response.ApplicationId ?? applicationId, - configurationProfileId: response.ConfigurationProfileId ?? configurationProfileId, - versionNumber: response.VersionNumber ?? null, - contentType: response.ContentType ?? null, - versionLabel: response.VersionLabel ?? null, - } -} - -export async function getHostedConfigurationVersion( - client: AppConfigClient, - applicationId: string, - configurationProfileId: string, - versionNumber: number -) { - const response = await client.send( - new GetHostedConfigurationVersionCommand({ - ApplicationId: applicationId, - ConfigurationProfileId: configurationProfileId, - VersionNumber: versionNumber, - }) - ) - - return { - applicationId: response.ApplicationId ?? applicationId, - configurationProfileId: response.ConfigurationProfileId ?? configurationProfileId, - versionNumber: response.VersionNumber ?? null, - description: response.Description ?? null, - content: decodeContent(response.Content), - contentType: response.ContentType ?? null, - versionLabel: response.VersionLabel ?? null, - } -} - -export async function listHostedConfigurationVersions( - client: AppConfigClient, - applicationId: string, - configurationProfileId: string, - maxResults?: number | null, - nextToken?: string | null -) { - const response = await client.send( - new ListHostedConfigurationVersionsCommand({ - ApplicationId: applicationId, - ConfigurationProfileId: configurationProfileId, - ...(maxResults ? { MaxResults: maxResults } : {}), - ...(nextToken ? { NextToken: nextToken } : {}), - }) - ) - - const versions = (response.Items ?? []).map((item) => ({ - applicationId: item.ApplicationId ?? null, - configurationProfileId: item.ConfigurationProfileId ?? null, - versionNumber: item.VersionNumber ?? null, - description: item.Description ?? null, - contentType: item.ContentType ?? null, - versionLabel: item.VersionLabel ?? null, - })) - - return { - versions, - nextToken: response.NextToken ?? null, - count: versions.length, - } -} - -export async function listDeploymentStrategies( - client: AppConfigClient, - maxResults?: number | null, - nextToken?: string | null -) { - const response = await client.send( - new ListDeploymentStrategiesCommand({ - ...(maxResults ? { MaxResults: maxResults } : {}), - ...(nextToken ? { NextToken: nextToken } : {}), - }) - ) - - const deploymentStrategies = (response.Items ?? []).map((item) => ({ - id: item.Id ?? '', - name: item.Name ?? '', - description: item.Description ?? null, - deploymentDurationInMinutes: item.DeploymentDurationInMinutes ?? null, - growthType: item.GrowthType ?? null, - growthFactor: item.GrowthFactor ?? null, - finalBakeTimeInMinutes: item.FinalBakeTimeInMinutes ?? null, - replicateTo: item.ReplicateTo ?? null, - })) - - return { - deploymentStrategies, - nextToken: response.NextToken ?? null, - count: deploymentStrategies.length, - } -} - -export async function startDeployment( - client: AppConfigClient, - applicationId: string, - environmentId: string, - deploymentStrategyId: string, - configurationProfileId: string, - configurationVersion: string, - description?: string | null -) { - const response = await client.send( - new StartDeploymentCommand({ - ApplicationId: applicationId, - EnvironmentId: environmentId, - DeploymentStrategyId: deploymentStrategyId, - ConfigurationProfileId: configurationProfileId, - ConfigurationVersion: configurationVersion, - ...(description ? { Description: description } : {}), - }) - ) - - return { - message: `Deployment ${response.DeploymentNumber ?? ''} started`, - deploymentNumber: response.DeploymentNumber ?? null, - state: response.State ?? null, - percentageComplete: response.PercentageComplete ?? null, - } -} - -export async function getDeployment( - client: AppConfigClient, - applicationId: string, - environmentId: string, - deploymentNumber: number -) { - const response = await client.send( - new GetDeploymentCommand({ - ApplicationId: applicationId, - EnvironmentId: environmentId, - DeploymentNumber: deploymentNumber, - }) - ) - - return { - applicationId: response.ApplicationId ?? applicationId, - environmentId: response.EnvironmentId ?? environmentId, - deploymentStrategyId: response.DeploymentStrategyId ?? '', - configurationProfileId: response.ConfigurationProfileId ?? '', - deploymentNumber: response.DeploymentNumber ?? null, - configurationName: response.ConfigurationName ?? null, - configurationVersion: response.ConfigurationVersion ?? null, - description: response.Description ?? null, - state: response.State ?? null, - percentageComplete: response.PercentageComplete ?? null, - startedAt: response.StartedAt?.toISOString() ?? null, - completedAt: response.CompletedAt?.toISOString() ?? null, - } -} - -export async function listDeployments( - client: AppConfigClient, - applicationId: string, - environmentId: string, - maxResults?: number | null, - nextToken?: string | null -) { - const response = await client.send( - new ListDeploymentsCommand({ - ApplicationId: applicationId, - EnvironmentId: environmentId, - ...(maxResults ? { MaxResults: maxResults } : {}), - ...(nextToken ? { NextToken: nextToken } : {}), - }) - ) - - const deployments = (response.Items ?? []).map((item) => ({ - deploymentNumber: item.DeploymentNumber ?? null, - configurationName: item.ConfigurationName ?? null, - configurationVersion: item.ConfigurationVersion ?? null, - deploymentDurationInMinutes: item.DeploymentDurationInMinutes ?? null, - growthType: item.GrowthType ?? null, - growthFactor: item.GrowthFactor ?? null, - finalBakeTimeInMinutes: item.FinalBakeTimeInMinutes ?? null, - state: item.State ?? null, - percentageComplete: item.PercentageComplete ?? null, - startedAt: item.StartedAt?.toISOString() ?? null, - completedAt: item.CompletedAt?.toISOString() ?? null, - versionLabel: item.VersionLabel ?? null, - })) - - return { - deployments, - nextToken: response.NextToken ?? null, - count: deployments.length, - } -} - -export async function stopDeployment( - client: AppConfigClient, - applicationId: string, - environmentId: string, - deploymentNumber: number -) { - const response = await client.send( - new StopDeploymentCommand({ - ApplicationId: applicationId, - EnvironmentId: environmentId, - DeploymentNumber: deploymentNumber, - }) - ) - - return { - message: `Deployment ${response.DeploymentNumber ?? deploymentNumber} stopped`, - deploymentNumber: response.DeploymentNumber ?? null, - state: response.State ?? null, - } -} - -export async function getConfiguration( - client: AppConfigDataClient, - applicationId: string, - environmentId: string, - configurationProfileId: string -) { - const session = await client.send( - new StartConfigurationSessionCommand({ - ApplicationIdentifier: applicationId, - EnvironmentIdentifier: environmentId, - ConfigurationProfileIdentifier: configurationProfileId, - }) - ) - - const response = await client.send( - new GetLatestConfigurationCommand({ - ConfigurationToken: session.InitialConfigurationToken, - }) - ) - - return { - configuration: decodeContent(response.Configuration), - contentType: response.ContentType ?? null, - versionLabel: response.VersionLabel ?? null, - } -} - -export async function getApplication(client: AppConfigClient, applicationId: string) { - const response = await client.send(new GetApplicationCommand({ ApplicationId: applicationId })) - - return { - id: response.Id ?? '', - name: response.Name ?? '', - description: response.Description ?? null, - } -} - -export async function updateApplication( - client: AppConfigClient, - applicationId: string, - name?: string | null, - description?: string | null -) { - const response = await client.send( - new UpdateApplicationCommand({ - ApplicationId: applicationId, - ...(name ? { Name: name } : {}), - ...(description != null ? { Description: description } : {}), - }) - ) - - return { - message: `Application "${response.Name ?? applicationId}" updated`, - id: response.Id ?? '', - name: response.Name ?? '', - description: response.Description ?? null, - } -} - -export async function deleteApplication(client: AppConfigClient, applicationId: string) { - await client.send(new DeleteApplicationCommand({ ApplicationId: applicationId })) - - return { - message: `Application ${applicationId} deleted`, - id: applicationId, - } -} - -export async function getEnvironment( - client: AppConfigClient, - applicationId: string, - environmentId: string -) { - const response = await client.send( - new GetEnvironmentCommand({ ApplicationId: applicationId, EnvironmentId: environmentId }) - ) - - return { - applicationId: response.ApplicationId ?? applicationId, - id: response.Id ?? '', - name: response.Name ?? '', - description: response.Description ?? null, - state: response.State ?? null, - monitors: (response.Monitors ?? []).map((monitor) => ({ - alarmArn: monitor.AlarmArn ?? '', - alarmRoleArn: monitor.AlarmRoleArn ?? null, - })), - } -} - -export async function updateEnvironment( - client: AppConfigClient, - applicationId: string, - environmentId: string, - name?: string | null, - description?: string | null -) { - const response = await client.send( - new UpdateEnvironmentCommand({ - ApplicationId: applicationId, - EnvironmentId: environmentId, - ...(name ? { Name: name } : {}), - ...(description != null ? { Description: description } : {}), - }) - ) - - return { - message: `Environment "${response.Name ?? environmentId}" updated`, - applicationId: response.ApplicationId ?? applicationId, - id: response.Id ?? '', - name: response.Name ?? '', - state: response.State ?? null, - } -} - -export async function deleteEnvironment( - client: AppConfigClient, - applicationId: string, - environmentId: string -) { - await client.send( - new DeleteEnvironmentCommand({ ApplicationId: applicationId, EnvironmentId: environmentId }) - ) - - return { - message: `Environment ${environmentId} deleted`, - applicationId, - id: environmentId, - } -} - -export async function getConfigurationProfile( - client: AppConfigClient, - applicationId: string, - configurationProfileId: string -) { - const response = await client.send( - new GetConfigurationProfileCommand({ - ApplicationId: applicationId, - ConfigurationProfileId: configurationProfileId, - }) - ) - - return { - applicationId: response.ApplicationId ?? applicationId, - id: response.Id ?? '', - name: response.Name ?? '', - description: response.Description ?? null, - locationUri: response.LocationUri ?? null, - retrievalRoleArn: response.RetrievalRoleArn ?? null, - type: response.Type ?? null, - validators: (response.Validators ?? []).map((validator) => ({ - type: validator.Type ?? '', - })), - } -} - -export async function updateConfigurationProfile( - client: AppConfigClient, - applicationId: string, - configurationProfileId: string, - name?: string | null, - description?: string | null, - retrievalRoleArn?: string | null -) { - const response = await client.send( - new UpdateConfigurationProfileCommand({ - ApplicationId: applicationId, - ConfigurationProfileId: configurationProfileId, - ...(name ? { Name: name } : {}), - ...(description != null ? { Description: description } : {}), - ...(retrievalRoleArn != null ? { RetrievalRoleArn: retrievalRoleArn } : {}), - }) - ) - - return { - message: `Configuration profile "${response.Name ?? configurationProfileId}" updated`, - applicationId: response.ApplicationId ?? applicationId, - id: response.Id ?? '', - name: response.Name ?? '', - description: response.Description ?? null, - type: response.Type ?? null, - } -} - -export async function deleteConfigurationProfile( - client: AppConfigClient, - applicationId: string, - configurationProfileId: string -) { - await client.send( - new DeleteConfigurationProfileCommand({ - ApplicationId: applicationId, - ConfigurationProfileId: configurationProfileId, - }) - ) - - return { - message: `Configuration profile ${configurationProfileId} deleted`, - applicationId, - id: configurationProfileId, - } -} - -export async function deleteHostedConfigurationVersion( - client: AppConfigClient, - applicationId: string, - configurationProfileId: string, - versionNumber: number -) { - await client.send( - new DeleteHostedConfigurationVersionCommand({ - ApplicationId: applicationId, - ConfigurationProfileId: configurationProfileId, - VersionNumber: versionNumber, - }) - ) - - return { - message: `Hosted configuration version ${versionNumber} deleted`, - applicationId, - configurationProfileId, - versionNumber, - } -} diff --git a/apps/sim/app/api/tools/asana/add-comment/route.ts b/apps/sim/app/api/tools/asana/add-comment/route.ts deleted file mode 100644 index bbe40e66584..00000000000 --- a/apps/sim/app/api/tools/asana/add-comment/route.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { asanaAddCommentContract } from '@/lib/api/contracts/tools/asana' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AsanaAddCommentAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(asanaAddCommentContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, taskGid, text } = parsed.data.body - - const taskGidValidation = validateAlphanumericId(taskGid, 'taskGid', 100) - if (!taskGidValidation.isValid) { - return NextResponse.json({ error: taskGidValidation.error }, { status: 400 }) - } - - const url = `https://app.asana.com/api/1.0/tasks/${taskGid}/stories` - - const body = { - data: { - text, - }, - } - - const response = await fetch(url, { - method: 'POST', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify(body), - }) - - if (!response.ok) { - const errorText = await response.text() - let errorMessage = `Asana API error: ${response.status} ${response.statusText}` - - try { - const errorData = JSON.parse(errorText) - const asanaError = errorData.errors?.[0] - if (asanaError) { - errorMessage = `${asanaError.message || errorMessage} (${asanaError.help || ''})` - } - logger.error('Asana API error:', { - status: response.status, - statusText: response.statusText, - error: errorData, - }) - } catch (_e) { - logger.error('Asana API error (unparsed):', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - } - - return NextResponse.json( - { - success: false, - error: errorMessage, - details: errorText, - }, - { status: response.status } - ) - } - - const result = await response.json() - const story = result.data - - return NextResponse.json({ - success: true, - ts: new Date().toISOString(), - gid: story.gid, - text: story.text || '', - created_at: story.created_at, - created_by: story.created_by - ? { - gid: story.created_by.gid, - name: story.created_by.name, - } - : undefined, - }) - } catch (error) { - logger.error('Error processing request:', error) - return NextResponse.json( - { - error: 'Failed to add comment to Asana task', - details: (error as Error).message, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/asana/add-followers/route.ts b/apps/sim/app/api/tools/asana/add-followers/route.ts deleted file mode 100644 index d9ada412efd..00000000000 --- a/apps/sim/app/api/tools/asana/add-followers/route.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { asanaAddFollowersContract } from '@/lib/api/contracts/tools/asana' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AsanaAddFollowersAPI') - -interface AsanaFollower { - gid: string - name: string -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(asanaAddFollowersContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, taskGid, followers } = parsed.data.body - - const taskGidValidation = validateAlphanumericId(taskGid, 'taskGid', 100) - if (!taskGidValidation.isValid) { - return NextResponse.json({ error: taskGidValidation.error }, { status: 400 }) - } - - for (const follower of followers) { - const followerValidation = validateAlphanumericId(follower, 'follower', 100) - if (!followerValidation.isValid) { - return NextResponse.json({ error: followerValidation.error }, { status: 400 }) - } - } - - const url = `https://app.asana.com/api/1.0/tasks/${taskGid}/addFollowers?opt_fields=name,followers.name` - - const response = await fetch(url, { - method: 'POST', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ data: { followers } }), - }) - - if (!response.ok) { - const errorText = await response.text() - let errorMessage = `Asana API error: ${response.status} ${response.statusText}` - - try { - const errorData = JSON.parse(errorText) - const asanaError = errorData.errors?.[0] - if (asanaError) { - errorMessage = `${asanaError.message || errorMessage} (${asanaError.help || ''})` - } - logger.error('Asana API error:', { - status: response.status, - statusText: response.statusText, - error: errorData, - }) - } catch (_e) { - logger.error('Asana API error (unparsed):', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - } - - return NextResponse.json( - { success: false, error: errorMessage, details: errorText }, - { status: response.status } - ) - } - - const result = await response.json() - const task = result.data - const taskFollowers: AsanaFollower[] = Array.isArray(task.followers) ? task.followers : [] - - return NextResponse.json({ - success: true, - ts: new Date().toISOString(), - gid: task.gid, - name: task.name || '', - followers: taskFollowers.map((follower) => ({ - gid: follower.gid, - name: follower.name, - })), - }) - } catch (error) { - logger.error('Error adding followers to Asana task:', error) - return NextResponse.json( - { error: 'Failed to add followers to Asana task', details: (error as Error).message }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/asana/create-project/route.ts b/apps/sim/app/api/tools/asana/create-project/route.ts deleted file mode 100644 index 1af9133f375..00000000000 --- a/apps/sim/app/api/tools/asana/create-project/route.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { asanaCreateProjectContract } from '@/lib/api/contracts/tools/asana' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AsanaCreateProjectAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(asanaCreateProjectContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, workspace, name, notes } = parsed.data.body - - const workspaceValidation = validateAlphanumericId(workspace, 'workspace', 100) - if (!workspaceValidation.isValid) { - return NextResponse.json({ error: workspaceValidation.error }, { status: 400 }) - } - - const projectData: Record = { name, workspace } - if (notes) { - projectData.notes = notes - } - - const response = await fetch( - 'https://app.asana.com/api/1.0/projects?opt_fields=name,notes,archived,color,created_at,modified_at,permalink_url', - { - method: 'POST', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ data: projectData }), - } - ) - - if (!response.ok) { - const errorText = await response.text() - let errorMessage = `Asana API error: ${response.status} ${response.statusText}` - - try { - const errorData = JSON.parse(errorText) - const asanaError = errorData.errors?.[0] - if (asanaError) { - errorMessage = `${asanaError.message || errorMessage} (${asanaError.help || ''})` - } - logger.error('Asana API error:', { - status: response.status, - statusText: response.statusText, - error: errorData, - }) - } catch (_e) { - logger.error('Asana API error (unparsed):', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - } - - return NextResponse.json( - { success: false, error: errorMessage, details: errorText }, - { status: response.status } - ) - } - - const result = await response.json() - const project = result.data - - return NextResponse.json({ - success: true, - ts: new Date().toISOString(), - gid: project.gid, - name: project.name, - notes: project.notes || '', - archived: project.archived ?? false, - color: project.color ?? null, - created_at: project.created_at, - modified_at: project.modified_at, - permalink_url: project.permalink_url, - }) - } catch (error) { - logger.error('Error creating Asana project:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { error: getErrorMessage(error, 'Internal server error'), success: false }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/asana/create-section/route.ts b/apps/sim/app/api/tools/asana/create-section/route.ts deleted file mode 100644 index 0d351d6da67..00000000000 --- a/apps/sim/app/api/tools/asana/create-section/route.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { asanaCreateSectionContract } from '@/lib/api/contracts/tools/asana' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AsanaCreateSectionAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(asanaCreateSectionContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, projectGid, name } = parsed.data.body - - const projectGidValidation = validateAlphanumericId(projectGid, 'projectGid', 100) - if (!projectGidValidation.isValid) { - return NextResponse.json({ error: projectGidValidation.error }, { status: 400 }) - } - - const url = `https://app.asana.com/api/1.0/projects/${projectGid}/sections` - - const response = await fetch(url, { - method: 'POST', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ data: { name } }), - }) - - if (!response.ok) { - const errorText = await response.text() - let errorMessage = `Asana API error: ${response.status} ${response.statusText}` - - try { - const errorData = JSON.parse(errorText) - const asanaError = errorData.errors?.[0] - if (asanaError) { - errorMessage = `${asanaError.message || errorMessage} (${asanaError.help || ''})` - } - logger.error('Asana API error:', { - status: response.status, - statusText: response.statusText, - error: errorData, - }) - } catch (_e) { - logger.error('Asana API error (unparsed):', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - } - - return NextResponse.json( - { success: false, error: errorMessage, details: errorText }, - { status: response.status } - ) - } - - const result = await response.json() - const section = result.data - - return NextResponse.json({ - success: true, - ts: new Date().toISOString(), - gid: section.gid, - name: section.name, - created_at: section.created_at, - }) - } catch (error) { - logger.error('Error creating Asana section:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { error: getErrorMessage(error, 'Internal server error'), success: false }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/asana/create-subtask/route.ts b/apps/sim/app/api/tools/asana/create-subtask/route.ts deleted file mode 100644 index a1ce4676a07..00000000000 --- a/apps/sim/app/api/tools/asana/create-subtask/route.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { asanaCreateSubtaskContract } from '@/lib/api/contracts/tools/asana' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AsanaCreateSubtaskAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(asanaCreateSubtaskContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, taskGid, name, notes, assignee, due_on } = parsed.data.body - - const taskGidValidation = validateAlphanumericId(taskGid, 'taskGid', 100) - if (!taskGidValidation.isValid) { - return NextResponse.json({ error: taskGidValidation.error }, { status: 400 }) - } - - const subtaskData: Record = { name } - if (notes) { - subtaskData.notes = notes - } - if (assignee) { - subtaskData.assignee = assignee - } - if (due_on) { - subtaskData.due_on = due_on - } - - const url = `https://app.asana.com/api/1.0/tasks/${taskGid}/subtasks?opt_fields=name,notes,completed,created_at,permalink_url` - - const response = await fetch(url, { - method: 'POST', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ data: subtaskData }), - }) - - if (!response.ok) { - const errorText = await response.text() - let errorMessage = `Asana API error: ${response.status} ${response.statusText}` - - try { - const errorData = JSON.parse(errorText) - const asanaError = errorData.errors?.[0] - if (asanaError) { - errorMessage = `${asanaError.message || errorMessage} (${asanaError.help || ''})` - } - logger.error('Asana API error:', { - status: response.status, - statusText: response.statusText, - error: errorData, - }) - } catch (_e) { - logger.error('Asana API error (unparsed):', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - } - - return NextResponse.json( - { success: false, error: errorMessage, details: errorText }, - { status: response.status } - ) - } - - const result = await response.json() - const task = result.data - - return NextResponse.json({ - success: true, - ts: new Date().toISOString(), - gid: task.gid, - name: task.name, - notes: task.notes || '', - completed: task.completed || false, - created_at: task.created_at, - permalink_url: task.permalink_url, - }) - } catch (error) { - logger.error('Error creating Asana subtask:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { error: getErrorMessage(error, 'Internal server error'), success: false }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/asana/create-task/route.ts b/apps/sim/app/api/tools/asana/create-task/route.ts deleted file mode 100644 index 39521ca9d04..00000000000 --- a/apps/sim/app/api/tools/asana/create-task/route.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { asanaCreateTaskContract } from '@/lib/api/contracts/tools/asana' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AsanaCreateTaskAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(asanaCreateTaskContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, workspace, name, notes, assignee, due_on } = parsed.data.body - - const workspaceValidation = validateAlphanumericId(workspace, 'workspace', 100) - if (!workspaceValidation.isValid) { - return NextResponse.json({ error: workspaceValidation.error }, { status: 400 }) - } - - const url = - 'https://app.asana.com/api/1.0/tasks?opt_fields=name,notes,completed,created_at,permalink_url' - - const taskData: Record = { - name, - workspace, - } - - if (notes) { - taskData.notes = notes - } - - if (assignee) { - taskData.assignee = assignee - } - - if (due_on) { - taskData.due_on = due_on - } - - const body = { data: taskData } - - const response = await fetch(url, { - method: 'POST', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify(body), - }) - - if (!response.ok) { - const errorText = await response.text() - let errorMessage = `Asana API error: ${response.status} ${response.statusText}` - - try { - const errorData = JSON.parse(errorText) - const asanaError = errorData.errors?.[0] - if (asanaError) { - errorMessage = `${asanaError.message || errorMessage} (${asanaError.help || ''})` - } - logger.error('Asana API error:', { - status: response.status, - statusText: response.statusText, - error: errorData, - }) - } catch (_e) { - logger.error('Asana API error (unparsed):', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - } - - return NextResponse.json( - { - success: false, - error: errorMessage, - details: errorText, - }, - { status: response.status } - ) - } - - const result = await response.json() - const task = result.data - - return NextResponse.json({ - success: true, - ts: new Date().toISOString(), - gid: task.gid, - name: task.name, - notes: task.notes || '', - completed: task.completed || false, - created_at: task.created_at, - permalink_url: task.permalink_url, - }) - } catch (error) { - logger.error('Error creating Asana task:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/asana/delete-task/route.ts b/apps/sim/app/api/tools/asana/delete-task/route.ts deleted file mode 100644 index 6ca717e4f62..00000000000 --- a/apps/sim/app/api/tools/asana/delete-task/route.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { asanaDeleteTaskContract } from '@/lib/api/contracts/tools/asana' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AsanaDeleteTaskAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(asanaDeleteTaskContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, taskGid } = parsed.data.body - - const taskGidValidation = validateAlphanumericId(taskGid, 'taskGid', 100) - if (!taskGidValidation.isValid) { - return NextResponse.json({ error: taskGidValidation.error }, { status: 400 }) - } - - const url = `https://app.asana.com/api/1.0/tasks/${taskGid}` - - const response = await fetch(url, { - method: 'DELETE', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) - - if (!response.ok) { - const errorText = await response.text() - let errorMessage = `Asana API error: ${response.status} ${response.statusText}` - - try { - const errorData = JSON.parse(errorText) - const asanaError = errorData.errors?.[0] - if (asanaError) { - errorMessage = `${asanaError.message || errorMessage} (${asanaError.help || ''})` - } - logger.error('Asana API error:', { - status: response.status, - statusText: response.statusText, - error: errorData, - }) - } catch (_e) { - logger.error('Asana API error (unparsed):', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - } - - return NextResponse.json( - { success: false, error: errorMessage, details: errorText }, - { status: response.status } - ) - } - - return NextResponse.json({ - success: true, - ts: new Date().toISOString(), - gid: taskGid, - deleted: true, - }) - } catch (error) { - logger.error('Error deleting Asana task:', error) - return NextResponse.json( - { error: 'Failed to delete Asana task', details: (error as Error).message }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/asana/get-project/route.ts b/apps/sim/app/api/tools/asana/get-project/route.ts deleted file mode 100644 index 3e7022a5a9b..00000000000 --- a/apps/sim/app/api/tools/asana/get-project/route.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { asanaGetProjectContract } from '@/lib/api/contracts/tools/asana' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AsanaGetProjectAPI') - -const PROJECT_OPT_FIELDS = 'name,notes,archived,color,created_at,modified_at,permalink_url' - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(asanaGetProjectContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, projectGid } = parsed.data.body - - const projectGidValidation = validateAlphanumericId(projectGid, 'projectGid', 100) - if (!projectGidValidation.isValid) { - return NextResponse.json({ error: projectGidValidation.error }, { status: 400 }) - } - - const url = `https://app.asana.com/api/1.0/projects/${projectGid}?opt_fields=${PROJECT_OPT_FIELDS}` - - const response = await fetch(url, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) - - if (!response.ok) { - const errorText = await response.text() - let errorMessage = `Asana API error: ${response.status} ${response.statusText}` - - try { - const errorData = JSON.parse(errorText) - const asanaError = errorData.errors?.[0] - if (asanaError) { - errorMessage = `${asanaError.message || errorMessage} (${asanaError.help || ''})` - } - logger.error('Asana API error:', { - status: response.status, - statusText: response.statusText, - error: errorData, - }) - } catch (_e) { - logger.error('Asana API error (unparsed):', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - } - - return NextResponse.json( - { success: false, error: errorMessage, details: errorText }, - { status: response.status } - ) - } - - const result = await response.json() - const project = result.data - - return NextResponse.json({ - success: true, - ts: new Date().toISOString(), - gid: project.gid, - name: project.name, - notes: project.notes || '', - archived: project.archived ?? false, - color: project.color ?? null, - created_at: project.created_at, - modified_at: project.modified_at, - permalink_url: project.permalink_url, - }) - } catch (error) { - logger.error('Error processing request:', error) - return NextResponse.json( - { error: 'Failed to retrieve Asana project', details: (error as Error).message }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/asana/get-projects/route.ts b/apps/sim/app/api/tools/asana/get-projects/route.ts deleted file mode 100644 index b05e9d2d4b8..00000000000 --- a/apps/sim/app/api/tools/asana/get-projects/route.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { asanaGetProjectsContract } from '@/lib/api/contracts/tools/asana' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AsanaGetProjectsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(asanaGetProjectsContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, workspace } = parsed.data.body - - const workspaceValidation = validateAlphanumericId(workspace, 'workspace', 100) - if (!workspaceValidation.isValid) { - return NextResponse.json({ error: workspaceValidation.error }, { status: 400 }) - } - - const url = `https://app.asana.com/api/1.0/projects?workspace=${workspace}` - - const response = await fetch(url, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) - - if (!response.ok) { - const errorText = await response.text() - let errorMessage = `Asana API error: ${response.status} ${response.statusText}` - - try { - const errorData = JSON.parse(errorText) - const asanaError = errorData.errors?.[0] - if (asanaError) { - errorMessage = `${asanaError.message || errorMessage} (${asanaError.help || ''})` - } - logger.error('Asana API error:', { - status: response.status, - statusText: response.statusText, - error: errorData, - }) - } catch (_e) { - logger.error('Asana API error (unparsed):', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - } - - return NextResponse.json( - { - success: false, - error: errorMessage, - details: errorText, - }, - { status: response.status } - ) - } - - const result = await response.json() - const projects = result.data - - return NextResponse.json({ - success: true, - ts: new Date().toISOString(), - projects: projects.map((project: { gid: string; name: string; resource_type: string }) => ({ - gid: project.gid, - name: project.name, - resource_type: project.resource_type, - })), - }) - } catch (error) { - logger.error('Error processing request:', error) - return NextResponse.json( - { - error: 'Failed to retrieve Asana projects', - details: (error as Error).message, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/asana/get-task/route.ts b/apps/sim/app/api/tools/asana/get-task/route.ts deleted file mode 100644 index 304308b52c4..00000000000 --- a/apps/sim/app/api/tools/asana/get-task/route.ts +++ /dev/null @@ -1,224 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { asanaGetTaskContract } from '@/lib/api/contracts/tools/asana' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AsanaGetTaskAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(asanaGetTaskContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, taskGid, workspace, project, limit } = parsed.data.body - - if (taskGid) { - const taskGidValidation = validateAlphanumericId(taskGid, 'taskGid', 100) - if (!taskGidValidation.isValid) { - return NextResponse.json({ error: taskGidValidation.error }, { status: 400 }) - } - - const url = `https://app.asana.com/api/1.0/tasks/${taskGid}?opt_fields=gid,name,notes,completed,assignee,assignee.name,due_on,created_at,modified_at,created_by,created_by.name,resource_type,resource_subtype` - - const response = await fetch(url, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) - - if (!response.ok) { - const errorText = await response.text() - let errorMessage = `Asana API error: ${response.status} ${response.statusText}` - - try { - const errorData = JSON.parse(errorText) - const asanaError = errorData.errors?.[0] - if (asanaError) { - errorMessage = `${asanaError.message || errorMessage} (${asanaError.help || ''})` - } - logger.error('Asana API error:', { - status: response.status, - statusText: response.statusText, - error: errorData, - }) - } catch (_e) { - logger.error('Asana API error (unparsed):', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - } - - return NextResponse.json( - { - success: false, - error: errorMessage, - details: errorText, - }, - { status: response.status } - ) - } - - const result = await response.json() - const task = result.data - - return NextResponse.json({ - success: true, - ts: new Date().toISOString(), - gid: task.gid, - resource_type: task.resource_type, - resource_subtype: task.resource_subtype, - name: task.name, - notes: task.notes || '', - completed: task.completed || false, - assignee: task.assignee - ? { - gid: task.assignee.gid, - name: task.assignee.name, - } - : undefined, - created_by: task.created_by - ? { - gid: task.created_by.gid, - resource_type: task.created_by.resource_type, - name: task.created_by.name, - } - : undefined, - due_on: task.due_on || undefined, - created_at: task.created_at, - modified_at: task.modified_at, - }) - } - - if (!workspace && !project) { - logger.error('Either taskGid or workspace/project must be provided') - return NextResponse.json( - { error: 'Either taskGid or workspace/project must be provided' }, - { status: 400 } - ) - } - - const params = new URLSearchParams() - - if (project) { - const projectValidation = validateAlphanumericId(project, 'project', 100) - if (!projectValidation.isValid) { - return NextResponse.json({ error: projectValidation.error }, { status: 400 }) - } - params.append('project', project) - } else if (workspace) { - const workspaceValidation = validateAlphanumericId(workspace, 'workspace', 100) - if (!workspaceValidation.isValid) { - return NextResponse.json({ error: workspaceValidation.error }, { status: 400 }) - } - params.append('workspace', workspace) - } - - if (limit) { - params.append('limit', String(limit)) - } else { - params.append('limit', '50') - } - - params.append( - 'opt_fields', - 'gid,name,notes,completed,assignee,assignee.name,due_on,created_at,modified_at,created_by,created_by.name,resource_type,resource_subtype' - ) - - const url = `https://app.asana.com/api/1.0/tasks?${params.toString()}` - - const response = await fetch(url, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) - - if (!response.ok) { - const errorText = await response.text() - let errorMessage = `Asana API error: ${response.status} ${response.statusText}` - - try { - const errorData = JSON.parse(errorText) - const asanaError = errorData.errors?.[0] - if (asanaError) { - errorMessage = `${asanaError.message || errorMessage} (${asanaError.help || ''})` - } - logger.error('Asana API error:', { - status: response.status, - statusText: response.statusText, - error: errorData, - }) - } catch (_e) { - logger.error('Asana API error (unparsed):', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - } - - return NextResponse.json( - { - success: false, - error: errorMessage, - details: errorText, - }, - { status: response.status } - ) - } - - const result = await response.json() - const tasks = result.data - - return NextResponse.json({ - success: true, - ts: new Date().toISOString(), - tasks: tasks.map((task: any) => ({ - gid: task.gid, - resource_type: task.resource_type, - resource_subtype: task.resource_subtype, - name: task.name, - notes: task.notes || '', - completed: task.completed || false, - assignee: task.assignee - ? { - gid: task.assignee.gid, - name: task.assignee.name, - } - : undefined, - created_by: task.created_by - ? { - gid: task.created_by.gid, - resource_type: task.created_by.resource_type, - name: task.created_by.name, - } - : undefined, - due_on: task.due_on || undefined, - created_at: task.created_at, - modified_at: task.modified_at, - })), - next_page: result.next_page, - }) - } catch (error) { - logger.error('Error processing request:', error) - return NextResponse.json( - { - error: 'Failed to retrieve Asana task(s)', - details: (error as Error).message, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/asana/list-sections/route.ts b/apps/sim/app/api/tools/asana/list-sections/route.ts deleted file mode 100644 index 05524251068..00000000000 --- a/apps/sim/app/api/tools/asana/list-sections/route.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { asanaListSectionsContract } from '@/lib/api/contracts/tools/asana' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AsanaListSectionsAPI') - -interface AsanaSection { - gid: string - name: string - resource_type?: string -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(asanaListSectionsContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, projectGid } = parsed.data.body - - const projectGidValidation = validateAlphanumericId(projectGid, 'projectGid', 100) - if (!projectGidValidation.isValid) { - return NextResponse.json({ error: projectGidValidation.error }, { status: 400 }) - } - - const url = `https://app.asana.com/api/1.0/projects/${projectGid}/sections` - - const response = await fetch(url, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) - - if (!response.ok) { - const errorText = await response.text() - let errorMessage = `Asana API error: ${response.status} ${response.statusText}` - - try { - const errorData = JSON.parse(errorText) - const asanaError = errorData.errors?.[0] - if (asanaError) { - errorMessage = `${asanaError.message || errorMessage} (${asanaError.help || ''})` - } - logger.error('Asana API error:', { - status: response.status, - statusText: response.statusText, - error: errorData, - }) - } catch (_e) { - logger.error('Asana API error (unparsed):', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - } - - return NextResponse.json( - { success: false, error: errorMessage, details: errorText }, - { status: response.status } - ) - } - - const result = await response.json() - const sections: AsanaSection[] = Array.isArray(result.data) ? result.data : [] - - return NextResponse.json({ - success: true, - ts: new Date().toISOString(), - sections: sections.map((section) => ({ - gid: section.gid, - name: section.name, - resource_type: section.resource_type, - })), - }) - } catch (error) { - logger.error('Error processing request:', error) - return NextResponse.json( - { error: 'Failed to retrieve Asana sections', details: (error as Error).message }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/asana/list-workspaces/route.ts b/apps/sim/app/api/tools/asana/list-workspaces/route.ts deleted file mode 100644 index 1c550e2c4d7..00000000000 --- a/apps/sim/app/api/tools/asana/list-workspaces/route.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { asanaListWorkspacesContract } from '@/lib/api/contracts/tools/asana' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AsanaListWorkspacesAPI') - -interface AsanaWorkspace { - gid: string - name: string - resource_type?: string -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(asanaListWorkspacesContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken } = parsed.data.body - - const url = 'https://app.asana.com/api/1.0/workspaces?limit=100' - - const response = await fetch(url, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) - - if (!response.ok) { - const errorText = await response.text() - let errorMessage = `Asana API error: ${response.status} ${response.statusText}` - - try { - const errorData = JSON.parse(errorText) - const asanaError = errorData.errors?.[0] - if (asanaError) { - errorMessage = `${asanaError.message || errorMessage} (${asanaError.help || ''})` - } - logger.error('Asana API error:', { - status: response.status, - statusText: response.statusText, - error: errorData, - }) - } catch (_e) { - logger.error('Asana API error (unparsed):', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - } - - return NextResponse.json( - { success: false, error: errorMessage, details: errorText }, - { status: response.status } - ) - } - - const result = await response.json() - const workspaces: AsanaWorkspace[] = Array.isArray(result.data) ? result.data : [] - - return NextResponse.json({ - success: true, - ts: new Date().toISOString(), - workspaces: workspaces.map((workspace) => ({ - gid: workspace.gid, - name: workspace.name, - resource_type: workspace.resource_type, - })), - }) - } catch (error) { - logger.error('Error processing request:', error) - return NextResponse.json( - { error: 'Failed to retrieve Asana workspaces', details: (error as Error).message }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/asana/search-tasks/route.ts b/apps/sim/app/api/tools/asana/search-tasks/route.ts deleted file mode 100644 index d3d0488f997..00000000000 --- a/apps/sim/app/api/tools/asana/search-tasks/route.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { asanaSearchTasksContract } from '@/lib/api/contracts/tools/asana' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AsanaSearchTasksAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(asanaSearchTasksContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, workspace, text, assignee, projects, completed } = parsed.data.body - - const workspaceValidation = validateAlphanumericId(workspace, 'workspace', 100) - if (!workspaceValidation.isValid) { - return NextResponse.json({ error: workspaceValidation.error }, { status: 400 }) - } - - const params = new URLSearchParams() - - if (text) { - params.append('text', text) - } - - if (assignee) { - params.append('assignee.any', assignee) - } - - if (projects && Array.isArray(projects) && projects.length > 0) { - params.append('projects.any', projects.join(',')) - } - - if (completed !== undefined) { - params.append('completed', String(completed)) - } - - params.append( - 'opt_fields', - 'gid,name,notes,completed,assignee,assignee.name,due_on,created_at,modified_at,created_by,created_by.name,resource_type,resource_subtype' - ) - - const url = `https://app.asana.com/api/1.0/workspaces/${workspace}/tasks/search?${params.toString()}` - - const response = await fetch(url, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) - - if (!response.ok) { - const errorText = await response.text() - let errorMessage = `Asana API error: ${response.status} ${response.statusText}` - - try { - const errorData = JSON.parse(errorText) - const asanaError = errorData.errors?.[0] - if (asanaError) { - errorMessage = `${asanaError.message || errorMessage} (${asanaError.help || ''})` - } - logger.error('Asana API error:', { - status: response.status, - statusText: response.statusText, - error: errorData, - }) - } catch (_e) { - logger.error('Asana API error (unparsed):', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - } - - return NextResponse.json( - { - success: false, - error: errorMessage, - details: errorText, - }, - { status: response.status } - ) - } - - const result = await response.json() - const tasks = result.data - - return NextResponse.json({ - success: true, - ts: new Date().toISOString(), - tasks: tasks.map((task: any) => ({ - gid: task.gid, - resource_type: task.resource_type, - resource_subtype: task.resource_subtype, - name: task.name, - notes: task.notes || '', - completed: task.completed || false, - assignee: task.assignee - ? { - gid: task.assignee.gid, - name: task.assignee.name, - } - : undefined, - created_by: task.created_by - ? { - gid: task.created_by.gid, - resource_type: task.created_by.resource_type, - name: task.created_by.name, - } - : undefined, - due_on: task.due_on || undefined, - created_at: task.created_at, - modified_at: task.modified_at, - })), - next_page: result.next_page, - }) - } catch (error) { - logger.error('Error processing request:', error) - return NextResponse.json( - { - error: 'Failed to search Asana tasks', - details: (error as Error).message, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/asana/update-task/route.ts b/apps/sim/app/api/tools/asana/update-task/route.ts deleted file mode 100644 index 44dabf2404a..00000000000 --- a/apps/sim/app/api/tools/asana/update-task/route.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { asanaUpdateTaskContract } from '@/lib/api/contracts/tools/asana' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AsanaUpdateTaskAPI') - -export const PUT = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(asanaUpdateTaskContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, taskGid, name, notes, assignee, completed, due_on } = parsed.data.body - - const taskGidValidation = validateAlphanumericId(taskGid, 'taskGid', 100) - if (!taskGidValidation.isValid) { - return NextResponse.json({ error: taskGidValidation.error }, { status: 400 }) - } - - const url = `https://app.asana.com/api/1.0/tasks/${taskGid}` - - const taskData: Record = {} - - if (name !== undefined) { - taskData.name = name - } - - if (notes !== undefined) { - taskData.notes = notes - } - - if (assignee !== undefined) { - taskData.assignee = assignee - } - - if (completed !== undefined) { - taskData.completed = completed - } - - if (due_on !== undefined) { - taskData.due_on = due_on - } - - const body = { data: taskData } - - const response = await fetch(url, { - method: 'PUT', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify(body), - }) - - if (!response.ok) { - const errorText = await response.text() - let errorMessage = `Asana API error: ${response.status} ${response.statusText}` - - try { - const errorData = JSON.parse(errorText) - const asanaError = errorData.errors?.[0] - if (asanaError) { - errorMessage = `${asanaError.message || errorMessage} (${asanaError.help || ''})` - } - logger.error('Asana API error:', { - status: response.status, - statusText: response.statusText, - error: errorData, - }) - } catch (_e) { - logger.error('Asana API error (unparsed):', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - } - - return NextResponse.json( - { - success: false, - error: errorMessage, - details: errorText, - }, - { status: response.status } - ) - } - - const result = await response.json() - const task = result.data - - return NextResponse.json({ - success: true, - ts: new Date().toISOString(), - gid: task.gid, - name: task.name, - notes: task.notes || '', - completed: task.completed || false, - modified_at: task.modified_at, - }) - } catch (error) { - logger.error('Error updating Asana task:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/athena/batch-get-query-execution/route.ts b/apps/sim/app/api/tools/athena/batch-get-query-execution/route.ts deleted file mode 100644 index 825a1b3efa3..00000000000 --- a/apps/sim/app/api/tools/athena/batch-get-query-execution/route.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { BatchGetQueryExecutionCommand } from '@aws-sdk/client-athena' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAthenaBatchGetQueryExecutionContract } from '@/lib/api/contracts/tools/aws/athena-batch-get-query-execution' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAthenaClient } from '@/app/api/tools/athena/utils' - -const logger = createLogger('AthenaBatchGetQueryExecution') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsAthenaBatchGetQueryExecutionContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - const client = createAthenaClient({ - region: data.region, - accessKeyId: data.accessKeyId, - secretAccessKey: data.secretAccessKey, - }) - - try { - const command = new BatchGetQueryExecutionCommand({ - QueryExecutionIds: data.queryExecutionIds, - }) - - const response = await client.send(command) - - return NextResponse.json({ - success: true, - output: { - queryExecutions: (response.QueryExecutions ?? []).map((execution) => ({ - queryExecutionId: execution.QueryExecutionId ?? '', - query: execution.Query ?? null, - state: execution.Status?.State ?? null, - stateChangeReason: execution.Status?.StateChangeReason ?? null, - statementType: execution.StatementType ?? null, - database: execution.QueryExecutionContext?.Database ?? null, - catalog: execution.QueryExecutionContext?.Catalog ?? null, - workGroup: execution.WorkGroup ?? null, - submissionDateTime: execution.Status?.SubmissionDateTime?.getTime() ?? null, - completionDateTime: execution.Status?.CompletionDateTime?.getTime() ?? null, - dataScannedInBytes: execution.Statistics?.DataScannedInBytes ?? null, - engineExecutionTimeInMillis: execution.Statistics?.EngineExecutionTimeInMillis ?? null, - queryPlanningTimeInMillis: execution.Statistics?.QueryPlanningTimeInMillis ?? null, - queryQueueTimeInMillis: execution.Statistics?.QueryQueueTimeInMillis ?? null, - totalExecutionTimeInMillis: execution.Statistics?.TotalExecutionTimeInMillis ?? null, - outputLocation: execution.ResultConfiguration?.OutputLocation ?? null, - })), - unprocessedQueryExecutionIds: (response.UnprocessedQueryExecutionIds ?? []).map( - (item) => ({ - queryExecutionId: item.QueryExecutionId ?? null, - errorCode: item.ErrorCode ?? null, - errorMessage: item.ErrorMessage ?? null, - }) - ), - }, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to batch get Athena query executions') - logger.error('BatchGetQueryExecution failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/athena/create-named-query/route.ts b/apps/sim/app/api/tools/athena/create-named-query/route.ts deleted file mode 100644 index 4eb3c56acf1..00000000000 --- a/apps/sim/app/api/tools/athena/create-named-query/route.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { CreateNamedQueryCommand } from '@aws-sdk/client-athena' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAthenaCreateNamedQueryContract } from '@/lib/api/contracts/tools/aws/athena-create-named-query' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAthenaClient } from '@/app/api/tools/athena/utils' - -const logger = createLogger('AthenaCreateNamedQuery') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsAthenaCreateNamedQueryContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - const client = createAthenaClient({ - region: data.region, - accessKeyId: data.accessKeyId, - secretAccessKey: data.secretAccessKey, - }) - - const command = new CreateNamedQueryCommand({ - Name: data.name, - Database: data.database, - QueryString: data.queryString, - ...(data.description && { Description: data.description }), - ...(data.workGroup && { WorkGroup: data.workGroup }), - }) - - const response = await client.send(command) - - if (!response.NamedQueryId) { - throw new Error('No named query ID returned') - } - - return NextResponse.json({ - success: true, - output: { - namedQueryId: response.NamedQueryId, - }, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to create Athena named query') - logger.error('CreateNamedQuery failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/athena/delete-named-query/route.ts b/apps/sim/app/api/tools/athena/delete-named-query/route.ts deleted file mode 100644 index 3026156a0fd..00000000000 --- a/apps/sim/app/api/tools/athena/delete-named-query/route.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { DeleteNamedQueryCommand } from '@aws-sdk/client-athena' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAthenaDeleteNamedQueryContract } from '@/lib/api/contracts/tools/aws/athena-delete-named-query' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAthenaClient } from '@/app/api/tools/athena/utils' - -const logger = createLogger('AthenaDeleteNamedQuery') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsAthenaDeleteNamedQueryContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - const client = createAthenaClient({ - region: data.region, - accessKeyId: data.accessKeyId, - secretAccessKey: data.secretAccessKey, - }) - - try { - const command = new DeleteNamedQueryCommand({ - NamedQueryId: data.namedQueryId, - }) - - await client.send(command) - - return NextResponse.json({ - success: true, - output: { - success: true, - }, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to delete Athena named query') - logger.error('DeleteNamedQuery failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/athena/get-named-query/route.ts b/apps/sim/app/api/tools/athena/get-named-query/route.ts deleted file mode 100644 index abf3b978387..00000000000 --- a/apps/sim/app/api/tools/athena/get-named-query/route.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { GetNamedQueryCommand } from '@aws-sdk/client-athena' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAthenaGetNamedQueryContract } from '@/lib/api/contracts/tools/aws/athena-get-named-query' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAthenaClient } from '@/app/api/tools/athena/utils' - -const logger = createLogger('AthenaGetNamedQuery') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsAthenaGetNamedQueryContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - const client = createAthenaClient({ - region: data.region, - accessKeyId: data.accessKeyId, - secretAccessKey: data.secretAccessKey, - }) - - const command = new GetNamedQueryCommand({ - NamedQueryId: data.namedQueryId, - }) - - const response = await client.send(command) - const namedQuery = response.NamedQuery - - if (!namedQuery) { - throw new Error('No named query data returned') - } - - return NextResponse.json({ - success: true, - output: { - namedQueryId: namedQuery.NamedQueryId ?? data.namedQueryId, - name: namedQuery.Name ?? '', - description: namedQuery.Description ?? null, - database: namedQuery.Database ?? '', - queryString: namedQuery.QueryString ?? '', - workGroup: namedQuery.WorkGroup ?? null, - }, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to get Athena named query') - logger.error('GetNamedQuery failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/athena/get-query-execution/route.ts b/apps/sim/app/api/tools/athena/get-query-execution/route.ts deleted file mode 100644 index 899bc096adf..00000000000 --- a/apps/sim/app/api/tools/athena/get-query-execution/route.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { GetQueryExecutionCommand } from '@aws-sdk/client-athena' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAthenaGetQueryExecutionContract } from '@/lib/api/contracts/tools/aws/athena-get-query-execution' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAthenaClient } from '@/app/api/tools/athena/utils' - -const logger = createLogger('AthenaGetQueryExecution') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsAthenaGetQueryExecutionContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - const client = createAthenaClient({ - region: data.region, - accessKeyId: data.accessKeyId, - secretAccessKey: data.secretAccessKey, - }) - - const command = new GetQueryExecutionCommand({ - QueryExecutionId: data.queryExecutionId, - }) - - const response = await client.send(command) - const execution = response.QueryExecution - - if (!execution) { - throw new Error('No query execution data returned') - } - - return NextResponse.json({ - success: true, - output: { - queryExecutionId: execution.QueryExecutionId ?? data.queryExecutionId, - query: execution.Query ?? '', - state: execution.Status?.State ?? 'UNKNOWN', - stateChangeReason: execution.Status?.StateChangeReason ?? null, - statementType: execution.StatementType ?? null, - database: execution.QueryExecutionContext?.Database ?? null, - catalog: execution.QueryExecutionContext?.Catalog ?? null, - workGroup: execution.WorkGroup ?? null, - submissionDateTime: execution.Status?.SubmissionDateTime?.getTime() ?? null, - completionDateTime: execution.Status?.CompletionDateTime?.getTime() ?? null, - dataScannedInBytes: execution.Statistics?.DataScannedInBytes ?? null, - engineExecutionTimeInMillis: execution.Statistics?.EngineExecutionTimeInMillis ?? null, - queryPlanningTimeInMillis: execution.Statistics?.QueryPlanningTimeInMillis ?? null, - queryQueueTimeInMillis: execution.Statistics?.QueryQueueTimeInMillis ?? null, - totalExecutionTimeInMillis: execution.Statistics?.TotalExecutionTimeInMillis ?? null, - outputLocation: execution.ResultConfiguration?.OutputLocation ?? null, - }, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to get Athena query execution') - logger.error('GetQueryExecution failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/athena/get-query-results/route.ts b/apps/sim/app/api/tools/athena/get-query-results/route.ts deleted file mode 100644 index 2f1289db064..00000000000 --- a/apps/sim/app/api/tools/athena/get-query-results/route.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { GetQueryResultsCommand } from '@aws-sdk/client-athena' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAthenaGetQueryResultsContract } from '@/lib/api/contracts/tools/aws/athena-get-query-results' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAthenaClient } from '@/app/api/tools/athena/utils' - -const logger = createLogger('AthenaGetQueryResults') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsAthenaGetQueryResultsContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - const client = createAthenaClient({ - region: data.region, - accessKeyId: data.accessKeyId, - secretAccessKey: data.secretAccessKey, - }) - - const isFirstPage = !data.nextToken - const adjustedMaxResults = - data.maxResults !== undefined && isFirstPage ? data.maxResults + 1 : data.maxResults - - const command = new GetQueryResultsCommand({ - QueryExecutionId: data.queryExecutionId, - ...(adjustedMaxResults !== undefined && { MaxResults: adjustedMaxResults }), - ...(data.nextToken && { NextToken: data.nextToken }), - }) - - const response = await client.send(command) - - const columnInfo = response.ResultSet?.ResultSetMetadata?.ColumnInfo ?? [] - const columns = columnInfo.map((col) => ({ - name: col.Name ?? '', - type: col.Type ?? 'varchar', - })) - - const rawRows = response.ResultSet?.Rows ?? [] - const dataRows = data.nextToken ? rawRows : rawRows.slice(1) - const rows = dataRows.map((row) => { - const record: Record = {} - const rowData = row.Data ?? [] - for (let i = 0; i < columns.length; i++) { - record[columns[i].name] = rowData[i]?.VarCharValue ?? '' - } - return record - }) - - return NextResponse.json({ - success: true, - output: { - columns, - rows, - nextToken: response.NextToken ?? null, - updateCount: response.UpdateCount ?? null, - }, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to get Athena query results') - logger.error('GetQueryResults failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/athena/list-databases/route.ts b/apps/sim/app/api/tools/athena/list-databases/route.ts deleted file mode 100644 index 28e7109c091..00000000000 --- a/apps/sim/app/api/tools/athena/list-databases/route.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { ListDatabasesCommand } from '@aws-sdk/client-athena' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAthenaListDatabasesContract } from '@/lib/api/contracts/tools/aws/athena-list-databases' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAthenaClient } from '@/app/api/tools/athena/utils' - -const logger = createLogger('AthenaListDatabases') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsAthenaListDatabasesContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - const client = createAthenaClient({ - region: data.region, - accessKeyId: data.accessKeyId, - secretAccessKey: data.secretAccessKey, - }) - - try { - const command = new ListDatabasesCommand({ - CatalogName: data.catalogName, - ...(data.workGroup && { WorkGroup: data.workGroup }), - ...(data.maxResults !== undefined && { MaxResults: data.maxResults }), - ...(data.nextToken && { NextToken: data.nextToken }), - }) - - const response = await client.send(command) - - return NextResponse.json({ - success: true, - output: { - databases: (response.DatabaseList ?? []).map((db) => ({ - name: db.Name ?? '', - description: db.Description ?? null, - })), - nextToken: response.NextToken ?? null, - }, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to list Athena databases') - logger.error('ListDatabases failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/athena/list-named-queries/route.ts b/apps/sim/app/api/tools/athena/list-named-queries/route.ts deleted file mode 100644 index 4b9c53d1a6e..00000000000 --- a/apps/sim/app/api/tools/athena/list-named-queries/route.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { ListNamedQueriesCommand } from '@aws-sdk/client-athena' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAthenaListNamedQueriesContract } from '@/lib/api/contracts/tools/aws/athena-list-named-queries' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAthenaClient } from '@/app/api/tools/athena/utils' - -const logger = createLogger('AthenaListNamedQueries') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsAthenaListNamedQueriesContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - const client = createAthenaClient({ - region: data.region, - accessKeyId: data.accessKeyId, - secretAccessKey: data.secretAccessKey, - }) - - const command = new ListNamedQueriesCommand({ - ...(data.workGroup && { WorkGroup: data.workGroup }), - ...(data.maxResults !== undefined && { MaxResults: data.maxResults }), - ...(data.nextToken && { NextToken: data.nextToken }), - }) - - const response = await client.send(command) - - return NextResponse.json({ - success: true, - output: { - namedQueryIds: response.NamedQueryIds ?? [], - nextToken: response.NextToken ?? null, - }, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to list Athena named queries') - logger.error('ListNamedQueries failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/athena/list-query-executions/route.ts b/apps/sim/app/api/tools/athena/list-query-executions/route.ts deleted file mode 100644 index 51602fdc937..00000000000 --- a/apps/sim/app/api/tools/athena/list-query-executions/route.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { ListQueryExecutionsCommand } from '@aws-sdk/client-athena' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAthenaListQueryExecutionsContract } from '@/lib/api/contracts/tools/aws/athena-list-query-executions' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAthenaClient } from '@/app/api/tools/athena/utils' - -const logger = createLogger('AthenaListQueryExecutions') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsAthenaListQueryExecutionsContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - const client = createAthenaClient({ - region: data.region, - accessKeyId: data.accessKeyId, - secretAccessKey: data.secretAccessKey, - }) - - const command = new ListQueryExecutionsCommand({ - ...(data.workGroup && { WorkGroup: data.workGroup }), - ...(data.maxResults !== undefined && { MaxResults: data.maxResults }), - ...(data.nextToken && { NextToken: data.nextToken }), - }) - - const response = await client.send(command) - - return NextResponse.json({ - success: true, - output: { - queryExecutionIds: response.QueryExecutionIds ?? [], - nextToken: response.NextToken ?? null, - }, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to list Athena query executions') - logger.error('ListQueryExecutions failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/athena/list-table-metadata/route.ts b/apps/sim/app/api/tools/athena/list-table-metadata/route.ts deleted file mode 100644 index 51db9af6be3..00000000000 --- a/apps/sim/app/api/tools/athena/list-table-metadata/route.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { ListTableMetadataCommand } from '@aws-sdk/client-athena' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAthenaListTableMetadataContract } from '@/lib/api/contracts/tools/aws/athena-list-table-metadata' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAthenaClient } from '@/app/api/tools/athena/utils' - -const logger = createLogger('AthenaListTableMetadata') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsAthenaListTableMetadataContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - const client = createAthenaClient({ - region: data.region, - accessKeyId: data.accessKeyId, - secretAccessKey: data.secretAccessKey, - }) - - try { - const command = new ListTableMetadataCommand({ - CatalogName: data.catalogName, - DatabaseName: data.databaseName, - ...(data.expression && { Expression: data.expression }), - ...(data.workGroup && { WorkGroup: data.workGroup }), - ...(data.maxResults !== undefined && { MaxResults: data.maxResults }), - ...(data.nextToken && { NextToken: data.nextToken }), - }) - - const response = await client.send(command) - - return NextResponse.json({ - success: true, - output: { - tables: (response.TableMetadataList ?? []).map((table) => ({ - name: table.Name ?? '', - tableType: table.TableType ?? null, - createTime: table.CreateTime?.getTime() ?? null, - lastAccessTime: table.LastAccessTime?.getTime() ?? null, - columns: (table.Columns ?? []).map((col) => ({ - name: col.Name ?? '', - type: col.Type ?? null, - comment: col.Comment ?? null, - })), - partitionKeys: (table.PartitionKeys ?? []).map((col) => ({ - name: col.Name ?? '', - type: col.Type ?? null, - comment: col.Comment ?? null, - })), - })), - nextToken: response.NextToken ?? null, - }, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to list Athena table metadata') - logger.error('ListTableMetadata failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/athena/start-query/route.ts b/apps/sim/app/api/tools/athena/start-query/route.ts deleted file mode 100644 index 4d4687d0a65..00000000000 --- a/apps/sim/app/api/tools/athena/start-query/route.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { StartQueryExecutionCommand } from '@aws-sdk/client-athena' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAthenaStartQueryContract } from '@/lib/api/contracts/tools/aws/athena-start-query' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAthenaClient } from '@/app/api/tools/athena/utils' - -const logger = createLogger('AthenaStartQuery') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsAthenaStartQueryContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - const client = createAthenaClient({ - region: data.region, - accessKeyId: data.accessKeyId, - secretAccessKey: data.secretAccessKey, - }) - - const command = new StartQueryExecutionCommand({ - QueryString: data.queryString, - ...(data.database || data.catalog - ? { - QueryExecutionContext: { - ...(data.database && { Database: data.database }), - ...(data.catalog && { Catalog: data.catalog }), - }, - } - : {}), - ...(data.outputLocation - ? { - ResultConfiguration: { - OutputLocation: data.outputLocation, - }, - } - : {}), - ...(data.workGroup && { WorkGroup: data.workGroup }), - }) - - const response = await client.send(command) - - if (!response.QueryExecutionId) { - throw new Error('No query execution ID returned') - } - - return NextResponse.json({ - success: true, - output: { - queryExecutionId: response.QueryExecutionId, - }, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to start Athena query') - logger.error('StartQuery failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/athena/stop-query/route.ts b/apps/sim/app/api/tools/athena/stop-query/route.ts deleted file mode 100644 index 186c8df8b90..00000000000 --- a/apps/sim/app/api/tools/athena/stop-query/route.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { StopQueryExecutionCommand } from '@aws-sdk/client-athena' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsAthenaStopQueryContract } from '@/lib/api/contracts/tools/aws/athena-stop-query' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAthenaClient } from '@/app/api/tools/athena/utils' - -const logger = createLogger('AthenaStopQuery') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsAthenaStopQueryContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - const client = createAthenaClient({ - region: data.region, - accessKeyId: data.accessKeyId, - secretAccessKey: data.secretAccessKey, - }) - - const command = new StopQueryExecutionCommand({ - QueryExecutionId: data.queryExecutionId, - }) - - await client.send(command) - - return NextResponse.json({ - success: true, - output: { - success: true, - }, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to stop Athena query') - logger.error('StopQuery failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/athena/utils.ts b/apps/sim/app/api/tools/athena/utils.ts deleted file mode 100644 index 1d52a2ffe64..00000000000 --- a/apps/sim/app/api/tools/athena/utils.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { AthenaClient } from '@aws-sdk/client-athena' - -interface AwsCredentials { - region: string - accessKeyId: string - secretAccessKey: string -} - -export function createAthenaClient(config: AwsCredentials): AthenaClient { - return new AthenaClient({ - region: config.region, - credentials: { - accessKeyId: config.accessKeyId, - secretAccessKey: config.secretAccessKey, - }, - }) -} diff --git a/apps/sim/app/api/tools/azure_data_explorer/proxy/route.test.ts b/apps/sim/app/api/tools/azure_data_explorer/proxy/route.test.ts deleted file mode 100644 index 010a68a239b..00000000000 --- a/apps/sim/app/api/tools/azure_data_explorer/proxy/route.test.ts +++ /dev/null @@ -1,321 +0,0 @@ -/** - * @vitest-environment node - */ -import { createMockRequest, hybridAuthMockFns } from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' -import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' - -const { mockSecureFetch, MOCK_MAX_JSON_BYTES } = vi.hoisted(() => ({ - mockSecureFetch: vi.fn(), - MOCK_MAX_JSON_BYTES: 10 * 1024 * 1024, -})) - -vi.mock('@/lib/core/security/input-validation.server', () => ({ - secureFetchWithValidation: mockSecureFetch, - MAX_JSON_API_RESPONSE_BYTES: MOCK_MAX_JSON_BYTES, -})) - -import { POST } from '@/app/api/tools/azure_data_explorer/proxy/route' - -const baseBody = { - clusterUri: 'https://mycluster.eastus.kusto.windows.net', - tenantId: 'tenant-1', - clientId: 'client-1', - clientSecret: 'secret-1', - endpoint: 'query', - database: 'Samples', - csl: 'print Test="Hello, World!"', -} - -function jsonResponse(body: unknown, status = 200) { - return { - ok: status >= 200 && status < 300, - status, - headers: new Headers(), - json: async () => body, - text: async () => JSON.stringify(body), - } -} - -/** Queues the Entra token response, then the cluster response. */ -function mockCluster(clusterBody: unknown, status = 200) { - mockSecureFetch.mockReset() - mockSecureFetch - .mockResolvedValueOnce(jsonResponse({ access_token: 'token-1', expires_in: 3600 })) - .mockResolvedValueOnce(jsonResponse(clusterBody, status)) -} - -let secretCounter = 0 - -/** - * The route caches Entra tokens per credential, so every test needs its own - * secret to exercise the token fetch rather than a warm cache entry. - */ -function post(body: Record) { - secretCounter += 1 - return POST( - createMockRequest('POST', { ...body, clientSecret: `secret-${secretCounter}` }) as never, - undefined as never - ) -} - -/** - * A v1 query answer. The primary result deliberately is NOT the first table, so - * a reader that ignores the table of contents picks the wrong one. - */ -function queryResponse(options: { severity: number; statusDescription: string }) { - return { - Tables: [ - { - TableName: 'Table_0', - Columns: [{ ColumnName: 'Value', DataType: 'String', ColumnType: 'string' }], - Rows: [['{"Visualization":null}']], - }, - { - TableName: 'Table_1', - Columns: [{ ColumnName: 'Test', DataType: 'String', ColumnType: 'string' }], - Rows: [['Hello, World!']], - }, - { - TableName: 'Table_2', - Columns: [ - { ColumnName: 'Severity', DataType: 'Int32', ColumnType: 'int' }, - { ColumnName: 'StatusCode', DataType: 'Int32', ColumnType: 'int' }, - { ColumnName: 'StatusDescription', DataType: 'String', ColumnType: 'string' }, - ], - Rows: [[options.severity, 0, options.statusDescription]], - }, - { - TableName: 'Table_3', - Columns: [ - { ColumnName: 'Ordinal', DataType: 'Int64', ColumnType: 'long' }, - { ColumnName: 'Kind', DataType: 'String', ColumnType: 'string' }, - { ColumnName: 'Name', DataType: 'String', ColumnType: 'string' }, - ], - Rows: [ - [0, 'QueryProperties', '@ExtendedProperties'], - [1, 'QueryResult', 'PrimaryResult'], - [2, 'QueryStatus', 'QueryStatus'], - ], - }, - ], - } -} - -describe('POST /api/tools/azure_data_explorer/proxy', () => { - beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ success: true, userId: 'user-1' }) - }) - - it('returns the primary result table named by the table of contents', async () => { - mockCluster(queryResponse({ severity: 4, statusDescription: 'Query completed successfully' })) - - const response = await post(baseBody) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.output.tableName).toBe('Table_1') - expect(data.output.columns).toEqual([{ name: 'Test', type: 'string', dataType: 'String' }]) - expect(data.output.rows).toEqual([['Hello, World!']]) - expect(data.output.records).toEqual([{ Test: 'Hello, World!' }]) - expect(data.output.rowCount).toBe(1) - }) - - it('reports a partial query failure even though the cluster answered 200', async () => { - mockCluster(queryResponse({ severity: 2, statusDescription: 'Query execution has exceeded' })) - - const response = await post(baseBody) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.success).toBe(false) - expect(data.error).toBe('Query execution has exceeded') - }) - - it('does not mistake a result column named Severity for a failed query', async () => { - const shadowed = queryResponse({ - severity: 4, - statusDescription: 'Query completed successfully', - }) - shadowed.Tables[1] = { - TableName: 'Table_1', - Columns: [ - { ColumnName: 'Severity', DataType: 'Int32', ColumnType: 'int' }, - { ColumnName: 'StatusDescription', DataType: 'String', ColumnType: 'string' }, - ], - Rows: [[1, 'disk almost full']], - } - mockCluster(shadowed) - - const response = await post(baseBody) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(data.output.records).toEqual([{ Severity: 1, StatusDescription: 'disk almost full' }]) - }) - - it('returns the first table for a management command, which has no table of contents', async () => { - mockCluster({ - Tables: [ - { - TableName: 'Table_0', - Columns: [{ ColumnName: 'TableName', DataType: 'String', ColumnType: 'string' }], - Rows: [['StormEvents'], ['Logs']], - }, - ], - }) - - const response = await post({ ...baseBody, endpoint: 'mgmt', csl: '.show tables' }) - const data = await response.json() - - expect(data.output.records).toEqual([{ TableName: 'StormEvents' }, { TableName: 'Logs' }]) - }) - - it('sends the KQL request to the cluster with a bearer token and read-only header', async () => { - mockCluster(queryResponse({ severity: 4, statusDescription: 'Query completed successfully' })) - - await post({ ...baseBody, readOnly: true }) - - const [url, options] = mockSecureFetch.mock.calls[1] - expect(url).toBe('https://mycluster.eastus.kusto.windows.net/v1/rest/query') - expect(options.headers.Authorization).toBe('Bearer token-1') - expect(options.headers['x-ms-readonly']).toBe('true') - expect(JSON.parse(options.body)).toEqual({ db: 'Samples', csl: baseBody.csl }) - }) - - it('requests an Entra token audience of the cluster origin by default', async () => { - mockCluster(queryResponse({ severity: 4, statusDescription: 'Query completed successfully' })) - - await post(baseBody) - - const [tokenUrl, tokenOptions] = mockSecureFetch.mock.calls[0] - expect(tokenUrl).toBe('https://login.microsoftonline.com/tenant-1/oauth2/token') - expect(Object.fromEntries(new URLSearchParams(tokenOptions.body))).toEqual({ - grant_type: 'client_credentials', - client_id: 'client-1', - client_secret: `secret-${secretCounter}`, - resource: 'https://mycluster.eastus.kusto.windows.net', - }) - }) - - it.each([ - ['https://c.usgovvirginia.kusto.usgovcloudapi.net', 'https://login.microsoftonline.us'], - ['https://c.chinanorth.kusto.chinacloudapi.cn', 'https://login.partner.microsoftonline.cn'], - ['https://c.eastus.kusto.windows.net', 'https://login.microsoftonline.com'], - ])('authenticates %s against its own cloud Entra authority', async (clusterUri, authority) => { - mockCluster(queryResponse({ severity: 4, statusDescription: 'Query completed successfully' })) - - const response = await post({ ...baseBody, clusterUri, resource: undefined }) - - expect(response.status).toBe(200) - const [tokenUrl] = mockSecureFetch.mock.calls[0] - expect(tokenUrl).toBe(`${authority}/tenant-1/oauth2/token`) - }) - - it('rejects a cluster URI outside the Kusto service domains', async () => { - mockCluster(queryResponse({ severity: 4, statusDescription: 'Query completed successfully' })) - - const response = await post({ ...baseBody, clusterUri: 'https://evil.example.com' }) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toContain('Azure Data Explorer or Fabric Eventhouse endpoint') - expect(mockSecureFetch).not.toHaveBeenCalled() - }) - - it('surfaces the Kusto error envelope on a failed request', async () => { - mockCluster({ error: { code: 'BadRequest_SyntaxError', message: "Syntax error: 'wher'" } }, 400) - - const response = await post(baseBody) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toBe("[BadRequest_SyntaxError] Syntax error: 'wher'") - }) - - it('rejects an entity name outside the documented Kusto identifier character set', async () => { - const response = await post({ ...baseBody, database: 'Samples"] | drop table X //' }) - - expect(response.status).toBe(400) - expect(mockSecureFetch).not.toHaveBeenCalled() - }) - - it('accepts an apex token audience, which is the Fabric Eventhouse form', async () => { - mockCluster(queryResponse({ severity: 4, statusDescription: 'Query completed successfully' })) - - const response = await post({ ...baseBody, resource: 'https://kusto.fabric.microsoft.com' }) - - expect(response.status).toBe(200) - const [, tokenOptions] = mockSecureFetch.mock.calls[0] - expect(Object.fromEntries(new URLSearchParams(tokenOptions.body)).resource).toBe( - 'https://kusto.fabric.microsoft.com' - ) - }) - - it('caps the rows it returns and reports what the cluster actually produced', async () => { - const wide = queryResponse({ severity: 4, statusDescription: 'Query completed successfully' }) - wide.Tables[1].Rows = Array.from({ length: 10_050 }, (_, i) => [`row-${i}`]) - mockCluster(wide) - - const response = await post(baseBody) - const data = await response.json() - - expect(data.output.rowCount).toBe(10_000) - expect(data.output.rows).toHaveLength(10_000) - expect(data.output.records).toHaveLength(10_000) - expect(data.output.totalRowCount).toBe(10_050) - expect(data.output.truncated).toBe(true) - }) - - it('reports truncated as false when every row fits', async () => { - mockCluster(queryResponse({ severity: 4, statusDescription: 'Query completed successfully' })) - - const response = await post(baseBody) - const data = await response.json() - - expect(data.output.truncated).toBe(false) - expect(data.output.totalRowCount).toBe(1) - }) - - it('bounds the cluster response body rather than reading it unlimited', async () => { - mockCluster(queryResponse({ severity: 4, statusDescription: 'Query completed successfully' })) - - await post(baseBody) - - const [, options] = mockSecureFetch.mock.calls[1] - expect(options.maxResponseBytes).toBe(MOCK_MAX_JSON_BYTES) - }) - - it('asks the caller to narrow the query when the response exceeds the cap', async () => { - mockSecureFetch.mockReset() - mockSecureFetch - .mockResolvedValueOnce(jsonResponse({ access_token: 'token-1', expires_in: 3600 })) - .mockRejectedValueOnce( - new PayloadSizeLimitError({ - label: 'Azure Data Explorer response', - maxBytes: MOCK_MAX_JSON_BYTES, - }) - ) - - const response = await post(baseBody) - const data = await response.json() - - expect(response.status).toBe(413) - expect(data.error).toContain('Narrow the query') - }) - - it('rejects an unauthenticated request before reaching the cluster', async () => { - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: false, - error: 'Authentication required', - }) - - const response = await post(baseBody) - - expect(response.status).toBe(401) - expect(mockSecureFetch).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/app/api/tools/azure_data_explorer/proxy/route.ts b/apps/sim/app/api/tools/azure_data_explorer/proxy/route.ts deleted file mode 100644 index ada24aa4784..00000000000 --- a/apps/sim/app/api/tools/azure_data_explorer/proxy/route.ts +++ /dev/null @@ -1,403 +0,0 @@ -import { createHash } from 'node:crypto' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { truncate } from '@sim/utils/string' -import { type NextRequest, NextResponse } from 'next/server' -import { - type AzureDataExplorerProxyRequest, - assertSafeAzureDataExplorerClusterUri, - azureDataExplorerProxyContract, - resolveEntraAuthority, -} from '@/lib/api/contracts/tools/azure_data_explorer' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - MAX_JSON_API_RESPONSE_BYTES, - secureFetchWithValidation, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('AzureDataExplorerProxyAPI') - -const OUTBOUND_FETCH_TIMEOUT_MS = 120_000 -const TOKEN_FETCH_TIMEOUT_MS = 30_000 -const TOKEN_CACHE_MAX_ENTRIES = 500 -const TOKEN_SAFETY_WINDOW_MS = 60_000 -const MAX_ERROR_MESSAGE_LENGTH = 2000 -const MAX_TOKEN_RESPONSE_BYTES = 256 * 1024 -/** Rows a single result may carry into a workflow value. */ -const MAX_PROJECTED_ROWS = 10_000 - -interface CachedToken { - accessToken: string - expiresAt: number -} - -const TOKEN_CACHE = new Map() - -/** The cluster's own origin is the documented default token audience. */ -function resolveResource(req: AzureDataExplorerProxyRequest, clusterUrl: URL): string { - return (req.resource || clusterUrl.origin).replace(/\/+$/, '') -} - -function tokenCacheKey( - req: AzureDataExplorerProxyRequest, - authority: string, - resource: string -): string { - const secretHash = createHash('sha256').update(req.clientSecret).digest('hex').slice(0, 16) - return `${authority}::${req.tenantId}::${req.clientId}::${secretHash}::${resource}` -} - -function rememberToken(key: string, token: CachedToken): void { - if (TOKEN_CACHE.has(key)) TOKEN_CACHE.delete(key) - TOKEN_CACHE.set(key, token) - while (TOKEN_CACHE.size > TOKEN_CACHE_MAX_ENTRIES) { - const oldestKey = TOKEN_CACHE.keys().next().value - if (oldestKey === undefined) break - TOKEN_CACHE.delete(oldestKey) - } -} - -async function fetchAccessToken( - req: AzureDataExplorerProxyRequest, - authority: string, - resource: string, - requestId: string -): Promise { - const cacheKey = tokenCacheKey(req, authority, resource) - const cached = TOKEN_CACHE.get(cacheKey) - if (cached && cached.expiresAt - TOKEN_SAFETY_WINDOW_MS > Date.now()) { - return cached.accessToken - } - - const body = new URLSearchParams({ - grant_type: 'client_credentials', - client_id: req.clientId, - client_secret: req.clientSecret, - resource, - }) - - const response = await secureFetchWithValidation( - `${authority}/${encodeURIComponent(req.tenantId)}/oauth2/token`, - { - method: 'POST', - headers: { - 'Content-Type': 'application/x-www-form-urlencoded', - Accept: 'application/json', - }, - body: body.toString(), - timeout: TOKEN_FETCH_TIMEOUT_MS, - maxResponseBytes: MAX_TOKEN_RESPONSE_BYTES, - }, - 'tokenUrl' - ) - - if (!response.ok) { - const text = await response.text().catch(() => '') - logger.warn(`[${requestId}] Entra token fetch failed (${response.status}): ${text}`) - throw new Error( - `Microsoft Entra token request failed: HTTP ${response.status}. Verify tenantId, clientId, clientSecret, and that the app has access to the cluster.` - ) - } - - const data = (await response.json()) as { access_token?: string; expires_in?: string | number } - if (!data.access_token) { - throw new Error('Microsoft Entra token response did not include an access token') - } - - const expiresInSeconds = Number(data.expires_in) - const expiresInMs = (Number.isFinite(expiresInSeconds) ? expiresInSeconds : 3600) * 1000 - rememberToken(cacheKey, { - accessToken: data.access_token, - expiresAt: Date.now() + expiresInMs, - }) - return data.access_token -} - -interface KustoColumn { - ColumnName?: string - DataType?: string - ColumnType?: string -} - -interface KustoTable { - TableName?: string - Columns?: KustoColumn[] - Rows?: unknown[][] -} - -function columnNames(table: KustoTable): string[] { - return (table.Columns ?? []).map((column) => column.ColumnName ?? '') -} - -interface TableOfContents { - primaryOrdinal: number | null - statusOrdinal: number | null -} - -/** - * Reads the trailing table of contents, which maps each ordinal in the response - * to a kind. It is the only thing that identifies which table holds the query's - * results and which holds its status — a management command has no table of - * contents, and returns `null` here. - */ -function readTableOfContents(tables: KustoTable[]): TableOfContents | null { - if (tables.length === 0) return null - - const contents = tables[tables.length - 1] - const names = columnNames(contents) - const ordinalIndex = names.indexOf('Ordinal') - const kindIndex = names.indexOf('Kind') - if (ordinalIndex < 0 || kindIndex < 0) return null - - let primaryOrdinal: number | null = null - let statusOrdinal: number | null = null - for (const row of contents.Rows ?? []) { - const ordinal = Number(row[ordinalIndex]) - if (!Number.isInteger(ordinal) || !tables[ordinal]) continue - if (row[kindIndex] === 'QueryResult' && primaryOrdinal === null) primaryOrdinal = ordinal - if (row[kindIndex] === 'QueryStatus' && statusOrdinal === null) statusOrdinal = ordinal - } - return { primaryOrdinal, statusOrdinal } -} - -/** - * Picks the table holding the query's own results — the first `QueryResult` - * ordinal the table of contents names. A management command returns a single - * table with no table of contents, so the first table is the answer. - */ -function selectPrimaryTable( - tables: KustoTable[], - contents: TableOfContents | null -): KustoTable | null { - if (tables.length === 0) return null - if (contents?.primaryOrdinal != null) return tables[contents.primaryOrdinal] ?? tables[0] - return tables[0] -} - -/** - * Finds a partial query failure. Kusto answers 200 as soon as it starts - * processing, then reports later failures through the QueryStatus table, where a - * severity of 2 or lower means the request did not succeed. - * - * Only the table the table of contents names as `QueryStatus` is inspected. - * Scanning every table for `Severity`/`StatusDescription` columns would - * misread an ordinary log query that happens to select columns of those names - * as a failed request. - */ -function findQueryFailure(tables: KustoTable[], contents: TableOfContents | null): string | null { - if (contents?.statusOrdinal == null) return null - const table = tables[contents.statusOrdinal] - if (!table) return null - - const names = columnNames(table) - const severityIndex = names.indexOf('Severity') - const descriptionIndex = names.indexOf('StatusDescription') - if (severityIndex < 0 || descriptionIndex < 0) return null - - for (const row of table.Rows ?? []) { - const severity = Number(row[severityIndex]) - if (!Number.isFinite(severity) || severity > 2) continue - const description = row[descriptionIndex] - return typeof description === 'string' && description.length > 0 - ? description - : 'Kusto reported a query failure' - } - return null -} - -interface ProjectedTable { - tableName: string | null - columns: Array<{ name: string; type: string | null; dataType: string | null }> - rows: unknown[][] - records: Array> - rowCount: number - totalRowCount: number - truncated: boolean -} - -const EMPTY_PROJECTION: ProjectedTable = { - tableName: null, - columns: [], - rows: [], - records: [], - rowCount: 0, - totalRowCount: 0, - truncated: false, -} - -/** - * Projects the result table into a bounded payload. - * - * Kusto's own result truncation is a request property the caller can raise or - * disable, so neither the row count nor the byte count of a response is bounded - * upstream. `MAX_PROJECTED_ROWS` is the ceiling on what a single workflow value - * may carry; `truncated` tells the caller to narrow the query rather than - * silently trusting a short answer. - */ -function projectTable(table: KustoTable | null): ProjectedTable { - if (!table) return EMPTY_PROJECTION - - const columns = (table.Columns ?? []).map((column) => ({ - name: column.ColumnName ?? '', - type: column.ColumnType ?? null, - dataType: column.DataType ?? null, - })) - const allRows = table.Rows ?? [] - const rows = allRows.length > MAX_PROJECTED_ROWS ? allRows.slice(0, MAX_PROJECTED_ROWS) : allRows - const records = rows.map((row) => { - const record: Record = {} - columns.forEach((column, index) => { - if (column.name) record[column.name] = row[index] ?? null - }) - return record - }) - - return { - tableName: table.TableName ?? null, - columns, - rows, - records, - rowCount: rows.length, - totalRowCount: allRows.length, - truncated: allRows.length > rows.length, - } -} - -/** - * Kusto failures follow the Microsoft REST guidelines envelope, but a request - * without a JSON body (or a gateway error) can answer with plain text. - */ -function extractKustoError(body: unknown, status: number): string { - if (body && typeof body === 'object') { - const error = (body as { error?: { code?: unknown; message?: unknown } }).error - if (error && typeof error === 'object') { - const message = typeof error.message === 'string' ? error.message : '' - const code = typeof error.code === 'string' ? error.code : '' - if (message) return code ? `[${code}] ${message}` : message - if (code) return code - } - } - if (typeof body === 'string' && body.length > 0) { - return truncate(body, MAX_ERROR_MESSAGE_LENGTH) - } - return `Azure Data Explorer request failed with HTTP ${status}` -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Azure Data Explorer request: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - azureDataExplorerProxyContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Validation failed') }, - { status: 400 } - ), - } - ) - if (!parsed.success) return parsed.response - const proxyReq = parsed.data.body - - const clusterUrl = assertSafeAzureDataExplorerClusterUri(proxyReq.clusterUri) - const resource = resolveResource(proxyReq, clusterUrl) - const authority = resolveEntraAuthority(clusterUrl.hostname) - const accessToken = await fetchAccessToken(proxyReq, authority, resource, requestId) - - const headers: Record = { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - 'Content-Type': 'application/json; charset=utf-8', - 'x-ms-client-request-id': `Sim.Workflow;${requestId}`, - 'x-ms-app': 'Sim', - } - if (proxyReq.readOnly) headers['x-ms-readonly'] = 'true' - - const response = await secureFetchWithValidation( - `${clusterUrl.origin}/v1/rest/${proxyReq.endpoint}`, - { - method: 'POST', - headers, - body: JSON.stringify({ - ...(proxyReq.database ? { db: proxyReq.database } : {}), - csl: proxyReq.csl, - ...(proxyReq.properties ? { properties: proxyReq.properties } : {}), - }), - timeout: OUTBOUND_FETCH_TIMEOUT_MS, - maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, - }, - 'clusterUri' - ) - - const raw = await response.text() - let body: unknown = null - if (raw.length > 0) { - try { - body = JSON.parse(raw) - } catch { - body = raw - } - } - - if (!response.ok) { - const message = extractKustoError(body, response.status) - logger.warn(`[${requestId}] Azure Data Explorer error (${response.status}): ${message}`) - return NextResponse.json( - { success: false, error: message, status: response.status }, - { status: response.status } - ) - } - - const tables = Array.isArray((body as { Tables?: KustoTable[] } | null)?.Tables) - ? ((body as { Tables: KustoTable[] }).Tables ?? []) - : [] - - const contents = readTableOfContents(tables) - - const failure = findQueryFailure(tables, contents) - if (failure) { - logger.warn(`[${requestId}] Azure Data Explorer partial query failure: ${failure}`) - return NextResponse.json( - { success: false, error: truncate(failure, MAX_ERROR_MESSAGE_LENGTH), status: 200 }, - { status: 400 } - ) - } - - return NextResponse.json({ - success: true, - output: projectTable(selectPrimaryTable(tables, contents)), - }) - } catch (error) { - if (isPayloadSizeLimitError(error)) { - logger.warn(`[${requestId}] Azure Data Explorer response exceeded the size cap`) - return NextResponse.json( - { - success: false, - error: - 'The Azure Data Explorer response was too large to return. Narrow the query — add a `where` filter, aggregate with `summarize`, or bound it with `take` or `top N by`.', - }, - { status: 413 } - ) - } - logger.error(`[${requestId}] Unexpected Azure Data Explorer proxy error:`, error) - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/box/upload/route.ts b/apps/sim/app/api/tools/box/upload/route.ts deleted file mode 100644 index f490fdc02a9..00000000000 --- a/apps/sim/app/api/tools/box/upload/route.ts +++ /dev/null @@ -1,145 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { boxUploadContract } from '@/lib/api/contracts/storage-transfer' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('BoxUploadAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Box upload attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated Box upload request via ${authResult.authType}`) - - const parsed = await parseRequest(boxUploadContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - let fileBuffer: Buffer - let fileName: string - - if (validatedData.file) { - const userFiles = processFilesToUserFiles( - [validatedData.file as RawFileInput], - requestId, - logger - ) - - if (userFiles.length === 0) { - return NextResponse.json({ success: false, error: 'Invalid file input' }, { status: 400 }) - } - - const userFile = userFiles[0] - logger.info(`[${requestId}] Downloading file: ${userFile.name} (${userFile.size} bytes)`) - - const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) - if (denied) return denied - try { - const result = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - }) - fileBuffer = result.buffer - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Failed to download file') }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } - fileName = validatedData.fileName || userFile.name - } else if (validatedData.fileContent) { - logger.info(`[${requestId}] Using legacy base64 content input`) - fileBuffer = Buffer.from(validatedData.fileContent, 'base64') - fileName = validatedData.fileName || 'file' - } else { - return NextResponse.json({ success: false, error: 'File is required' }, { status: 400 }) - } - - logger.info( - `[${requestId}] Uploading to Box folder ${validatedData.parentFolderId}: ${fileName} (${fileBuffer.length} bytes)` - ) - - const attributes = JSON.stringify({ - name: fileName, - parent: { id: validatedData.parentFolderId }, - }) - - const formData = new FormData() - formData.append('attributes', attributes) - formData.append( - 'file', - new Blob([new Uint8Array(fileBuffer)], { type: 'application/octet-stream' }), - fileName - ) - - const response = await fetch('https://upload.box.com/api/2.0/files/content', { - method: 'POST', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - }, - body: formData, - }) - - const data = await response.json() - - if (!response.ok) { - const errorMessage = data.message || 'Failed to upload file' - logger.error(`[${requestId}] Box API error:`, { status: response.status, data }) - return NextResponse.json({ success: false, error: errorMessage }, { status: response.status }) - } - - const file = data.entries?.[0] - - if (!file) { - return NextResponse.json( - { success: false, error: 'No file returned in upload response' }, - { status: 500 } - ) - } - - logger.info(`[${requestId}] File uploaded successfully: ${file.name} (ID: ${file.id})`) - - return NextResponse.json({ - success: true, - output: { - id: file.id ?? '', - name: file.name ?? '', - size: file.size ?? 0, - sha1: file.sha1 ?? null, - createdAt: file.created_at ?? null, - modifiedAt: file.modified_at ?? null, - parentId: file.parent?.id ?? null, - parentName: file.parent?.name ?? null, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Unexpected error:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/brex/upload-receipt/route.test.ts b/apps/sim/app/api/tools/brex/upload-receipt/route.test.ts deleted file mode 100644 index 1702e54cc86..00000000000 --- a/apps/sim/app/api/tools/brex/upload-receipt/route.test.ts +++ /dev/null @@ -1,266 +0,0 @@ -/** - * @vitest-environment node - */ -import { - createMockRequest, - hybridAuthMockFns, - inputValidationMock, - inputValidationMockFns, -} from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { mockProcessFilesToUserFiles, mockDownloadFileFromStorage, mockAssertToolFileAccess } = - vi.hoisted(() => ({ - mockProcessFilesToUserFiles: vi.fn(), - mockDownloadFileFromStorage: vi.fn(), - mockAssertToolFileAccess: vi.fn(), - })) - -vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) -vi.mock('@/lib/uploads/utils/file-utils', () => ({ - processFilesToUserFiles: mockProcessFilesToUserFiles, -})) -vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ - downloadServableFileFromStorage: mockDownloadFileFromStorage, -})) -vi.mock('@/app/api/files/authorization', () => ({ - assertToolFileAccess: mockAssertToolFileAccess, -})) - -import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { POST } from '@/app/api/tools/brex/upload-receipt/route' - -const mockFetch = vi.fn() - -const PINNED_IP = '52.216.0.1' - -const baseBody = { - apiKey: 'bxt_test_token', - expenseId: 'expense_123', - file: { key: 'uploads/receipt.pdf', name: 'receipt.pdf', size: 5, type: 'application/pdf' }, -} - -function jsonResponse(body: unknown, status = 200) { - return { - ok: status >= 200 && status < 300, - status, - text: async () => JSON.stringify(body), - json: async () => body, - } -} - -beforeEach(() => { - vi.clearAllMocks() - vi.stubGlobal('fetch', mockFetch) - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ - isValid: true, - resolvedIP: PINNED_IP, - }) - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValue(jsonResponse({})) - mockProcessFilesToUserFiles.mockReturnValue([ - { key: 'uploads/receipt.pdf', name: 'receipt.pdf', size: 5, type: 'application/pdf' }, - ]) - mockAssertToolFileAccess.mockResolvedValue(null) - mockDownloadFileFromStorage.mockResolvedValue({ - buffer: Buffer.from('receipt-bytes'), - contentType: 'application/pdf', - }) -}) - -describe('POST /api/tools/brex/upload-receipt', () => { - it('rejects unauthenticated requests', async () => { - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValueOnce({ - success: false, - error: 'unauthorized', - }) - - const response = await POST(createMockRequest('POST', baseBody)) - expect(response.status).toBe(401) - expect(mockFetch).not.toHaveBeenCalled() - }) - - it('creates a receipt upload for an expense and PUTs the file to the pre-signed URL', async () => { - mockFetch.mockResolvedValueOnce( - jsonResponse({ id: 'receipt_1', uri: 'https://s3.example.com/presigned' }) - ) - - const response = await POST(createMockRequest('POST', baseBody)) - expect(response.status).toBe(200) - const data = await response.json() - expect(data).toEqual({ - success: true, - output: { receiptId: 'receipt_1', receiptName: 'receipt.pdf', expenseId: 'expense_123' }, - }) - - expect(mockFetch).toHaveBeenCalledTimes(1) - const [createUrl, createInit] = mockFetch.mock.calls[0] - expect(createUrl).toBe('https://api.brex.com/v1/expenses/card/expense_123/receipt_upload') - expect(createInit.method).toBe('POST') - expect(createInit.headers.Authorization).toBe('Bearer bxt_test_token') - expect(JSON.parse(createInit.body)).toEqual({ receipt_name: 'receipt.pdf' }) - - expect(inputValidationMockFns.mockValidateUrlWithDNS).toHaveBeenCalledWith( - 'https://s3.example.com/presigned', - 'uri' - ) - const [uploadUrl, pinnedIP, uploadInit] = - inputValidationMockFns.mockSecureFetchWithPinnedIP.mock.calls[0] - expect(uploadUrl).toBe('https://s3.example.com/presigned') - expect(pinnedIP).toBe(PINNED_IP) - expect(uploadInit.method).toBe('PUT') - }) - - it('rejects a whitespace-only expense ID instead of falling back to receipt match', async () => { - const response = await POST(createMockRequest('POST', { ...baseBody, expenseId: ' ' })) - expect(response.status).toBe(400) - expect(mockFetch).not.toHaveBeenCalled() - }) - - it('trims a padded expense ID before building the upload URL', async () => { - mockFetch.mockResolvedValueOnce( - jsonResponse({ id: 'receipt_5', uri: 'https://s3.example.com/presigned' }) - ) - - const response = await POST( - createMockRequest('POST', { ...baseBody, expenseId: ' expense_123 ' }) - ) - expect(response.status).toBe(200) - const [createUrl] = mockFetch.mock.calls[0] - expect(createUrl).toBe('https://api.brex.com/v1/expenses/card/expense_123/receipt_upload') - const data = await response.json() - expect(data.output.expenseId).toBe('expense_123') - }) - - it('rejects a whitespace-only receipt name', async () => { - const response = await POST(createMockRequest('POST', { ...baseBody, receiptName: ' ' })) - expect(response.status).toBe(400) - expect(mockFetch).not.toHaveBeenCalled() - }) - - it('rejects an API key containing header-breaking characters', async () => { - const response = await POST( - createMockRequest('POST', { ...baseBody, apiKey: 'bxt_test\r\nX-Injected: 1' }) - ) - expect(response.status).toBe(400) - expect(mockFetch).not.toHaveBeenCalled() - }) - - it('uses receipt match when no expense ID is provided', async () => { - mockFetch.mockResolvedValueOnce( - jsonResponse({ id: 'receipt_2', uri: 'https://s3.example.com/presigned' }) - ) - - const response = await POST( - createMockRequest('POST', { apiKey: 'bxt_test_token', file: baseBody.file }) - ) - expect(response.status).toBe(200) - const data = await response.json() - expect(data.output).toEqual({ - receiptId: 'receipt_2', - receiptName: 'receipt.pdf', - expenseId: null, - }) - - const [createUrl] = mockFetch.mock.calls[0] - expect(createUrl).toBe('https://api.brex.com/v1/expenses/card/receipt_match') - }) - - it('honors a receipt name override', async () => { - mockFetch.mockResolvedValueOnce( - jsonResponse({ id: 'receipt_3', uri: 'https://s3.example.com/presigned' }) - ) - - const response = await POST( - createMockRequest('POST', { ...baseBody, receiptName: 'march-dinner.pdf' }) - ) - expect(response.status).toBe(200) - const [, createInit] = mockFetch.mock.calls[0] - expect(JSON.parse(createInit.body)).toEqual({ receipt_name: 'march-dinner.pdf' }) - }) - - it('propagates Brex API errors', async () => { - mockFetch.mockResolvedValueOnce(jsonResponse({ message: 'Expense not found' }, 404)) - - const response = await POST(createMockRequest('POST', baseBody)) - expect(response.status).toBe(404) - const data = await response.json() - expect(data.success).toBe(false) - expect(data.error).toContain('Expense not found') - expect(mockFetch).toHaveBeenCalledTimes(1) - expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() - }) - - it('asks the downloader for at most 50 MB', async () => { - await POST(createMockRequest('POST', baseBody)) - - expect(mockDownloadFileFromStorage).toHaveBeenCalledWith( - expect.anything(), - expect.any(String), - expect.anything(), - { maxBytes: 50 * 1024 * 1024 } - ) - }) - - it('rejects files over the 50 MB limit', async () => { - mockDownloadFileFromStorage.mockRejectedValueOnce( - new PayloadSizeLimitError({ - label: 'storage file download', - maxBytes: 50 * 1024 * 1024, - observedBytes: 50 * 1024 * 1024 + 1, - }) - ) - - const response = await POST(createMockRequest('POST', baseBody)) - expect(response.status).toBe(400) - const data = await response.json() - expect(data.error).toContain('50 MB') - expect(mockFetch).not.toHaveBeenCalled() - }) - - it('blocks pre-signed URLs that fail SSRF validation', async () => { - mockFetch.mockResolvedValueOnce( - jsonResponse({ id: 'receipt_6', uri: 'https://169.254.169.254/latest/meta-data' }) - ) - inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValueOnce({ - isValid: false, - error: 'uri resolves to a blocked IP address', - }) - - const response = await POST(createMockRequest('POST', baseBody)) - expect(response.status).toBe(502) - const data = await response.json() - expect(data.error).toContain('invalid upload URL') - expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() - }) - - it('fails when the pre-signed upload fails', async () => { - mockFetch.mockResolvedValueOnce( - jsonResponse({ id: 'receipt_4', uri: 'https://s3.example.com/presigned' }) - ) - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce(jsonResponse({}, 403)) - - const response = await POST(createMockRequest('POST', baseBody)) - expect(response.status).toBe(502) - const data = await response.json() - expect(data.success).toBe(false) - }) - - it('denies access to files the caller cannot read', async () => { - const deniedResponse = new Response( - JSON.stringify({ success: false, error: 'File not found' }), - { - status: 404, - } - ) - mockAssertToolFileAccess.mockResolvedValueOnce(deniedResponse) - - const response = await POST(createMockRequest('POST', baseBody)) - expect(response.status).toBe(404) - expect(mockFetch).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/app/api/tools/brex/upload-receipt/route.ts b/apps/sim/app/api/tools/brex/upload-receipt/route.ts deleted file mode 100644 index c63bb4374b7..00000000000 --- a/apps/sim/app/api/tools/brex/upload-receipt/route.ts +++ /dev/null @@ -1,163 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { brexUploadReceiptContract } from '@/lib/api/contracts/tools/brex' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { BREX_API_BASE, buildBrexHeaders } from '@/tools/brex/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('BrexUploadReceiptAPI') - -const MAX_RECEIPT_SIZE_BYTES = 50 * 1024 * 1024 - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Brex receipt upload attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(brexUploadReceiptContract, request, {}) - if (!parsed.success) return parsed.response - const { apiKey, expenseId, file, receiptName } = parsed.data.body - - const userFiles = processFilesToUserFiles([file as RawFileInput], requestId, logger) - if (userFiles.length === 0) { - return NextResponse.json({ success: false, error: 'Invalid file input' }, { status: 400 }) - } - - const userFile = userFiles[0] - const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) - if (denied) return denied - - let fileBuffer: Buffer - try { - const resolved = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_RECEIPT_SIZE_BYTES, - }) - fileBuffer = resolved.buffer - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - if (isPayloadSizeLimitError(error)) { - return NextResponse.json( - { success: false, error: 'Receipt file exceeds the 50 MB limit' }, - { status: 400 } - ) - } - logger.error(`[${requestId}] Failed to download receipt file:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error') }, - { status: 500 } - ) - } - - const effectiveReceiptName = receiptName || userFile.name - const endpoint = expenseId - ? `${BREX_API_BASE}/v1/expenses/card/${encodeURIComponent(expenseId)}/receipt_upload` - : `${BREX_API_BASE}/v1/expenses/card/receipt_match` - - logger.info( - `[${requestId}] Creating Brex ${expenseId ? 'receipt upload' : 'receipt match'}: ${effectiveReceiptName} (${fileBuffer.length} bytes)` - ) - - const createResponse = await fetch(endpoint, { - method: 'POST', - headers: buildBrexHeaders(apiKey), - body: JSON.stringify({ receipt_name: effectiveReceiptName }), - }) - - if (!createResponse.ok) { - const errorText = await createResponse.text() - logger.error(`[${requestId}] Brex API error:`, { - status: createResponse.status, - error: errorText, - }) - let message = errorText - try { - message = JSON.parse(errorText).message ?? errorText - } catch { - message = errorText - } - return NextResponse.json( - { success: false, error: `Brex API error (${createResponse.status}): ${message}` }, - { status: createResponse.status } - ) - } - - const createData = await createResponse.json() - if (!createData.uri || !createData.id) { - return NextResponse.json( - { success: false, error: 'Brex did not return an upload URL' }, - { status: 502 } - ) - } - - const uriValidation = await validateUrlWithDNS(createData.uri, 'uri') - if (!uriValidation.isValid) { - logger.error(`[${requestId}] Pre-signed upload URL failed SSRF validation:`, { - error: uriValidation.error, - }) - return NextResponse.json( - { success: false, error: 'Brex returned an invalid upload URL' }, - { status: 502 } - ) - } - - const uploadResponse = await secureFetchWithPinnedIP( - createData.uri, - uriValidation.resolvedIP!, - { - method: 'PUT', - body: new Uint8Array(fileBuffer), - } - ) - - if (!uploadResponse.ok) { - logger.error(`[${requestId}] Receipt upload to pre-signed URL failed:`, { - status: uploadResponse.status, - }) - return NextResponse.json( - { success: false, error: `Failed to upload receipt file (${uploadResponse.status})` }, - { status: 502 } - ) - } - - logger.info(`[${requestId}] Receipt uploaded successfully (ID: ${createData.id})`) - - return NextResponse.json({ - success: true, - output: { - receiptId: createData.id, - receiptName: effectiveReceiptName, - expenseId: expenseId ?? null, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Unexpected error:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/buffer/create-post/route.ts b/apps/sim/app/api/tools/buffer/create-post/route.ts deleted file mode 100644 index ef0ab658659..00000000000 --- a/apps/sim/app/api/tools/buffer/create-post/route.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { bufferCreatePostContract } from '@/lib/api/contracts/tools/buffer' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { forwardPostMutation } from '@/app/api/tools/buffer/server-utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('BufferCreatePostAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Buffer create post attempt`, { - error: authResult.error || 'Missing userId', - }) - return NextResponse.json( - { success: false, error: authResult.error || 'Unauthorized' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - bufferCreatePostContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const body = parsed.data.body - - return await forwardPostMutation({ - apiKey: body.apiKey, - channelId: body.channelId, - text: body.text, - mode: body.mode, - schedulingType: body.schedulingType, - dueAt: body.dueAt, - saveToDraft: body.saveToDraft, - media: body.media, - mediaType: body.mediaType, - mediaAltText: body.mediaAltText, - userId: authResult.userId, - requestId, - logger, - }) - } catch (error) { - const message = getErrorMessage(error, 'Failed to create post') - logger.error(`[${requestId}] Buffer create post failed`, { error: message }) - return NextResponse.json({ success: false, error: message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/buffer/edit-post/route.ts b/apps/sim/app/api/tools/buffer/edit-post/route.ts deleted file mode 100644 index 30fddcd6b22..00000000000 --- a/apps/sim/app/api/tools/buffer/edit-post/route.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { bufferEditPostContract } from '@/lib/api/contracts/tools/buffer' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { forwardPostMutation } from '@/app/api/tools/buffer/server-utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('BufferEditPostAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Buffer edit post attempt`, { - error: authResult.error || 'Missing userId', - }) - return NextResponse.json( - { success: false, error: authResult.error || 'Unauthorized' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - bufferEditPostContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const body = parsed.data.body - - return await forwardPostMutation({ - apiKey: body.apiKey, - postId: body.postId, - text: body.text, - mode: body.mode, - schedulingType: body.schedulingType, - dueAt: body.dueAt, - saveToDraft: body.saveToDraft, - media: body.media, - mediaType: body.mediaType, - mediaAltText: body.mediaAltText, - userId: authResult.userId, - requestId, - logger, - }) - } catch (error) { - const message = getErrorMessage(error, 'Failed to edit post') - logger.error(`[${requestId}] Buffer edit post failed`, { error: message }) - return NextResponse.json({ success: false, error: message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/buffer/server-utils.ts b/apps/sim/app/api/tools/buffer/server-utils.ts deleted file mode 100644 index fee163e1568..00000000000 --- a/apps/sim/app/api/tools/buffer/server-utils.ts +++ /dev/null @@ -1,294 +0,0 @@ -import type { Logger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { NextResponse } from 'next/server' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' -import { resolveFileInputToUrl } from '@/lib/uploads/utils/file-utils.server' -import { - BUFFER_API_URL, - BUFFER_POST_SELECTION, - bufferHeaders, - mapBufferPost, - parseBufferGraphQLResponse, -} from '@/tools/buffer/types' - -const VIDEO_EXTENSIONS = ['.mp4', '.mov', '.m4v', '.webm', '.avi'] -const IMAGE_EXTENSIONS = ['.png', '.jpg', '.jpeg', '.gif', '.webp'] -const MEDIA_PROBE_TIMEOUT_MS = 5000 -/** - * Buffer fetches asset URLs at publish time, which for queued or scheduled - * posts can be days after createPost. Presign stored files for the S3 maximum - * of 7 days so scheduled posts within that window can still be published. - * The effective lifetime is additionally bounded by the signing credentials: - * Sim's storage clients sign with static keys (AWS_ACCESS_KEY_ID / - * AWS_SECRET_ACCESS_KEY), which support the full 7 days; deployments signing - * with temporary session credentials cap every presigned URL at the session - * lifetime platform-wide. - */ -const MEDIA_PRESIGN_EXPIRY_SECONDS = 7 * 24 * 60 * 60 - -const CREATE_POST_MUTATION = ` - mutation CreatePost($input: CreatePostInput!) { - createPost(input: $input) { - __typename - ... on PostActionSuccess { - post { - ${BUFFER_POST_SELECTION} - } - } - ... on MutationError { - message - } - } - } -` - -const EDIT_POST_MUTATION = ` - mutation EditPost($input: EditPostInput!) { - editPost(input: $input) { - __typename - ... on PostActionSuccess { - post { - ${BUFFER_POST_SELECTION} - } - } - ... on MutationError { - message - } - } - } -` - -interface ResolveMediaAssetOptions { - media: RawFileInput | string - mediaType?: 'auto' | 'image' | 'video' | null - mediaAltText?: string | null - userId: string - requestId: string - logger: Logger -} - -interface ResolvedMediaAsset { - asset?: Record - errorResponse?: NextResponse -} - -/** - * Classifies media by extension: 'video', 'image', or null when the - * path/URL has no recognizable media extension. - */ -function mediaKindFromExtension(pathOrName: string): 'image' | 'video' | null { - const lowered = pathOrName.toLowerCase().split(/[?#]/)[0] - if (VIDEO_EXTENSIONS.some((extension) => lowered.endsWith(extension))) return 'video' - if (IMAGE_EXTENSIONS.some((extension) => lowered.endsWith(extension))) return 'image' - return null -} - -/** - * Determines whether media should be attached as a video or image asset. - * Prefers the file's MIME type, then the path/URL extension, and for - * extensionless URLs falls back to a DNS-pinned HEAD probe of the resolved - * URL's Content-Type. Returns null when nothing is conclusive so the caller - * can ask for an explicit media type instead of guessing. - */ -async function resolveMediaKind( - mimeType: string | undefined, - pathOrName: string, - fileUrl: string, - requestId: string, - logger: Logger -): Promise<'image' | 'video' | null> { - if (mimeType?.startsWith('video/')) return 'video' - if (mimeType?.startsWith('image/')) return 'image' - - const extensionKind = mediaKindFromExtension(pathOrName) - if (extensionKind) return extensionKind - - try { - const validation = await validateUrlWithDNS(fileUrl, 'media') - if (validation.isValid && validation.resolvedIP) { - const probe = await secureFetchWithPinnedIP(fileUrl, validation.resolvedIP, { - method: 'HEAD', - timeout: MEDIA_PROBE_TIMEOUT_MS, - }) - const contentType = probe.headers.get('content-type') || '' - if (contentType.startsWith('video/')) return 'video' - if (contentType.startsWith('image/')) return 'image' - } - } catch (error) { - logger.warn(`[${requestId}] Media content-type probe was inconclusive`, { - error: getErrorMessage(error, 'probe failed'), - }) - } - return null -} - -/** - * Resolves a media input (uploaded file, file reference, or external URL) to a - * Buffer AssetInput. Buffer downloads assets from publicly accessible URLs, so - * stored files are verified for access and resolved to short-lived presigned - * URLs. - */ -export async function resolveMediaAsset( - options: ResolveMediaAssetOptions -): Promise { - const { media, mediaType, mediaAltText, userId, requestId, logger } = options - - const isFileInput = typeof media === 'object' - const resolution = await resolveFileInputToUrl({ - file: isFileInput ? media : undefined, - filePath: isFileInput ? undefined : media, - userId, - requestId, - logger, - presignExpirySeconds: MEDIA_PRESIGN_EXPIRY_SECONDS, - }) - if (resolution.error || !resolution.fileUrl) { - return { - errorResponse: NextResponse.json( - { success: false, error: resolution.error?.message || 'Failed to resolve media file' }, - { status: resolution.error?.status || 400 } - ), - } - } - - const mimeType = isFileInput ? media.type : undefined - const pathOrName = isFileInput ? media.name || '' : media - const kind = - mediaType === 'image' || mediaType === 'video' - ? mediaType - : await resolveMediaKind(mimeType, pathOrName, resolution.fileUrl, requestId, logger) - if (!kind) { - return { - errorResponse: NextResponse.json( - { - success: false, - error: - 'Could not determine whether the media is an image or a video. Set mediaType to "image" or "video".', - }, - { status: 400 } - ), - } - } - if (kind === 'video') { - return { asset: { video: { url: resolution.fileUrl } } } - } - - const image: Record = { url: resolution.fileUrl } - if (mediaAltText?.trim()) { - image.metadata = { altText: mediaAltText.trim() } - } - return { asset: { image } } -} - -interface ExecutePostMutationOptions { - apiKey: string - mutation: typeof CREATE_POST_MUTATION | typeof EDIT_POST_MUTATION - input: Record - requestId: string - logger: Logger -} - -/** - * Executes a createPost/editPost mutation against the Buffer GraphQL API and - * maps the PostActionPayload union onto the route's response envelope. - */ -async function executePostMutation(options: ExecutePostMutationOptions): Promise { - const { apiKey, mutation, input, requestId, logger } = options - - let result: Record - try { - const response = await fetch(BUFFER_API_URL, { - method: 'POST', - headers: bufferHeaders(apiKey), - body: JSON.stringify({ query: mutation, variables: { input } }), - }) - const data = await parseBufferGraphQLResponse(response) - result = data.createPost ?? data.editPost - } catch (error) { - const message = getErrorMessage(error, 'Buffer API request failed') - logger.error(`[${requestId}] Buffer post mutation failed`, { error: message }) - return NextResponse.json({ success: false, error: message }, { status: 502 }) - } - - if (result?.__typename !== 'PostActionSuccess' || !result.post) { - const message = result?.message || 'Buffer rejected the post' - logger.warn(`[${requestId}] Buffer rejected post mutation`, { - typename: result?.__typename, - error: message, - }) - return NextResponse.json({ success: false, error: message }, { status: 400 }) - } - - return NextResponse.json({ - success: true, - output: { post: mapBufferPost(result.post) }, - }) -} - -interface ForwardPostMutationOptions { - apiKey: string - postId?: string - channelId?: string - text?: string | null - mode: string - schedulingType: string - dueAt?: string | null - saveToDraft?: boolean | null - media?: RawFileInput | string | null - mediaType?: 'auto' | 'image' | 'video' | null - mediaAltText?: string | null - userId: string - requestId: string - logger: Logger -} - -/** - * Builds the CreatePostInput/EditPostInput from a validated route body - * (resolving media to a fetchable URL) and forwards the mutation to Buffer. - * Passing `postId` selects the editPost mutation; otherwise createPost runs. - */ -export async function forwardPostMutation( - options: ForwardPostMutationOptions -): Promise { - const { apiKey, postId, channelId, media, mediaType, mediaAltText, userId, requestId, logger } = - options - - const input: Record = { - mode: options.mode, - schedulingType: options.schedulingType, - } - if (postId) { - input.id = postId - } else { - input.channelId = channelId - input.assets = [] - } - if (options.text != null && options.text !== '') input.text = options.text - if (options.dueAt) input.dueAt = options.dueAt - if (options.saveToDraft != null) input.saveToDraft = options.saveToDraft - - if (media) { - const { asset, errorResponse } = await resolveMediaAsset({ - media, - mediaType, - mediaAltText, - userId, - requestId, - logger, - }) - if (errorResponse) return errorResponse - input.assets = [asset] - } - - return executePostMutation({ - apiKey, - mutation: postId ? EDIT_POST_MUTATION : CREATE_POST_MUTATION, - input, - requestId, - logger, - }) -} diff --git a/apps/sim/app/api/tools/clickhouse/count-rows/route.ts b/apps/sim/app/api/tools/clickhouse/count-rows/route.ts deleted file mode 100644 index 5b7b90821ca..00000000000 --- a/apps/sim/app/api/tools/clickhouse/count-rows/route.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseCountRowsContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseCountRows } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseCountRowsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse count rows attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseCountRowsContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const count = await executeClickHouseCountRows(params, params.table, params.where) - - return NextResponse.json({ - message: `Table contains ${count} row(s).`, - count, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse count rows failed:`, error) - - return NextResponse.json( - { error: `ClickHouse count rows failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/create-database/route.ts b/apps/sim/app/api/tools/clickhouse/create-database/route.ts deleted file mode 100644 index b748a20595e..00000000000 --- a/apps/sim/app/api/tools/clickhouse/create-database/route.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseCreateDatabaseContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseCreateDatabase } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseCreateDatabaseAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse create database attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseCreateDatabaseContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - await executeClickHouseCreateDatabase(params, params.name) - - return NextResponse.json({ - message: `Database '${params.name}' created.`, - rows: [], - rowCount: 0, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse create database failed:`, error) - - return NextResponse.json( - { error: `ClickHouse create database failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/create-table/route.ts b/apps/sim/app/api/tools/clickhouse/create-table/route.ts deleted file mode 100644 index 47cc3ff5f7f..00000000000 --- a/apps/sim/app/api/tools/clickhouse/create-table/route.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseCreateTableContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseCreateTable } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseCreateTableAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse create table attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseCreateTableContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - await executeClickHouseCreateTable( - params, - params.table, - params.columns, - params.engine, - params.orderBy, - params.partitionBy - ) - - return NextResponse.json({ - message: `Table '${params.table}' created.`, - rows: [], - rowCount: 0, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse create table failed:`, error) - - return NextResponse.json( - { error: `ClickHouse create table failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/delete/route.ts b/apps/sim/app/api/tools/clickhouse/delete/route.ts deleted file mode 100644 index f773aabba4a..00000000000 --- a/apps/sim/app/api/tools/clickhouse/delete/route.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseDeleteContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseDelete } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseDeleteAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse delete attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseDeleteContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Deleting data from ${params.table} on ${params.host}:${params.port}/${params.database}` - ) - - const result = await executeClickHouseDelete(params, params.table, params.where) - - logger.info(`[${requestId}] Delete mutation submitted, ${result.rowCount} row(s) affected`) - - return NextResponse.json({ - message: `Delete mutation submitted. ClickHouse mutations run asynchronously. ${result.rowCount} row(s) affected.`, - rows: result.rows, - rowCount: result.rowCount, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse delete failed:`, error) - - return NextResponse.json( - { error: `ClickHouse delete failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/describe-table/route.ts b/apps/sim/app/api/tools/clickhouse/describe-table/route.ts deleted file mode 100644 index e258d781bc1..00000000000 --- a/apps/sim/app/api/tools/clickhouse/describe-table/route.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseDescribeTableContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseDescribeTable } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseDescribeTableAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse describe table attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseDescribeTableContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const result = await executeClickHouseDescribeTable(params, params.table) - - return NextResponse.json({ - message: `Described table with ${result.rowCount} column(s).`, - rows: result.rows, - rowCount: result.rowCount, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse describe table failed:`, error) - - return NextResponse.json( - { error: `ClickHouse describe table failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/drop-database/route.ts b/apps/sim/app/api/tools/clickhouse/drop-database/route.ts deleted file mode 100644 index e06f897b337..00000000000 --- a/apps/sim/app/api/tools/clickhouse/drop-database/route.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseDropDatabaseContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseDropDatabase } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseDropDatabaseAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse drop database attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseDropDatabaseContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - await executeClickHouseDropDatabase(params, params.name) - - return NextResponse.json({ - message: `Database '${params.name}' dropped.`, - rows: [], - rowCount: 0, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse drop database failed:`, error) - - return NextResponse.json( - { error: `ClickHouse drop database failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/drop-partition/route.ts b/apps/sim/app/api/tools/clickhouse/drop-partition/route.ts deleted file mode 100644 index 790526586ba..00000000000 --- a/apps/sim/app/api/tools/clickhouse/drop-partition/route.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseDropPartitionContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseDropPartition } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseDropPartitionAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse drop partition attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseDropPartitionContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - await executeClickHouseDropPartition(params, params.table, params.partition) - - return NextResponse.json({ - message: `Dropped partition from table '${params.table}'.`, - rows: [], - rowCount: 0, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse drop partition failed:`, error) - - return NextResponse.json( - { error: `ClickHouse drop partition failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/drop-table/route.ts b/apps/sim/app/api/tools/clickhouse/drop-table/route.ts deleted file mode 100644 index 1ae9f6832a8..00000000000 --- a/apps/sim/app/api/tools/clickhouse/drop-table/route.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseDropTableContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseDropTable } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseDropTableAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse drop table attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseDropTableContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - await executeClickHouseDropTable(params, params.table) - - return NextResponse.json({ - message: `Table '${params.table}' dropped.`, - rows: [], - rowCount: 0, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse drop table failed:`, error) - - return NextResponse.json( - { error: `ClickHouse drop table failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/execute/route.ts b/apps/sim/app/api/tools/clickhouse/execute/route.ts deleted file mode 100644 index 3e2c4baacf6..00000000000 --- a/apps/sim/app/api/tools/clickhouse/execute/route.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseExecuteContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseQuery } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseExecuteAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse execute attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseExecuteContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Executing ClickHouse statement on ${params.host}:${params.port}/${params.database}` - ) - - const result = await executeClickHouseQuery(params, params.query) - - logger.info(`[${requestId}] Statement executed successfully, ${result.rowCount} row(s)`) - - return NextResponse.json({ - message: `Statement executed successfully. ${result.rowCount} row(s) returned or affected.`, - rows: result.rows, - rowCount: result.rowCount, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse execute failed:`, error) - - return NextResponse.json( - { error: `ClickHouse execute failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/insert-rows/route.ts b/apps/sim/app/api/tools/clickhouse/insert-rows/route.ts deleted file mode 100644 index fb4f90b8634..00000000000 --- a/apps/sim/app/api/tools/clickhouse/insert-rows/route.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseInsertRowsContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseInsertRows } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseInsertRowsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse insert rows attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseInsertRowsContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const result = await executeClickHouseInsertRows(params, params.table, params.rows) - - return NextResponse.json({ - message: `Inserted ${result.rowCount} row(s) into '${params.table}'.`, - rows: result.rows, - rowCount: result.rowCount, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse insert rows failed:`, error) - - return NextResponse.json( - { error: `ClickHouse insert rows failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/insert/route.ts b/apps/sim/app/api/tools/clickhouse/insert/route.ts deleted file mode 100644 index a7cc4ed908f..00000000000 --- a/apps/sim/app/api/tools/clickhouse/insert/route.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseInsertContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseInsert } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseInsertAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse insert attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseInsertContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Inserting data into ${params.table} on ${params.host}:${params.port}/${params.database}` - ) - - const result = await executeClickHouseInsert(params, params.table, params.data) - - logger.info(`[${requestId}] Insert executed successfully, ${result.rowCount} row(s) inserted`) - - return NextResponse.json({ - message: `Data inserted successfully. ${result.rowCount} row(s) affected.`, - rows: result.rows, - rowCount: result.rowCount, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse insert failed:`, error) - - return NextResponse.json( - { error: `ClickHouse insert failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/introspect/route.ts b/apps/sim/app/api/tools/clickhouse/introspect/route.ts deleted file mode 100644 index cd3257c6275..00000000000 --- a/apps/sim/app/api/tools/clickhouse/introspect/route.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseIntrospectContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseIntrospect } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseIntrospectAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse introspect attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseIntrospectContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Introspecting ClickHouse schema on ${params.host}:${params.port}/${params.database}` - ) - - const result = await executeClickHouseIntrospect(params) - - logger.info( - `[${requestId}] Introspection completed successfully, found ${result.tables.length} tables` - ) - - return NextResponse.json({ - message: `Schema introspection completed. Found ${result.tables.length} table(s) in database '${params.database}'.`, - tables: result.tables, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse introspection failed:`, error) - - return NextResponse.json( - { error: `ClickHouse introspection failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/kill-query/route.ts b/apps/sim/app/api/tools/clickhouse/kill-query/route.ts deleted file mode 100644 index c46f6d1393c..00000000000 --- a/apps/sim/app/api/tools/clickhouse/kill-query/route.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseKillQueryContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseKillQuery } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseKillQueryAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse kill query attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseKillQueryContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const result = await executeClickHouseKillQuery(params, params.queryId) - - return NextResponse.json({ - message: `Kill command executed for query '${params.queryId}'.`, - rows: result.rows, - rowCount: result.rowCount, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse kill query failed:`, error) - - return NextResponse.json( - { error: `ClickHouse kill query failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/list-clusters/route.ts b/apps/sim/app/api/tools/clickhouse/list-clusters/route.ts deleted file mode 100644 index 643c7be9621..00000000000 --- a/apps/sim/app/api/tools/clickhouse/list-clusters/route.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseListClustersContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseListClusters } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseListClustersAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse list clusters attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseListClustersContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const result = await executeClickHouseListClusters(params) - - return NextResponse.json({ - message: `Found ${result.rowCount} cluster node(s).`, - rows: result.rows, - rowCount: result.rowCount, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse list clusters failed:`, error) - - return NextResponse.json( - { error: `ClickHouse list clusters failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/list-databases/route.ts b/apps/sim/app/api/tools/clickhouse/list-databases/route.ts deleted file mode 100644 index c524b162474..00000000000 --- a/apps/sim/app/api/tools/clickhouse/list-databases/route.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseListDatabasesContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseListDatabases } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseListDatabasesAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse list databases attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseListDatabasesContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const result = await executeClickHouseListDatabases(params) - - return NextResponse.json({ - message: `Found ${result.rowCount} database(s).`, - rows: result.rows, - rowCount: result.rowCount, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse list databases failed:`, error) - - return NextResponse.json( - { error: `ClickHouse list databases failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/list-mutations/route.ts b/apps/sim/app/api/tools/clickhouse/list-mutations/route.ts deleted file mode 100644 index 84034b42436..00000000000 --- a/apps/sim/app/api/tools/clickhouse/list-mutations/route.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseListMutationsContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseListMutations } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseListMutationsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse list mutations attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseListMutationsContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const result = await executeClickHouseListMutations(params, params.table, params.onlyRunning) - - return NextResponse.json({ - message: `Found ${result.rowCount} mutation(s).`, - rows: result.rows, - rowCount: result.rowCount, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse list mutations failed:`, error) - - return NextResponse.json( - { error: `ClickHouse list mutations failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/list-partitions/route.ts b/apps/sim/app/api/tools/clickhouse/list-partitions/route.ts deleted file mode 100644 index d064850ad1f..00000000000 --- a/apps/sim/app/api/tools/clickhouse/list-partitions/route.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseListPartitionsContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseListPartitions } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseListPartitionsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse list partitions attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseListPartitionsContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const result = await executeClickHouseListPartitions(params, params.table) - - return NextResponse.json({ - message: `Found ${result.rowCount} partition(s).`, - rows: result.rows, - rowCount: result.rowCount, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse list partitions failed:`, error) - - return NextResponse.json( - { error: `ClickHouse list partitions failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/list-running-queries/route.ts b/apps/sim/app/api/tools/clickhouse/list-running-queries/route.ts deleted file mode 100644 index d542966d5d0..00000000000 --- a/apps/sim/app/api/tools/clickhouse/list-running-queries/route.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseListRunningQueriesContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseListRunningQueries } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseListRunningQueriesAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse list running queries attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseListRunningQueriesContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const result = await executeClickHouseListRunningQueries(params) - - return NextResponse.json({ - message: `Found ${result.rowCount} running query(ies).`, - rows: result.rows, - rowCount: result.rowCount, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse list running queries failed:`, error) - - return NextResponse.json( - { error: `ClickHouse list running queries failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/list-tables/route.ts b/apps/sim/app/api/tools/clickhouse/list-tables/route.ts deleted file mode 100644 index 4d9df7a2dc7..00000000000 --- a/apps/sim/app/api/tools/clickhouse/list-tables/route.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseListTablesContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseListTables } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseListTablesAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse list tables attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseListTablesContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const result = await executeClickHouseListTables(params) - - return NextResponse.json({ - message: `Found ${result.rowCount} table(s).`, - rows: result.rows, - rowCount: result.rowCount, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse list tables failed:`, error) - - return NextResponse.json( - { error: `ClickHouse list tables failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/optimize-table/route.ts b/apps/sim/app/api/tools/clickhouse/optimize-table/route.ts deleted file mode 100644 index 3d22b8b3788..00000000000 --- a/apps/sim/app/api/tools/clickhouse/optimize-table/route.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseOptimizeTableContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseOptimizeTable } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseOptimizeTableAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse optimize table attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseOptimizeTableContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - await executeClickHouseOptimizeTable(params, params.table, params.final) - - return NextResponse.json({ - message: `Optimize submitted for table '${params.table}'.`, - rows: [], - rowCount: 0, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse optimize table failed:`, error) - - return NextResponse.json( - { error: `ClickHouse optimize table failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/query/route.ts b/apps/sim/app/api/tools/clickhouse/query/route.ts deleted file mode 100644 index 4d70b48b55b..00000000000 --- a/apps/sim/app/api/tools/clickhouse/query/route.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseQueryContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseQuery } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseQueryAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse query attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseQueryContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Executing ClickHouse query on ${params.host}:${params.port}/${params.database}` - ) - - const result = await executeClickHouseQuery(params, params.query, { enforceReadOnly: true }) - - logger.info(`[${requestId}] Query executed successfully, returned ${result.rowCount} rows`) - - return NextResponse.json({ - message: `Query executed successfully. ${result.rowCount} row(s) returned.`, - rows: result.rows, - rowCount: result.rowCount, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse query failed:`, error) - - return NextResponse.json({ error: `ClickHouse query failed: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/rename-table/route.ts b/apps/sim/app/api/tools/clickhouse/rename-table/route.ts deleted file mode 100644 index eec1f7ec436..00000000000 --- a/apps/sim/app/api/tools/clickhouse/rename-table/route.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseRenameTableContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseRenameTable } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseRenameTableAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse rename table attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseRenameTableContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - await executeClickHouseRenameTable(params, params.table, params.newTable) - - return NextResponse.json({ - message: `Renamed table '${params.table}' to '${params.newTable}'.`, - rows: [], - rowCount: 0, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse rename table failed:`, error) - - return NextResponse.json( - { error: `ClickHouse rename table failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/show-create-table/route.ts b/apps/sim/app/api/tools/clickhouse/show-create-table/route.ts deleted file mode 100644 index 8c93d402803..00000000000 --- a/apps/sim/app/api/tools/clickhouse/show-create-table/route.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseShowCreateTableContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseShowCreateTable } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseShowCreateTableAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse show create table attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseShowCreateTableContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const ddl = await executeClickHouseShowCreateTable(params, params.table) - - return NextResponse.json({ - message: 'Retrieved CREATE statement.', - ddl, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse show create table failed:`, error) - - return NextResponse.json( - { error: `ClickHouse show create table failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/table-stats/route.ts b/apps/sim/app/api/tools/clickhouse/table-stats/route.ts deleted file mode 100644 index 405fbaf06cc..00000000000 --- a/apps/sim/app/api/tools/clickhouse/table-stats/route.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseTableStatsContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseTableStats } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseTableStatsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse table stats attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseTableStatsContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const result = await executeClickHouseTableStats(params, params.table) - - return NextResponse.json({ - message: `Retrieved stats for ${result.rowCount} table(s).`, - rows: result.rows, - rowCount: result.rowCount, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse table stats failed:`, error) - - return NextResponse.json( - { error: `ClickHouse table stats failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/truncate-table/route.ts b/apps/sim/app/api/tools/clickhouse/truncate-table/route.ts deleted file mode 100644 index 27452eb9849..00000000000 --- a/apps/sim/app/api/tools/clickhouse/truncate-table/route.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseTruncateTableContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseTruncateTable } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseTruncateTableAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse truncate table attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseTruncateTableContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - await executeClickHouseTruncateTable(params, params.table) - - return NextResponse.json({ - message: `Table '${params.table}' truncated.`, - rows: [], - rowCount: 0, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse truncate table failed:`, error) - - return NextResponse.json( - { error: `ClickHouse truncate table failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/update/route.ts b/apps/sim/app/api/tools/clickhouse/update/route.ts deleted file mode 100644 index 9d43755da4c..00000000000 --- a/apps/sim/app/api/tools/clickhouse/update/route.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { clickhouseUpdateContract } from '@/lib/api/contracts/tools/databases/clickhouse' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeClickHouseUpdate } from '@/app/api/tools/clickhouse/utils' - -const logger = createLogger('ClickHouseUpdateAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized ClickHouse update attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(clickhouseUpdateContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Updating data in ${params.table} on ${params.host}:${params.port}/${params.database}` - ) - - const result = await executeClickHouseUpdate(params, params.table, params.data, params.where) - - logger.info(`[${requestId}] Update mutation submitted, ${result.rowCount} row(s) written`) - - return NextResponse.json({ - message: `Update mutation submitted. ClickHouse mutations run asynchronously. ${result.rowCount} row(s) written.`, - rows: result.rows, - rowCount: result.rowCount, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] ClickHouse update failed:`, error) - - return NextResponse.json( - { error: `ClickHouse update failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/clickhouse/utils.test.ts b/apps/sim/app/api/tools/clickhouse/utils.test.ts deleted file mode 100644 index 23e45da05c5..00000000000 --- a/apps/sim/app/api/tools/clickhouse/utils.test.ts +++ /dev/null @@ -1,127 +0,0 @@ -/** - * @vitest-environment node - */ -import { beforeEach, describe, expect, it, vi } from 'vitest' -import type { ClickHouseConnectionConfig } from '@/tools/clickhouse/types' - -const { mockValidateDatabaseHost, mockSecureFetchWithPinnedIP, mockValidateSqlWhereClause } = - vi.hoisted(() => ({ - mockValidateDatabaseHost: vi.fn(), - mockSecureFetchWithPinnedIP: vi.fn(), - mockValidateSqlWhereClause: vi.fn(), - })) - -vi.mock('@/lib/core/security/input-validation.server', () => ({ - MAX_JSON_API_RESPONSE_BYTES: 10 * 1024 * 1024, - validateDatabaseHost: mockValidateDatabaseHost, - secureFetchWithPinnedIP: mockSecureFetchWithPinnedIP, - validateSqlWhereClause: mockValidateSqlWhereClause, -})) - -import { executeClickHouseInsert, executeClickHouseQuery } from '@/app/api/tools/clickhouse/utils' - -function makeConfig( - overrides: Partial = {} -): ClickHouseConnectionConfig { - return { - host: 'clickhouse.example.com', - port: 8123, - database: 'default', - username: 'default', - password: 'secret', - secure: false, - ...overrides, - } -} - -function okResponse(body: string, summary?: string) { - return { - ok: true, - status: 200, - statusText: 'OK', - text: async () => body, - headers: { - get: (name: string) => - name.toLowerCase() === 'x-clickhouse-summary' ? (summary ?? null) : null, - }, - } -} - -describe('clickhouseRequest DNS pinning', () => { - beforeEach(() => { - vi.clearAllMocks() - mockValidateDatabaseHost.mockResolvedValue({ - isValid: true, - resolvedIP: '93.184.216.34', - originalHostname: 'clickhouse.example.com', - }) - mockValidateSqlWhereClause.mockReturnValue({ isValid: true }) - mockSecureFetchWithPinnedIP.mockResolvedValue(okResponse('{"data":[{"x":1}],"rows":1}')) - }) - - it('pins the connection to the validated IP, not the attacker-controlled hostname', async () => { - await executeClickHouseQuery(makeConfig({ host: 'rebind.attacker.example' }), 'SELECT 1') - - expect(mockValidateDatabaseHost).toHaveBeenCalledWith('rebind.attacker.example', 'host') - expect(mockSecureFetchWithPinnedIP).toHaveBeenCalledTimes(1) - - const [url, pinnedIP, options] = mockSecureFetchWithPinnedIP.mock.calls[0] - // The actual TCP target is the validated IP — re-resolution of the hostname can never happen. - expect(pinnedIP).toBe('93.184.216.34') - // The hostname is preserved only in the URL (for Host header / TLS SNI), never used to connect. - expect(url).toContain('rebind.attacker.example') - expect(options.method).toBe('POST') - }) - - it('never issues the request when host validation fails (no SSRF window)', async () => { - mockValidateDatabaseHost.mockResolvedValue({ - isValid: false, - error: 'host resolves to a blocked IP address', - }) - - await expect(executeClickHouseQuery(makeConfig(), 'SELECT 1')).rejects.toThrow( - 'host resolves to a blocked IP address' - ) - expect(mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() - }) - - it('uses https and disallows http redirects when secure is true', async () => { - await executeClickHouseQuery(makeConfig({ secure: true, port: 8443 }), 'SELECT 1') - - const [url, , options] = mockSecureFetchWithPinnedIP.mock.calls[0] - expect(url).toMatch(/^https:\/\//) - expect(options.allowHttp).toBe(false) - }) - - it('allows http for the initial request when secure is false', async () => { - await executeClickHouseQuery(makeConfig({ secure: false }), 'SELECT 1') - - const [url, , options] = mockSecureFetchWithPinnedIP.mock.calls[0] - expect(url).toMatch(/^http:\/\//) - expect(options.allowHttp).toBe(true) - }) - - it('sends the statement as the body with a matching Content-Length and auth headers', async () => { - await executeClickHouseInsert(makeConfig(), 'events', { id: 1 }) - - const [, , options] = mockSecureFetchWithPinnedIP.mock.calls[0] - expect(options.body).toContain('INSERT INTO `events` FORMAT JSONEachRow') - expect(options.headers['Content-Length']).toBe(String(Buffer.byteLength(options.body, 'utf-8'))) - expect(options.headers['X-ClickHouse-User']).toBe('default') - expect(options.headers['X-ClickHouse-Key']).toBe('secret') - }) - - it('propagates non-ok responses as errors with the body text', async () => { - mockSecureFetchWithPinnedIP.mockResolvedValue({ - ok: false, - status: 400, - statusText: 'Bad Request', - text: async () => 'Code: 62. DB::Exception: Syntax error', - headers: { get: () => null }, - }) - - await expect(executeClickHouseQuery(makeConfig(), 'SELECT 1')).rejects.toThrow( - 'Code: 62. DB::Exception: Syntax error' - ) - }) -}) diff --git a/apps/sim/app/api/tools/clickhouse/utils.ts b/apps/sim/app/api/tools/clickhouse/utils.ts deleted file mode 100644 index 5b613285449..00000000000 --- a/apps/sim/app/api/tools/clickhouse/utils.ts +++ /dev/null @@ -1,852 +0,0 @@ -import { - MAX_JSON_API_RESPONSE_BYTES, - secureFetchWithPinnedIP, - validateDatabaseHost, - validateSqlWhereClause, -} from '@/lib/core/security/input-validation.server' -import type { ClickHouseConnectionConfig } from '@/tools/clickhouse/types' - -const REQUEST_TIMEOUT_MS = 30_000 - -interface ClickHouseSummary { - read_rows?: string - written_rows?: string - result_rows?: string -} - -interface ClickHouseHttpResult { - text: string - summary: ClickHouseSummary | null -} - -export interface ClickHouseRowsResult { - rows: unknown[] - rowCount: number -} - -interface ClickHouseColumnRow { - table: string - name: string - type: string - default_kind?: string - default_expression?: string - is_in_primary_key?: number | string - is_in_sorting_key?: number | string - position?: number | string -} - -interface ClickHouseTableRow { - name: string - engine?: string - total_rows?: number | string | null -} - -export interface ClickHouseIntrospectionResult { - tables: Array<{ - name: string - database: string - engine: string - totalRows?: number - columns: Array<{ - name: string - type: string - defaultKind?: string - defaultExpression?: string - isInPrimaryKey: boolean - isInSortingKey: boolean - }> - }> -} - -/** - * Sends a single statement to the ClickHouse HTTP interface and returns the raw - * response body alongside the parsed `X-ClickHouse-Summary` header. - * @see https://clickhouse.com/docs/interfaces/http - */ -async function clickhouseRequest( - config: ClickHouseConnectionConfig, - statement: string, - options: { readOnly?: boolean } = {} -): Promise { - const hostValidation = await validateDatabaseHost(config.host, 'host') - if (!hostValidation.isValid) { - throw new Error(hostValidation.error) - } - - const protocol = config.secure ? 'https' : 'http' - const url = new URL(`${protocol}://${config.host}:${config.port}/`) - url.searchParams.set('database', config.database) - if (options.readOnly) { - // Server-enforced read-only: ClickHouse rejects any write/DDL and forbids the - // query from re-enabling writes via `SET readonly=0`. This is the real boundary - // for the query operation; the SQL-shape checks below are defense-in-depth. - url.searchParams.set('readonly', '1') - } - - // Pin the connection to the IP that passed validation. Without this, fetch() - // would re-resolve `config.host` and a DNS-rebinding hostname could point the - // actual request at an internal/private address after validation succeeded. - const response = await secureFetchWithPinnedIP(url.toString(), hostValidation.resolvedIP!, { - method: 'POST', - headers: { - 'X-ClickHouse-User': config.username, - 'X-ClickHouse-Key': config.password, - 'Content-Type': 'text/plain; charset=utf-8', - 'Content-Length': String(Buffer.byteLength(statement, 'utf-8')), - }, - body: statement, - timeout: REQUEST_TIMEOUT_MS, - allowHttp: !config.secure, - maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, - }) - - const text = await response.text() - - if (!response.ok) { - throw new Error(text.trim() || `ClickHouse request failed with status ${response.status}`) - } - - return { text, summary: parseSummary(response.headers.get('x-clickhouse-summary')) } -} - -function parseSummary(header: string | null): ClickHouseSummary | null { - if (!header) return null - try { - return JSON.parse(header) as ClickHouseSummary - } catch { - return null - } -} - -/** - * Parses a ClickHouse `FORMAT JSON` response body into rows, falling back to the - * summary header's row counts for statements that do not return a result set. - */ -function parseRowsResult(result: ClickHouseHttpResult): ClickHouseRowsResult { - const trimmed = result.text.trim() - if (trimmed) { - try { - const parsed = JSON.parse(trimmed) as { data?: unknown[]; rows?: number } - if (parsed && Array.isArray(parsed.data)) { - const rowCount = typeof parsed.rows === 'number' ? parsed.rows : parsed.data.length - return { rows: parsed.data, rowCount } - } - } catch { - // Body was not JSON (e.g. a non-SELECT statement); fall through to summary. - } - } - - const written = Number(result.summary?.written_rows ?? 0) - const read = Number(result.summary?.read_rows ?? 0) - return { rows: [], rowCount: written || read || 0 } -} - -/** Read-only statement leaders that return a result set and never mutate data. */ -const READ_ONLY_STATEMENT = /^(select|with|show|describe|desc|explain|exists)\b/i - -/** - * Normalizes the output format of a read statement to JSON so the HTTP response - * can always be parsed into rows. Strips every `FORMAT ` clause — wherever - * it sits relative to a trailing `SETTINGS` clause — and appends a single canonical - * `FORMAT JSON`. The `format()` function and `FORMAT`/format names appearing inside - * strings or comments are ignored (the scan runs on comment/string-masked SQL). - * Non-read statements are returned untouched (their own FORMAT, e.g. JSONEachRow - * for inserts, is preserved). - */ -function ensureJsonFormat(query: string): string { - const trimmed = query.trim().replace(/;+\s*$/, '') - if (!READ_ONLY_STATEMENT.test(trimmed)) { - return trimmed - } - const masked = maskSqlNoise(trimmed) - const formatClause = /\bformat\s+[a-z0-9_]+\b/gi - const spans: Array<[number, number]> = [] - for (let match = formatClause.exec(masked); match !== null; match = formatClause.exec(masked)) { - spans.push([match.index, match.index + match[0].length]) - } - let result = trimmed - for (let i = spans.length - 1; i >= 0; i--) { - result = result.slice(0, spans[i][0]) + result.slice(spans[i][1]) - } - return `${result.replace(/\s+$/, '')}\nFORMAT JSON` -} - -/** - * Replaces string literals ('...'), quoted identifiers ("..." / `...`), and SQL - * comments (`-- …` and `/* … *​/`) with spaces so that structural scans (e.g. for - * statement-chaining semicolons) only see actual SQL code, not data or comments. - */ -function maskSqlNoise(sql: string): string { - let out = '' - let i = 0 - while (i < sql.length) { - const ch = sql[i] - if (ch === "'" || ch === '"' || ch === '`') { - out += ' ' - i++ - while (i < sql.length && sql[i] !== ch) { - if (ch !== '`' && sql[i] === '\\') { - out += ' ' - i += 2 - continue - } - out += ' ' - i++ - } - if (i < sql.length) { - out += ' ' - i++ - } - continue - } - if (ch === '-' && sql[i + 1] === '-') { - const newline = sql.indexOf('\n', i + 2) - const end = newline === -1 ? sql.length : newline - out += ' '.repeat(end - i) - i = end - continue - } - if (ch === '/' && sql[i + 1] === '*') { - const close = sql.indexOf('*/', i + 2) - const end = close === -1 ? sql.length : close + 2 - out += ' '.repeat(end - i) - i = end - continue - } - out += ch - i++ - } - return out -} - -/** - * Detects whether a statement chains a second statement after a `;`, ignoring - * semicolons inside string literals, quoted identifiers, and comments. A trailing - * semicolon (with only whitespace/comments after it) is allowed. - */ -function hasChainedStatement(sql: string): boolean { - return /;\s*\S/.test(maskSqlNoise(sql)) -} - -/** - * Write/DDL statement shapes that must never run under the read-only query - * operation, even when wrapped by a leading `WITH` CTE (e.g. `WITH … INSERT INTO …`). - * Patterns require the keyword's statement context (e.g. `insert into`, `alter table`) - * so SQL functions/columns like `truncate(x)` or `created_at` are not false-positives. - */ -const MUTATING_STATEMENT = [ - /\binsert\s+into\b/i, - /\bdelete\s+from\b/i, - /\bupdate\s+[\w.`"]+\s+set\b/i, - /\balter\s+table\b/i, - /\b(?:create|attach)\s+(?:or\s+replace\s+)?(?:temporary\s+)?(?:table|database|dictionary|view|materialized\s+view|live\s+view|function|user|role)\b/i, - /\bdrop\s+(?:table|database|dictionary|view|column|partition|index|function|user|role)\b/i, - /\btruncate\s+table\b/i, - /\brename\s+(?:table|database|dictionary)\b/i, - /\bdetach\s+(?:table|database|dictionary|view|permanently)\b/i, - /\b(?:grant|revoke)\b/i, - /\boptimize\s+table\b/i, -] - -/** Whether a statement performs a write/DDL anywhere (comments and strings masked out). */ -function isMutatingStatement(sql: string): boolean { - const masked = maskSqlNoise(sql) - return MUTATING_STATEMENT.some((pattern) => pattern.test(masked)) -} - -/** - * Strips leading whitespace, `--`/`/* … *​/` comments, and opening parens from a - * statement so the read-only leader keyword can be detected even when a query - * starts with a comment (e.g. `-- note\nSELECT …`) or wrapping parens. - */ -function stripLeadingNoise(sql: string): string { - let s = sql.trim() - for (;;) { - if (s.startsWith('--')) { - const newline = s.indexOf('\n') - s = (newline === -1 ? '' : s.slice(newline + 1)).trim() - } else if (s.startsWith('/*')) { - const close = s.indexOf('*/') - s = (close === -1 ? '' : s.slice(close + 2)).trim() - } else if (s.startsWith('(')) { - s = s.slice(1).trim() - } else { - return s - } - } -} - -export async function executeClickHouseQuery( - config: ClickHouseConnectionConfig, - query: string, - options: { enforceReadOnly?: boolean } = {} -): Promise { - if (options.enforceReadOnly) { - // Strip leading comments/parens so wrapped or commented selects still validate. - const leader = stripLeadingNoise(query) - if (!READ_ONLY_STATEMENT.test(leader)) { - throw new Error( - 'The query operation only allows read-only statements (SELECT, WITH, SHOW, DESCRIBE, EXPLAIN, EXISTS). Use the Execute Raw SQL operation to run writes or DDL.' - ) - } - if (hasChainedStatement(query)) { - throw new Error( - 'The query operation only allows a single statement; chained statements separated by ";" are not allowed. Use the Execute Raw SQL operation to run multiple statements.' - ) - } - if (isMutatingStatement(query)) { - throw new Error( - 'The query operation only allows read-only statements; a write or DDL statement (e.g. INSERT/ALTER/DROP, including after a WITH clause) was detected. Use the Execute Raw SQL operation instead.' - ) - } - } - const result = await clickhouseRequest(config, ensureJsonFormat(query), { - readOnly: options.enforceReadOnly, - }) - return parseRowsResult(result) -} - -export async function executeClickHouseInsert( - config: ClickHouseConnectionConfig, - table: string, - data: Record -): Promise { - const sanitizedTable = sanitizeIdentifier(table) - const statement = `INSERT INTO ${sanitizedTable} FORMAT JSONEachRow\n${JSON.stringify(data)}` - const result = await clickhouseRequest(config, statement) - const written = Number(result.summary?.written_rows ?? 0) - return { rows: [], rowCount: written || 1 } -} - -export async function executeClickHouseUpdate( - config: ClickHouseConnectionConfig, - table: string, - data: Record, - where: string -): Promise { - validateWhereClause(where) - const sanitizedTable = sanitizeIdentifier(table) - const assignments = Object.entries(data) - .map(([column, value]) => `${sanitizeIdentifier(column)} = ${formatValue(value)}`) - .join(', ') - - if (!assignments) { - throw new Error('Update data object cannot be empty') - } - - const statement = `ALTER TABLE ${sanitizedTable} UPDATE ${assignments} WHERE ${where}` - const result = await clickhouseRequest(config, statement) - return { rows: [], rowCount: Number(result.summary?.written_rows ?? 0) } -} - -export async function executeClickHouseDelete( - config: ClickHouseConnectionConfig, - table: string, - where: string -): Promise { - validateWhereClause(where) - const sanitizedTable = sanitizeIdentifier(table) - const statement = `ALTER TABLE ${sanitizedTable} DELETE WHERE ${where}` - const result = await clickhouseRequest(config, statement) - return { rows: [], rowCount: Number(result.summary?.written_rows ?? 0) } -} - -export async function executeClickHouseIntrospect( - config: ClickHouseConnectionConfig -): Promise { - const database = quoteString(config.database) - - const tablesResult = await clickhouseRequest( - config, - `SELECT name, engine, total_rows FROM system.tables WHERE database = ${database} ORDER BY name FORMAT JSON` - ) - const tableRows = parseDataArray(tablesResult.text) - - const columnsResult = await clickhouseRequest( - config, - `SELECT table, name, type, default_kind, default_expression, is_in_primary_key, is_in_sorting_key, position FROM system.columns WHERE database = ${database} ORDER BY table, position FORMAT JSON` - ) - const columnRows = parseDataArray(columnsResult.text) - - const columnsByTable = new Map< - string, - ClickHouseIntrospectionResult['tables'][number]['columns'] - >() - for (const column of columnRows) { - const columns = columnsByTable.get(column.table) ?? [] - columns.push({ - name: column.name, - type: column.type, - defaultKind: column.default_kind || undefined, - defaultExpression: column.default_expression || undefined, - isInPrimaryKey: toBoolean(column.is_in_primary_key), - isInSortingKey: toBoolean(column.is_in_sorting_key), - }) - columnsByTable.set(column.table, columns) - } - - const tables = tableRows.map((table) => ({ - name: table.name, - database: config.database, - engine: table.engine ?? '', - totalRows: table.total_rows != null ? Number(table.total_rows) : undefined, - columns: columnsByTable.get(table.name) ?? [], - })) - - return { tables } -} - -function parseDataArray(text: string): T[] { - const trimmed = text.trim() - if (!trimmed) return [] - try { - const parsed = JSON.parse(trimmed) as { data?: T[] } - return Array.isArray(parsed.data) ? parsed.data : [] - } catch { - return [] - } -} - -function toBoolean(value: number | string | undefined): boolean { - return value === 1 || value === '1' -} - -/** - * Quotes and escapes a value for inline use in a ClickHouse statement. - * Strings use ClickHouse's backslash escaping for single quotes and backslashes. - */ -function formatValue(value: unknown): string { - if (value === null || value === undefined) { - return 'NULL' - } - if (typeof value === 'number') { - return Number.isFinite(value) ? String(value) : 'NULL' - } - if (typeof value === 'boolean') { - return value ? '1' : '0' - } - if (typeof value === 'object') { - return quoteString(JSON.stringify(value)) - } - return quoteString(String(value)) -} - -function quoteString(value: string): string { - return `'${value.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'` -} - -/** - * Validates and backtick-quotes a ClickHouse identifier, supporting - * `database.table` qualified names. - */ -export function sanitizeIdentifier(identifier: string): string { - if (identifier.includes('.')) { - return identifier - .split('.') - .map((part) => sanitizeSingleIdentifier(part)) - .join('.') - } - return sanitizeSingleIdentifier(identifier) -} - -function sanitizeSingleIdentifier(identifier: string): string { - const cleaned = identifier.replace(/`/g, '') - if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(cleaned)) { - throw new Error( - `Invalid identifier: ${identifier}. Identifiers must start with a letter or underscore and contain only letters, numbers, and underscores.` - ) - } - return `\`${cleaned}\`` -} - -/** - * Rejects WHERE clauses containing SQL-injection or always-true tautology - * patterns so user-supplied conditions cannot broaden a mutation to every row. - * Delegates to the shared {@link validateSqlWhereClause} guard (defense-in-depth). - */ -function validateWhereClause(where: string): void { - const result = validateSqlWhereClause(where, 'WHERE clause') - if (!result.isValid) { - throw new Error(result.error) - } -} - -/** - * Runs a SELECT statement (which must already include `FORMAT JSON`) and returns - * the parsed rows and row count. - */ -async function runSelect( - config: ClickHouseConnectionConfig, - statement: string -): Promise { - const result = await clickhouseRequest(config, statement) - return parseRowsResult(result) -} - -/** - * Runs a statement that does not return a result set (DDL or mutation) and - * returns the number of written rows reported by the summary header. - */ -async function runStatement( - config: ClickHouseConnectionConfig, - statement: string -): Promise { - const result = await clickhouseRequest(config, statement) - return Number(result.summary?.written_rows ?? 0) -} - -/** - * Validates a free-form SQL expression (ORDER BY, PARTITION BY, engine args) - * rejecting statement terminators and comment sequences. - */ -function validateExpression(expression: string, label: string): void { - if (/;|--|\/\*|\*\//.test(expression)) { - throw new Error(`${label} contains a disallowed character`) - } -} - -/** - * Validates an ORDER BY / PARTITION BY expression that is spliced inside wrapping - * parentheses in the generated DDL. In addition to rejecting terminators/comments, - * it requires balanced parentheses (quote-aware) so the expression cannot close - * the wrapping `(...)` early and append extra clauses (e.g. `id) SETTINGS …`). - */ -function validateClauseExpression(expression: string, label: string): void { - const trimmed = expression.trim() - if (!trimmed) { - throw new Error(`${label} is required`) - } - if (/;|--|\/\*|\*\//.test(trimmed)) { - throw new Error(`${label} contains a disallowed sequence`) - } - let depth = 0 - let inString = false - for (let i = 0; i < trimmed.length; i++) { - const ch = trimmed[i] - if (inString) { - if (ch === '\\') i++ - else if (ch === "'") inString = false - continue - } - if (ch === "'") inString = true - else if (ch === '(') depth++ - else if (ch === ')') { - depth-- - if (depth < 0) { - throw new Error(`${label} has unbalanced parentheses`) - } - } - } - if (inString || depth !== 0) { - throw new Error(`${label} has unbalanced parentheses or quotes`) - } -} - -/** - * Validates a partition value for `DROP PARTITION`. ClickHouse partition values - * are literals (signed numbers or single-quoted strings) or a parenthesised tuple - * of such literals, so anything else is rejected — barewords like `ALL`, function - * calls, operators, and extra tokens that could broaden the statement beyond - * dropping a single partition. - */ -function validatePartitionExpression(partition: string): void { - const partitionPattern = - /^\(?\s*(?:'(?:[^'\\]|\\.)*'|-?\d+(?:\.\d+)?)(?:\s*,\s*(?:'(?:[^'\\]|\\.)*'|-?\d+(?:\.\d+)?))*\s*\)?$/ - if (!partitionPattern.test(partition.trim())) { - throw new Error( - "Partition must be a literal value or a tuple of literals (number or single-quoted string), e.g. 202401, '2024-01', or (2024, 'EU')" - ) - } -} - -export function executeClickHouseListDatabases( - config: ClickHouseConnectionConfig -): Promise { - return runSelect( - config, - 'SELECT name, engine, comment FROM system.databases ORDER BY name FORMAT JSON' - ) -} - -export function executeClickHouseListTables( - config: ClickHouseConnectionConfig -): Promise { - return runSelect( - config, - `SELECT name, engine, total_rows AS totalRows, total_bytes AS totalBytes, comment FROM system.tables WHERE database = ${quoteString(config.database)} ORDER BY name FORMAT JSON` - ) -} - -export function executeClickHouseDescribeTable( - config: ClickHouseConnectionConfig, - table: string -): Promise { - const tableName = stripDatabasePrefix(table) - return runSelect( - config, - `SELECT name, type, default_kind AS defaultKind, default_expression AS defaultExpression, comment, is_in_primary_key AS isInPrimaryKey, is_in_sorting_key AS isInSortingKey FROM system.columns WHERE database = ${quoteString(config.database)} AND table = ${quoteString(tableName)} ORDER BY position FORMAT JSON` - ) -} - -export async function executeClickHouseShowCreateTable( - config: ClickHouseConnectionConfig, - table: string -): Promise { - const result = await runSelect( - config, - `SHOW CREATE TABLE ${sanitizeIdentifier(table)} FORMAT JSON` - ) - const firstRow = result.rows[0] as Record | undefined - if (!firstRow) { - return '' - } - // ClickHouse returns the DDL in a single String column (named `statement`); - // fall back to the first column value to stay robust to column-name changes. - const value = firstRow.statement ?? Object.values(firstRow)[0] - return typeof value === 'string' ? value : '' -} - -export async function executeClickHouseCountRows( - config: ClickHouseConnectionConfig, - table: string, - where?: string -): Promise { - let statement = `SELECT count() AS count FROM ${sanitizeIdentifier(table)}` - if (where?.trim()) { - validateWhereClause(where) - statement += ` WHERE ${where}` - } - const result = await runSelect(config, `${statement} FORMAT JSON`) - const firstRow = result.rows[0] as { count?: number | string } | undefined - return firstRow?.count != null ? Number(firstRow.count) : 0 -} - -export function executeClickHouseListPartitions( - config: ClickHouseConnectionConfig, - table: string -): Promise { - const tableName = stripDatabasePrefix(table) - return runSelect( - config, - `SELECT partition, count() AS parts, sum(rows) AS rows, sum(bytes_on_disk) AS bytesOnDisk FROM system.parts WHERE database = ${quoteString(config.database)} AND table = ${quoteString(tableName)} AND active GROUP BY partition ORDER BY partition FORMAT JSON` - ) -} - -export function executeClickHouseListMutations( - config: ClickHouseConnectionConfig, - table?: string, - onlyRunning = false -): Promise { - const filters = [`database = ${quoteString(config.database)}`] - if (table?.trim()) { - filters.push(`table = ${quoteString(stripDatabasePrefix(table))}`) - } - if (onlyRunning) { - filters.push('is_done = 0') - } - return runSelect( - config, - `SELECT table, mutation_id AS mutationId, command, create_time AS createTime, is_done AS isDone, parts_to_do AS partsToDo, latest_fail_reason AS latestFailReason FROM system.mutations WHERE ${filters.join(' AND ')} ORDER BY create_time DESC FORMAT JSON` - ) -} - -export function executeClickHouseListRunningQueries( - config: ClickHouseConnectionConfig -): Promise { - return runSelect( - config, - 'SELECT query_id AS queryId, user, toFloat64(elapsed) AS elapsedSeconds, formatReadableSize(memory_usage) AS memoryUsage, query FROM system.processes ORDER BY elapsed DESC FORMAT JSON' - ) -} - -export function executeClickHouseTableStats( - config: ClickHouseConnectionConfig, - table?: string -): Promise { - const filters = ['active', `database = ${quoteString(config.database)}`] - if (table?.trim()) { - filters.push(`table = ${quoteString(stripDatabasePrefix(table))}`) - } - return runSelect( - config, - `SELECT database, table, sum(rows) AS rows, sum(bytes_on_disk) AS bytesOnDisk, formatReadableSize(sum(bytes_on_disk)) AS sizeOnDisk, count() AS parts FROM system.parts WHERE ${filters.join(' AND ')} GROUP BY database, table ORDER BY sum(bytes_on_disk) DESC FORMAT JSON` - ) -} - -export function executeClickHouseListClusters( - config: ClickHouseConnectionConfig -): Promise { - return runSelect( - config, - 'SELECT cluster, shard_num AS shardNum, replica_num AS replicaNum, host_name AS hostName, port, is_local AS isLocal FROM system.clusters ORDER BY cluster, shard_num, replica_num FORMAT JSON' - ) -} - -export async function executeClickHouseCreateDatabase( - config: ClickHouseConnectionConfig, - name: string -): Promise { - await clickhouseRequest(config, `CREATE DATABASE IF NOT EXISTS ${sanitizeIdentifier(name)}`) -} - -export async function executeClickHouseDropDatabase( - config: ClickHouseConnectionConfig, - name: string -): Promise { - await clickhouseRequest(config, `DROP DATABASE IF EXISTS ${sanitizeIdentifier(name)}`) -} - -/** - * Validates a single ClickHouse column type. Types may legitimately contain - * commas, single-quoted strings, `=`, and `-` inside their parameter parentheses - * (e.g. `Decimal(10, 2)`, `Enum8('a' = 1, 'b' = -2)`, `Map(String, UInt64)`, - * `Array(Tuple(a UInt8, b String))`). We allow those but reject anything that - * could break out of the single type literal and inject another column or SQL: - * comment/terminator sequences, a top-level (unparenthesised) comma, or an - * unbalanced closing paren. - */ -function validateColumnType(type: string): void { - const trimmed = type.trim() - if (!trimmed || !/^[A-Za-z_]/.test(trimmed)) { - throw new Error(`Invalid column type: ${type}`) - } - if (!/^[A-Za-z0-9_(),.\s'"=-]+$/.test(trimmed) || /--|;/.test(trimmed)) { - throw new Error(`Invalid column type: ${type}`) - } - let depth = 0 - let inString = false - for (let i = 0; i < trimmed.length; i++) { - const ch = trimmed[i] - if (inString) { - if (ch === '\\') i++ - else if (ch === "'") inString = false - continue - } - if (ch === "'") inString = true - else if (ch === '(') depth++ - else if (ch === ')') { - depth-- - if (depth < 0) throw new Error(`Invalid column type: ${type}`) - } else if (ch === ',' && depth === 0) { - throw new Error(`Invalid column type: ${type}`) - } - } - if (inString || depth !== 0) { - throw new Error(`Invalid column type: ${type}`) - } -} - -export async function executeClickHouseCreateTable( - config: ClickHouseConnectionConfig, - table: string, - columns: Array<{ name: string; type: string }>, - engine: string, - orderBy: string, - partitionBy?: string -): Promise { - if (!Array.isArray(columns) || columns.length === 0) { - throw new Error('At least one column definition is required') - } - - const columnDefs = columns.map((column) => { - if (!column?.name || !column?.type) { - throw new Error('Each column requires a name and type') - } - validateColumnType(column.type) - return `${sanitizeIdentifier(column.name)} ${column.type.trim()}` - }) - - if (!/^[A-Za-z][A-Za-z0-9]*(\(.*\))?$/.test(engine.trim())) { - throw new Error(`Invalid table engine: ${engine}`) - } - validateExpression(engine, 'Engine') - - if (!orderBy?.trim()) { - throw new Error('ORDER BY expression is required') - } - validateClauseExpression(orderBy, 'ORDER BY') - - let statement = `CREATE TABLE IF NOT EXISTS ${sanitizeIdentifier(table)} (${columnDefs.join(', ')}) ENGINE = ${engine.trim()}` - if (partitionBy?.trim()) { - validateClauseExpression(partitionBy, 'PARTITION BY') - statement += ` PARTITION BY (${partitionBy.trim()})` - } - statement += ` ORDER BY (${orderBy.trim()})` - - await clickhouseRequest(config, statement) -} - -export async function executeClickHouseDropTable( - config: ClickHouseConnectionConfig, - table: string -): Promise { - await clickhouseRequest(config, `DROP TABLE IF EXISTS ${sanitizeIdentifier(table)}`) -} - -export async function executeClickHouseTruncateTable( - config: ClickHouseConnectionConfig, - table: string -): Promise { - await clickhouseRequest(config, `TRUNCATE TABLE IF EXISTS ${sanitizeIdentifier(table)}`) -} - -export async function executeClickHouseRenameTable( - config: ClickHouseConnectionConfig, - fromTable: string, - toTable: string -): Promise { - await clickhouseRequest( - config, - `RENAME TABLE ${sanitizeIdentifier(fromTable)} TO ${sanitizeIdentifier(toTable)}` - ) -} - -export async function executeClickHouseOptimizeTable( - config: ClickHouseConnectionConfig, - table: string, - final: boolean -): Promise { - await clickhouseRequest( - config, - `OPTIMIZE TABLE ${sanitizeIdentifier(table)}${final ? ' FINAL' : ''}` - ) -} - -export async function executeClickHouseDropPartition( - config: ClickHouseConnectionConfig, - table: string, - partition: string -): Promise { - validatePartitionExpression(partition) - await clickhouseRequest( - config, - `ALTER TABLE ${sanitizeIdentifier(table)} DROP PARTITION ${partition.trim()}` - ) -} - -export function executeClickHouseKillQuery( - config: ClickHouseConnectionConfig, - queryId: string -): Promise { - return runSelect(config, `KILL QUERY WHERE query_id = ${quoteString(queryId)} SYNC FORMAT JSON`) -} - -export async function executeClickHouseInsertRows( - config: ClickHouseConnectionConfig, - table: string, - rows: Array> -): Promise { - if (!Array.isArray(rows) || rows.length === 0) { - throw new Error('At least one row is required') - } - const sanitizedTable = sanitizeIdentifier(table) - const payload = rows.map((row) => JSON.stringify(row)).join('\n') - const statement = `INSERT INTO ${sanitizedTable} FORMAT JSONEachRow\n${payload}` - const written = await runStatement(config, statement) - return { rows: [], rowCount: written || rows.length } -} - -function stripDatabasePrefix(table: string): string { - const parts = table.split('.') - return parts[parts.length - 1].replace(/`/g, '') -} diff --git a/apps/sim/app/api/tools/clickup/upload-attachment/route.ts b/apps/sim/app/api/tools/clickup/upload-attachment/route.ts deleted file mode 100644 index bcb4d8ab435..00000000000 --- a/apps/sim/app/api/tools/clickup/upload-attachment/route.ts +++ /dev/null @@ -1,131 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { clickupUploadAttachmentContract } from '@/lib/api/contracts/tools/clickup' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { - CLICKUP_API_BASE_URL, - clickupAuthorizationHeader, - extractClickUpErrorMessage, - mapClickUpAttachment, -} from '@/tools/clickup/shared' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('ClickUpUploadAttachmentAPI') - -const MAX_UPLOAD_SIZE_BYTES = 100 * 1024 * 1024 - -function uploadSizeError(bytes: number): NextResponse { - const sizeMB = (bytes / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { success: false, error: `File size (${sizeMB}MB) exceeds upload limit of 100MB` }, - { status: 400 } - ) -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json( - { success: false, error: authResult.error || 'Unauthorized' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(clickupUploadAttachmentContract, request, {}) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const userFiles = processFilesToUserFiles([params.file as RawFileInput], requestId, logger) - if (userFiles.length === 0) { - return NextResponse.json( - { success: false, error: 'No valid file provided for upload' }, - { status: 400 } - ) - } - - const userFile = userFiles[0] - const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) - if (denied) return denied - - if (userFile.size > MAX_UPLOAD_SIZE_BYTES) { - return uploadSizeError(userFile.size) - } - - let buffer: Buffer - let downloadedContentType = '' - try { - const result = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_UPLOAD_SIZE_BYTES, - }) - buffer = result.buffer - downloadedContentType = result.contentType - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - if (error instanceof PayloadSizeLimitError) { - return uploadSizeError(error.observedBytes ?? userFile.size) - } - throw error - } - - if (buffer.length > MAX_UPLOAD_SIZE_BYTES) { - return uploadSizeError(buffer.length) - } - - const formData = new FormData() - const blob = new Blob([new Uint8Array(buffer)], { - type: downloadedContentType || userFile.type || 'application/octet-stream', - }) - formData.append('attachment', blob, userFile.name) - - const url = `${CLICKUP_API_BASE_URL}/task/${encodeURIComponent(params.taskId)}/attachment` - const response = await fetch(url, { - method: 'POST', - headers: { - Authorization: clickupAuthorizationHeader(params.accessToken), - }, - body: formData, - }) - - const data: unknown = await response.json().catch(() => null) - if (!response.ok) { - const message = extractClickUpErrorMessage( - response, - data, - 'Failed to upload ClickUp attachment' - ) - logger.error(`[${requestId}] ClickUp attachment upload failed`, { - status: response.status, - message, - }) - return NextResponse.json({ success: false, error: message }, { status: response.status }) - } - - return NextResponse.json({ - success: true, - output: { - attachment: mapClickUpAttachment(data), - files: userFiles, - }, - }) - } catch (error) { - logger.error(`[${requestId}] ClickUp attachment upload error`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Internal server error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/cloudformation/cancel-update-stack/route.ts b/apps/sim/app/api/tools/cloudformation/cancel-update-stack/route.ts deleted file mode 100644 index 5158bfcb125..00000000000 --- a/apps/sim/app/api/tools/cloudformation/cancel-update-stack/route.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { CancelUpdateStackCommand, CloudFormationClient } from '@aws-sdk/client-cloudformation' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudformationCancelUpdateStackContract } from '@/lib/api/contracts/tools/aws/cloudformation-cancel-update-stack' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -const logger = createLogger('CloudFormationCancelUpdateStack') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudformationCancelUpdateStackContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const client = new CloudFormationClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - logger.info(`Cancelling update for CloudFormation stack "${validatedData.stackName}"`) - - try { - const command = new CancelUpdateStackCommand({ - StackName: validatedData.stackName, - }) - - await client.send(command) - - return NextResponse.json({ - success: true, - output: { - message: `Update for stack "${validatedData.stackName}" is being cancelled and rolled back`, - }, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to cancel CloudFormation stack update') - logger.error('CancelUpdateStack failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/cloudformation/create-change-set/route.ts b/apps/sim/app/api/tools/cloudformation/create-change-set/route.ts deleted file mode 100644 index cb3bff94456..00000000000 --- a/apps/sim/app/api/tools/cloudformation/create-change-set/route.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { CloudFormationClient, CreateChangeSetCommand } from '@aws-sdk/client-cloudformation' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudformationCreateChangeSetContract } from '@/lib/api/contracts/tools/aws/cloudformation-create-change-set' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { parseCapabilities, toStackParameters } from '../utils' - -const logger = createLogger('CloudFormationCreateChangeSet') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudformationCreateChangeSetContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const client = new CloudFormationClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - logger.info( - `Creating change set "${validatedData.changeSetName}" for stack "${validatedData.stackName}"` - ) - - try { - const command = new CreateChangeSetCommand({ - StackName: validatedData.stackName, - ChangeSetName: validatedData.changeSetName, - TemplateBody: validatedData.templateBody, - UsePreviousTemplate: validatedData.usePreviousTemplate, - Parameters: toStackParameters(validatedData.parameters), - Capabilities: parseCapabilities(validatedData.capabilities), - ChangeSetType: validatedData.changeSetType, - Description: validatedData.description, - }) - - const response = await client.send(command) - - return NextResponse.json({ - success: true, - output: { - changeSetId: response.Id ?? '', - stackId: response.StackId ?? '', - }, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to create CloudFormation change set') - logger.error('CreateChangeSet failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/cloudformation/create-stack/route.ts b/apps/sim/app/api/tools/cloudformation/create-stack/route.ts deleted file mode 100644 index a08b04683df..00000000000 --- a/apps/sim/app/api/tools/cloudformation/create-stack/route.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { CloudFormationClient, CreateStackCommand } from '@aws-sdk/client-cloudformation' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudformationCreateStackContract } from '@/lib/api/contracts/tools/aws/cloudformation-create-stack' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { parseCapabilities, toStackParameters, toStackTags } from '../utils' - -const logger = createLogger('CloudFormationCreateStack') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudformationCreateStackContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const client = new CloudFormationClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - logger.info(`Creating CloudFormation stack "${validatedData.stackName}"`) - - try { - const command = new CreateStackCommand({ - StackName: validatedData.stackName, - TemplateBody: validatedData.templateBody, - Parameters: toStackParameters(validatedData.parameters), - Capabilities: parseCapabilities(validatedData.capabilities), - Tags: toStackTags(validatedData.tags), - OnFailure: validatedData.onFailure, - TimeoutInMinutes: validatedData.timeoutInMinutes, - }) - - const response = await client.send(command) - - return NextResponse.json({ - success: true, - output: { - stackId: response.StackId ?? '', - }, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to create CloudFormation stack') - logger.error('CreateStack failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/cloudformation/delete-stack/route.ts b/apps/sim/app/api/tools/cloudformation/delete-stack/route.ts deleted file mode 100644 index 4b176ffd679..00000000000 --- a/apps/sim/app/api/tools/cloudformation/delete-stack/route.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { CloudFormationClient, DeleteStackCommand } from '@aws-sdk/client-cloudformation' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudformationDeleteStackContract } from '@/lib/api/contracts/tools/aws/cloudformation-delete-stack' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -const logger = createLogger('CloudFormationDeleteStack') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudformationDeleteStackContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const client = new CloudFormationClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - logger.info(`Deleting CloudFormation stack "${validatedData.stackName}"`) - - try { - const retainResources = validatedData.retainResources - ?.split(',') - .map((r) => r.trim()) - .filter(Boolean) - - const command = new DeleteStackCommand({ - StackName: validatedData.stackName, - ...(retainResources && retainResources.length > 0 && { RetainResources: retainResources }), - }) - - await client.send(command) - - logger.info( - `Successfully requested deletion of CloudFormation stack "${validatedData.stackName}"` - ) - - return NextResponse.json({ - success: true, - output: { - message: `Deletion of stack "${validatedData.stackName}" has been initiated`, - }, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to delete CloudFormation stack') - logger.error('DeleteStack failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/cloudformation/describe-change-set/route.ts b/apps/sim/app/api/tools/cloudformation/describe-change-set/route.ts deleted file mode 100644 index 16d81a82862..00000000000 --- a/apps/sim/app/api/tools/cloudformation/describe-change-set/route.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { CloudFormationClient, DescribeChangeSetCommand } from '@aws-sdk/client-cloudformation' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudformationDescribeChangeSetContract } from '@/lib/api/contracts/tools/aws/cloudformation-describe-change-set' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -const logger = createLogger('CloudFormationDescribeChangeSet') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudformationDescribeChangeSetContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const client = new CloudFormationClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - try { - const command = new DescribeChangeSetCommand({ - ChangeSetName: validatedData.changeSetName, - ...(validatedData.stackName && { StackName: validatedData.stackName }), - }) - - const response = await client.send(command) - - const changes = (response.Changes ?? []).map((c) => ({ - action: c.ResourceChange?.Action, - logicalResourceId: c.ResourceChange?.LogicalResourceId, - physicalResourceId: c.ResourceChange?.PhysicalResourceId, - resourceType: c.ResourceChange?.ResourceType, - replacement: c.ResourceChange?.Replacement, - })) - - return NextResponse.json({ - success: true, - output: { - changeSetName: response.ChangeSetName, - changeSetId: response.ChangeSetId, - stackId: response.StackId, - stackName: response.StackName, - description: response.Description, - executionStatus: response.ExecutionStatus, - status: response.Status, - statusReason: response.StatusReason, - creationTime: response.CreationTime?.getTime(), - capabilities: response.Capabilities ?? [], - changes, - }, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to describe CloudFormation change set') - logger.error('DescribeChangeSet failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/cloudformation/describe-stack-drift-detection-status/route.ts b/apps/sim/app/api/tools/cloudformation/describe-stack-drift-detection-status/route.ts deleted file mode 100644 index a46a22deb57..00000000000 --- a/apps/sim/app/api/tools/cloudformation/describe-stack-drift-detection-status/route.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { - CloudFormationClient, - DescribeStackDriftDetectionStatusCommand, -} from '@aws-sdk/client-cloudformation' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudformationDescribeStackDriftDetectionStatusContract } from '@/lib/api/contracts/tools/aws/cloudformation-describe-stack-drift-detection-status' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -const logger = createLogger('CloudFormationDescribeStackDriftDetectionStatus') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest( - awsCloudformationDescribeStackDriftDetectionStatusContract, - request, - { - errorFormat: 'details', - logger, - } - ) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const client = new CloudFormationClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - const command = new DescribeStackDriftDetectionStatusCommand({ - StackDriftDetectionId: validatedData.stackDriftDetectionId, - }) - - const response = await client.send(command) - - return NextResponse.json({ - success: true, - output: { - stackId: response.StackId ?? '', - stackDriftDetectionId: response.StackDriftDetectionId ?? '', - stackDriftStatus: response.StackDriftStatus, - detectionStatus: response.DetectionStatus ?? 'UNKNOWN', - detectionStatusReason: response.DetectionStatusReason, - driftedStackResourceCount: response.DriftedStackResourceCount, - timestamp: response.Timestamp?.getTime(), - }, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to describe stack drift detection status') - logger.error('DescribeStackDriftDetectionStatus failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/cloudformation/describe-stack-events/route.ts b/apps/sim/app/api/tools/cloudformation/describe-stack-events/route.ts deleted file mode 100644 index 64775f37125..00000000000 --- a/apps/sim/app/api/tools/cloudformation/describe-stack-events/route.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { - CloudFormationClient, - DescribeStackEventsCommand, - type StackEvent, -} from '@aws-sdk/client-cloudformation' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudformationDescribeStackEventsContract } from '@/lib/api/contracts/tools/aws/cloudformation-describe-stack-events' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -const logger = createLogger('CloudFormationDescribeStackEvents') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudformationDescribeStackEventsContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const client = new CloudFormationClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - const limit = validatedData.limit ?? 50 - - const allEvents: StackEvent[] = [] - let nextToken: string | undefined - do { - const command = new DescribeStackEventsCommand({ - StackName: validatedData.stackName, - ...(nextToken && { NextToken: nextToken }), - }) - const response = await client.send(command) - allEvents.push(...(response.StackEvents ?? [])) - nextToken = allEvents.length >= limit ? undefined : response.NextToken - } while (nextToken) - - const events = allEvents.slice(0, limit).map((e) => ({ - stackId: e.StackId ?? '', - eventId: e.EventId ?? '', - stackName: e.StackName ?? '', - logicalResourceId: e.LogicalResourceId, - physicalResourceId: e.PhysicalResourceId, - resourceType: e.ResourceType, - resourceStatus: e.ResourceStatus, - resourceStatusReason: e.ResourceStatusReason, - timestamp: e.Timestamp?.getTime(), - })) - - return NextResponse.json({ - success: true, - output: { events }, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to describe CloudFormation stack events') - logger.error('DescribeStackEvents failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/cloudformation/describe-stacks/route.ts b/apps/sim/app/api/tools/cloudformation/describe-stacks/route.ts deleted file mode 100644 index b2ff65c9723..00000000000 --- a/apps/sim/app/api/tools/cloudformation/describe-stacks/route.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { - CloudFormationClient, - DescribeStacksCommand, - type Stack, -} from '@aws-sdk/client-cloudformation' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudformationDescribeStacksContract } from '@/lib/api/contracts/tools/aws/cloudformation-describe-stacks' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -const logger = createLogger('CloudFormationDescribeStacks') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudformationDescribeStacksContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const client = new CloudFormationClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - const allStacks: Stack[] = [] - let nextToken: string | undefined - do { - const command = new DescribeStacksCommand({ - ...(validatedData.stackName && { StackName: validatedData.stackName }), - ...(nextToken && { NextToken: nextToken }), - }) - const response = await client.send(command) - allStacks.push(...(response.Stacks ?? [])) - nextToken = response.NextToken - } while (nextToken) - - const stacks = allStacks.map((s) => ({ - stackName: s.StackName ?? '', - stackId: s.StackId ?? '', - stackStatus: s.StackStatus ?? 'UNKNOWN', - stackStatusReason: s.StackStatusReason, - creationTime: s.CreationTime?.getTime(), - lastUpdatedTime: s.LastUpdatedTime?.getTime(), - description: s.Description, - enableTerminationProtection: s.EnableTerminationProtection, - driftInformation: s.DriftInformation - ? { - stackDriftStatus: s.DriftInformation.StackDriftStatus, - lastCheckTimestamp: s.DriftInformation.LastCheckTimestamp?.getTime(), - } - : null, - outputs: (s.Outputs ?? []).map((o) => ({ - outputKey: o.OutputKey ?? '', - outputValue: o.OutputValue ?? '', - description: o.Description, - })), - tags: (s.Tags ?? []).map((t) => ({ - key: t.Key ?? '', - value: t.Value ?? '', - })), - })) - - return NextResponse.json({ - success: true, - output: { stacks }, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to describe CloudFormation stacks') - logger.error('DescribeStacks failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/cloudformation/detect-stack-drift/route.ts b/apps/sim/app/api/tools/cloudformation/detect-stack-drift/route.ts deleted file mode 100644 index d0c8a719574..00000000000 --- a/apps/sim/app/api/tools/cloudformation/detect-stack-drift/route.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { CloudFormationClient, DetectStackDriftCommand } from '@aws-sdk/client-cloudformation' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudformationDetectStackDriftContract } from '@/lib/api/contracts/tools/aws/cloudformation-detect-stack-drift' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -const logger = createLogger('CloudFormationDetectStackDrift') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudformationDetectStackDriftContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const client = new CloudFormationClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - const command = new DetectStackDriftCommand({ - StackName: validatedData.stackName, - }) - - const response = await client.send(command) - - if (!response.StackDriftDetectionId) { - throw new Error('No drift detection ID returned') - } - - return NextResponse.json({ - success: true, - output: { - stackDriftDetectionId: response.StackDriftDetectionId, - }, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to detect CloudFormation stack drift') - logger.error('DetectStackDrift failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/cloudformation/execute-change-set/route.ts b/apps/sim/app/api/tools/cloudformation/execute-change-set/route.ts deleted file mode 100644 index 3229e0b73e9..00000000000 --- a/apps/sim/app/api/tools/cloudformation/execute-change-set/route.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { CloudFormationClient, ExecuteChangeSetCommand } from '@aws-sdk/client-cloudformation' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudformationExecuteChangeSetContract } from '@/lib/api/contracts/tools/aws/cloudformation-execute-change-set' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -const logger = createLogger('CloudFormationExecuteChangeSet') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudformationExecuteChangeSetContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const client = new CloudFormationClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - logger.info(`Executing change set "${validatedData.changeSetName}"`) - - try { - const command = new ExecuteChangeSetCommand({ - ChangeSetName: validatedData.changeSetName, - ...(validatedData.stackName && { StackName: validatedData.stackName }), - }) - - await client.send(command) - - return NextResponse.json({ - success: true, - output: { - message: `Change set "${validatedData.changeSetName}" execution has been initiated`, - }, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to execute CloudFormation change set') - logger.error('ExecuteChangeSet failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/cloudformation/get-template-summary/route.ts b/apps/sim/app/api/tools/cloudformation/get-template-summary/route.ts deleted file mode 100644 index 555db0118a5..00000000000 --- a/apps/sim/app/api/tools/cloudformation/get-template-summary/route.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { CloudFormationClient, GetTemplateSummaryCommand } from '@aws-sdk/client-cloudformation' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudformationGetTemplateSummaryContract } from '@/lib/api/contracts/tools/aws/cloudformation-get-template-summary' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -const logger = createLogger('CloudFormationGetTemplateSummary') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudformationGetTemplateSummaryContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const client = new CloudFormationClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - try { - const command = new GetTemplateSummaryCommand({ - ...(validatedData.templateBody && { TemplateBody: validatedData.templateBody }), - ...(validatedData.stackName && { StackName: validatedData.stackName }), - }) - - const response = await client.send(command) - - return NextResponse.json({ - success: true, - output: { - description: response.Description, - parameters: (response.Parameters ?? []).map((p) => ({ - parameterKey: p.ParameterKey, - defaultValue: p.DefaultValue, - parameterType: p.ParameterType, - noEcho: p.NoEcho, - description: p.Description, - })), - capabilities: response.Capabilities ?? [], - capabilitiesReason: response.CapabilitiesReason, - resourceTypes: response.ResourceTypes ?? [], - version: response.Version, - declaredTransforms: response.DeclaredTransforms ?? [], - }, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to get CloudFormation template summary') - logger.error('GetTemplateSummary failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/cloudformation/get-template/route.ts b/apps/sim/app/api/tools/cloudformation/get-template/route.ts deleted file mode 100644 index ed1617c2504..00000000000 --- a/apps/sim/app/api/tools/cloudformation/get-template/route.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { CloudFormationClient, GetTemplateCommand } from '@aws-sdk/client-cloudformation' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudformationGetTemplateContract } from '@/lib/api/contracts/tools/aws/cloudformation-get-template' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -const logger = createLogger('CloudFormationGetTemplate') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudformationGetTemplateContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const client = new CloudFormationClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - const command = new GetTemplateCommand({ - StackName: validatedData.stackName, - ...(validatedData.templateStage && { TemplateStage: validatedData.templateStage }), - }) - - const response = await client.send(command) - - return NextResponse.json({ - success: true, - output: { - templateBody: response.TemplateBody ?? '', - stagesAvailable: response.StagesAvailable ?? [], - }, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to get CloudFormation template') - logger.error('GetTemplate failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/cloudformation/list-stack-resources/route.ts b/apps/sim/app/api/tools/cloudformation/list-stack-resources/route.ts deleted file mode 100644 index b10c02872ce..00000000000 --- a/apps/sim/app/api/tools/cloudformation/list-stack-resources/route.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { - CloudFormationClient, - ListStackResourcesCommand, - type StackResourceSummary, -} from '@aws-sdk/client-cloudformation' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudformationListStackResourcesContract } from '@/lib/api/contracts/tools/aws/cloudformation-list-stack-resources' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -const logger = createLogger('CloudFormationListStackResources') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudformationListStackResourcesContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const client = new CloudFormationClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - const allSummaries: StackResourceSummary[] = [] - let nextToken: string | undefined - do { - const command = new ListStackResourcesCommand({ - StackName: validatedData.stackName, - ...(nextToken && { NextToken: nextToken }), - }) - const response = await client.send(command) - allSummaries.push(...(response.StackResourceSummaries ?? [])) - nextToken = response.NextToken - } while (nextToken) - - const resources = allSummaries.map((r) => ({ - logicalResourceId: r.LogicalResourceId ?? '', - physicalResourceId: r.PhysicalResourceId, - resourceType: r.ResourceType ?? '', - resourceStatus: r.ResourceStatus ?? 'UNKNOWN', - resourceStatusReason: r.ResourceStatusReason, - lastUpdatedTimestamp: r.LastUpdatedTimestamp?.getTime(), - driftInformation: r.DriftInformation - ? { - stackResourceDriftStatus: r.DriftInformation.StackResourceDriftStatus, - lastCheckTimestamp: r.DriftInformation.LastCheckTimestamp?.getTime(), - } - : null, - })) - - return NextResponse.json({ - success: true, - output: { resources }, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to list CloudFormation stack resources') - logger.error('ListStackResources failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/cloudformation/update-stack/route.ts b/apps/sim/app/api/tools/cloudformation/update-stack/route.ts deleted file mode 100644 index 0377e188371..00000000000 --- a/apps/sim/app/api/tools/cloudformation/update-stack/route.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { CloudFormationClient, UpdateStackCommand } from '@aws-sdk/client-cloudformation' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudformationUpdateStackContract } from '@/lib/api/contracts/tools/aws/cloudformation-update-stack' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { parseCapabilities, toStackParameters, toStackTags } from '../utils' - -const logger = createLogger('CloudFormationUpdateStack') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudformationUpdateStackContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const client = new CloudFormationClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - logger.info(`Updating CloudFormation stack "${validatedData.stackName}"`) - - try { - const command = new UpdateStackCommand({ - StackName: validatedData.stackName, - TemplateBody: validatedData.templateBody, - UsePreviousTemplate: validatedData.usePreviousTemplate, - Parameters: toStackParameters(validatedData.parameters), - Capabilities: parseCapabilities(validatedData.capabilities), - Tags: toStackTags(validatedData.tags), - }) - - const response = await client.send(command) - - return NextResponse.json({ - success: true, - output: { - stackId: response.StackId ?? '', - }, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to update CloudFormation stack') - logger.error('UpdateStack failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/cloudformation/utils.ts b/apps/sim/app/api/tools/cloudformation/utils.ts deleted file mode 100644 index 44d3611afc8..00000000000 --- a/apps/sim/app/api/tools/cloudformation/utils.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { Capability, Parameter, Tag } from '@aws-sdk/client-cloudformation' - -/** - * Parses a comma-separated capabilities string (e.g. "CAPABILITY_IAM,CAPABILITY_NAMED_IAM") - * into the array shape the CloudFormation SDK expects. - */ -export function parseCapabilities(value?: string): Capability[] | undefined { - if (!value) return undefined - const capabilities = value - .split(',') - .map((c) => c.trim()) - .filter(Boolean) - return capabilities.length > 0 ? (capabilities as Capability[]) : undefined -} - -/** - * Maps camelCase stack parameter inputs to the PascalCase `Parameter` shape CloudFormation expects. - */ -export function toStackParameters( - parameters?: { parameterKey: string; parameterValue?: string; usePreviousValue?: boolean }[] -): Parameter[] | undefined { - if (!parameters || parameters.length === 0) return undefined - return parameters.map((p) => ({ - ParameterKey: p.parameterKey, - ParameterValue: p.parameterValue, - UsePreviousValue: p.usePreviousValue, - })) -} - -/** - * Maps camelCase tag inputs to the PascalCase `Tag` shape CloudFormation expects. - */ -export function toStackTags(tags?: { key: string; value: string }[]): Tag[] | undefined { - if (!tags || tags.length === 0) return undefined - return tags.map((t) => ({ Key: t.key, Value: t.value })) -} diff --git a/apps/sim/app/api/tools/cloudformation/validate-template/route.ts b/apps/sim/app/api/tools/cloudformation/validate-template/route.ts deleted file mode 100644 index da1af83142f..00000000000 --- a/apps/sim/app/api/tools/cloudformation/validate-template/route.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { CloudFormationClient, ValidateTemplateCommand } from '@aws-sdk/client-cloudformation' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudformationValidateTemplateContract } from '@/lib/api/contracts/tools/aws/cloudformation-validate-template' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -const logger = createLogger('CloudFormationValidateTemplate') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudformationValidateTemplateContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const client = new CloudFormationClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - const command = new ValidateTemplateCommand({ - TemplateBody: validatedData.templateBody, - }) - - const response = await client.send(command) - - return NextResponse.json({ - success: true, - output: { - description: response.Description, - parameters: (response.Parameters ?? []).map((p) => ({ - parameterKey: p.ParameterKey, - defaultValue: p.DefaultValue, - noEcho: p.NoEcho, - description: p.Description, - })), - capabilities: response.Capabilities ?? [], - capabilitiesReason: response.CapabilitiesReason, - declaredTransforms: response.DeclaredTransforms ?? [], - }, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Failed to validate CloudFormation template') - logger.error('ValidateTemplate failed', { error: errorMessage }) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/cloudwatch/describe-alarm-history/route.ts b/apps/sim/app/api/tools/cloudwatch/describe-alarm-history/route.ts deleted file mode 100644 index 25fbc4e18a0..00000000000 --- a/apps/sim/app/api/tools/cloudwatch/describe-alarm-history/route.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { - type AlarmType, - CloudWatchClient, - DescribeAlarmHistoryCommand, -} from '@aws-sdk/client-cloudwatch' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudwatchDescribeAlarmHistoryContract } from '@/lib/api/contracts/tools/aws/cloudwatch-describe-alarm-history' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -const logger = createLogger('CloudWatchDescribeAlarmHistory') - -/** AWS DescribeAlarmHistory caps `MaxRecords` at 100 items per page. */ -const ALARM_HISTORY_PAGE_SIZE = 100 - -/** Upper bound on pages drained to avoid unbounded loops on long-lived alarms. */ -const MAX_ALARM_HISTORY_PAGES = 20 - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudwatchDescribeAlarmHistoryContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info('Describing CloudWatch alarm history') - - const client = new CloudWatchClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - try { - const totalLimit = validatedData.limit - const alarmHistoryItems: { - alarmName: string | undefined - alarmType: string | undefined - timestamp: number | undefined - historyItemType: string | undefined - historySummary: string | undefined - }[] = [] - let nextToken: string | undefined - - for (let page = 0; page < MAX_ALARM_HISTORY_PAGES; page++) { - const pageLimit = - totalLimit !== undefined - ? Math.min(ALARM_HISTORY_PAGE_SIZE, totalLimit - alarmHistoryItems.length) - : ALARM_HISTORY_PAGE_SIZE - - const command = new DescribeAlarmHistoryCommand({ - ...(validatedData.alarmName && { AlarmName: validatedData.alarmName }), - // AWS defaults AlarmTypes to MetricAlarm-only, so always request both kinds explicitly. - AlarmTypes: ['MetricAlarm', 'CompositeAlarm'] as AlarmType[], - ...(validatedData.historyItemType && { - HistoryItemType: validatedData.historyItemType, - }), - ...(validatedData.startDate !== undefined && { - StartDate: new Date(validatedData.startDate * 1000), - }), - ...(validatedData.endDate !== undefined && { - EndDate: new Date(validatedData.endDate * 1000), - }), - ScanBy: validatedData.scanBy ?? 'TimestampDescending', - MaxRecords: pageLimit, - ...(nextToken && { NextToken: nextToken }), - }) - - const response = await client.send(command) - - for (const item of response.AlarmHistoryItems ?? []) { - alarmHistoryItems.push({ - alarmName: item.AlarmName, - alarmType: item.AlarmType, - timestamp: item.Timestamp?.getTime(), - historyItemType: item.HistoryItemType, - historySummary: item.HistorySummary, - }) - } - - nextToken = response.NextToken - if (!nextToken) break - if (totalLimit !== undefined && alarmHistoryItems.length >= totalLimit) break - - if (page === MAX_ALARM_HISTORY_PAGES - 1) { - logger.warn( - `DescribeAlarmHistory hit pagination cap of ${MAX_ALARM_HISTORY_PAGES} pages; history may be incomplete` - ) - } - } - - const cappedItems = - totalLimit !== undefined ? alarmHistoryItems.slice(0, totalLimit) : alarmHistoryItems - - logger.info(`Successfully described ${cappedItems.length} alarm history items`) - - return NextResponse.json({ - success: true, - output: { alarmHistoryItems: cappedItems }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('DescribeAlarmHistory failed', { error: toError(error).message }) - return NextResponse.json( - { error: `Failed to describe CloudWatch alarm history: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/cloudwatch/describe-alarms/route.ts b/apps/sim/app/api/tools/cloudwatch/describe-alarms/route.ts deleted file mode 100644 index 1d7686266ec..00000000000 --- a/apps/sim/app/api/tools/cloudwatch/describe-alarms/route.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { - type AlarmType, - CloudWatchClient, - DescribeAlarmsCommand, - type StateValue, -} from '@aws-sdk/client-cloudwatch' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudwatchDescribeAlarmsContract } from '@/lib/api/contracts/tools/aws/cloudwatch-describe-alarms' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -const logger = createLogger('CloudWatchDescribeAlarms') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudwatchDescribeAlarmsContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info('Describing CloudWatch alarms') - - const client = new CloudWatchClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - try { - const command = new DescribeAlarmsCommand({ - ...(validatedData.alarmNamePrefix && { AlarmNamePrefix: validatedData.alarmNamePrefix }), - ...(validatedData.stateValue && { StateValue: validatedData.stateValue as StateValue }), - AlarmTypes: validatedData.alarmType - ? [validatedData.alarmType as AlarmType] - : (['MetricAlarm', 'CompositeAlarm'] as AlarmType[]), - ...(validatedData.limit !== undefined && { MaxRecords: validatedData.limit }), - }) - - const response = await client.send(command) - - const metricAlarms = (response.MetricAlarms ?? []).map((a) => ({ - alarmName: a.AlarmName ?? '', - alarmArn: a.AlarmArn ?? '', - stateValue: a.StateValue ?? 'UNKNOWN', - stateReason: a.StateReason ?? '', - metricName: a.MetricName, - namespace: a.Namespace, - comparisonOperator: a.ComparisonOperator, - threshold: a.Threshold, - evaluationPeriods: a.EvaluationPeriods, - stateUpdatedTimestamp: a.StateUpdatedTimestamp?.getTime(), - })) - - const compositeAlarms = (response.CompositeAlarms ?? []).map((a) => ({ - alarmName: a.AlarmName ?? '', - alarmArn: a.AlarmArn ?? '', - stateValue: a.StateValue ?? 'UNKNOWN', - stateReason: a.StateReason ?? '', - metricName: undefined, - namespace: undefined, - comparisonOperator: undefined, - threshold: undefined, - evaluationPeriods: undefined, - stateUpdatedTimestamp: a.StateUpdatedTimestamp?.getTime(), - })) - - const alarms = [...metricAlarms, ...compositeAlarms] - - logger.info(`Successfully described ${alarms.length} alarms`) - - return NextResponse.json({ - success: true, - output: { alarms }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('DescribeAlarms failed', { error: toError(error).message }) - return NextResponse.json( - { error: `Failed to describe CloudWatch alarms: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/cloudwatch/describe-log-groups/route.ts b/apps/sim/app/api/tools/cloudwatch/describe-log-groups/route.ts index 4a2aabea1c9..3911351f96e 100644 --- a/apps/sim/app/api/tools/cloudwatch/describe-log-groups/route.ts +++ b/apps/sim/app/api/tools/cloudwatch/describe-log-groups/route.ts @@ -1,105 +1,19 @@ -import { DescribeLogGroupsCommand } from '@aws-sdk/client-cloudwatch-logs' import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' import { cloudwatchLogGroupsSelectorContract } from '@/lib/api/contracts/selectors/cloudwatch' import { parseToolRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createCloudWatchLogsClient } from '@/app/api/tools/cloudwatch/utils' +import { createCloudWatchHttpRoute } from '@/lib/internal/cloudwatch/http-route' +import { executeCloudwatchDescribeLogGroups } from '@/lib/internal/cloudwatch/operations' const logger = createLogger('CloudWatchDescribeLogGroups') -/** AWS DescribeLogGroups caps `limit` at 50 items per page. */ -const LOG_GROUPS_PAGE_SIZE = 50 - -/** Upper bound on pages drained to avoid unbounded loops on very large accounts. */ -const MAX_LOG_GROUPS_PAGES = 20 - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(cloudwatchLogGroupsSelectorContract, request, { +export const POST = createCloudWatchHttpRoute({ + logger, + parse: (request) => + parseToolRequest(cloudwatchLogGroupsSelectorContract, request, { errorFormat: 'firstError', logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info('Describing CloudWatch log groups') - - const client = createCloudWatchLogsClient({ - region: validatedData.region, - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }) - - try { - const totalLimit = validatedData.limit - const logGroups: { - logGroupName: string - arn: string - storedBytes: number - retentionInDays: number | undefined - creationTime: number | undefined - }[] = [] - let nextToken: string | undefined - - for (let page = 0; page < MAX_LOG_GROUPS_PAGES; page++) { - const pageLimit = - totalLimit !== undefined - ? Math.min(LOG_GROUPS_PAGE_SIZE, totalLimit - logGroups.length) - : LOG_GROUPS_PAGE_SIZE - - const command = new DescribeLogGroupsCommand({ - ...(validatedData.prefix && { logGroupNamePrefix: validatedData.prefix }), - limit: pageLimit, - ...(nextToken && { nextToken }), - }) - - const response = await client.send(command) - - for (const lg of response.logGroups ?? []) { - logGroups.push({ - logGroupName: lg.logGroupName ?? '', - arn: lg.arn ?? '', - storedBytes: lg.storedBytes ?? 0, - retentionInDays: lg.retentionInDays, - creationTime: lg.creationTime, - }) - } - - nextToken = response.nextToken - if (!nextToken) break - if (totalLimit !== undefined && logGroups.length >= totalLimit) break - - if (page === MAX_LOG_GROUPS_PAGES - 1) { - logger.warn( - `DescribeLogGroups hit pagination cap of ${MAX_LOG_GROUPS_PAGES} pages; log group list may be incomplete` - ) - } - } - - const cappedLogGroups = totalLimit !== undefined ? logGroups.slice(0, totalLimit) : logGroups - - logger.info(`Successfully described ${cappedLogGroups.length} log groups`) - - return NextResponse.json({ - success: true, - output: { logGroups: cappedLogGroups }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('DescribeLogGroups failed', { error: toError(error).message }) - return NextResponse.json( - { error: `Failed to describe CloudWatch log groups: ${toError(error).message}` }, - { status: 500 } - ) - } + }), + execute: executeCloudwatchDescribeLogGroups, + errorMessage: 'Failed to describe CloudWatch log groups', + auth: 'session-or-internal', }) diff --git a/apps/sim/app/api/tools/cloudwatch/describe-log-streams/route.ts b/apps/sim/app/api/tools/cloudwatch/describe-log-streams/route.ts index 8a8a5617797..3cbde9e151d 100644 --- a/apps/sim/app/api/tools/cloudwatch/describe-log-streams/route.ts +++ b/apps/sim/app/api/tools/cloudwatch/describe-log-streams/route.ts @@ -1,56 +1,19 @@ import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' import { cloudwatchLogStreamsSelectorContract } from '@/lib/api/contracts/selectors/cloudwatch' import { parseToolRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createCloudWatchLogsClient, describeLogStreams } from '@/app/api/tools/cloudwatch/utils' +import { createCloudWatchHttpRoute } from '@/lib/internal/cloudwatch/http-route' +import { executeCloudwatchDescribeLogStreams } from '@/lib/internal/cloudwatch/operations' const logger = createLogger('CloudWatchDescribeLogStreams') -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(cloudwatchLogStreamsSelectorContract, request, { +export const POST = createCloudWatchHttpRoute({ + logger, + parse: (request) => + parseToolRequest(cloudwatchLogStreamsSelectorContract, request, { errorFormat: 'firstError', logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`Describing log streams for group: ${validatedData.logGroupName}`) - - const client = createCloudWatchLogsClient({ - region: validatedData.region, - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }) - - try { - const result = await describeLogStreams(client, validatedData.logGroupName, { - prefix: validatedData.prefix, - limit: validatedData.limit, - }) - - logger.info(`Successfully described ${result.logStreams.length} log streams`) - - return NextResponse.json({ - success: true, - output: { logStreams: result.logStreams }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('DescribeLogStreams failed', { error: toError(error).message }) - return NextResponse.json( - { error: `Failed to describe CloudWatch log streams: ${toError(error).message}` }, - { status: 500 } - ) - } + }), + execute: executeCloudwatchDescribeLogStreams, + errorMessage: 'Failed to describe CloudWatch log streams', + auth: 'session-or-internal', }) diff --git a/apps/sim/app/api/tools/cloudwatch/filter-log-events/route.ts b/apps/sim/app/api/tools/cloudwatch/filter-log-events/route.ts deleted file mode 100644 index 318d0890dc4..00000000000 --- a/apps/sim/app/api/tools/cloudwatch/filter-log-events/route.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudwatchFilterLogEventsContract } from '@/lib/api/contracts/tools/aws/cloudwatch-filter-log-events' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createCloudWatchLogsClient, filterLogEvents } from '@/app/api/tools/cloudwatch/utils' - -const logger = createLogger('CloudWatchFilterLogEvents') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudwatchFilterLogEventsContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`Filtering log events in ${validatedData.logGroupName}`) - - const client = createCloudWatchLogsClient({ - region: validatedData.region, - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }) - - try { - const result = await filterLogEvents(client, validatedData.logGroupName, { - filterPattern: validatedData.filterPattern, - logStreamNamePrefix: validatedData.logStreamNamePrefix, - // CloudWatch Logs timestamps are epoch milliseconds; our params are epoch seconds. - startTime: - validatedData.startTime !== undefined ? validatedData.startTime * 1000 : undefined, - endTime: validatedData.endTime !== undefined ? validatedData.endTime * 1000 : undefined, - startFromHead: validatedData.startFromHead, - limit: validatedData.limit, - }) - - logger.info(`Successfully filtered ${result.events.length} log events`) - - return NextResponse.json({ - success: true, - output: { events: result.events }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('FilterLogEvents failed', { error: toError(error).message }) - return NextResponse.json( - { error: `Failed to filter CloudWatch log events: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/cloudwatch/get-log-events/route.ts b/apps/sim/app/api/tools/cloudwatch/get-log-events/route.ts deleted file mode 100644 index 57d8a14523a..00000000000 --- a/apps/sim/app/api/tools/cloudwatch/get-log-events/route.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudwatchGetLogEventsContract } from '@/lib/api/contracts/tools/aws/cloudwatch-get-log-events' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createCloudWatchLogsClient, getLogEvents } from '@/app/api/tools/cloudwatch/utils' - -const logger = createLogger('CloudWatchGetLogEvents') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudwatchGetLogEventsContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info( - `Getting log events from ${validatedData.logGroupName}/${validatedData.logStreamName}` - ) - - const client = createCloudWatchLogsClient({ - region: validatedData.region, - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }) - - try { - const result = await getLogEvents( - client, - validatedData.logGroupName, - validatedData.logStreamName, - { - startTime: validatedData.startTime, - endTime: validatedData.endTime, - limit: validatedData.limit, - } - ) - - logger.info(`Successfully retrieved ${result.events.length} log events`) - - return NextResponse.json({ - success: true, - output: { events: result.events }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('GetLogEvents failed', { error: toError(error).message }) - return NextResponse.json( - { error: `Failed to get CloudWatch log events: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/cloudwatch/get-metric-statistics/route.ts b/apps/sim/app/api/tools/cloudwatch/get-metric-statistics/route.ts deleted file mode 100644 index 615122c0da8..00000000000 --- a/apps/sim/app/api/tools/cloudwatch/get-metric-statistics/route.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { CloudWatchClient, GetMetricStatisticsCommand } from '@aws-sdk/client-cloudwatch' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudwatchGetMetricStatisticsContract } from '@/lib/api/contracts/tools/aws/cloudwatch-get-metric-statistics' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -const logger = createLogger('CloudWatchGetMetricStatistics') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudwatchGetMetricStatisticsContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info( - `Getting metric statistics for ${validatedData.namespace}/${validatedData.metricName}` - ) - - const client = new CloudWatchClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - try { - let parsedDimensions: { Name: string; Value: string }[] | undefined - if (validatedData.dimensions) { - try { - const dims = JSON.parse(validatedData.dimensions) - if (Array.isArray(dims)) { - parsedDimensions = dims.map((d: Record) => ({ - Name: d.name, - Value: d.value, - })) - } else if (typeof dims === 'object') { - parsedDimensions = Object.entries(dims).map(([name, value]) => ({ - Name: name, - Value: String(value), - })) - } - } catch { - return NextResponse.json({ error: 'Invalid dimensions JSON format' }, { status: 400 }) - } - } - - const command = new GetMetricStatisticsCommand({ - Namespace: validatedData.namespace, - MetricName: validatedData.metricName, - StartTime: new Date(validatedData.startTime * 1000), - EndTime: new Date(validatedData.endTime * 1000), - Period: validatedData.period, - Statistics: validatedData.statistics, - ...(parsedDimensions && { Dimensions: parsedDimensions }), - }) - - const response = await client.send(command) - - const datapoints = (response.Datapoints ?? []) - .sort((a, b) => (a.Timestamp?.getTime() ?? 0) - (b.Timestamp?.getTime() ?? 0)) - .map((dp) => ({ - timestamp: dp.Timestamp ? dp.Timestamp.getTime() : 0, - average: dp.Average, - sum: dp.Sum, - minimum: dp.Minimum, - maximum: dp.Maximum, - sampleCount: dp.SampleCount, - unit: dp.Unit, - })) - - logger.info(`Successfully retrieved ${datapoints.length} datapoints`) - - return NextResponse.json({ - success: true, - output: { - label: response.Label ?? validatedData.metricName, - datapoints, - }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('GetMetricStatistics failed', { error: toError(error).message }) - return NextResponse.json( - { error: `Failed to get CloudWatch metric statistics: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/cloudwatch/list-metrics/route.ts b/apps/sim/app/api/tools/cloudwatch/list-metrics/route.ts deleted file mode 100644 index 3aafe8b8c37..00000000000 --- a/apps/sim/app/api/tools/cloudwatch/list-metrics/route.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { CloudWatchClient, ListMetricsCommand } from '@aws-sdk/client-cloudwatch' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudwatchListMetricsContract } from '@/lib/api/contracts/tools/aws/cloudwatch-list-metrics' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -const logger = createLogger('CloudWatchListMetrics') - -/** AWS ListMetrics returns up to 500 results per page. */ -const METRICS_PAGE_SIZE = 500 - -/** Upper bound on pages drained to avoid unbounded loops on accounts with many metrics. */ -const MAX_METRICS_PAGES = 20 - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudwatchListMetricsContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info('Listing CloudWatch metrics') - - const client = new CloudWatchClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - try { - const totalLimit = validatedData.limit ?? METRICS_PAGE_SIZE - const metrics: { - namespace: string - metricName: string - dimensions: { name: string; value: string }[] - }[] = [] - let nextToken: string | undefined - - for (let page = 0; page < MAX_METRICS_PAGES; page++) { - const command = new ListMetricsCommand({ - ...(validatedData.namespace && { Namespace: validatedData.namespace }), - ...(validatedData.metricName && { MetricName: validatedData.metricName }), - ...(validatedData.recentlyActive && { RecentlyActive: 'PT3H' }), - ...(nextToken && { NextToken: nextToken }), - }) - - const response = await client.send(command) - - for (const m of response.Metrics ?? []) { - metrics.push({ - namespace: m.Namespace ?? '', - metricName: m.MetricName ?? '', - dimensions: (m.Dimensions ?? []).map((d) => ({ - name: d.Name ?? '', - value: d.Value ?? '', - })), - }) - } - - nextToken = response.NextToken - if (!nextToken) break - if (metrics.length >= totalLimit) break - - if (page === MAX_METRICS_PAGES - 1) { - logger.warn( - `ListMetrics hit pagination cap of ${MAX_METRICS_PAGES} pages; metric list may be incomplete` - ) - } - } - - const cappedMetrics = metrics.slice(0, totalLimit) - - logger.info(`Successfully listed ${cappedMetrics.length} metrics`) - - return NextResponse.json({ - success: true, - output: { metrics: cappedMetrics }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('ListMetrics failed', { error: toError(error).message }) - return NextResponse.json( - { error: `Failed to list CloudWatch metrics: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/cloudwatch/mute-alarm/route.ts b/apps/sim/app/api/tools/cloudwatch/mute-alarm/route.ts deleted file mode 100644 index 016b0e922bb..00000000000 --- a/apps/sim/app/api/tools/cloudwatch/mute-alarm/route.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { CloudWatchClient, PutAlarmMuteRuleCommand } from '@aws-sdk/client-cloudwatch' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudwatchMuteAlarmContract } from '@/lib/api/contracts/tools/aws/cloudwatch-mute-alarm' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -const logger = createLogger('CloudWatchMuteAlarm') - -function toAtExpression(date: Date): string { - const yyyy = date.getUTCFullYear() - const mm = String(date.getUTCMonth() + 1).padStart(2, '0') - const dd = String(date.getUTCDate()).padStart(2, '0') - const hh = String(date.getUTCHours()).padStart(2, '0') - const min = String(date.getUTCMinutes()).padStart(2, '0') - return `at(${yyyy}-${mm}-${dd}T${hh}:${min})` -} - -function toIsoDuration(value: number, unit: 'minutes' | 'hours' | 'days'): string { - switch (unit) { - case 'minutes': - return `PT${value}M` - case 'hours': - return `PT${value}H` - case 'days': - return `P${value}D` - } -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudwatchMuteAlarmContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const startDate = - validatedData.startDate !== undefined ? new Date(validatedData.startDate * 1000) : new Date() - const expression = toAtExpression(startDate) - const duration = toIsoDuration(validatedData.durationValue, validatedData.durationUnit) - - logger.info( - `Creating CloudWatch alarm mute rule "${validatedData.muteRuleName}" for ${validatedData.alarmNames.length} alarm(s) (${expression}, duration ${duration})` - ) - - const client = new CloudWatchClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - try { - const command = new PutAlarmMuteRuleCommand({ - Name: validatedData.muteRuleName, - ...(validatedData.description && { Description: validatedData.description }), - Rule: { - Schedule: { - Expression: expression, - Duration: duration, - }, - }, - MuteTargets: { AlarmNames: validatedData.alarmNames }, - }) - - await client.send(command) - - logger.info(`Successfully created mute rule "${validatedData.muteRuleName}"`) - - return NextResponse.json({ - success: true, - output: { - success: true, - muteRuleName: validatedData.muteRuleName, - alarmNames: validatedData.alarmNames, - expression, - duration, - }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('MuteAlarm failed', { error: toError(error).message }) - return NextResponse.json( - { error: `Failed to create CloudWatch alarm mute rule: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/cloudwatch/put-log-group-retention/route.ts b/apps/sim/app/api/tools/cloudwatch/put-log-group-retention/route.ts deleted file mode 100644 index ae3cef9fd35..00000000000 --- a/apps/sim/app/api/tools/cloudwatch/put-log-group-retention/route.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { - DeleteRetentionPolicyCommand, - PutRetentionPolicyCommand, -} from '@aws-sdk/client-cloudwatch-logs' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudwatchPutLogGroupRetentionContract } from '@/lib/api/contracts/tools/aws/cloudwatch-put-log-group-retention' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createCloudWatchLogsClient } from '@/app/api/tools/cloudwatch/utils' - -const logger = createLogger('CloudWatchPutLogGroupRetention') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudwatchPutLogGroupRetentionContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const client = createCloudWatchLogsClient({ - region: validatedData.region, - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }) - - try { - if (validatedData.retentionInDays !== undefined) { - logger.info( - `Setting retention for log group "${validatedData.logGroupName}" to ${validatedData.retentionInDays} days` - ) - await client.send( - new PutRetentionPolicyCommand({ - logGroupName: validatedData.logGroupName, - retentionInDays: validatedData.retentionInDays, - }) - ) - } else { - logger.info( - `Removing retention policy for log group "${validatedData.logGroupName}" (events never expire)` - ) - await client.send( - new DeleteRetentionPolicyCommand({ logGroupName: validatedData.logGroupName }) - ) - } - - return NextResponse.json({ - success: true, - output: { - success: true, - logGroupName: validatedData.logGroupName, - retentionInDays: validatedData.retentionInDays ?? null, - }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('PutLogGroupRetention failed', { error: toError(error).message }) - return NextResponse.json( - { error: `Failed to set CloudWatch log group retention: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/cloudwatch/put-metric-data/retry-semantics.test.ts b/apps/sim/app/api/tools/cloudwatch/put-metric-data/retry-semantics.test.ts deleted file mode 100644 index 5926af4a066..00000000000 --- a/apps/sim/app/api/tools/cloudwatch/put-metric-data/retry-semantics.test.ts +++ /dev/null @@ -1,73 +0,0 @@ -/** - * Pins what `maxAttempts` *means*, using the real AWS SDK. - * - * `route.test.ts` asserts the route configures `maxAttempts: 1`; on its own that - * only pins a number. This file counts how many datapoints a real - * `CloudWatchClient` actually hands to a peer that accepts the request and then - * dies before writing a response byte -- the ambiguous transport failure the - * SDK's retry layer replays, and the one that makes CloudWatch aggregate a - * duplicate. It fails if a future SDK bump changes the default budget or stops - * honouring the pin. - * - * Deliberately not mocking `@aws-sdk/client-cloudwatch` here: the SDK's retry - * middleware is the subject under test, so it must be the real one. - * - * @vitest-environment node - */ -import http from 'node:http' -import type { AddressInfo } from 'node:net' -import { CloudWatchClient, PutMetricDataCommand } from '@aws-sdk/client-cloudwatch' -import { describe, expect, it } from 'vitest' - -/** `@smithy/util-retry`'s `DEFAULT_MAX_ATTEMPTS`, which every unpinned client inherits. */ -const SDK_DEFAULT_MAX_ATTEMPTS = 3 - -async function countDeliveries(clientConfig: Record): Promise { - let received = 0 - const server = http.createServer((req, res) => { - req.on('data', () => {}) - req.on('end', () => { - if (String(req.headers['x-amz-target'] ?? '').endsWith('PutMetricData')) { - received++ - req.socket.destroy() - return - } - res.writeHead(200, { 'content-type': 'application/x-amz-json-1.0' }) - res.end('{}') - }) - }) - await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) - const { port } = server.address() as AddressInfo - - const client = new CloudWatchClient({ - region: 'us-east-1', - endpoint: `http://127.0.0.1:${port}`, - credentials: { accessKeyId: 'AKIAEXAMPLE', secretAccessKey: 'secret' }, - ...clientConfig, - }) - - try { - await client.send( - new PutMetricDataCommand({ - Namespace: 'Sim/Test', - MetricData: [{ MetricName: 'Requests', Value: 1 }], - }) - ) - } catch { - /* Every attempt fails by design; the delivery count is the assertion. */ - } finally { - client.destroy() - await new Promise((resolve) => server.close(() => resolve())) - } - return received -} - -describe('aws sdk retry semantics for PutMetricData', () => { - it('delivers the datapoint exactly once when maxAttempts is pinned to 1', async () => { - await expect(countDeliveries({ maxAttempts: 1 })).resolves.toBe(1) - }) - - it('aggregates a duplicate for every retry the default budget allows', async () => { - await expect(countDeliveries({})).resolves.toBe(SDK_DEFAULT_MAX_ATTEMPTS) - }) -}) diff --git a/apps/sim/app/api/tools/cloudwatch/put-metric-data/route.test.ts b/apps/sim/app/api/tools/cloudwatch/put-metric-data/route.test.ts deleted file mode 100644 index 080d0d30b7f..00000000000 --- a/apps/sim/app/api/tools/cloudwatch/put-metric-data/route.test.ts +++ /dev/null @@ -1,65 +0,0 @@ -/** - * @vitest-environment node - */ -import { createMockRequest, hybridAuthMockFns } from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { mockSend, mockDestroy, capturedConfigs } = vi.hoisted(() => ({ - mockSend: vi.fn(), - mockDestroy: vi.fn(), - capturedConfigs: [] as Record[], -})) - -vi.mock('@aws-sdk/client-cloudwatch', () => ({ - CloudWatchClient: class { - constructor(config: Record) { - capturedConfigs.push(config) - } - send = mockSend - destroy = mockDestroy - }, - PutMetricDataCommand: class { - constructor(readonly input: Record) {} - }, -})) - -import { POST } from '@/app/api/tools/cloudwatch/put-metric-data/route' - -const body = { - region: 'us-east-1', - accessKeyId: 'AKIAEXAMPLE', - secretAccessKey: 'secret', - namespace: 'Sim/Test', - metricName: 'Requests', - value: 1, -} - -function postRoute() { - return POST(createMockRequest('POST', body)) -} - -describe('cloudwatch put-metric-data delivery class', () => { - beforeEach(() => { - vi.clearAllMocks() - capturedConfigs.length = 0 - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ success: true, userId: 'user-1' }) - mockSend.mockResolvedValue({}) - }) - - it('pins the client to a single attempt so the SDK cannot replay the datapoint', async () => { - const response = await postRoute() - - expect(response.status).toBe(200) - expect(capturedConfigs).toHaveLength(1) - expect(capturedConfigs[0].maxAttempts).toBe(1) - }) - - it('never lets an ambiguous failure turn into a second PutMetricData', async () => { - mockSend.mockRejectedValue(Object.assign(new Error('socket hang up'), { name: 'TimeoutError' })) - - const response = await postRoute() - - expect(response.status).toBe(500) - expect(mockSend).toHaveBeenCalledTimes(1) - }) -}) diff --git a/apps/sim/app/api/tools/cloudwatch/put-metric-data/route.ts b/apps/sim/app/api/tools/cloudwatch/put-metric-data/route.ts deleted file mode 100644 index 0c304e50804..00000000000 --- a/apps/sim/app/api/tools/cloudwatch/put-metric-data/route.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { - CloudWatchClient, - PutMetricDataCommand, - type StandardUnit, -} from '@aws-sdk/client-cloudwatch' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudwatchPutMetricDataContract } from '@/lib/api/contracts/tools/aws/cloudwatch-put-metric-data' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import type { DeliveryDeclaration } from '@/lib/core/http/classes' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -const logger = createLogger('CloudWatchPutMetricData') - -/** - * PutMetricData is additive, not last-write-wins: CloudWatch folds every - * datapoint it receives for a (namespace, metric, dimensions, timestamp) tuple - * into the same statistic set. Two deliveries of one user call therefore - * publish `SampleCount=2, Sum=2 x value` instead of `SampleCount=1, Sum=value`, - * silently doubling the customer's series and any alarm threshold read off it. - * Nothing on the wire distinguishes this from a correct write, and the datapoint - * cannot be retracted -- CloudWatch has no delete-datapoint API. - */ -const PUT_METRIC_DATA_DELIVERY = { - deliveryClass: 'once', - why: "the customer's metric series would double-count this datapoint, skewing every statistic and alarm derived from it", - userVisibleEffect: - 'a datapoint with SampleCount=2 and Sum=2x the reported value, permanently, with no way to retract it', -} satisfies DeliveryDeclaration - -/** - * The AWS SDK's own retry layer replays a request whenever the transport fails - * ambiguously (ECONNRESET, socket hangup, 500/502/503/504) -- exactly the cases - * where the peer may already have committed. `@smithy/util-retry` defaults to - * `DEFAULT_MAX_ATTEMPTS = 3`, so an unpinned client turns one severed socket - * into three aggregated datapoints. Pinning to 1 trades a lost datapoint for a - * correct series, which is the right trade for an additive metric: a gap is - * visible and self-healing, a doubled value is neither. - */ -const NON_IDEMPOTENT_MAX_ATTEMPTS = 1 - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudwatchPutMetricDataContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`Publishing metric ${validatedData.namespace}/${validatedData.metricName}`) - - const client = new CloudWatchClient({ - region: validatedData.region, - maxAttempts: NON_IDEMPOTENT_MAX_ATTEMPTS, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - try { - const timestamp = new Date() - - const dimensions: { Name: string; Value: string }[] = [] - if (validatedData.dimensions) { - const parsed = JSON.parse(validatedData.dimensions) - for (const [name, value] of Object.entries(parsed)) { - dimensions.push({ Name: name, Value: String(value) }) - } - } - - const command = new PutMetricDataCommand({ - Namespace: validatedData.namespace, - MetricData: [ - { - MetricName: validatedData.metricName, - Value: validatedData.value, - Timestamp: timestamp, - ...(validatedData.unit && { Unit: validatedData.unit as StandardUnit }), - ...(dimensions.length > 0 && { Dimensions: dimensions }), - }, - ], - }) - - await client.send(command) - - logger.info('Successfully published metric') - - return NextResponse.json({ - success: true, - output: { - success: true, - namespace: validatedData.namespace, - metricName: validatedData.metricName, - value: validatedData.value, - unit: validatedData.unit ?? 'None', - timestamp: timestamp.toISOString(), - }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('PutMetricData failed', { - error: toError(error).message, - deliveryClass: PUT_METRIC_DATA_DELIVERY.deliveryClass, - outcome: 'indeterminate', - duplicateEffect: PUT_METRIC_DATA_DELIVERY.userVisibleEffect, - }) - return NextResponse.json( - { error: `Failed to publish CloudWatch metric: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/cloudwatch/query-logs/route.ts b/apps/sim/app/api/tools/cloudwatch/query-logs/route.ts deleted file mode 100644 index 8fb74b75351..00000000000 --- a/apps/sim/app/api/tools/cloudwatch/query-logs/route.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { StartQueryCommand } from '@aws-sdk/client-cloudwatch-logs' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudwatchQueryLogsContract } from '@/lib/api/contracts/tools/aws/cloudwatch-query-logs' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createCloudWatchLogsClient, pollQueryResults } from '@/app/api/tools/cloudwatch/utils' - -const logger = createLogger('CloudWatchQueryLogs') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudwatchQueryLogsContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info('Running CloudWatch Log Insights query') - - const client = createCloudWatchLogsClient({ - region: validatedData.region, - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }) - - try { - const startQueryCommand = new StartQueryCommand({ - logGroupNames: validatedData.logGroupNames, - queryString: validatedData.queryString, - startTime: validatedData.startTime, - endTime: validatedData.endTime, - ...(validatedData.limit !== undefined && { limit: validatedData.limit }), - }) - - const startQueryResponse = await client.send(startQueryCommand) - const queryId = startQueryResponse.queryId - - if (!queryId) { - throw new Error('Failed to start CloudWatch Log Insights query: no queryId returned') - } - - const result = await pollQueryResults(client, queryId) - - logger.info(`Query completed with status: ${result.status}`) - - return NextResponse.json({ - success: true, - output: { - results: result.results, - statistics: result.statistics, - status: result.status, - }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('QueryLogs failed', { error: toError(error).message }) - return NextResponse.json( - { error: `CloudWatch Log Insights query failed: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/cloudwatch/unmute-alarm/route.ts b/apps/sim/app/api/tools/cloudwatch/unmute-alarm/route.ts deleted file mode 100644 index f950d9146af..00000000000 --- a/apps/sim/app/api/tools/cloudwatch/unmute-alarm/route.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { CloudWatchClient, DeleteAlarmMuteRuleCommand } from '@aws-sdk/client-cloudwatch' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCloudwatchUnmuteAlarmContract } from '@/lib/api/contracts/tools/aws/cloudwatch-unmute-alarm' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -const logger = createLogger('CloudWatchUnmuteAlarm') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCloudwatchUnmuteAlarmContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`Deleting CloudWatch alarm mute rule "${validatedData.muteRuleName}"`) - - const client = new CloudWatchClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - try { - const command = new DeleteAlarmMuteRuleCommand({ - AlarmMuteRuleName: validatedData.muteRuleName, - }) - - await client.send(command) - - logger.info(`Successfully deleted mute rule "${validatedData.muteRuleName}"`) - - return NextResponse.json({ - success: true, - output: { - success: true, - muteRuleName: validatedData.muteRuleName, - }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('UnmuteAlarm failed', { error: toError(error).message }) - return NextResponse.json( - { error: `Failed to delete CloudWatch alarm mute rule: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/cloudwatch/utils.ts b/apps/sim/app/api/tools/cloudwatch/utils.ts deleted file mode 100644 index a477fda9e9f..00000000000 --- a/apps/sim/app/api/tools/cloudwatch/utils.ts +++ /dev/null @@ -1,291 +0,0 @@ -import { - CloudWatchLogsClient, - DescribeLogStreamsCommand, - FilterLogEventsCommand, - GetLogEventsCommand, - GetQueryResultsCommand, - type ResultField, -} from '@aws-sdk/client-cloudwatch-logs' -import { createLogger } from '@sim/logger' -import { sleep } from '@sim/utils/helpers' -import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' - -interface AwsCredentials { - region: string - accessKeyId: string - secretAccessKey: string -} - -export function createCloudWatchLogsClient(config: AwsCredentials): CloudWatchLogsClient { - return new CloudWatchLogsClient({ - region: config.region, - credentials: { - accessKeyId: config.accessKeyId, - secretAccessKey: config.secretAccessKey, - }, - }) -} - -interface PollOptions { - maxWaitMs?: number - pollIntervalMs?: number -} - -interface PollResult { - results: Record[] - statistics: { - bytesScanned: number - recordsMatched: number - recordsScanned: number - } - status: string -} - -function parseResultFields(fields: ResultField[] | undefined): Record { - const record: Record = {} - if (!fields) return record - for (const field of fields) { - if (field.field && field.value !== undefined) { - record[field.field] = field.value ?? '' - } - } - return record -} - -export async function pollQueryResults( - client: CloudWatchLogsClient, - queryId: string, - options: PollOptions = {} -): Promise { - const { maxWaitMs = DEFAULT_EXECUTION_TIMEOUT_MS, pollIntervalMs = 1_000 } = options - const startTime = Date.now() - - while (Date.now() - startTime < maxWaitMs) { - const command = new GetQueryResultsCommand({ queryId }) - const response = await client.send(command) - - const status = response.status ?? 'Unknown' - - if (status === 'Complete') { - return { - results: (response.results ?? []).map(parseResultFields), - statistics: { - bytesScanned: response.statistics?.bytesScanned ?? 0, - recordsMatched: response.statistics?.recordsMatched ?? 0, - recordsScanned: response.statistics?.recordsScanned ?? 0, - }, - status, - } - } - - if (status === 'Failed' || status === 'Cancelled') { - throw new Error(`CloudWatch Log Insights query ${status.toLowerCase()}`) - } - - await sleep(pollIntervalMs) - } - - // Timeout -- fetch one last time for partial results - const finalResponse = await client.send(new GetQueryResultsCommand({ queryId })) - return { - results: (finalResponse.results ?? []).map(parseResultFields), - statistics: { - bytesScanned: finalResponse.statistics?.bytesScanned ?? 0, - recordsMatched: finalResponse.statistics?.recordsMatched ?? 0, - recordsScanned: finalResponse.statistics?.recordsScanned ?? 0, - }, - status: `Timeout (last status: ${finalResponse.status ?? 'Unknown'})`, - } -} - -/** AWS DescribeLogStreams caps `limit` at 50 items per page. */ -const LOG_STREAMS_PAGE_SIZE = 50 - -/** Upper bound on pages drained to avoid unbounded loops on log groups with many streams. */ -const MAX_LOG_STREAMS_PAGES = 20 - -const logger = createLogger('CloudWatchUtils') - -interface DescribedLogStream { - logStreamName: string - lastEventTimestamp: number | undefined - firstEventTimestamp: number | undefined - creationTime: number | undefined - storedBytes: number -} - -/** - * Lists log streams for a log group, following `nextToken` so the complete set - * is returned rather than just the first page. Bounded by - * `MAX_LOG_STREAMS_PAGES`; logs a warning rather than silently dropping streams - * when the cap is hit. Ordering/prefix inputs are preserved across all pages. - * - * When `limit` is provided it is treated as a total result cap: draining stops - * once enough streams have been collected. When omitted, every page is drained. - */ -export async function describeLogStreams( - client: CloudWatchLogsClient, - logGroupName: string, - options?: { prefix?: string; limit?: number } -): Promise<{ logStreams: DescribedLogStream[] }> { - const hasPrefix = Boolean(options?.prefix) - const totalLimit = options?.limit - const logStreams: DescribedLogStream[] = [] - let nextToken: string | undefined - - for (let page = 0; page < MAX_LOG_STREAMS_PAGES; page++) { - const pageLimit = - totalLimit !== undefined - ? Math.min(LOG_STREAMS_PAGE_SIZE, totalLimit - logStreams.length) - : LOG_STREAMS_PAGE_SIZE - - const command = new DescribeLogStreamsCommand({ - logGroupName, - ...(hasPrefix - ? { orderBy: 'LogStreamName', logStreamNamePrefix: options!.prefix } - : { orderBy: 'LastEventTime', descending: true }), - limit: pageLimit, - ...(nextToken && { nextToken }), - }) - - const response = await client.send(command) - - for (const ls of response.logStreams ?? []) { - logStreams.push({ - logStreamName: ls.logStreamName ?? '', - lastEventTimestamp: ls.lastEventTimestamp, - firstEventTimestamp: ls.firstEventTimestamp, - creationTime: ls.creationTime, - storedBytes: ls.storedBytes ?? 0, - }) - } - - nextToken = response.nextToken - if (!nextToken) break - if (totalLimit !== undefined && logStreams.length >= totalLimit) break - - if (page === MAX_LOG_STREAMS_PAGES - 1) { - logger.warn( - `DescribeLogStreams hit pagination cap of ${MAX_LOG_STREAMS_PAGES} pages; log stream list may be incomplete`, - { logGroupName } - ) - } - } - - return { - logStreams: totalLimit !== undefined ? logStreams.slice(0, totalLimit) : logStreams, - } -} - -/** AWS FilterLogEvents caps `limit` at 10,000 events per page. */ -const FILTER_LOG_EVENTS_PAGE_SIZE = 10_000 - -/** Upper bound on pages drained to avoid unbounded loops on very active log groups. */ -const MAX_FILTER_LOG_EVENTS_PAGES = 20 - -interface FilteredLogEventResult { - logStreamName: string | undefined - timestamp: number | undefined - message: string | undefined - ingestionTime: number | undefined -} - -/** - * Searches log events across all streams (or a prefix-matched subset) in a log - * group, following `nextToken` so the complete matching set is returned rather - * than just the first page. Bounded by `MAX_FILTER_LOG_EVENTS_PAGES`. - * - * When `limit` is provided it is treated as a total result cap: draining stops - * once enough events have been collected. When omitted, every page is drained. - */ -export async function filterLogEvents( - client: CloudWatchLogsClient, - logGroupName: string, - options?: { - filterPattern?: string - logStreamNamePrefix?: string - startTime?: number - endTime?: number - startFromHead?: boolean - limit?: number - } -): Promise<{ events: FilteredLogEventResult[] }> { - const totalLimit = options?.limit - const events: FilteredLogEventResult[] = [] - let nextToken: string | undefined - - for (let page = 0; page < MAX_FILTER_LOG_EVENTS_PAGES; page++) { - const pageLimit = - totalLimit !== undefined - ? Math.min(FILTER_LOG_EVENTS_PAGE_SIZE, totalLimit - events.length) - : FILTER_LOG_EVENTS_PAGE_SIZE - - const command = new FilterLogEventsCommand({ - logGroupName, - ...(options?.filterPattern && { filterPattern: options.filterPattern }), - ...(options?.logStreamNamePrefix && { logStreamNamePrefix: options.logStreamNamePrefix }), - ...(options?.startTime !== undefined && { startTime: options.startTime }), - ...(options?.endTime !== undefined && { endTime: options.endTime }), - ...(options?.startFromHead !== undefined && { startFromHead: options.startFromHead }), - limit: pageLimit, - ...(nextToken && { nextToken }), - }) - - const response = await client.send(command) - - for (const e of response.events ?? []) { - events.push({ - logStreamName: e.logStreamName, - timestamp: e.timestamp, - message: e.message, - ingestionTime: e.ingestionTime, - }) - } - - nextToken = response.nextToken - if (!nextToken) break - if (totalLimit !== undefined && events.length >= totalLimit) break - - if (page === MAX_FILTER_LOG_EVENTS_PAGES - 1) { - logger.warn( - `FilterLogEvents hit pagination cap of ${MAX_FILTER_LOG_EVENTS_PAGES} pages; event list may be incomplete`, - { logGroupName } - ) - } - } - - return { - events: totalLimit !== undefined ? events.slice(0, totalLimit) : events, - } -} - -export async function getLogEvents( - client: CloudWatchLogsClient, - logGroupName: string, - logStreamName: string, - options?: { startTime?: number; endTime?: number; limit?: number } -): Promise<{ - events: { - timestamp: number | undefined - message: string | undefined - ingestionTime: number | undefined - }[] -}> { - const command = new GetLogEventsCommand({ - logGroupName, - logStreamName, - ...(options?.startTime !== undefined && { startTime: options.startTime * 1000 }), - ...(options?.endTime !== undefined && { endTime: options.endTime * 1000 }), - ...(options?.limit !== undefined && { limit: options.limit }), - startFromHead: true, - }) - - const response = await client.send(command) - return { - events: (response.events ?? []).map((e) => ({ - timestamp: e.timestamp, - message: e.message, - ingestionTime: e.ingestionTime, - })), - } -} diff --git a/apps/sim/app/api/tools/codepipeline/disable-stage-transition/route.ts b/apps/sim/app/api/tools/codepipeline/disable-stage-transition/route.ts deleted file mode 100644 index 3d2027d8df0..00000000000 --- a/apps/sim/app/api/tools/codepipeline/disable-stage-transition/route.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { - CodePipelineClient, - DisableStageTransitionCommand, - type StageTransitionType, -} from '@aws-sdk/client-codepipeline' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCodepipelineDisableStageTransitionContract } from '@/lib/api/contracts/tools/aws/codepipeline-disable-stage-transition' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { awsErrorStatus } from '@/app/api/tools/codepipeline/utils' - -const logger = createLogger('CodePipelineDisableStageTransition') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCodepipelineDisableStageTransitionContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info('Disabling CodePipeline stage transition') - - const client = new CodePipelineClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - try { - const command = new DisableStageTransitionCommand({ - pipelineName: validatedData.pipelineName, - stageName: validatedData.stageName, - transitionType: validatedData.transitionType as StageTransitionType, - reason: validatedData.reason, - }) - - await client.send(command) - - logger.info('Successfully disabled stage transition') - - return NextResponse.json({ - success: true, - output: { - pipelineName: validatedData.pipelineName, - stageName: validatedData.stageName, - transitionType: validatedData.transitionType, - }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('DisableStageTransition failed', { error: toError(error).message }) - return NextResponse.json( - { error: `Failed to disable CodePipeline stage transition: ${toError(error).message}` }, - { status: awsErrorStatus(error) } - ) - } -}) diff --git a/apps/sim/app/api/tools/codepipeline/enable-stage-transition/route.ts b/apps/sim/app/api/tools/codepipeline/enable-stage-transition/route.ts deleted file mode 100644 index a96c4b993c7..00000000000 --- a/apps/sim/app/api/tools/codepipeline/enable-stage-transition/route.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { - CodePipelineClient, - EnableStageTransitionCommand, - type StageTransitionType, -} from '@aws-sdk/client-codepipeline' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCodepipelineEnableStageTransitionContract } from '@/lib/api/contracts/tools/aws/codepipeline-enable-stage-transition' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { awsErrorStatus } from '@/app/api/tools/codepipeline/utils' - -const logger = createLogger('CodePipelineEnableStageTransition') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCodepipelineEnableStageTransitionContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info('Enabling CodePipeline stage transition') - - const client = new CodePipelineClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - try { - const command = new EnableStageTransitionCommand({ - pipelineName: validatedData.pipelineName, - stageName: validatedData.stageName, - transitionType: validatedData.transitionType as StageTransitionType, - }) - - await client.send(command) - - logger.info('Successfully enabled stage transition') - - return NextResponse.json({ - success: true, - output: { - pipelineName: validatedData.pipelineName, - stageName: validatedData.stageName, - transitionType: validatedData.transitionType, - }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('EnableStageTransition failed', { error: toError(error).message }) - return NextResponse.json( - { error: `Failed to enable CodePipeline stage transition: ${toError(error).message}` }, - { status: awsErrorStatus(error) } - ) - } -}) diff --git a/apps/sim/app/api/tools/codepipeline/get-pipeline-execution/route.ts b/apps/sim/app/api/tools/codepipeline/get-pipeline-execution/route.ts deleted file mode 100644 index f56a8153714..00000000000 --- a/apps/sim/app/api/tools/codepipeline/get-pipeline-execution/route.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { CodePipelineClient, GetPipelineExecutionCommand } from '@aws-sdk/client-codepipeline' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCodepipelineGetPipelineExecutionContract } from '@/lib/api/contracts/tools/aws/codepipeline-get-pipeline-execution' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { awsErrorStatus } from '@/app/api/tools/codepipeline/utils' - -const logger = createLogger('CodePipelineGetPipelineExecution') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCodepipelineGetPipelineExecutionContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info('Getting CodePipeline pipeline execution') - - const client = new CodePipelineClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - try { - const command = new GetPipelineExecutionCommand({ - pipelineName: validatedData.pipelineName, - pipelineExecutionId: validatedData.pipelineExecutionId, - }) - - const response = await client.send(command) - const execution = response.pipelineExecution - - if (!execution) { - throw new Error('Pipeline execution not found in response') - } - - logger.info('Successfully got pipeline execution') - - return NextResponse.json({ - success: true, - output: { - pipelineExecutionId: execution.pipelineExecutionId ?? validatedData.pipelineExecutionId, - pipelineName: execution.pipelineName ?? validatedData.pipelineName, - pipelineVersion: execution.pipelineVersion, - status: execution.status ?? 'Unknown', - statusSummary: execution.statusSummary, - executionMode: execution.executionMode, - executionType: execution.executionType, - triggerType: execution.trigger?.triggerType, - triggerDetail: execution.trigger?.triggerDetail, - artifactRevisions: (execution.artifactRevisions ?? []).map((r) => ({ - name: r.name ?? '', - revisionId: r.revisionId, - revisionSummary: r.revisionSummary, - revisionUrl: r.revisionUrl, - created: r.created?.getTime(), - })), - variables: (execution.variables ?? []).map((v) => ({ - name: v.name ?? '', - resolvedValue: v.resolvedValue ?? '', - })), - }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('GetPipelineExecution failed', { error: toError(error).message }) - return NextResponse.json( - { error: `Failed to get CodePipeline pipeline execution: ${toError(error).message}` }, - { status: awsErrorStatus(error) } - ) - } -}) diff --git a/apps/sim/app/api/tools/codepipeline/get-pipeline-state/route.ts b/apps/sim/app/api/tools/codepipeline/get-pipeline-state/route.ts deleted file mode 100644 index 8dcbc8dcf6d..00000000000 --- a/apps/sim/app/api/tools/codepipeline/get-pipeline-state/route.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { CodePipelineClient, GetPipelineStateCommand } from '@aws-sdk/client-codepipeline' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCodepipelineGetPipelineStateContract } from '@/lib/api/contracts/tools/aws/codepipeline-get-pipeline-state' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { awsErrorStatus } from '@/app/api/tools/codepipeline/utils' - -const logger = createLogger('CodePipelineGetPipelineState') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCodepipelineGetPipelineStateContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info('Getting CodePipeline pipeline state') - - const client = new CodePipelineClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - try { - const command = new GetPipelineStateCommand({ name: validatedData.pipelineName }) - const response = await client.send(command) - - const stageStates = (response.stageStates ?? []).map((stage) => ({ - stageName: stage.stageName ?? '', - status: stage.latestExecution?.status, - pipelineExecutionId: stage.latestExecution?.pipelineExecutionId, - inboundTransitionEnabled: stage.inboundTransitionState?.enabled, - actionStates: (stage.actionStates ?? []).map((action) => ({ - actionName: action.actionName ?? '', - status: action.latestExecution?.status, - summary: action.latestExecution?.summary, - lastStatusChange: action.latestExecution?.lastStatusChange?.getTime(), - externalExecutionId: action.latestExecution?.externalExecutionId, - externalExecutionUrl: action.latestExecution?.externalExecutionUrl, - errorCode: action.latestExecution?.errorDetails?.code, - errorMessage: action.latestExecution?.errorDetails?.message, - percentComplete: action.latestExecution?.percentComplete, - token: action.latestExecution?.token, - revisionId: action.currentRevision?.revisionId, - entityUrl: action.entityUrl, - })), - })) - - logger.info(`Successfully got pipeline state with ${stageStates.length} stages`) - - return NextResponse.json({ - success: true, - output: { - pipelineName: response.pipelineName ?? validatedData.pipelineName, - pipelineVersion: response.pipelineVersion, - created: response.created?.getTime(), - updated: response.updated?.getTime(), - stageStates, - }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('GetPipelineState failed', { error: toError(error).message }) - return NextResponse.json( - { error: `Failed to get CodePipeline pipeline state: ${toError(error).message}` }, - { status: awsErrorStatus(error) } - ) - } -}) diff --git a/apps/sim/app/api/tools/codepipeline/get-pipeline/route.ts b/apps/sim/app/api/tools/codepipeline/get-pipeline/route.ts deleted file mode 100644 index 4db9a2e5326..00000000000 --- a/apps/sim/app/api/tools/codepipeline/get-pipeline/route.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { CodePipelineClient, GetPipelineCommand } from '@aws-sdk/client-codepipeline' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCodepipelineGetPipelineContract } from '@/lib/api/contracts/tools/aws/codepipeline-get-pipeline' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { awsErrorStatus } from '@/app/api/tools/codepipeline/utils' - -const logger = createLogger('CodePipelineGetPipeline') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCodepipelineGetPipelineContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info('Getting CodePipeline pipeline structure') - - const client = new CodePipelineClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - try { - const command = new GetPipelineCommand({ - name: validatedData.pipelineName, - ...(validatedData.version !== undefined && { version: validatedData.version }), - }) - const response = await client.send(command) - const pipeline = response.pipeline - - if (!pipeline) { - throw new Error('Pipeline structure not found in response') - } - - const stages = (pipeline.stages ?? []).map((stage) => ({ - stageName: stage.name ?? '', - actions: (stage.actions ?? []).map((action) => ({ - name: action.name ?? '', - category: action.actionTypeId?.category ?? '', - owner: action.actionTypeId?.owner ?? '', - provider: action.actionTypeId?.provider ?? '', - version: action.actionTypeId?.version ?? '', - runOrder: action.runOrder, - configuration: action.configuration ?? {}, - inputArtifacts: (action.inputArtifacts ?? []).map((a) => a.name ?? ''), - outputArtifacts: (action.outputArtifacts ?? []).map((a) => a.name ?? ''), - })), - })) - - logger.info(`Successfully got pipeline structure with ${stages.length} stages`) - - return NextResponse.json({ - success: true, - output: { - pipelineName: pipeline.name ?? validatedData.pipelineName, - pipelineArn: response.metadata?.pipelineArn, - roleArn: pipeline.roleArn ?? '', - version: pipeline.version, - pipelineType: pipeline.pipelineType, - executionMode: pipeline.executionMode, - artifactStoreType: pipeline.artifactStore?.type, - artifactStoreLocation: pipeline.artifactStore?.location, - stages, - variables: (pipeline.variables ?? []).map((v) => ({ - name: v.name ?? '', - defaultValue: v.defaultValue, - description: v.description, - })), - created: response.metadata?.created?.getTime(), - updated: response.metadata?.updated?.getTime(), - }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('GetPipeline failed', { error: toError(error).message }) - return NextResponse.json( - { error: `Failed to get CodePipeline pipeline: ${toError(error).message}` }, - { status: awsErrorStatus(error) } - ) - } -}) diff --git a/apps/sim/app/api/tools/codepipeline/list-action-executions/route.ts b/apps/sim/app/api/tools/codepipeline/list-action-executions/route.ts deleted file mode 100644 index 56cc7324500..00000000000 --- a/apps/sim/app/api/tools/codepipeline/list-action-executions/route.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { CodePipelineClient, ListActionExecutionsCommand } from '@aws-sdk/client-codepipeline' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCodepipelineListActionExecutionsContract } from '@/lib/api/contracts/tools/aws/codepipeline-list-action-executions' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { awsErrorStatus } from '@/app/api/tools/codepipeline/utils' - -const logger = createLogger('CodePipelineListActionExecutions') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCodepipelineListActionExecutionsContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info('Listing CodePipeline action executions') - - const client = new CodePipelineClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - try { - const command = new ListActionExecutionsCommand({ - pipelineName: validatedData.pipelineName, - ...(validatedData.pipelineExecutionId && { - filter: { pipelineExecutionId: validatedData.pipelineExecutionId }, - }), - ...(validatedData.maxResults !== undefined && { maxResults: validatedData.maxResults }), - ...(validatedData.nextToken && { nextToken: validatedData.nextToken }), - }) - - const response = await client.send(command) - - const actionExecutionDetails = (response.actionExecutionDetails ?? []).map((d) => ({ - pipelineExecutionId: d.pipelineExecutionId, - actionExecutionId: d.actionExecutionId, - pipelineVersion: d.pipelineVersion, - stageName: d.stageName, - actionName: d.actionName, - startTime: d.startTime?.getTime(), - lastUpdateTime: d.lastUpdateTime?.getTime(), - updatedBy: d.updatedBy, - status: d.status, - externalExecutionId: d.output?.executionResult?.externalExecutionId, - externalExecutionSummary: d.output?.executionResult?.externalExecutionSummary, - externalExecutionUrl: d.output?.executionResult?.externalExecutionUrl, - errorCode: d.output?.executionResult?.errorDetails?.code, - errorMessage: d.output?.executionResult?.errorDetails?.message, - })) - - logger.info(`Successfully listed ${actionExecutionDetails.length} action executions`) - - return NextResponse.json({ - success: true, - output: { - actionExecutionDetails, - ...(response.nextToken && { nextToken: response.nextToken }), - }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('ListActionExecutions failed', { error: toError(error).message }) - return NextResponse.json( - { error: `Failed to list CodePipeline action executions: ${toError(error).message}` }, - { status: awsErrorStatus(error) } - ) - } -}) diff --git a/apps/sim/app/api/tools/codepipeline/list-pipeline-executions/route.ts b/apps/sim/app/api/tools/codepipeline/list-pipeline-executions/route.ts deleted file mode 100644 index f7083b7b619..00000000000 --- a/apps/sim/app/api/tools/codepipeline/list-pipeline-executions/route.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { CodePipelineClient, ListPipelineExecutionsCommand } from '@aws-sdk/client-codepipeline' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCodepipelineListPipelineExecutionsContract } from '@/lib/api/contracts/tools/aws/codepipeline-list-pipeline-executions' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { awsErrorStatus } from '@/app/api/tools/codepipeline/utils' - -const logger = createLogger('CodePipelineListPipelineExecutions') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCodepipelineListPipelineExecutionsContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info('Listing CodePipeline pipeline executions') - - const client = new CodePipelineClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - try { - const command = new ListPipelineExecutionsCommand({ - pipelineName: validatedData.pipelineName, - ...(validatedData.maxResults !== undefined && { maxResults: validatedData.maxResults }), - ...(validatedData.nextToken && { nextToken: validatedData.nextToken }), - ...(validatedData.succeededInStage && { - filter: { succeededInStage: { stageName: validatedData.succeededInStage } }, - }), - }) - - const response = await client.send(command) - - const executions = (response.pipelineExecutionSummaries ?? []).map((e) => ({ - pipelineExecutionId: e.pipelineExecutionId ?? '', - status: e.status ?? 'Unknown', - statusSummary: e.statusSummary, - startTime: e.startTime?.getTime(), - lastUpdateTime: e.lastUpdateTime?.getTime(), - executionMode: e.executionMode, - executionType: e.executionType, - stopTriggerReason: e.stopTrigger?.reason, - triggerType: e.trigger?.triggerType, - triggerDetail: e.trigger?.triggerDetail, - rollbackTargetPipelineExecutionId: e.rollbackMetadata?.rollbackTargetPipelineExecutionId, - sourceRevisions: (e.sourceRevisions ?? []).map((r) => ({ - actionName: r.actionName ?? '', - revisionId: r.revisionId, - revisionSummary: r.revisionSummary, - revisionUrl: r.revisionUrl, - })), - })) - - logger.info(`Successfully listed ${executions.length} pipeline executions`) - - return NextResponse.json({ - success: true, - output: { - executions, - ...(response.nextToken && { nextToken: response.nextToken }), - }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('ListPipelineExecutions failed', { error: toError(error).message }) - return NextResponse.json( - { error: `Failed to list CodePipeline pipeline executions: ${toError(error).message}` }, - { status: awsErrorStatus(error) } - ) - } -}) diff --git a/apps/sim/app/api/tools/codepipeline/list-pipelines/route.ts b/apps/sim/app/api/tools/codepipeline/list-pipelines/route.ts deleted file mode 100644 index bf3cb9e96d9..00000000000 --- a/apps/sim/app/api/tools/codepipeline/list-pipelines/route.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { CodePipelineClient, ListPipelinesCommand } from '@aws-sdk/client-codepipeline' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCodepipelineListPipelinesContract } from '@/lib/api/contracts/tools/aws/codepipeline-list-pipelines' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { awsErrorStatus } from '@/app/api/tools/codepipeline/utils' - -const logger = createLogger('CodePipelineListPipelines') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCodepipelineListPipelinesContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info('Listing CodePipeline pipelines') - - const client = new CodePipelineClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - try { - const command = new ListPipelinesCommand({ - ...(validatedData.maxResults !== undefined && { maxResults: validatedData.maxResults }), - ...(validatedData.nextToken && { nextToken: validatedData.nextToken }), - }) - - const response = await client.send(command) - - const pipelines = (response.pipelines ?? []).map((p) => ({ - name: p.name ?? '', - version: p.version, - pipelineType: p.pipelineType, - executionMode: p.executionMode, - created: p.created?.getTime(), - updated: p.updated?.getTime(), - })) - - logger.info(`Successfully listed ${pipelines.length} pipelines`) - - return NextResponse.json({ - success: true, - output: { - pipelines, - ...(response.nextToken && { nextToken: response.nextToken }), - }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('ListPipelines failed', { error: toError(error).message }) - return NextResponse.json( - { error: `Failed to list CodePipeline pipelines: ${toError(error).message}` }, - { status: awsErrorStatus(error) } - ) - } -}) diff --git a/apps/sim/app/api/tools/codepipeline/put-approval-result/route.ts b/apps/sim/app/api/tools/codepipeline/put-approval-result/route.ts deleted file mode 100644 index c58d2c27dc6..00000000000 --- a/apps/sim/app/api/tools/codepipeline/put-approval-result/route.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { - type ApprovalStatus, - CodePipelineClient, - PutApprovalResultCommand, -} from '@aws-sdk/client-codepipeline' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCodepipelinePutApprovalResultContract } from '@/lib/api/contracts/tools/aws/codepipeline-put-approval-result' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { awsErrorStatus } from '@/app/api/tools/codepipeline/utils' - -const logger = createLogger('CodePipelinePutApprovalResult') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCodepipelinePutApprovalResultContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info('Submitting CodePipeline approval result') - - const client = new CodePipelineClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - try { - const command = new PutApprovalResultCommand({ - pipelineName: validatedData.pipelineName, - stageName: validatedData.stageName, - actionName: validatedData.actionName, - token: validatedData.token, - result: { - status: validatedData.status as ApprovalStatus, - summary: validatedData.summary, - }, - }) - - const response = await client.send(command) - - logger.info('Successfully submitted approval result') - - return NextResponse.json({ - success: true, - output: { - approvedAt: response.approvedAt?.getTime(), - status: validatedData.status, - }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('PutApprovalResult failed', { error: toError(error).message }) - return NextResponse.json( - { error: `Failed to submit CodePipeline approval result: ${toError(error).message}` }, - { status: awsErrorStatus(error) } - ) - } -}) diff --git a/apps/sim/app/api/tools/codepipeline/retry-stage-execution/route.ts b/apps/sim/app/api/tools/codepipeline/retry-stage-execution/route.ts deleted file mode 100644 index 0886a5409f9..00000000000 --- a/apps/sim/app/api/tools/codepipeline/retry-stage-execution/route.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { - CodePipelineClient, - RetryStageExecutionCommand, - type StageRetryMode, -} from '@aws-sdk/client-codepipeline' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCodepipelineRetryStageExecutionContract } from '@/lib/api/contracts/tools/aws/codepipeline-retry-stage-execution' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { awsErrorStatus } from '@/app/api/tools/codepipeline/utils' - -const logger = createLogger('CodePipelineRetryStageExecution') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCodepipelineRetryStageExecutionContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info('Retrying CodePipeline stage execution') - - const client = new CodePipelineClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - try { - const command = new RetryStageExecutionCommand({ - pipelineName: validatedData.pipelineName, - stageName: validatedData.stageName, - pipelineExecutionId: validatedData.pipelineExecutionId, - retryMode: validatedData.retryMode as StageRetryMode, - }) - - const response = await client.send(command) - - logger.info('Successfully retried stage execution') - - return NextResponse.json({ - success: true, - output: { - pipelineExecutionId: response.pipelineExecutionId ?? validatedData.pipelineExecutionId, - }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('RetryStageExecution failed', { error: toError(error).message }) - return NextResponse.json( - { error: `Failed to retry CodePipeline stage execution: ${toError(error).message}` }, - { status: awsErrorStatus(error) } - ) - } -}) diff --git a/apps/sim/app/api/tools/codepipeline/start-execution/route.ts b/apps/sim/app/api/tools/codepipeline/start-execution/route.ts deleted file mode 100644 index 6d438d1bd7e..00000000000 --- a/apps/sim/app/api/tools/codepipeline/start-execution/route.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { CodePipelineClient, StartPipelineExecutionCommand } from '@aws-sdk/client-codepipeline' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCodepipelineStartExecutionContract } from '@/lib/api/contracts/tools/aws/codepipeline-start-execution' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { awsErrorStatus } from '@/app/api/tools/codepipeline/utils' - -const logger = createLogger('CodePipelineStartExecution') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCodepipelineStartExecutionContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info('Starting CodePipeline pipeline execution') - - const client = new CodePipelineClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - try { - const command = new StartPipelineExecutionCommand({ - name: validatedData.pipelineName, - ...(validatedData.clientRequestToken && { - clientRequestToken: validatedData.clientRequestToken, - }), - ...(validatedData.variables && - validatedData.variables.length > 0 && { variables: validatedData.variables }), - }) - - const response = await client.send(command) - - if (!response.pipelineExecutionId) { - throw new Error('No pipeline execution ID returned') - } - - logger.info('Successfully started pipeline execution') - - return NextResponse.json({ - success: true, - output: { pipelineExecutionId: response.pipelineExecutionId }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('StartPipelineExecution failed', { error: toError(error).message }) - return NextResponse.json( - { error: `Failed to start CodePipeline pipeline execution: ${toError(error).message}` }, - { status: awsErrorStatus(error) } - ) - } -}) diff --git a/apps/sim/app/api/tools/codepipeline/stop-execution/route.ts b/apps/sim/app/api/tools/codepipeline/stop-execution/route.ts deleted file mode 100644 index bf01f37945d..00000000000 --- a/apps/sim/app/api/tools/codepipeline/stop-execution/route.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { CodePipelineClient, StopPipelineExecutionCommand } from '@aws-sdk/client-codepipeline' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsCodepipelineStopExecutionContract } from '@/lib/api/contracts/tools/aws/codepipeline-stop-execution' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { awsErrorStatus } from '@/app/api/tools/codepipeline/utils' - -const logger = createLogger('CodePipelineStopExecution') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsCodepipelineStopExecutionContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info('Stopping CodePipeline pipeline execution') - - const client = new CodePipelineClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - try { - const command = new StopPipelineExecutionCommand({ - pipelineName: validatedData.pipelineName, - pipelineExecutionId: validatedData.pipelineExecutionId, - ...(validatedData.abandon !== undefined && { abandon: validatedData.abandon }), - ...(validatedData.reason && { reason: validatedData.reason }), - }) - - const response = await client.send(command) - - logger.info('Successfully stopped pipeline execution') - - return NextResponse.json({ - success: true, - output: { - pipelineExecutionId: response.pipelineExecutionId ?? validatedData.pipelineExecutionId, - }, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('StopPipelineExecution failed', { error: toError(error).message }) - return NextResponse.json( - { error: `Failed to stop CodePipeline pipeline execution: ${toError(error).message}` }, - { status: awsErrorStatus(error) } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/attachment/route.ts b/apps/sim/app/api/tools/confluence/attachment/route.ts deleted file mode 100644 index 81e2c1465af..00000000000 --- a/apps/sim/app/api/tools/confluence/attachment/route.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { confluenceDeleteAttachmentContract } from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluenceAttachmentAPI') - -export const dynamic = 'force-dynamic' - -// Delete an attachment -export const DELETE = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceDeleteAttachmentContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, cloudId: providedCloudId, attachmentId } = parsed.data.body - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!attachmentId) { - return NextResponse.json({ error: 'Attachment ID is required' }, { status: 400 }) - } - - const attachmentIdValidation = validateAlphanumericId(attachmentId, 'attachmentId', 255) - if (!attachmentIdValidation.isValid) { - return NextResponse.json({ error: attachmentIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/attachments/${attachmentId}` - - const response = await fetch(url, { - method: 'DELETE', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - return NextResponse.json({ attachmentId, deleted: true }) - } catch (error) { - logger.error('Error deleting Confluence attachment:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/attachments/route.ts b/apps/sim/app/api/tools/confluence/attachments/route.ts deleted file mode 100644 index e9a0afa691b..00000000000 --- a/apps/sim/app/api/tools/confluence/attachments/route.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { confluenceListAttachmentsContract } from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluenceAttachmentsAPI') - -export const dynamic = 'force-dynamic' - -// List attachments on a page -export const GET = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceListAttachmentsContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - pageId, - cloudId: providedCloudId, - limit, - cursor, - } = parsed.data.query - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!pageId) { - return NextResponse.json({ error: 'Page ID is required' }, { status: 400 }) - } - - const pageIdValidation = validateAlphanumericId(pageId, 'pageId', 255) - if (!pageIdValidation.isValid) { - return NextResponse.json({ error: pageIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const queryParams = new URLSearchParams() - queryParams.append('limit', String(Math.min(Number(limit), 250))) - if (cursor) { - queryParams.append('cursor', cursor) - } - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${pageId}/attachments?${queryParams.toString()}` - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - - const attachments = (data.results || []).map((attachment: any) => ({ - id: attachment.id, - title: attachment.title, - fileSize: attachment.fileSize || 0, - mediaType: attachment.mediaType || '', - downloadUrl: attachment.downloadLink || attachment._links?.download || '', - status: attachment.status ?? null, - webuiUrl: attachment._links?.webui ?? null, - pageId: attachment.pageId ?? null, - blogPostId: attachment.blogPostId ?? null, - comment: attachment.comment ?? null, - version: attachment.version ?? null, - })) - - return NextResponse.json({ - attachments, - nextCursor: data._links?.next - ? new URL(data._links.next, 'https://placeholder').searchParams.get('cursor') - : null, - }) - } catch (error) { - logger.error('Error listing Confluence attachments:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/blogposts/route.ts b/apps/sim/app/api/tools/confluence/blogposts/route.ts deleted file mode 100644 index 458f2c7cba8..00000000000 --- a/apps/sim/app/api/tools/confluence/blogposts/route.ts +++ /dev/null @@ -1,413 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { - confluenceBlogPostOperationContract, - confluenceDeleteBlogPostContract, - confluenceListBlogPostsContract, - confluenceUpdateBlogPostContract, -} from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluenceBlogPostsAPI') - -export const dynamic = 'force-dynamic' - -/** - * List all blog posts or get a specific blog post - */ -export const GET = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceListBlogPostsContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: providedCloudId, - limit, - status, - sort: sortOrder, - cursor, - } = parsed.data.query - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const queryParams = new URLSearchParams() - queryParams.append('limit', String(Math.min(Number(limit), 250))) - - if (status) { - queryParams.append('status', status) - } - - if (sortOrder) { - queryParams.append('sort', sortOrder) - } - - if (cursor) { - queryParams.append('cursor', cursor) - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/blogposts?${queryParams.toString()}` - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - - const blogPosts = (data.results || []).map((post: any) => ({ - id: post.id, - title: post.title, - status: post.status ?? null, - spaceId: post.spaceId ?? null, - authorId: post.authorId ?? null, - createdAt: post.createdAt ?? null, - version: post.version ?? null, - webUrl: post._links?.webui ?? null, - })) - - return NextResponse.json({ - blogPosts, - nextCursor: data._links?.next - ? new URL(data._links.next, 'https://placeholder').searchParams.get('cursor') - : null, - }) - } catch (error) { - logger.error('Error listing blog posts:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) - -/** - * Get a specific blog post by ID - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceBlogPostOperationContract, request, {}) - if (!parsed.success) return parsed.response - const body = parsed.data.body - - if ('title' in body && 'content' in body && 'spaceId' in body) { - // Create blog post - const { - domain, - accessToken, - cloudId: providedCloudId, - spaceId, - title, - content, - status, - } = body - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/blogposts` - - const createBody = { - spaceId, - status: status || 'current', - title, - body: { - representation: 'storage', - value: content, - }, - } - - const response = await fetch(url, { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - body: JSON.stringify(createBody), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - return NextResponse.json({ - id: data.id, - title: data.title, - spaceId: data.spaceId, - webUrl: data._links?.webui ?? null, - }) - } - // Get blog post by ID - const { domain, accessToken, cloudId: providedCloudId, blogPostId, bodyFormat } = body - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const queryParams = new URLSearchParams() - if (bodyFormat) { - queryParams.append('body-format', bodyFormat) - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/blogposts/${blogPostId}${queryParams.toString() ? `?${queryParams.toString()}` : ''}` - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - return NextResponse.json({ - id: data.id, - title: data.title, - status: data.status ?? null, - spaceId: data.spaceId ?? null, - authorId: data.authorId ?? null, - createdAt: data.createdAt ?? null, - version: data.version ?? null, - body: data.body ?? null, - webUrl: data._links?.webui ?? null, - }) - } catch (error) { - logger.error('Error with blog post operation:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) - -/** - * Update a blog post - */ -export const PUT = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceUpdateBlogPostContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - blogPostId, - title, - content, - cloudId: providedCloudId, - } = parsed.data.body - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - // Fetch current blog post to get version number - const currentUrl = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/blogposts/${blogPostId}?body-format=storage` - const currentResponse = await fetch(currentUrl, { - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!currentResponse.ok) { - const errorText = await currentResponse.text() - throw new Error( - parseAtlassianErrorMessage(currentResponse.status, currentResponse.statusText, errorText) - ) - } - - const currentPost = await currentResponse.json() - - if (!currentPost.version?.number) { - return NextResponse.json( - { error: 'Unable to determine current blog post version' }, - { status: 422 } - ) - } - - const currentVersion = currentPost.version.number - - const updateBody: Record = { - id: blogPostId, - version: { number: currentVersion + 1 }, - status: 'current', - title: title || currentPost.title, - body: { - representation: 'storage', - value: content || currentPost.body?.storage?.value || '', - }, - } - - const response = await fetch(currentUrl, { - method: 'PUT', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - body: JSON.stringify(updateBody), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - return NextResponse.json(data) - } catch (error) { - logger.error('Error updating blog post:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) - -/** - * Delete a blog post - */ -export const DELETE = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceDeleteBlogPostContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, blogPostId, cloudId: providedCloudId } = parsed.data.body - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/blogposts/${blogPostId}` - - const response = await fetch(url, { - method: 'DELETE', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - return NextResponse.json({ blogPostId, deleted: true }) - } catch (error) { - logger.error('Error deleting blog post:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/comment/route.ts b/apps/sim/app/api/tools/confluence/comment/route.ts deleted file mode 100644 index 07ddef573d1..00000000000 --- a/apps/sim/app/api/tools/confluence/comment/route.ts +++ /dev/null @@ -1,205 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { - confluenceDeleteCommentContract, - confluenceUpdateCommentContract, -} from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluenceCommentAPI') - -export const dynamic = 'force-dynamic' - -// Update a comment -export const PUT = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceUpdateCommentContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, cloudId: providedCloudId, commentId, comment } = parsed.data.body - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - // Detect comment type — try footer-comments first, fall back to inline-comments - const apiBase = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2` - let commentEndpoint = 'footer-comments' - let getResponse = await fetch(`${apiBase}/footer-comments/${commentId}`, { - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (getResponse.status === 404) { - commentEndpoint = 'inline-comments' - getResponse = await fetch(`${apiBase}/inline-comments/${commentId}`, { - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - } - - if (!getResponse.ok) { - const errorText = await getResponse.text() - throw new Error( - parseAtlassianErrorMessage(getResponse.status, getResponse.statusText, errorText) - ) - } - - const currentComment = await getResponse.json() - const currentVersion = currentComment.version?.number || 1 - - const url = `${apiBase}/${commentEndpoint}/${commentId}` - - const updateBody = { - body: { - representation: 'storage', - value: comment, - }, - version: { - number: currentVersion + 1, - message: 'Updated via Sim', - }, - } - - const response = await fetch(url, { - method: 'PUT', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - body: JSON.stringify(updateBody), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - return NextResponse.json(data) - } catch (error) { - logger.error('Error updating Confluence comment:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) - -// Delete a comment -export const DELETE = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceDeleteCommentContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, cloudId: providedCloudId, commentId } = parsed.data.body - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const apiBase = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2` - - // Detect comment type with a non-destructive GET so a 404 from a prior - // deletion isn't masked by a second DELETE attempt against the wrong endpoint. - let commentEndpoint = 'footer-comments' - let detectResponse = await fetch(`${apiBase}/footer-comments/${commentId}`, { - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (detectResponse.status === 404) { - commentEndpoint = 'inline-comments' - detectResponse = await fetch(`${apiBase}/inline-comments/${commentId}`, { - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - } - - if (!detectResponse.ok) { - const errorText = await detectResponse.text() - logger.error('Confluence API error response:', { - status: detectResponse.status, - statusText: detectResponse.statusText, - error: errorText, - }) - return NextResponse.json( - { - error: parseAtlassianErrorMessage( - detectResponse.status, - detectResponse.statusText, - errorText - ), - }, - { status: detectResponse.status } - ) - } - - const response = await fetch(`${apiBase}/${commentEndpoint}/${commentId}`, { - method: 'DELETE', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - return NextResponse.json({ commentId, deleted: true }) - } catch (error) { - logger.error('Error deleting Confluence comment:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/comments/route.ts b/apps/sim/app/api/tools/confluence/comments/route.ts deleted file mode 100644 index 88484f7853f..00000000000 --- a/apps/sim/app/api/tools/confluence/comments/route.ts +++ /dev/null @@ -1,213 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { - confluenceCreateCommentContract, - confluenceListCommentsContract, -} from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluenceCommentsAPI') - -export const dynamic = 'force-dynamic' - -// Create a comment -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceCreateCommentContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, cloudId: providedCloudId, pageId, comment } = parsed.data.body - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!pageId) { - return NextResponse.json({ error: 'Page ID is required' }, { status: 400 }) - } - - if (!comment) { - return NextResponse.json({ error: 'Comment is required' }, { status: 400 }) - } - - const pageIdValidation = validateAlphanumericId(pageId, 'pageId', 255) - if (!pageIdValidation.isValid) { - return NextResponse.json({ error: pageIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/footer-comments` - - logger.info('Calling Confluence API', { url }) - - const body = { - pageId, - body: { - representation: 'storage', - value: comment, - }, - } - - const response = await fetch(url, { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - body: JSON.stringify(body), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - return NextResponse.json({ ...data, pageId }) - } catch (error) { - logger.error('Error creating Confluence comment:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) - -// List comments on a page -export const GET = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceListCommentsContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - pageId, - cloudId: providedCloudId, - limit, - bodyFormat, - cursor, - } = parsed.data.query - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!pageId) { - return NextResponse.json({ error: 'Page ID is required' }, { status: 400 }) - } - - const pageIdValidation = validateAlphanumericId(pageId, 'pageId', 255) - if (!pageIdValidation.isValid) { - return NextResponse.json({ error: pageIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const queryParams = new URLSearchParams() - queryParams.append('limit', String(Math.min(Number(limit), 250))) - queryParams.append('body-format', bodyFormat) - if (cursor) { - queryParams.append('cursor', cursor) - } - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${pageId}/footer-comments?${queryParams.toString()}` - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - - const comments = (data.results || []).map((comment: any) => { - const bodyValue = comment.body?.storage?.value || comment.body?.view?.value || '' - return { - id: comment.id, - body: { - value: bodyValue, - representation: bodyFormat, - }, - createdAt: comment.createdAt || '', - authorId: comment.authorId || '', - status: comment.status ?? null, - title: comment.title ?? null, - pageId: comment.pageId ?? null, - blogPostId: comment.blogPostId ?? null, - parentCommentId: comment.parentCommentId ?? null, - version: comment.version ?? null, - } - }) - - return NextResponse.json({ - comments, - nextCursor: data._links?.next - ? new URL(data._links.next, 'https://placeholder').searchParams.get('cursor') - : null, - }) - } catch (error) { - logger.error('Error listing Confluence comments:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/create-page/route.ts b/apps/sim/app/api/tools/confluence/create-page/route.ts deleted file mode 100644 index 1fa8fd95836..00000000000 --- a/apps/sim/app/api/tools/confluence/create-page/route.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { confluenceCreatePageContract } from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluenceCreatePageAPI') - -export const dynamic = 'force-dynamic' - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceCreatePageContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: providedCloudId, - spaceId, - title, - content, - parentId, - } = parsed.data.body - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!spaceId) { - return NextResponse.json({ error: 'Space ID is required' }, { status: 400 }) - } - - if (!/^\d+$/.test(String(spaceId))) { - return NextResponse.json( - { - error: - 'Invalid Space ID. The Space ID must be a numeric value, not the space key from the URL. Use the "list" operation to get all spaces with their numeric IDs.', - }, - { status: 400 } - ) - } - - if (!title) { - return NextResponse.json({ error: 'Title is required' }, { status: 400 }) - } - - if (!content) { - return NextResponse.json({ error: 'Content is required' }, { status: 400 }) - } - - const spaceIdValidation = validateAlphanumericId(spaceId, 'spaceId', 255) - if (!spaceIdValidation.isValid) { - return NextResponse.json({ error: spaceIdValidation.error }, { status: 400 }) - } - - if (parentId) { - const parentIdValidation = validateAlphanumericId(parentId, 'parentId', 255) - if (!parentIdValidation.isValid) { - return NextResponse.json({ error: parentIdValidation.error }, { status: 400 }) - } - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const createBody: any = { - spaceId, - status: 'current', - title, - body: { - representation: 'storage', - value: content, - }, - } - - if (parentId !== undefined && parentId !== null && parentId !== '') { - createBody.parentId = parentId - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages` - - const response = await fetch(url, { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - body: JSON.stringify(createBody), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - let errorMessage = parseAtlassianErrorMessage(response.status, response.statusText, errorText) - if (errorMessage.includes("'spaceId'") && errorMessage.includes('Long')) { - errorMessage = 'Invalid Space ID. Use the list spaces operation to find valid space IDs.' - } - return NextResponse.json({ error: errorMessage }, { status: response.status }) - } - - const data = await response.json() - return NextResponse.json(data) - } catch (error) { - logger.error('Error creating Confluence page:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/labels/route.ts b/apps/sim/app/api/tools/confluence/labels/route.ts deleted file mode 100644 index 4d7316fb424..00000000000 --- a/apps/sim/app/api/tools/confluence/labels/route.ts +++ /dev/null @@ -1,289 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { - confluenceDeleteLabelContract, - confluenceLabelMutationContract, - confluenceListLabelsContract, -} from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluenceLabelsAPI') - -export const dynamic = 'force-dynamic' - -// Add a label to a page -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceLabelMutationContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: providedCloudId, - pageId, - labelName, - prefix: labelPrefix, - } = parsed.data.body - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!pageId) { - return NextResponse.json({ error: 'Page ID is required' }, { status: 400 }) - } - - if (!labelName) { - return NextResponse.json({ error: 'Label name is required' }, { status: 400 }) - } - - const pageIdValidation = validateAlphanumericId(pageId, 'pageId', 255) - if (!pageIdValidation.isValid) { - return NextResponse.json({ error: pageIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/rest/api/content/${pageId}/label` - - const body = [ - { - prefix: labelPrefix || 'global', - name: labelName, - }, - ] - - const response = await fetch(url, { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - body: JSON.stringify(body), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - const addedLabel = data.results?.[0] || data[0] || data - return NextResponse.json({ - id: addedLabel.id ?? '', - name: addedLabel.name ?? labelName, - prefix: addedLabel.prefix ?? labelPrefix ?? 'global', - pageId, - labelName, - }) - } catch (error) { - logger.error('Error adding Confluence label:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) - -// List labels on a page -export const GET = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceListLabelsContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - pageId, - cloudId: providedCloudId, - limit, - cursor, - } = parsed.data.query - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!pageId) { - return NextResponse.json({ error: 'Page ID is required' }, { status: 400 }) - } - - const pageIdValidation = validateAlphanumericId(pageId, 'pageId', 255) - if (!pageIdValidation.isValid) { - return NextResponse.json({ error: pageIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const queryParams = new URLSearchParams() - queryParams.append('limit', String(Math.min(Number(limit), 250))) - if (cursor) { - queryParams.append('cursor', cursor) - } - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${pageId}/labels?${queryParams.toString()}` - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - - const labels = (data.results || []).map((label: any) => ({ - id: label.id, - name: label.name, - prefix: label.prefix || 'global', - })) - - return NextResponse.json({ - labels, - nextCursor: data._links?.next - ? new URL(data._links.next, 'https://placeholder').searchParams.get('cursor') - : null, - }) - } catch (error) { - logger.error('Error listing Confluence labels:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) - -// Delete a label from a page -export const DELETE = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceDeleteLabelContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, cloudId: providedCloudId, pageId, labelName } = parsed.data.body - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!pageId) { - return NextResponse.json({ error: 'Page ID is required' }, { status: 400 }) - } - - if (!labelName) { - return NextResponse.json({ error: 'Label name is required' }, { status: 400 }) - } - - const pageIdValidation = validateAlphanumericId(pageId, 'pageId', 255) - if (!pageIdValidation.isValid) { - return NextResponse.json({ error: pageIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const encodedLabel = encodeURIComponent(labelName.trim()) - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/rest/api/content/${pageId}/label?name=${encodedLabel}` - - const response = await fetch(url, { - method: 'DELETE', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - return NextResponse.json({ - pageId, - labelName, - deleted: true, - }) - } catch (error) { - logger.error('Error deleting Confluence label:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/page-ancestors/route.ts b/apps/sim/app/api/tools/confluence/page-ancestors/route.ts deleted file mode 100644 index dd982c12afe..00000000000 --- a/apps/sim/app/api/tools/confluence/page-ancestors/route.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { confluencePageAncestorsContract } from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluencePageAncestorsAPI') - -export const dynamic = 'force-dynamic' - -/** - * Get ancestors (parent pages) of a specific Confluence page. - * Uses GET /wiki/api/v2/pages/{id}/ancestors - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluencePageAncestorsContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, pageId, cloudId: providedCloudId, limit } = parsed.data.body - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!pageId) { - return NextResponse.json({ error: 'Page ID is required' }, { status: 400 }) - } - - const pageIdValidation = validateAlphanumericId(pageId, 'pageId', 255) - if (!pageIdValidation.isValid) { - return NextResponse.json({ error: pageIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const queryParams = new URLSearchParams() - queryParams.append('limit', String(Math.min(limit, 250))) - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${pageId}/ancestors?${queryParams.toString()}` - - logger.info(`Fetching ancestors for page ${pageId}`) - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - - const ancestors = (data.results || []).map((page: any) => ({ - id: page.id, - title: page.title, - status: page.status ?? null, - spaceId: page.spaceId ?? null, - webUrl: page._links?.webui ?? null, - })) - - return NextResponse.json({ - ancestors, - pageId, - }) - } catch (error) { - logger.error('Error getting page ancestors:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/page-children/route.ts b/apps/sim/app/api/tools/confluence/page-children/route.ts deleted file mode 100644 index e3b0f18bef2..00000000000 --- a/apps/sim/app/api/tools/confluence/page-children/route.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { confluencePageChildrenContract } from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluencePageChildrenAPI') - -export const dynamic = 'force-dynamic' - -/** - * Get child pages of a specific Confluence page. - * Uses GET /wiki/api/v2/pages/{id}/children - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluencePageChildrenContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - pageId, - cloudId: providedCloudId, - limit, - cursor, - } = parsed.data.body - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!pageId) { - return NextResponse.json({ error: 'Page ID is required' }, { status: 400 }) - } - - const pageIdValidation = validateAlphanumericId(pageId, 'pageId', 255) - if (!pageIdValidation.isValid) { - return NextResponse.json({ error: pageIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const queryParams = new URLSearchParams() - queryParams.append('limit', String(Math.min(limit, 250))) - - if (cursor) { - queryParams.append('cursor', cursor) - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${pageId}/children?${queryParams.toString()}` - - logger.info(`Fetching child pages for page ${pageId}`) - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - - const children = (data.results || []).map((page: any) => ({ - id: page.id, - title: page.title, - status: page.status ?? null, - spaceId: page.spaceId ?? null, - childPosition: page.childPosition ?? null, - webUrl: page._links?.webui ?? null, - })) - - return NextResponse.json({ - children, - parentId: pageId, - nextCursor: data._links?.next - ? new URL(data._links.next, 'https://placeholder').searchParams.get('cursor') - : null, - }) - } catch (error) { - logger.error('Error getting child pages:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/page-descendants/route.ts b/apps/sim/app/api/tools/confluence/page-descendants/route.ts deleted file mode 100644 index 3e021760653..00000000000 --- a/apps/sim/app/api/tools/confluence/page-descendants/route.ts +++ /dev/null @@ -1,129 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { confluencePageDescendantsContract } from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { - validateAlphanumericId, - validateJiraCloudId, - validatePaginationCursor, -} from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluencePageDescendantsAPI') - -export const dynamic = 'force-dynamic' - -/** - * Get all descendants of a Confluence page recursively. - * Uses GET /wiki/api/v2/pages/{id}/descendants - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluencePageDescendantsContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - pageId, - cloudId: providedCloudId, - limit, - cursor, - } = parsed.data.body - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!pageId) { - return NextResponse.json({ error: 'Page ID is required' }, { status: 400 }) - } - - const pageIdValidation = validateAlphanumericId(pageId, 'pageId', 255) - if (!pageIdValidation.isValid) { - return NextResponse.json({ error: pageIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const queryParams = new URLSearchParams() - queryParams.append('limit', String(Math.min(limit, 250))) - - if (cursor) { - const cursorValidation = validatePaginationCursor(cursor, 'cursor') - if (!cursorValidation.isValid) { - return NextResponse.json({ error: cursorValidation.error }, { status: 400 }) - } - queryParams.append('cursor', cursor) - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${pageId}/descendants?${queryParams.toString()}` - - logger.info(`Fetching descendants for page ${pageId}`) - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - - const descendants = (data.results || []).map((page: any) => ({ - id: page.id, - title: page.title, - type: page.type ?? null, - status: page.status ?? null, - spaceId: page.spaceId ?? null, - parentId: page.parentId ?? null, - childPosition: page.childPosition ?? null, - depth: page.depth ?? null, - })) - - return NextResponse.json({ - descendants, - pageId, - nextCursor: data._links?.next - ? new URL(data._links.next, 'https://placeholder').searchParams.get('cursor') - : null, - }) - } catch (error) { - logger.error('Error getting page descendants:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/page-properties/route.ts b/apps/sim/app/api/tools/confluence/page-properties/route.ts deleted file mode 100644 index 03ee8d1630e..00000000000 --- a/apps/sim/app/api/tools/confluence/page-properties/route.ts +++ /dev/null @@ -1,371 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { - confluenceCreatePagePropertyContract, - confluenceDeletePagePropertyContract, - confluenceListPagePropertiesContract, - confluenceUpdatePagePropertyContract, -} from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluencePagePropertiesAPI') - -export const dynamic = 'force-dynamic' - -/** - * List all content properties on a page. - */ -export const GET = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceListPagePropertiesContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - pageId, - cloudId: providedCloudId, - limit, - cursor, - } = parsed.data.query - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!pageId) { - return NextResponse.json({ error: 'Page ID is required' }, { status: 400 }) - } - - const pageIdValidation = validateAlphanumericId(pageId, 'pageId', 255) - if (!pageIdValidation.isValid) { - return NextResponse.json({ error: pageIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const queryParams = new URLSearchParams() - queryParams.append('limit', String(Math.min(Number(limit), 250))) - if (cursor) { - queryParams.append('cursor', cursor) - } - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${pageId}/properties?${queryParams.toString()}` - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - - const properties = (data.results || []).map((prop: any) => ({ - id: prop.id, - key: prop.key, - value: prop.value ?? null, - version: prop.version ?? null, - })) - - return NextResponse.json({ - properties, - pageId, - nextCursor: data._links?.next - ? new URL(data._links.next, 'https://placeholder').searchParams.get('cursor') - : null, - }) - } catch (error) { - logger.error('Error listing page properties:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) - -/** - * Create a new content property on a page. - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceCreatePagePropertyContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, cloudId: providedCloudId, pageId, key, value } = parsed.data.body - - const pageIdValidation = validateAlphanumericId(pageId, 'pageId', 255) - if (!pageIdValidation.isValid) { - return NextResponse.json({ error: pageIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${pageId}/properties` - - const response = await fetch(url, { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - body: JSON.stringify({ key, value }), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - return NextResponse.json({ - id: data.id, - key: data.key, - value: data.value, - version: data.version, - pageId, - }) - } catch (error) { - logger.error('Error creating page property:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) - -/** - * Update a content property on a page. - */ -export const PUT = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceUpdatePagePropertyContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: providedCloudId, - pageId, - propertyId, - key, - value, - versionNumber, - } = parsed.data.body - - const pageIdValidation = validateAlphanumericId(pageId, 'pageId', 255) - if (!pageIdValidation.isValid) { - return NextResponse.json({ error: pageIdValidation.error }, { status: 400 }) - } - - const propertyIdValidation = validateAlphanumericId(propertyId, 'propertyId', 255) - if (!propertyIdValidation.isValid) { - return NextResponse.json({ error: propertyIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${pageId}/properties/${propertyId}` - - let nextVersion = versionNumber - if (nextVersion === undefined) { - const lookupResponse = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - if (!lookupResponse.ok) { - const errorText = await lookupResponse.text() - return NextResponse.json( - { - error: parseAtlassianErrorMessage( - lookupResponse.status, - lookupResponse.statusText, - errorText - ), - }, - { status: lookupResponse.status } - ) - } - const current = await lookupResponse.json() - const currentNumber = current?.version?.number - if (typeof currentNumber !== 'number') { - return NextResponse.json( - { error: 'Could not determine current property version' }, - { status: 500 } - ) - } - nextVersion = currentNumber + 1 - } - - const response = await fetch(url, { - method: 'PUT', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - body: JSON.stringify({ - key, - value, - version: { number: nextVersion }, - }), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - return NextResponse.json({ - id: data.id, - key: data.key, - value: data.value, - version: data.version, - pageId, - }) - } catch (error) { - logger.error('Error updating page property:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) - -/** - * Delete a content property from a page. - */ -export const DELETE = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceDeletePagePropertyContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, cloudId: providedCloudId, pageId, propertyId } = parsed.data.body - - const pageIdValidation = validateAlphanumericId(pageId, 'pageId', 255) - if (!pageIdValidation.isValid) { - return NextResponse.json({ error: pageIdValidation.error }, { status: 400 }) - } - - const propertyIdValidation = validateAlphanumericId(propertyId, 'propertyId', 255) - if (!propertyIdValidation.isValid) { - return NextResponse.json({ error: propertyIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${pageId}/properties/${propertyId}` - - const response = await fetch(url, { - method: 'DELETE', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - return NextResponse.json({ propertyId, pageId, deleted: true }) - } catch (error) { - logger.error('Error deleting page property:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/page-versions/route.ts b/apps/sim/app/api/tools/confluence/page-versions/route.ts deleted file mode 100644 index 7c424baaf06..00000000000 --- a/apps/sim/app/api/tools/confluence/page-versions/route.ts +++ /dev/null @@ -1,218 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { confluencePageVersionsContract } from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { - validateAlphanumericId, - validateJiraCloudId, - validateNumericId, - validatePaginationCursor, -} from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { cleanHtmlContent, getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluencePageVersionsAPI') - -export const dynamic = 'force-dynamic' - -/** - * List all versions of a page or get a specific version. - * Uses GET /wiki/api/v2/pages/{id}/versions - * and GET /wiki/api/v2/pages/{page-id}/versions/{version-number} - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluencePageVersionsContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - pageId, - versionNumber, - cloudId: providedCloudId, - limit = 50, - cursor, - } = parsed.data.body - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!pageId) { - return NextResponse.json({ error: 'Page ID is required' }, { status: 400 }) - } - - const pageIdValidation = validateAlphanumericId(pageId, 'pageId', 255) - if (!pageIdValidation.isValid) { - return NextResponse.json({ error: pageIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - // If versionNumber is provided, get specific version with page content - if (versionNumber !== undefined && versionNumber !== null) { - const versionValidation = validateNumericId(versionNumber, 'versionNumber', { min: 1 }) - if (!versionValidation.isValid) { - return NextResponse.json({ error: versionValidation.error }, { status: 400 }) - } - const safeVersion = versionValidation.sanitized - - const versionUrl = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${pageId}/versions/${safeVersion}` - const pageUrl = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${pageId}?version=${safeVersion}&body-format=storage` - - logger.info(`Fetching version ${versionNumber} for page ${pageId}`) - - const [versionResponse, pageResponse] = await Promise.all([ - fetch(versionUrl, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }), - fetch(pageUrl, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }), - ]) - - if (!versionResponse.ok) { - const errorText = await versionResponse.text() - logger.error('Confluence API error response:', { - status: versionResponse.status, - statusText: versionResponse.statusText, - error: errorText, - }) - return NextResponse.json( - { - error: parseAtlassianErrorMessage( - versionResponse.status, - versionResponse.statusText, - errorText - ), - }, - { status: versionResponse.status } - ) - } - - const versionData = await versionResponse.json() - - let title: string | null = null - let content: string | null = null - let body: Record | null = null - - if (pageResponse.ok) { - const pageData = await pageResponse.json() - title = pageData.title ?? null - body = pageData.body ?? null - - const rawContent = - pageData.body?.storage?.value || - pageData.body?.view?.value || - pageData.body?.atlas_doc_format?.value || - '' - if (rawContent) { - content = cleanHtmlContent(rawContent) - } - } else { - logger.warn( - `Could not fetch page content for version ${versionNumber}: ${pageResponse.status}` - ) - } - - return NextResponse.json({ - version: { - number: versionData.number, - message: versionData.message ?? null, - minorEdit: versionData.minorEdit ?? false, - authorId: versionData.authorId ?? null, - createdAt: versionData.createdAt ?? null, - }, - pageId, - title, - content, - body, - }) - } - // List all versions - const queryParams = new URLSearchParams() - queryParams.append('limit', String(Math.min(limit, 250))) - - if (cursor) { - const cursorValidation = validatePaginationCursor(cursor, 'cursor') - if (!cursorValidation.isValid) { - return NextResponse.json({ error: cursorValidation.error }, { status: 400 }) - } - queryParams.append('cursor', cursor) - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${pageId}/versions?${queryParams.toString()}` - - logger.info(`Fetching versions for page ${pageId}`) - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - - const versions = (data.results || []).map((version: any) => ({ - number: version.number, - message: version.message ?? null, - minorEdit: version.minorEdit ?? false, - authorId: version.authorId ?? null, - createdAt: version.createdAt ?? null, - })) - - return NextResponse.json({ - versions, - pageId, - nextCursor: data._links?.next - ? new URL(data._links.next, 'https://placeholder').searchParams.get('cursor') - : null, - }) - } catch (error) { - logger.error('Error with page versions:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/page/route.ts b/apps/sim/app/api/tools/confluence/page/route.ts index 5adfe362bc4..fa0f77eea52 100644 --- a/apps/sim/app/api/tools/confluence/page/route.ts +++ b/apps/sim/app/api/tools/confluence/page/route.ts @@ -1,254 +1,15 @@ import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { - confluenceDeletePageContract, - confluencePageSelectorContract, - confluenceUpdatePageContract, -} from '@/lib/api/contracts/selectors/confluence' +import { confluencePageSelectorContract } from '@/lib/api/contracts/selectors/confluence' import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluencePageAPI') +import { createConfluenceHttpRoute } from '@/lib/internal/confluence/http-route' +import { executeConfluenceRetrievePage } from '@/lib/internal/confluence/operations' export const dynamic = 'force-dynamic' -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluencePageSelectorContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, cloudId: providedCloudId, pageId } = parsed.data.body - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${pageId}?body-format=storage` - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - id: data.id, - title: data.title, - body: { - storage: { - value: data.body?.storage?.value ?? null, - representation: 'storage', - }, - }, - status: data.status ?? null, - spaceId: data.spaceId ?? null, - parentId: data.parentId ?? null, - authorId: data.authorId ?? null, - createdAt: data.createdAt ?? null, - version: data.version ?? null, - _links: data._links ?? null, - }) - } catch (error) { - logger.error('Error fetching Confluence page:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) - -export const PUT = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceUpdatePageContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - pageId, - cloudId: providedCloudId, - title, - body: pageBody, - version, - } = parsed.data.body - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const currentPageUrl = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${pageId}?body-format=storage` - const currentPageResponse = await fetch(currentPageUrl, { - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!currentPageResponse.ok) { - const errorText = await currentPageResponse.text() - throw new Error( - parseAtlassianErrorMessage( - currentPageResponse.status, - currentPageResponse.statusText, - errorText - ) - ) - } - - const currentPage = await currentPageResponse.json() - const currentVersion = currentPage.version.number - - const updateBody: any = { - id: pageId, - version: { - number: currentVersion + 1, - message: version?.message || 'Updated via API', - }, - status: 'current', - } - - if (title !== undefined && title !== null && title !== '') { - updateBody.title = title - } else { - updateBody.title = currentPage.title - } - - if (pageBody?.value !== undefined && pageBody?.value !== null && pageBody?.value !== '') { - updateBody.body = { - representation: 'storage', - value: pageBody.value, - } - } else { - updateBody.body = { - representation: 'storage', - value: currentPage.body?.storage?.value || '', - } - } - - const response = await fetch(currentPageUrl, { - method: 'PUT', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - body: JSON.stringify(updateBody), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - return NextResponse.json(data) - } catch (error) { - logger.error('Error updating Confluence page:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) - -export const DELETE = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceDeletePageContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, cloudId: providedCloudId, pageId, purge } = parsed.data.body - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const queryParams = new URLSearchParams() - if (purge) { - queryParams.append('purge', 'true') - } - const queryString = queryParams.toString() - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${pageId}${queryString ? `?${queryString}` : ''}` - - const response = await fetch(url, { - method: 'DELETE', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } +const logger = createLogger('ConfluencePageAPI') - return NextResponse.json({ pageId, deleted: true }) - } catch (error) { - logger.error('Error deleting Confluence page:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } +export const POST = createConfluenceHttpRoute({ + logger, + parse: (request) => parseRequest(confluencePageSelectorContract, request, {}), + execute: executeConfluenceRetrievePage, }) diff --git a/apps/sim/app/api/tools/confluence/pages-by-label/route.ts b/apps/sim/app/api/tools/confluence/pages-by-label/route.ts deleted file mode 100644 index 5d50f49cb3b..00000000000 --- a/apps/sim/app/api/tools/confluence/pages-by-label/route.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { confluencePagesByLabelContract } from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluencePagesByLabelAPI') - -export const dynamic = 'force-dynamic' - -export const GET = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluencePagesByLabelContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - labelId, - cloudId: providedCloudId, - limit, - cursor, - } = parsed.data.query - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!labelId) { - return NextResponse.json({ error: 'Label ID is required' }, { status: 400 }) - } - - const labelIdValidation = validateAlphanumericId(labelId, 'labelId', 255) - if (!labelIdValidation.isValid) { - return NextResponse.json({ error: labelIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const queryParams = new URLSearchParams() - queryParams.append('limit', String(Math.min(Number(limit), 250))) - if (cursor) { - queryParams.append('cursor', cursor) - } - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/labels/${labelId}/pages?${queryParams.toString()}` - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - - const pages = (data.results || []).map((page: any) => ({ - id: page.id, - title: page.title, - status: page.status ?? null, - spaceId: page.spaceId ?? null, - parentId: page.parentId ?? null, - authorId: page.authorId ?? null, - createdAt: page.createdAt ?? null, - version: page.version ?? null, - })) - - return NextResponse.json({ - pages, - labelId, - nextCursor: data._links?.next - ? new URL(data._links.next, 'https://placeholder').searchParams.get('cursor') - : null, - }) - } catch (error) { - logger.error('Error getting pages by label:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/search-in-space/route.ts b/apps/sim/app/api/tools/confluence/search-in-space/route.ts deleted file mode 100644 index 3be5877f6ed..00000000000 --- a/apps/sim/app/api/tools/confluence/search-in-space/route.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { confluenceSearchInSpaceContract } from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluenceSearchInSpaceAPI') - -export const dynamic = 'force-dynamic' - -/** - * Search for content within a specific Confluence space using CQL. - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceSearchInSpaceContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - spaceKey, - query, - cloudId: providedCloudId, - limit = 25, - contentType, - } = parsed.data.body - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!spaceKey) { - return NextResponse.json({ error: 'Space key is required' }, { status: 400 }) - } - - const spaceKeyValidation = validateAlphanumericId(spaceKey, 'spaceKey', 255) - if (!spaceKeyValidation.isValid) { - return NextResponse.json({ error: spaceKeyValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const escapeCqlValue = (value: string) => value.replace(/\\/g, '\\\\').replace(/"/g, '\\"') - - let cql = `space = "${escapeCqlValue(spaceKey)}"` - - if (query) { - cql += ` AND text ~ "${escapeCqlValue(query)}"` - } - - if (contentType) { - cql += ` AND type = "${escapeCqlValue(contentType)}"` - } - - const searchParams = new URLSearchParams({ - cql, - limit: String(Math.min(limit, 250)), - }) - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/rest/api/search?${searchParams.toString()}` - - logger.info(`Searching in space ${spaceKey} with CQL: ${cql}`) - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - - const results = (data.results || []).map((result: any) => ({ - id: result.content?.id ?? result.id, - title: result.content?.title ?? result.title, - type: result.content?.type ?? result.type, - status: result.content?.status ?? null, - url: result.url ?? result._links?.webui ?? '', - excerpt: result.excerpt ?? '', - lastModified: result.lastModified ?? null, - })) - - return NextResponse.json({ - results, - spaceKey, - totalSize: data.totalSize ?? results.length, - }) - } catch (error) { - logger.error('Error searching in space:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/search/route.ts b/apps/sim/app/api/tools/confluence/search/route.ts deleted file mode 100644 index a78ecdfc01a..00000000000 --- a/apps/sim/app/api/tools/confluence/search/route.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { confluenceSearchContract } from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('Confluence Search') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceSearchContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, cloudId: providedCloudId, query, limit } = parsed.data.body - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!query) { - return NextResponse.json({ error: 'Search query is required' }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const escapeCqlValue = (value: string) => value.replace(/\\/g, '\\\\').replace(/"/g, '\\"') - - const searchParams = new URLSearchParams({ - cql: `text ~ "${escapeCqlValue(query)}"`, - limit: limit.toString(), - }) - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/rest/api/search?${searchParams.toString()}` - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - - const results = (data.results || []).map((result: any) => { - const spaceData = result.resultGlobalContainer || result.content?.space - return { - id: result.content?.id || result.id, - title: result.content?.title || result.title, - type: result.content?.type || result.type, - url: result.url || result._links?.webui || '', - excerpt: result.excerpt || '', - status: result.content?.status ?? null, - spaceKey: result.resultGlobalContainer?.key ?? result.content?.space?.key ?? null, - space: spaceData - ? { - id: spaceData.id ?? null, - key: spaceData.key ?? null, - name: spaceData.name ?? spaceData.title ?? null, - } - : null, - lastModified: result.lastModified ?? result.content?.history?.lastUpdated?.when ?? null, - entityType: result.entityType ?? null, - } - }) - - return NextResponse.json({ results }) - } catch (error) { - logger.error('Error searching Confluence:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/space-blogposts/route.ts b/apps/sim/app/api/tools/confluence/space-blogposts/route.ts deleted file mode 100644 index 291af150d89..00000000000 --- a/apps/sim/app/api/tools/confluence/space-blogposts/route.ts +++ /dev/null @@ -1,131 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { confluenceSpaceBlogPostsContract } from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluenceSpaceBlogPostsAPI') - -export const dynamic = 'force-dynamic' - -/** - * List all blog posts in a specific Confluence space. - * Uses GET /wiki/api/v2/spaces/{id}/blogposts - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceSpaceBlogPostsContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - spaceId, - cloudId: providedCloudId, - limit = 25, - status, - bodyFormat, - cursor, - } = parsed.data.body - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!spaceId) { - return NextResponse.json({ error: 'Space ID is required' }, { status: 400 }) - } - - const spaceIdValidation = validateAlphanumericId(spaceId, 'spaceId', 255) - if (!spaceIdValidation.isValid) { - return NextResponse.json({ error: spaceIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const queryParams = new URLSearchParams() - queryParams.append('limit', String(Math.min(limit, 250))) - - if (status) { - queryParams.append('status', status) - } - - if (bodyFormat) { - queryParams.append('body-format', bodyFormat) - } - - if (cursor) { - queryParams.append('cursor', cursor) - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/spaces/${spaceId}/blogposts?${queryParams.toString()}` - - logger.info(`Fetching blog posts in space ${spaceId}`) - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - - const blogPosts = (data.results || []).map((post: any) => ({ - id: post.id, - title: post.title, - status: post.status ?? null, - spaceId: post.spaceId ?? null, - authorId: post.authorId ?? null, - createdAt: post.createdAt ?? null, - version: post.version ?? null, - body: post.body ?? null, - webUrl: post._links?.webui ?? null, - })) - - return NextResponse.json({ - blogPosts, - nextCursor: data._links?.next - ? new URL(data._links.next, 'https://placeholder').searchParams.get('cursor') - : null, - }) - } catch (error) { - logger.error('Error listing blog posts in space:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/space-labels/route.ts b/apps/sim/app/api/tools/confluence/space-labels/route.ts deleted file mode 100644 index 3a1cb43b15b..00000000000 --- a/apps/sim/app/api/tools/confluence/space-labels/route.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { confluenceSpaceLabelsContract } from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluenceSpaceLabelsAPI') - -export const dynamic = 'force-dynamic' - -export const GET = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceSpaceLabelsContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - spaceId, - cloudId: providedCloudId, - limit, - cursor, - } = parsed.data.query - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!spaceId) { - return NextResponse.json({ error: 'Space ID is required' }, { status: 400 }) - } - - const spaceIdValidation = validateAlphanumericId(spaceId, 'spaceId', 255) - if (!spaceIdValidation.isValid) { - return NextResponse.json({ error: spaceIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const queryParams = new URLSearchParams() - queryParams.append('limit', String(Math.min(Number(limit), 250))) - if (cursor) { - queryParams.append('cursor', cursor) - } - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/spaces/${spaceId}/labels?${queryParams.toString()}` - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - - const labels = (data.results || []).map((label: any) => ({ - id: label.id, - name: label.name, - prefix: label.prefix || 'global', - })) - - return NextResponse.json({ - labels, - spaceId, - nextCursor: data._links?.next - ? new URL(data._links.next, 'https://placeholder').searchParams.get('cursor') - : null, - }) - } catch (error) { - logger.error('Error listing space labels:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/space-pages/route.ts b/apps/sim/app/api/tools/confluence/space-pages/route.ts deleted file mode 100644 index 05b07b2bb68..00000000000 --- a/apps/sim/app/api/tools/confluence/space-pages/route.ts +++ /dev/null @@ -1,132 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { confluenceSpacePagesContract } from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluenceSpacePagesAPI') - -export const dynamic = 'force-dynamic' - -/** - * List all pages in a specific Confluence space. - * Uses GET /wiki/api/v2/spaces/{id}/pages - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceSpacePagesContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - spaceId, - cloudId: providedCloudId, - limit = 50, - status, - bodyFormat, - cursor, - } = parsed.data.body - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!spaceId) { - return NextResponse.json({ error: 'Space ID is required' }, { status: 400 }) - } - - const spaceIdValidation = validateAlphanumericId(spaceId, 'spaceId', 255) - if (!spaceIdValidation.isValid) { - return NextResponse.json({ error: spaceIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const queryParams = new URLSearchParams() - queryParams.append('limit', String(Math.min(limit, 250))) - - if (status) { - queryParams.append('status', status) - } - - if (bodyFormat) { - queryParams.append('body-format', bodyFormat) - } - - if (cursor) { - queryParams.append('cursor', cursor) - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/spaces/${spaceId}/pages?${queryParams.toString()}` - - logger.info(`Fetching pages in space ${spaceId}`) - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - - const pages = (data.results || []).map((page: any) => ({ - id: page.id, - title: page.title, - status: page.status ?? null, - spaceId: page.spaceId ?? null, - parentId: page.parentId ?? null, - authorId: page.authorId ?? null, - createdAt: page.createdAt ?? null, - version: page.version ?? null, - body: page.body ?? null, - webUrl: page._links?.webui ?? null, - })) - - return NextResponse.json({ - pages, - nextCursor: data._links?.next - ? new URL(data._links.next, 'https://placeholder').searchParams.get('cursor') - : null, - }) - } catch (error) { - logger.error('Error listing pages in space:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/space-permissions/route.ts b/apps/sim/app/api/tools/confluence/space-permissions/route.ts deleted file mode 100644 index c18179eca80..00000000000 --- a/apps/sim/app/api/tools/confluence/space-permissions/route.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { confluenceSpacePermissionsContract } from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { - validateAlphanumericId, - validateJiraCloudId, - validatePaginationCursor, -} from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluenceSpacePermissionsAPI') - -export const dynamic = 'force-dynamic' - -/** - * List permissions for a Confluence space. - * Uses GET /wiki/api/v2/spaces/{id}/permissions - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceSpacePermissionsContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - spaceId, - cloudId: providedCloudId, - limit, - cursor, - } = parsed.data.body - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!spaceId) { - return NextResponse.json({ error: 'Space ID is required' }, { status: 400 }) - } - - const spaceIdValidation = validateAlphanumericId(spaceId, 'spaceId', 255) - if (!spaceIdValidation.isValid) { - return NextResponse.json({ error: spaceIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const queryParams = new URLSearchParams() - queryParams.append('limit', String(Math.min(limit, 250))) - - if (cursor) { - const cursorValidation = validatePaginationCursor(cursor, 'cursor') - if (!cursorValidation.isValid) { - return NextResponse.json({ error: cursorValidation.error }, { status: 400 }) - } - queryParams.append('cursor', cursor) - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/spaces/${spaceId}/permissions?${queryParams.toString()}` - - logger.info(`Fetching permissions for space ${spaceId}`) - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - - const permissions = (data.results || []).map((perm: any) => ({ - id: perm.id, - principalType: perm.principal?.type ?? null, - principalId: perm.principal?.id ?? null, - operationKey: perm.operation?.key ?? null, - operationTargetType: perm.operation?.targetType ?? null, - anonymousAccess: perm.anonymousAccess ?? false, - unlicensedAccess: perm.unlicensedAccess ?? false, - })) - - return NextResponse.json({ - permissions, - spaceId, - nextCursor: data._links?.next - ? new URL(data._links.next, 'https://placeholder').searchParams.get('cursor') - : null, - }) - } catch (error) { - logger.error('Error listing space permissions:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/space-properties/route.ts b/apps/sim/app/api/tools/confluence/space-properties/route.ts deleted file mode 100644 index 3fb5a64c2cf..00000000000 --- a/apps/sim/app/api/tools/confluence/space-properties/route.ts +++ /dev/null @@ -1,218 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { confluenceSpacePropertiesContract } from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { - validateAlphanumericId, - validateJiraCloudId, - validatePaginationCursor, -} from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluenceSpacePropertiesAPI') - -export const dynamic = 'force-dynamic' - -/** - * List, create, or delete space properties. - * Uses GET/POST /wiki/api/v2/spaces/{id}/properties - * and DELETE /wiki/api/v2/spaces/{id}/properties/{propertyId} - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceSpacePropertiesContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - spaceId, - cloudId: providedCloudId, - action, - key, - value, - propertyId, - limit = 50, - cursor, - } = parsed.data.body - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!spaceId) { - return NextResponse.json({ error: 'Space ID is required' }, { status: 400 }) - } - - const spaceIdValidation = validateAlphanumericId(spaceId, 'spaceId', 255) - if (!spaceIdValidation.isValid) { - return NextResponse.json({ error: spaceIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const baseUrl = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/spaces/${spaceId}/properties` - - // Validate required params for specific actions - if (action === 'delete' && !propertyId) { - return NextResponse.json( - { error: 'Property ID is required for delete action' }, - { status: 400 } - ) - } - - if (action === 'create' && !key) { - return NextResponse.json( - { error: 'Property key is required for create action' }, - { status: 400 } - ) - } - - // Delete a property - if (action === 'delete' && propertyId) { - const propertyIdValidation = validateAlphanumericId(propertyId, 'propertyId', 255) - if (!propertyIdValidation.isValid) { - return NextResponse.json({ error: propertyIdValidation.error }, { status: 400 }) - } - - const url = `${baseUrl}/${encodeURIComponent(propertyId)}` - - logger.info(`Deleting space property ${propertyId} from space ${spaceId}`) - - const response = await fetch(url, { - method: 'DELETE', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - return NextResponse.json({ spaceId, propertyId, deleted: true }) - } - - // Create a property - if (action === 'create' && key) { - logger.info(`Creating space property '${key}' on space ${spaceId}`) - - const response = await fetch(baseUrl, { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - body: JSON.stringify({ key, value: value ?? {} }), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - return NextResponse.json({ - propertyId: data.id, - key: data.key, - value: data.value ?? null, - spaceId, - }) - } - - // List properties - const queryParams = new URLSearchParams() - queryParams.append('limit', String(Math.min(limit, 250))) - - if (cursor) { - const cursorValidation = validatePaginationCursor(cursor, 'cursor') - if (!cursorValidation.isValid) { - return NextResponse.json({ error: cursorValidation.error }, { status: 400 }) - } - queryParams.append('cursor', cursor) - } - - const url = `${baseUrl}?${queryParams.toString()}` - - logger.info(`Fetching properties for space ${spaceId}`) - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - - const properties = (data.results || []).map((prop: any) => ({ - id: prop.id, - key: prop.key, - value: prop.value ?? null, - })) - - return NextResponse.json({ - properties, - spaceId, - nextCursor: data._links?.next - ? new URL(data._links.next, 'https://placeholder').searchParams.get('cursor') - : null, - }) - } catch (error) { - logger.error('Error with space properties:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/space/route.ts b/apps/sim/app/api/tools/confluence/space/route.ts deleted file mode 100644 index e7dc345ef56..00000000000 --- a/apps/sim/app/api/tools/confluence/space/route.ts +++ /dev/null @@ -1,416 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { - confluenceCreateSpaceContract, - confluenceDeleteSpaceContract, - confluenceGetSpaceContract, - confluenceUpdateSpaceContract, -} from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluenceSpaceAPI') - -export const dynamic = 'force-dynamic' - -// Get a specific space -export const GET = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceGetSpaceContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, spaceId, cloudId: providedCloudId } = parsed.data.query - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!spaceId) { - return NextResponse.json({ error: 'Space ID is required' }, { status: 400 }) - } - - const spaceIdValidation = validateAlphanumericId(spaceId, 'spaceId', 255) - if (!spaceIdValidation.isValid) { - return NextResponse.json({ error: spaceIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/spaces/${spaceId}` - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - return NextResponse.json(data) - } catch (error) { - logger.error('Error getting Confluence space:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) - -/** - * Create a new Confluence space. - * Uses POST /wiki/api/v2/spaces - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceCreateSpaceContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - name, - key, - description, - cloudId: providedCloudId, - } = parsed.data.body - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!name) { - return NextResponse.json({ error: 'Space name is required' }, { status: 400 }) - } - - if (!key) { - return NextResponse.json({ error: 'Space key is required' }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/spaces` - - const createBody: Record = { name, key } - if (description) { - createBody.description = { value: description, representation: 'plain' } - } - - logger.info(`Creating space with key ${key}`) - - const response = await fetch(url, { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - body: JSON.stringify(createBody), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - return NextResponse.json(data) - } catch (error) { - logger.error('Error creating Confluence space:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) - -/** - * Update a Confluence space. - * Uses PUT /wiki/api/v2/spaces/{id} - */ -export const PUT = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceUpdateSpaceContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - spaceId, - name, - description, - cloudId: providedCloudId, - } = parsed.data.body - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!spaceId) { - return NextResponse.json({ error: 'Space ID is required' }, { status: 400 }) - } - - const spaceIdValidation = validateAlphanumericId(spaceId, 'spaceId', 255) - if (!spaceIdValidation.isValid) { - return NextResponse.json({ error: spaceIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - if (!name && description === undefined) { - return NextResponse.json( - { error: 'At least one of name or description is required for update' }, - { status: 400 } - ) - } - - const lookupUrl = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/spaces/${spaceId}` - const lookupResponse = await fetch(lookupUrl, { - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - if (!lookupResponse.ok) { - const errorText = await lookupResponse.text() - return NextResponse.json( - { - error: parseAtlassianErrorMessage( - lookupResponse.status, - lookupResponse.statusText, - errorText - ), - }, - { status: lookupResponse.status } - ) - } - const currentSpace = await lookupResponse.json() - const spaceKey = currentSpace.key - - const updateBody: Record = { - name: name || currentSpace.name, - } - if (description !== undefined) { - updateBody.description = { plain: { value: description, representation: 'plain' } } - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/rest/api/space/${encodeURIComponent(spaceKey)}` - logger.info(`Updating space ${spaceKey}`) - - const response = await fetch(url, { - method: 'PUT', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - body: JSON.stringify(updateBody), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - return NextResponse.json(data) - } catch (error) { - logger.error('Error updating Confluence space:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) - -/** - * Delete a Confluence space. - * Uses DELETE /wiki/api/v2/spaces/{id} - */ -export const DELETE = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceDeleteSpaceContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, spaceId, cloudId: providedCloudId } = parsed.data.body - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!spaceId) { - return NextResponse.json({ error: 'Space ID is required' }, { status: 400 }) - } - - const spaceIdValidation = validateAlphanumericId(spaceId, 'spaceId', 255) - if (!spaceIdValidation.isValid) { - return NextResponse.json({ error: spaceIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const lookupUrl = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/spaces/${spaceId}` - const lookupResponse = await fetch(lookupUrl, { - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - if (!lookupResponse.ok) { - const errorText = await lookupResponse.text() - return NextResponse.json( - { - error: parseAtlassianErrorMessage( - lookupResponse.status, - lookupResponse.statusText, - errorText - ), - }, - { status: lookupResponse.status } - ) - } - const currentSpace = await lookupResponse.json() - const spaceKey = currentSpace.key - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/rest/api/space/${encodeURIComponent(spaceKey)}` - - logger.info(`Deleting space ${spaceKey}`) - - const response = await fetch(url, { - method: 'DELETE', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - let longTask: { id?: string; statusLink?: string } = {} - try { - const text = await response.text() - if (text) { - const data = JSON.parse(text) - longTask = { - id: data?.id, - statusLink: data?.links?.status, - } - } - } catch { - // 204 No Content or non-JSON body — ignore - } - - return NextResponse.json({ - spaceId, - deleted: true, - longTaskId: longTask.id, - longTaskStatusLink: longTask.statusLink, - }) - } catch (error) { - logger.error('Error deleting Confluence space:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/spaces/route.ts b/apps/sim/app/api/tools/confluence/spaces/route.ts deleted file mode 100644 index 6346c1345b3..00000000000 --- a/apps/sim/app/api/tools/confluence/spaces/route.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { confluenceListSpacesContract } from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluenceSpacesAPI') - -export const dynamic = 'force-dynamic' - -// List all spaces -export const GET = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceListSpacesContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, cloudId: providedCloudId, limit, cursor } = parsed.data.query - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const queryParams = new URLSearchParams() - queryParams.append('limit', String(Math.min(Number(limit), 250))) - if (cursor) { - queryParams.append('cursor', cursor) - } - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/spaces?${queryParams.toString()}` - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - - const spaces = (data.results || []).map((space: any) => ({ - id: space.id, - name: space.name, - key: space.key, - type: space.type, - status: space.status, - authorId: space.authorId ?? null, - createdAt: space.createdAt ?? null, - homepageId: space.homepageId ?? null, - description: space.description ?? null, - })) - - return NextResponse.json({ - spaces, - nextCursor: data._links?.next - ? new URL(data._links.next, 'https://placeholder').searchParams.get('cursor') - : null, - }) - } catch (error) { - logger.error('Error listing Confluence spaces:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/tasks/route.ts b/apps/sim/app/api/tools/confluence/tasks/route.ts deleted file mode 100644 index 923209fcbf7..00000000000 --- a/apps/sim/app/api/tools/confluence/tasks/route.ts +++ /dev/null @@ -1,298 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { confluenceTasksContract } from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { - validateAlphanumericId, - validateJiraCloudId, - validatePaginationCursor, - validatePathSegment, -} from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluenceTasksAPI') - -export const dynamic = 'force-dynamic' - -/** - * List, get, or update Confluence inline tasks. - * Uses GET /wiki/api/v2/tasks, GET /wiki/api/v2/tasks/{id}, PUT /wiki/api/v2/tasks/{id} - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceTasksContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: providedCloudId, - action, - taskId, - status: taskStatus, - pageId, - spaceId, - assignedTo, - limit = 50, - cursor, - } = parsed.data.body - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - // Update a task - if (action === 'update' && taskId) { - const taskIdValidation = validateAlphanumericId(taskId, 'taskId', 255) - if (!taskIdValidation.isValid) { - return NextResponse.json({ error: taskIdValidation.error }, { status: 400 }) - } - - // First fetch the current task to get required fields - const getUrl = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/tasks/${taskId}` - const getResponse = await fetch(getUrl, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!getResponse.ok) { - const errorText = await getResponse.text() - return NextResponse.json( - { - error: parseAtlassianErrorMessage( - getResponse.status, - getResponse.statusText, - errorText - ), - }, - { status: getResponse.status } - ) - } - - const currentTask = await getResponse.json() - - const updateBody: Record = { - id: taskId, - status: taskStatus || currentTask.status, - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/tasks/${taskId}` - - logger.info(`Updating task ${taskId}`) - - const response = await fetch(url, { - method: 'PUT', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - body: JSON.stringify(updateBody), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - return NextResponse.json({ - task: { - id: data.id, - localId: data.localId ?? null, - spaceId: data.spaceId ?? null, - pageId: data.pageId ?? null, - blogPostId: data.blogPostId ?? null, - status: data.status, - body: data.body?.storage?.value ?? null, - createdBy: data.createdBy ?? null, - assignedTo: data.assignedTo ?? null, - completedBy: data.completedBy ?? null, - createdAt: data.createdAt ?? null, - updatedAt: data.updatedAt ?? null, - dueAt: data.dueAt ?? null, - completedAt: data.completedAt ?? null, - }, - }) - } - - // Get a specific task - if (taskId) { - const taskIdValidation = validateAlphanumericId(taskId, 'taskId', 255) - if (!taskIdValidation.isValid) { - return NextResponse.json({ error: taskIdValidation.error }, { status: 400 }) - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/tasks/${taskId}` - - logger.info(`Fetching task ${taskId}`) - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - return NextResponse.json({ - task: { - id: data.id, - localId: data.localId ?? null, - spaceId: data.spaceId ?? null, - pageId: data.pageId ?? null, - blogPostId: data.blogPostId ?? null, - status: data.status, - body: data.body?.storage?.value ?? null, - createdBy: data.createdBy ?? null, - assignedTo: data.assignedTo ?? null, - completedBy: data.completedBy ?? null, - createdAt: data.createdAt ?? null, - updatedAt: data.updatedAt ?? null, - dueAt: data.dueAt ?? null, - completedAt: data.completedAt ?? null, - }, - }) - } - - // List tasks - const queryParams = new URLSearchParams() - queryParams.append('limit', String(Math.min(limit, 250))) - - if (cursor) { - const cursorValidation = validatePaginationCursor(cursor, 'cursor') - if (!cursorValidation.isValid) { - return NextResponse.json({ error: cursorValidation.error }, { status: 400 }) - } - queryParams.append('cursor', cursor) - } - if (taskStatus) queryParams.append('status', taskStatus) - if (pageId) { - const pageIdValidation = validateAlphanumericId(pageId, 'pageId', 255) - if (!pageIdValidation.isValid) { - return NextResponse.json({ error: pageIdValidation.error }, { status: 400 }) - } - queryParams.append('page-id', pageId) - } - if (spaceId) { - const spaceIdValidation = validateAlphanumericId(spaceId, 'spaceId', 255) - if (!spaceIdValidation.isValid) { - return NextResponse.json({ error: spaceIdValidation.error }, { status: 400 }) - } - queryParams.append('space-id', spaceId) - } - if (assignedTo) { - // Atlassian account IDs: 5d5bd05c3aee0123abc or 557058:6b9c9931-4693-49c1-8b3a-931f1af98134 - const assignedToValidation = validatePathSegment(assignedTo, { - paramName: 'assignedTo', - maxLength: 128, - customPattern: /^[a-zA-Z0-9_|:-]+$/, - }) - if (!assignedToValidation.isValid) { - return NextResponse.json({ error: assignedToValidation.error }, { status: 400 }) - } - queryParams.append('assigned-to', assignedTo) - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/tasks?${queryParams.toString()}` - - logger.info('Fetching tasks') - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - - const tasks = (data.results || []).map((task: any) => ({ - id: task.id, - localId: task.localId ?? null, - spaceId: task.spaceId ?? null, - pageId: task.pageId ?? null, - blogPostId: task.blogPostId ?? null, - status: task.status, - body: task.body?.storage?.value ?? null, - createdBy: task.createdBy ?? null, - assignedTo: task.assignedTo ?? null, - completedBy: task.completedBy ?? null, - createdAt: task.createdAt ?? null, - updatedAt: task.updatedAt ?? null, - dueAt: task.dueAt ?? null, - completedAt: task.completedAt ?? null, - })) - - return NextResponse.json({ - tasks, - nextCursor: data._links?.next - ? new URL(data._links.next, 'https://placeholder').searchParams.get('cursor') - : null, - }) - } catch (error) { - logger.error('Error with tasks:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/upload-attachment/route.ts b/apps/sim/app/api/tools/confluence/upload-attachment/route.ts deleted file mode 100644 index 8e0b14f5d4f..00000000000 --- a/apps/sim/app/api/tools/confluence/upload-attachment/route.ts +++ /dev/null @@ -1,178 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { confluenceUploadAttachmentContract } from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import { processSingleFileToUserFile, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluenceUploadAttachmentAPI') - -export const dynamic = 'force-dynamic' - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceUploadAttachmentContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: providedCloudId, - pageId, - file, - fileName, - comment, - } = parsed.data.body - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!pageId) { - return NextResponse.json({ error: 'Page ID is required' }, { status: 400 }) - } - - if (!file) { - return NextResponse.json({ error: 'File is required' }, { status: 400 }) - } - - const pageIdValidation = validateAlphanumericId(pageId, 'pageId', 255) - if (!pageIdValidation.isValid) { - return NextResponse.json({ error: pageIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - let fileToProcess = file as RawFileInput - if (Array.isArray(file)) { - if (file.length === 0) { - return NextResponse.json({ error: 'No file provided' }, { status: 400 }) - } - fileToProcess = file[0] as RawFileInput - } - - let userFile - try { - userFile = processSingleFileToUserFile(fileToProcess, 'confluence-upload', logger) - } catch (error) { - return NextResponse.json( - { error: getErrorMessage(error, 'Failed to process file') }, - { status: 400 } - ) - } - - const denied = await assertToolFileAccess( - userFile.key, - auth.userId, - 'confluence-upload', - logger - ) - if (denied) return denied - - let fileBuffer: Buffer - let resolvedContentType: string - try { - const servable = await downloadServableFileFromStorage( - userFile, - 'confluence-upload', - logger, - { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - } - ) - fileBuffer = servable.buffer - resolvedContentType = servable.contentType - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - logger.error('Failed to download file from storage:', error) - return NextResponse.json( - { - error: `Failed to download file: ${getErrorMessage(error, 'Unknown error')}`, - }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } - - const uploadFileName = fileName || userFile.name || 'attachment' - const mimeType = resolvedContentType || userFile.type || 'application/octet-stream' - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/rest/api/content/${pageId}/child/attachment` - - const formData = new FormData() - const blob = new Blob([new Uint8Array(fileBuffer)], { type: mimeType }) - formData.append('file', blob, uploadFileName) - - if (comment) { - formData.append('comment', comment) - } - - // Add minorEdit field as required by Confluence API - formData.append('minorEdit', 'false') - - const response = await fetch(url, { - method: 'POST', - headers: { - Authorization: `Bearer ${accessToken}`, - 'X-Atlassian-Token': 'nocheck', - }, - body: formData, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - - const attachment = data.results?.[0] || data - - return NextResponse.json({ - attachmentId: attachment.id, - title: attachment.title, - fileSize: attachment.extensions?.fileSize || 0, - mediaType: attachment.extensions?.mediaType || mimeType, - downloadUrl: attachment._links?.download || '', - pageId: pageId, - }) - } catch (error) { - logger.error('Error uploading Confluence attachment:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/confluence/user/route.ts b/apps/sim/app/api/tools/confluence/user/route.ts deleted file mode 100644 index f761d0286e6..00000000000 --- a/apps/sim/app/api/tools/confluence/user/route.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { confluenceUserContract } from '@/lib/api/contracts/selectors/confluence' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId, validatePathSegment } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getConfluenceCloudId } from '@/tools/confluence/utils' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('ConfluenceUserAPI') - -export const dynamic = 'force-dynamic' - -/** - * Get a Confluence user by account ID. - * Uses GET /wiki/rest/api/user?accountId={accountId} - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(confluenceUserContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, accountId, cloudId: providedCloudId } = parsed.data.body - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!accountId) { - return NextResponse.json({ error: 'Account ID is required' }, { status: 400 }) - } - - // Atlassian account IDs: 5d5bd05c3aee0123abc or 557058:6b9c9931-4693-49c1-8b3a-931f1af98134 - const accountIdValidation = validatePathSegment(accountId, { - paramName: 'accountId', - maxLength: 128, - customPattern: /^[a-zA-Z0-9_|:-]+$/, - }) - if (!accountIdValidation.isValid) { - return NextResponse.json({ error: accountIdValidation.error }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getConfluenceCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/rest/api/user?accountId=${encodeURIComponent(accountId)}` - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - - const data = await response.json() - return NextResponse.json(data) - } catch (error) { - logger.error('Error getting Confluence user:', error) - return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/crowdstrike/query/falcon.ts b/apps/sim/app/api/tools/crowdstrike/query/falcon.ts deleted file mode 100644 index fa21e3181b7..00000000000 --- a/apps/sim/app/api/tools/crowdstrike/query/falcon.ts +++ /dev/null @@ -1,261 +0,0 @@ -import { isRecordLike } from '@sim/utils/object' -import type { CrowdStrikeBaseParams, CrowdStrikeCloud } from '@/tools/crowdstrike/types' - -export type JsonRecord = Record - -const CLOUD_BASE_URLS: Record = { - 'eu-1': 'https://api.eu-1.crowdstrike.com', - 'us-1': 'https://api.crowdstrike.com', - 'us-2': 'https://api.us-2.crowdstrike.com', - 'us-3': 'https://api.us-3.crowdstrike.com', - 'us-gov-1': 'https://api.laggar.gcw.crowdstrike.com', - 'us-gov-2': 'https://api.us-gov-2.crowdstrike.mil', -} - -export function getCloudBaseUrl(cloud: CrowdStrikeCloud): string { - return CLOUD_BASE_URLS[cloud] -} - -export function getString(value: unknown): string | null { - return typeof value === 'string' ? value : null -} - -export function getNumber(value: unknown): number | null { - return typeof value === 'number' ? value : null -} - -export function getBoolean(value: unknown): boolean | null { - return typeof value === 'boolean' ? value : null -} - -export function getStringArray(value: unknown): string[] { - if (!Array.isArray(value)) { - return [] - } - - return value.filter((entry): entry is string => typeof entry === 'string') -} - -export function getRecordArray(value: unknown): JsonRecord[] { - if (!Array.isArray(value)) { - return [] - } - - return value.filter(isRecordLike) -} - -export function getRecord(value: unknown): JsonRecord | null { - return isRecordLike(value) ? value : null -} - -/** - * Every Falcon endpoint this integration calls answers with a flat - * `{ meta, resources, errors }` envelope, so the envelope readers below and - * `getFalconErrorMessage` both read the payload root directly. - */ -export function getResourcesArray(data: unknown): unknown[] { - if (!isRecordLike(data) || !Array.isArray(data.resources)) { - return [] - } - - return data.resources -} - -export function getRecordResources(data: unknown): JsonRecord[] { - return getResourcesArray(data).filter(isRecordLike) -} - -export function getStringResources(data: unknown): string[] { - return getStringArray(getResourcesArray(data)) -} - -export function getFirstRecordResource(data: unknown): JsonRecord | null { - return getRecordResources(data)[0] ?? null -} - -export function getPagination(data: unknown) { - if (!isRecordLike(data) || !isRecordLike(data.meta) || !isRecordLike(data.meta.pagination)) { - return null - } - - const { pagination } = data.meta - - return { - limit: getNumber(pagination.limit), - offset: getNumber(pagination.offset), - total: getNumber(pagination.total), - } -} - -/** Offset pagination plus the `after` cursor the IOC Management API returns. */ -export function getCursorPagination(data: unknown) { - if (!isRecordLike(data) || !isRecordLike(data.meta) || !isRecordLike(data.meta.pagination)) { - return null - } - - const { pagination } = data.meta - - return { - after: getString(pagination.after), - limit: getNumber(pagination.limit), - offset: getNumber(pagination.offset), - total: getNumber(pagination.total), - } -} - -/** Spotlight paginates by cursor only — it returns no offset. */ -export function getSpotlightPagination(data: unknown) { - if (!isRecordLike(data) || !isRecordLike(data.meta) || !isRecordLike(data.meta.pagination)) { - return null - } - - const { pagination } = data.meta - - return { - after: getString(pagination.after), - limit: getNumber(pagination.limit), - total: getNumber(pagination.total), - } -} - -/** - * CrowdStrike returns `{ meta, resources, errors }` on every endpoint, and a 200 - * can still carry a populated `errors` array for the IDs that failed. - */ -export function getEnvelopeErrors(data: unknown) { - if (!isRecordLike(data)) { - return [] - } - - return getRecordArray(data.errors).map((entry) => ({ - code: getNumber(entry.code), - id: getString(entry.id), - message: getString(entry.message), - })) -} - -export function getFalconErrorMessage(data: unknown, fallback: string): string { - if (!isRecordLike(data)) { - return fallback - } - - const errors = Array.isArray(data.errors) ? data.errors : [] - const firstError = errors[0] - if (isRecordLike(firstError)) { - const firstMessage = getString(firstError.message) ?? getString(firstError.code) - if (firstMessage) { - return firstMessage - } - } - - return ( - getString(data.message) ?? - getString(data.error_description) ?? - getString(data.error) ?? - fallback - ) -} - -/** - * Raised when the Falcon OAuth2 token exchange fails. Carries the Falcon status - * so the route can answer with the real cause (401 for bad credentials) instead - * of letting a credential problem fall through to a generic 500. - */ -export class CrowdStrikeAuthError extends Error { - readonly status: number - - constructor(message: string, status: number) { - super(message) - this.name = 'CrowdStrikeAuthError' - this.status = status >= 400 && status <= 599 ? status : 502 - } -} - -export async function getAccessToken(params: CrowdStrikeBaseParams): Promise { - const baseUrl = getCloudBaseUrl(params.cloud) - const response = await fetch(`${baseUrl}/oauth2/token`, { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/x-www-form-urlencoded', - }, - body: new URLSearchParams({ - client_id: params.clientId, - client_secret: params.clientSecret, - grant_type: 'client_credentials', - }).toString(), - cache: 'no-store', - }) - - const data: unknown = await response.json().catch(() => null) - if (!response.ok) { - throw new CrowdStrikeAuthError( - getFalconErrorMessage(data, 'Failed to authenticate with CrowdStrike'), - response.status - ) - } - - if (!isRecordLike(data) || typeof data.access_token !== 'string') { - throw new CrowdStrikeAuthError('CrowdStrike authentication did not return an access token', 502) - } - - return data.access_token -} - -interface CrowdStrikeRequestOptions { - method: 'GET' | 'POST' | 'PATCH' | 'DELETE' - path: string - query?: Record - repeatedQuery?: Record - body?: unknown -} - -export interface CrowdStrikeCallResult { - ok: boolean - status: number - data: unknown -} - -export function buildUrl(baseUrl: string, options: CrowdStrikeRequestOptions): string { - const url = new URL(options.path, baseUrl) - - for (const [key, value] of Object.entries(options.query ?? {})) { - if (value !== undefined) { - url.searchParams.set(key, String(value)) - } - } - - for (const [key, values] of Object.entries(options.repeatedQuery ?? {})) { - for (const value of values ?? []) { - url.searchParams.append(key, value) - } - } - - return url.toString() -} - -export async function callCrowdStrike( - baseUrl: string, - accessToken: string, - options: CrowdStrikeRequestOptions -): Promise { - const headers: Record = { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - } - - if (options.body !== undefined) { - headers['Content-Type'] = 'application/json' - } - - const response = await fetch(buildUrl(baseUrl, options), { - method: options.method, - headers, - body: options.body === undefined ? undefined : JSON.stringify(options.body), - cache: 'no-store', - }) - - const data: unknown = await response.json().catch(() => null) - - return { ok: response.ok, status: response.status, data } -} diff --git a/apps/sim/app/api/tools/crowdstrike/query/normalize.ts b/apps/sim/app/api/tools/crowdstrike/query/normalize.ts deleted file mode 100644 index d8784d077ef..00000000000 --- a/apps/sim/app/api/tools/crowdstrike/query/normalize.ts +++ /dev/null @@ -1,266 +0,0 @@ -import { - getBoolean, - getNumber, - getRecord, - getRecordArray, - getString, - getStringArray, - type JsonRecord, -} from '@/app/api/tools/crowdstrike/query/falcon' -import type { - CrowdStrikeAffectedEntity, - CrowdStrikeAlert, - CrowdStrikeCase, - CrowdStrikeFalconUser, - CrowdStrikeHostGroup, - CrowdStrikeIndicator, - CrowdStrikeVulnerability, -} from '@/tools/crowdstrike/types' - -export function normalizeAlert(resource: JsonRecord): CrowdStrikeAlert { - const device = getRecord(resource.device) - - return { - compositeId: getString(resource.composite_id), - id: getString(resource.id), - cid: getString(resource.cid), - aggregateId: getString(resource.aggregate_id), - agentId: getString(resource.agent_id), - deviceId: device ? getString(device.device_id) : null, - hostname: device ? getString(device.hostname) : null, - name: getString(resource.name), - displayName: getString(resource.display_name), - description: getString(resource.description), - type: getString(resource.type), - product: getString(resource.product), - platform: getString(resource.platform), - severity: getNumber(resource.severity), - severityName: getString(resource.severity_name), - confidence: getNumber(resource.confidence), - status: getString(resource.status), - assignedToName: getString(resource.assigned_to_name), - assignedToUid: getString(resource.assigned_to_uid), - assignedToUuid: getString(resource.assigned_to_uuid), - tactic: getString(resource.tactic), - tacticId: getString(resource.tactic_id), - technique: getString(resource.technique), - techniqueId: getString(resource.technique_id), - scenario: getString(resource.scenario), - objective: getString(resource.objective), - resolution: getString(resource.resolution), - showInUi: getBoolean(resource.show_in_ui), - tags: getStringArray(resource.tags), - filename: getString(resource.filename), - filepath: getString(resource.filepath), - cmdline: getString(resource.cmdline), - sha256: getString(resource.sha256), - sha1: getString(resource.sha1), - md5: getString(resource.md5), - userName: getString(resource.user_name), - userId: getString(resource.user_id), - patternId: getNumber(resource.pattern_id), - falconHostLink: getString(resource.falcon_host_link), - controlGraphId: getString(resource.control_graph_id), - external: getBoolean(resource.external), - emailSent: getBoolean(resource.email_sent), - isAggregated: getBoolean(resource.is_aggregated), - isFalconPlatformIoa: getBoolean(resource.is_falcon_platform_ioa), - dataDomains: getStringArray(resource.data_domains), - iocValues: getStringArray(resource.ioc_values), - linkedCaseIds: getStringArray(resource.linked_case_ids), - linkedBehavioralDetections: getStringArray(resource.linked_behavioral_detections), - timestamp: getString(resource.timestamp), - createdTimestamp: getString(resource.created_timestamp), - updatedTimestamp: getString(resource.updated_timestamp), - crawledTimestamp: getString(resource.crawled_timestamp), - contextTimestamp: getString(resource.context_timestamp), - } -} - -export function normalizeAffectedEntity(resource: JsonRecord): CrowdStrikeAffectedEntity { - return { - id: getString(resource.id), - path: getString(resource.path), - } -} - -export function normalizeHostGroup(resource: JsonRecord): CrowdStrikeHostGroup { - return { - id: getString(resource.id), - name: getString(resource.name), - description: getString(resource.description), - groupType: getString(resource.group_type), - assignmentRule: getString(resource.assignment_rule), - createdBy: getString(resource.created_by), - createdTimestamp: getString(resource.created_timestamp), - modifiedBy: getString(resource.modified_by), - modifiedTimestamp: getString(resource.modified_timestamp), - } -} - -export function normalizeIndicator(resource: JsonRecord): CrowdStrikeIndicator { - const metadata = getRecord(resource.metadata) - - return { - id: getString(resource.id), - type: getString(resource.type), - value: getString(resource.value), - action: getString(resource.action), - mobileAction: getString(resource.mobile_action), - severity: getString(resource.severity), - description: getString(resource.description), - source: getString(resource.source), - appliedGlobally: getBoolean(resource.applied_globally), - platforms: getStringArray(resource.platforms), - hostGroups: getStringArray(resource.host_groups), - tags: getStringArray(resource.tags), - expiration: getString(resource.expiration), - expired: getBoolean(resource.expired), - deleted: getBoolean(resource.deleted), - fromParent: getBoolean(resource.from_parent), - parentCidName: getString(resource.parent_cid_name), - createdBy: getString(resource.created_by), - createdOn: getString(resource.created_on), - modifiedBy: getString(resource.modified_by), - modifiedOn: getString(resource.modified_on), - metadata: metadata - ? { - avHits: getNumber(metadata.av_hits), - companyName: getString(metadata.company_name), - fileDescription: getString(metadata.file_description), - fileVersion: getString(metadata.file_version), - filename: getString(metadata.filename), - originalFilename: getString(metadata.original_filename), - productName: getString(metadata.product_name), - productVersion: getString(metadata.product_version), - signed: getBoolean(metadata.signed), - } - : null, - } -} - -export function normalizeVulnerability(resource: JsonRecord): CrowdStrikeVulnerability { - const cve = getRecord(resource.cve) - const cisaInfo = cve ? getRecord(cve.cisa_info) : null - const app = getRecord(resource.app) - const hostInfo = getRecord(resource.host_info) - const remediation = getRecord(resource.remediation) - const suppressionInfo = getRecord(resource.suppression_info) - - return { - id: getString(resource.id), - aid: getString(resource.aid), - cid: getString(resource.cid), - status: getString(resource.status), - confidence: getString(resource.confidence), - vulnerabilityId: getString(resource.vulnerability_id), - createdTimestamp: getString(resource.created_timestamp), - updatedTimestamp: getString(resource.updated_timestamp), - closedTimestamp: getString(resource.closed_timestamp), - cve: cve - ? { - id: getString(cve.id), - baseScore: getNumber(cve.base_score), - severity: getString(cve.severity), - exprtRating: getString(cve.exprt_rating), - exploitStatus: getNumber(cve.exploit_status), - exploitabilityScore: getNumber(cve.exploitability_score), - impactScore: getNumber(cve.impact_score), - remediationLevel: getString(cve.remediation_level), - description: getString(cve.description), - publishedDate: getString(cve.published_date), - vector: getString(cve.vector), - types: getStringArray(cve.types), - isCisaKev: cisaInfo ? getBoolean(cisaInfo.is_cisa_kev) : null, - cisaDueDate: cisaInfo ? getString(cisaInfo.due_date) : null, - } - : null, - app: app - ? { - productNameNormalized: getString(app.product_name_normalized), - productNameVersion: getString(app.product_name_version), - vendorNormalized: getString(app.vendor_normalized), - } - : null, - hostInfo: hostInfo - ? { - hostname: getString(hostInfo.hostname), - localIp: getString(hostInfo.local_ip), - machineDomain: getString(hostInfo.machine_domain), - osVersion: getString(hostInfo.os_version), - platform: getString(hostInfo.platform), - productTypeDesc: getString(hostInfo.product_type_desc), - assetCriticality: getString(hostInfo.asset_criticality), - internetExposure: getString(hostInfo.internet_exposure), - tags: getStringArray(hostInfo.tags), - groups: getRecordArray(hostInfo.groups) - .map((group) => getString(group.name)) - .filter((name): name is string => name !== null), - } - : null, - remediationIds: remediation ? getStringArray(remediation.ids) : [], - remediations: remediation - ? getRecordArray(remediation.entities).map((entity) => ({ - id: getString(entity.id), - title: getString(entity.title), - action: getString(entity.action), - type: getString(entity.type), - link: getString(entity.link), - reference: getString(entity.reference), - vendorUrl: getString(entity.vendor_url), - })) - : [], - suppressionInfo: suppressionInfo - ? { - isSuppressed: getBoolean(suppressionInfo.is_suppressed), - reason: getString(suppressionInfo.reason), - } - : null, - } -} - -function normalizeFalconUser(value: unknown): CrowdStrikeFalconUser | null { - const user = getRecord(value) - if (!user) { - return null - } - - return { - uuid: getString(user.uuid), - email: getString(user.email), - fullName: getString(user.full_name), - } -} - -export function normalizeCase(resource: JsonRecord): CrowdStrikeCase { - const severityInfo = getRecord(resource.severity_info) - const template = getRecord(resource.template) - const sla = getRecord(resource.sla) - const readOnly = getRecord(resource.read_only) - - return { - id: getString(resource.id), - cid: getString(resource.cid), - name: getString(resource.name), - description: getString(resource.description), - descriptionFormat: getString(resource.description_format), - status: getString(resource.status), - severity: getNumber(resource.severity), - severityLevel: severityInfo ? getString(severityInfo.level) : null, - referenceId: getString(resource.reference_id), - version: getNumber(resource.version), - tags: getStringArray(resource.tags), - assignedTo: normalizeFalconUser(resource.assigned_to), - createdBy: normalizeFalconUser(resource.created_by), - lastUpdatedBy: normalizeFalconUser(resource.last_updated_by), - createdTimestamp: getString(resource.created_timestamp), - updatedTimestamp: getString(resource.updated_timestamp), - startTimestamp: getString(resource.start_timestamp), - endTimestamp: getString(resource.end_timestamp), - templateId: template ? getString(template.id) : null, - templateName: template ? getString(template.name) : null, - slaId: sla ? getString(sla.id) : null, - slaName: sla ? getString(sla.name) : null, - isReadOnly: readOnly ? getBoolean(readOnly.is_read_only) : null, - } -} diff --git a/apps/sim/app/api/tools/crowdstrike/query/operations.test.ts b/apps/sim/app/api/tools/crowdstrike/query/operations.test.ts deleted file mode 100644 index 263462da8d8..00000000000 --- a/apps/sim/app/api/tools/crowdstrike/query/operations.test.ts +++ /dev/null @@ -1,1212 +0,0 @@ -/** - * @vitest-environment node - */ -import { createMockRequest, hybridAuthMockFns } from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { fetchMock } = vi.hoisted(() => ({ - fetchMock: vi.fn(), -})) - -import { MAX_ID_URL_BYTES } from '@/app/api/tools/crowdstrike/query/operations' -import { POST } from '@/app/api/tools/crowdstrike/query/route' - -function jsonResponse(body: unknown, status = 200): Response { - return new Response(JSON.stringify(body), { - status, - headers: { 'Content-Type': 'application/json' }, - }) -} - -const credentials = { - clientId: 'client-id', - clientSecret: 'client-secret', - cloud: 'us-1' as const, -} - -function requestFor(body: Record) { - return createMockRequest('POST', { ...credentials, ...body }) -} - -describe('CrowdStrike extended operations', () => { - beforeEach(() => { - vi.clearAllMocks() - fetchMock.mockReset() - vi.stubGlobal('fetch', fetchMock) - - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-123', - authType: 'internal_jwt', - }) - - fetchMock.mockResolvedValueOnce(jsonResponse({ access_token: 'token-123' })) - }) - - it('queries alerts and returns composite ids with pagination', async () => { - fetchMock.mockResolvedValueOnce( - jsonResponse({ - meta: { pagination: { limit: 2, offset: 0, total: 7 } }, - resources: ['cid:aid:alert-1', 'cid:aid:alert-2'], - }) - ) - - const response = await POST( - requestFor({ operation: 'crowdstrike_query_alerts', filter: 'status:"new"', limit: 2 }) - ) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.output).toEqual({ - alertIds: ['cid:aid:alert-1', 'cid:aid:alert-2'], - count: 2, - pagination: { limit: 2, offset: 0, total: 7 }, - }) - - const queryUrl = new URL(fetchMock.mock.calls[1][0]) - expect(queryUrl.pathname).toBe('/alerts/queries/alerts/v2') - expect(queryUrl.searchParams.get('filter')).toBe('status:"new"') - expect(queryUrl.searchParams.get('limit')).toBe('2') - }) - - it('normalizes alert details from documented fields', async () => { - fetchMock.mockResolvedValueOnce( - jsonResponse({ - resources: [ - { - composite_id: 'cid:aid:alert-1', - id: 'alert-1', - severity: 70, - severity_name: 'High', - status: 'new', - tags: ['triage'], - device: { device_id: 'a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1', hostname: 'web-01' }, - }, - ], - }) - ) - - const response = await POST( - requestFor({ - operation: 'crowdstrike_get_alert_details', - compositeIds: ['cid:aid:alert-1'], - }) - ) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.output.count).toBe(1) - expect(data.output.alerts[0]).toMatchObject({ - compositeId: 'cid:aid:alert-1', - id: 'alert-1', - severity: 70, - severityName: 'High', - status: 'new', - tags: ['triage'], - deviceId: 'a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1', - hostname: 'web-01', - }) - - const [, detailsCall] = fetchMock.mock.calls - expect(JSON.parse(detailsCall[1].body)).toEqual({ composite_ids: ['cid:aid:alert-1'] }) - }) - - it('builds documented action parameters when updating alerts', async () => { - fetchMock.mockResolvedValueOnce(jsonResponse({ meta: {}, errors: [] })) - - const response = await POST( - requestFor({ - operation: 'crowdstrike_update_alerts', - compositeIds: ['cid:aid:alert-1'], - updateStatus: 'closed', - appendComment: 'Resolved by automation', - showInUi: false, - }) - ) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.output.updatedIds).toEqual(['cid:aid:alert-1']) - - const [, updateCall] = fetchMock.mock.calls - expect(updateCall[1].method).toBe('PATCH') - expect(JSON.parse(updateCall[1].body)).toEqual({ - action_parameters: [ - { name: 'update_status', value: 'closed' }, - { name: 'append_comment', value: 'Resolved by automation' }, - { name: 'show_in_ui', value: 'false' }, - ], - composite_ids: ['cid:aid:alert-1'], - }) - }) - - it('rejects an alert update that carries no action', async () => { - const response = await POST( - requestFor({ - operation: 'crowdstrike_update_alerts', - compositeIds: ['cid:aid:alert-1'], - }) - ) - - expect(response.status).toBe(400) - expect(fetchMock).toHaveBeenCalledTimes(0) - }) - - it('contains hosts through the documented action endpoint', async () => { - fetchMock.mockResolvedValueOnce( - jsonResponse( - { - resources: [ - { id: 'a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1', path: '/devices/entities/devices/v1' }, - ], - }, - 202 - ) - ) - - const response = await POST( - requestFor({ - operation: 'crowdstrike_perform_host_action', - actionName: 'contain', - deviceIds: ['a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1'], - }) - ) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.output.affected).toEqual([ - { id: 'a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1', path: '/devices/entities/devices/v1' }, - ]) - - const actionUrl = new URL(fetchMock.mock.calls[1][0]) - expect(actionUrl.pathname).toBe('/devices/entities/devices-actions/v2') - expect(actionUrl.searchParams.get('action_name')).toBe('contain') - expect(JSON.parse(fetchMock.mock.calls[1][1].body)).toEqual({ - ids: ['a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1'], - }) - }) - - it('rejects an unsupported host action', async () => { - const response = await POST( - requestFor({ - operation: 'crowdstrike_perform_host_action', - actionName: 'delete_host', - deviceIds: ['a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1'], - }) - ) - - expect(response.status).toBe(400) - expect(fetchMock).toHaveBeenCalledTimes(0) - }) - - it('adds hosts to a group with a device_id FQL filter', async () => { - fetchMock.mockResolvedValueOnce(jsonResponse({ resources: [{ id: 'group-1', name: 'SOC' }] })) - - const response = await POST( - requestFor({ - operation: 'crowdstrike_perform_host_group_action', - actionName: 'add-hosts', - hostGroupId: 'group-1', - deviceIds: ['a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1', 'b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2'], - }) - ) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.output.hostGroups[0]).toMatchObject({ id: 'group-1', name: 'SOC' }) - - const groupUrl = new URL(fetchMock.mock.calls[1][0]) - expect(groupUrl.pathname).toBe('/devices/entities/host-group-actions/v1') - expect(groupUrl.searchParams.get('action_name')).toBe('add-hosts') - expect(JSON.parse(fetchMock.mock.calls[1][1].body)).toEqual({ - action_parameters: [ - { - name: 'filter', - value: - "(device_id:['a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1','b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2'])", - }, - ], - ids: ['group-1'], - }) - }) - - it('treats a 200 with only envelope errors as a failure', async () => { - fetchMock.mockResolvedValueOnce( - jsonResponse({ - resources: [], - errors: [{ code: 404, id: 'ioc-1', message: 'Indicator not found' }], - }) - ) - - const response = await POST( - requestFor({ operation: 'crowdstrike_get_indicator_details', indicatorIds: ['ioc-1'] }) - ) - const data = await response.json() - - expect(response.status).toBe(404) - expect(data.success).toBe(false) - expect(data.error).toBe('Indicator not found') - }) - - it('falls back to 502 when a 200 carries errors without a usable code', async () => { - fetchMock.mockResolvedValueOnce( - jsonResponse({ resources: [], errors: [{ message: 'Upstream unavailable' }] }) - ) - - const response = await POST( - requestFor({ operation: 'crowdstrike_get_indicator_details', indicatorIds: ['ioc-1'] }) - ) - const data = await response.json() - - expect(response.status).toBe(502) - expect(data.success).toBe(false) - expect(data.error).toBe('Upstream unavailable') - }) - - it('fails an alert update when the meta-only envelope reports any error', async () => { - fetchMock.mockResolvedValueOnce( - jsonResponse({ meta: {}, errors: [{ code: 403, message: 'Alert is read only' }] }) - ) - - const response = await POST( - requestFor({ - operation: 'crowdstrike_update_alerts', - compositeIds: ['cid:aid:alert-1'], - updateStatus: 'closed', - }) - ) - const data = await response.json() - - expect(response.status).toBe(403) - expect(data.success).toBe(false) - expect(data.error).toBe('Alert is read only') - }) - - it('sends remove_tags_by_prefix using the documented action parameter name', async () => { - fetchMock.mockResolvedValueOnce(jsonResponse({ meta: {}, errors: [] })) - - const response = await POST( - requestFor({ - operation: 'crowdstrike_update_alerts', - compositeIds: ['cid:aid:alert-1'], - removeTagsByPrefix: 'auto-', - }) - ) - - expect(response.status).toBe(200) - - const [, updateCall] = fetchMock.mock.calls - expect(JSON.parse(updateCall[1].body).action_parameters).toEqual([ - { name: 'remove_tags_by_prefix', value: 'auto-' }, - ]) - }) - - it('surfaces envelope errors alongside partial indicator results', async () => { - fetchMock.mockResolvedValueOnce( - jsonResponse({ - resources: [{ id: 'ioc-1', type: 'sha256', value: 'abc', action: 'prevent' }], - errors: [{ code: 404, id: 'ioc-2', message: 'Indicator not found' }], - }) - ) - - const response = await POST( - requestFor({ - operation: 'crowdstrike_get_indicator_details', - indicatorIds: ['ioc-1', 'ioc-2'], - }) - ) - const data = await response.json() - - expect(data.success).toBe(true) - expect(data.output.count).toBe(1) - expect(data.output.errors).toEqual([{ code: 404, id: 'ioc-2', message: 'Indicator not found' }]) - }) - - it('deletes indicators by filter without an ids list', async () => { - fetchMock.mockResolvedValueOnce(jsonResponse({ resources: ['ioc-1'] })) - - const response = await POST( - requestFor({ - operation: 'crowdstrike_delete_indicators', - filter: "source:'automation'", - comment: 'cleanup', - }) - ) - const data = await response.json() - - expect(data.output.deletedIds).toEqual(['ioc-1']) - - const deleteUrl = new URL(fetchMock.mock.calls[1][0]) - expect(fetchMock.mock.calls[1][1].method).toBe('DELETE') - expect(deleteUrl.searchParams.get('filter')).toBe("source:'automation'") - expect(deleteUrl.searchParams.getAll('ids')).toEqual([]) - expect(deleteUrl.searchParams.get('comment')).toBe('cleanup') - }) - - it('rejects a delete that supplies both ids and a filter', async () => { - const response = await POST( - requestFor({ - operation: 'crowdstrike_delete_indicators', - filter: "source:'automation'", - indicatorIds: ['ioc-9'], - }) - ) - - expect(response.status).toBe(400) - expect(fetchMock).toHaveBeenCalledTimes(0) - }) - - it('rejects a host agent ID that is not a 32-character AID', async () => { - const response = await POST( - requestFor({ - operation: 'crowdstrike_perform_host_action', - actionName: 'contain', - deviceIds: ["not-an-aid') or (device_id:['*'"], - }) - ) - - expect(response.status).toBe(400) - expect(fetchMock).toHaveBeenCalledTimes(0) - }) - - it('rejects an alert update that both assigns and unassigns', async () => { - const response = await POST( - requestFor({ - operation: 'crowdstrike_update_alerts', - compositeIds: ['cid:aid:alert-1'], - assignToUuid: 'user-uuid', - unassign: true, - }) - ) - - expect(response.status).toBe(400) - expect(fetchMock).toHaveBeenCalledTimes(0) - }) - - it('rejects an indicator update whose entry carries no id', async () => { - const response = await POST( - requestFor({ - operation: 'crowdstrike_update_indicators', - indicators: [{ action: 'prevent' }], - }) - ) - - expect(response.status).toBe(400) - expect(await response.json()).toMatchObject({ error: expect.stringContaining('id') }) - expect(fetchMock).toHaveBeenCalledTimes(0) - }) - - it('rejects an indicator update that tries to change the immutable type or value', async () => { - const response = await POST( - requestFor({ - operation: 'crowdstrike_update_indicators', - indicators: [{ id: 'ioc-1', value: 'evil.example' }], - }) - ) - - expect(response.status).toBe(400) - expect(fetchMock).toHaveBeenCalledTimes(0) - }) - - it('rejects an indicator create that omits the required applied_globally scope', async () => { - const response = await POST( - requestFor({ - operation: 'crowdstrike_create_indicators', - indicators: [{ type: 'sha256', value: 'a'.repeat(64), action: 'prevent' }], - }) - ) - - expect(response.status).toBe(400) - expect(fetchMock).toHaveBeenCalledTimes(0) - }) - - it('rejects a write-tier RTR base command on the read-scoped endpoint', async () => { - const response = await POST( - requestFor({ - operation: 'crowdstrike_execute_rtr_command', - sessionId: 'session-1', - baseCommand: 'eventlog backup', - commandString: 'eventlog backup Security', - }) - ) - - expect(response.status).toBe(400) - expect(fetchMock).toHaveBeenCalledTimes(0) - }) - - it('routes an aggregate request to the US-3 cloud', async () => { - fetchMock.mockResolvedValueOnce(jsonResponse({ resources: [] })) - - const response = await POST( - createMockRequest('POST', { - clientId: 'client-id', - clientSecret: 'client-secret', - cloud: 'us-3', - operation: 'crowdstrike_query_sensors', - }) - ) - - expect(response.status).toBe(200) - expect(String(fetchMock.mock.calls[0][0])).toBe('https://api.us-3.crowdstrike.com/oauth2/token') - expect(new URL(fetchMock.mock.calls[1][0]).host).toBe('api.us-3.crowdstrike.com') - }) - - it('rejects an aggregate query that names neither a field nor a type', async () => { - const response = await POST( - requestFor({ - operation: 'crowdstrike_get_sensor_aggregates', - aggregateQuery: { size: 10 }, - }) - ) - - expect(response.status).toBe(400) - expect(fetchMock).toHaveBeenCalledTimes(0) - }) - - it('forwards the percents and filters_spec aggregate fields instead of stripping them', async () => { - fetchMock.mockResolvedValueOnce(jsonResponse({ resources: [] })) - - const aggregateQuery = { - field: 'status', - name: 'by-status', - percents: [50, 95], - filters_spec: { filters: { stale: "status:'inactive'" }, other_bucket: true }, - } - - await POST(requestFor({ operation: 'crowdstrike_get_sensor_aggregates', aggregateQuery })) - - expect(JSON.parse(fetchMock.mock.calls[1][1].body)).toEqual(aggregateQuery) - }) - - it('rejects a delete with neither ids nor a filter', async () => { - const response = await POST(requestFor({ operation: 'crowdstrike_delete_indicators' })) - - expect(response.status).toBe(400) - expect(fetchMock).toHaveBeenCalledTimes(0) - }) - - it('requires a filter for Spotlight vulnerability queries', async () => { - const response = await POST(requestFor({ operation: 'crowdstrike_query_vulnerabilities' })) - - expect(response.status).toBe(400) - expect(fetchMock).toHaveBeenCalledTimes(0) - }) - - it('returns Spotlight cursor pagination', async () => { - fetchMock.mockResolvedValueOnce( - jsonResponse({ - meta: { pagination: { after: 'cursor-1', limit: 1, total: 12 } }, - resources: ['vuln-1'], - }) - ) - - const response = await POST( - requestFor({ operation: 'crowdstrike_query_vulnerabilities', filter: 'status:"open"' }) - ) - const data = await response.json() - - expect(data.output).toEqual({ - vulnerabilityIds: ['vuln-1'], - count: 1, - pagination: { after: 'cursor-1', limit: 1, total: 12 }, - }) - }) - - it('normalizes nested vulnerability details', async () => { - fetchMock.mockResolvedValueOnce( - jsonResponse({ - resources: [ - { - id: 'vuln-1', - aid: 'a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1', - status: 'open', - cve: { - id: 'CVE-2026-0001', - base_score: 9.8, - severity: 'CRITICAL', - cisa_info: { is_cisa_kev: true, due_date: '2026-09-01' }, - }, - host_info: { hostname: 'web-01', groups: [{ id: 'g1', name: 'SOC' }], tags: ['prod'] }, - remediation: { ids: ['rem-1'], entities: [{ id: 'rem-1', title: 'Patch now' }] }, - }, - ], - }) - ) - - const response = await POST( - requestFor({ - operation: 'crowdstrike_get_vulnerability_details', - vulnerabilityIds: ['vuln-1'], - }) - ) - const data = await response.json() - - const vulnerability = data.output.vulnerabilities[0] - expect(vulnerability.cve).toMatchObject({ - id: 'CVE-2026-0001', - baseScore: 9.8, - severity: 'CRITICAL', - isCisaKev: true, - cisaDueDate: '2026-09-01', - }) - expect(vulnerability.hostInfo).toMatchObject({ hostname: 'web-01', groups: ['SOC'] }) - expect(vulnerability.remediationIds).toEqual(['rem-1']) - expect(vulnerability.remediations[0]).toMatchObject({ id: 'rem-1', title: 'Patch now' }) - }) - - it('opens and closes a Real Time Response session', async () => { - fetchMock.mockResolvedValueOnce( - jsonResponse( - { - resources: [ - { - session_id: 'session-1', - device_id: 'a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1', - pwd: 'C:\\', - offline_queued: false, - existing_aid_sessions: 0, - created_at: '2026-08-15T00:00:00Z', - }, - ], - }, - 201 - ) - ) - - const initResponse = await POST( - requestFor({ - operation: 'crowdstrike_init_rtr_session', - deviceId: 'a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1', - }) - ) - const initData = await initResponse.json() - - expect(initData.output).toMatchObject({ - sessionId: 'session-1', - deviceId: 'a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1', - pwd: 'C:\\', - }) - expect(JSON.parse(fetchMock.mock.calls[1][1].body)).toEqual({ - device_id: 'a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1', - }) - - fetchMock.mockResolvedValueOnce(jsonResponse({ access_token: 'token-123' })) - fetchMock.mockResolvedValueOnce(jsonResponse({ meta: {} })) - - const deleteResponse = await POST( - requestFor({ operation: 'crowdstrike_delete_rtr_session', sessionId: 'session-1' }) - ) - const deleteData = await deleteResponse.json() - - expect(deleteData.output).toMatchObject({ sessionId: 'session-1', deleted: true }) - const deleteUrl = new URL(fetchMock.mock.calls[3][0]) - expect(deleteUrl.pathname).toBe('/real-time-response/entities/sessions/v1') - expect(deleteUrl.searchParams.get('session_id')).toBe('session-1') - }) - - it('defaults the RTR command status sequence to zero', async () => { - fetchMock.mockResolvedValueOnce( - jsonResponse({ - resources: [ - { - session_id: 'session-1', - complete: true, - stdout: 'Directory listing', - stderr: '', - base_command: 'ls', - sequence_id: 0, - }, - ], - }) - ) - - const response = await POST( - requestFor({ operation: 'crowdstrike_get_rtr_command_status', cloudRequestId: 'req-1' }) - ) - const data = await response.json() - - expect(data.output).toMatchObject({ complete: true, stdout: 'Directory listing' }) - const statusUrl = new URL(fetchMock.mock.calls[1][0]) - expect(statusUrl.searchParams.get('cloud_request_id')).toBe('req-1') - expect(statusUrl.searchParams.get('sequence_id')).toBe('0') - }) - - it('normalizes Case Management case details', async () => { - fetchMock.mockResolvedValueOnce( - jsonResponse({ - resources: [ - { - id: 'case-1', - name: 'Suspicious login', - status: 'In Progress', - severity: 3, - severity_info: { level: 'High' }, - reference_id: 'CASE-42', - assigned_to: { uuid: 'u-1', email: 'a@example.com', full_name: 'Analyst One' }, - template: { id: 't-1', name: 'Triage' }, - read_only: { is_read_only: false }, - tags: ['phishing'], - }, - ], - }) - ) - - const response = await POST( - requestFor({ operation: 'crowdstrike_get_case_details', caseIds: ['case-1'] }) - ) - const data = await response.json() - - expect(data.output.cases[0]).toMatchObject({ - id: 'case-1', - name: 'Suspicious login', - status: 'In Progress', - severity: 3, - severityLevel: 'High', - referenceId: 'CASE-42', - assignedTo: { uuid: 'u-1', email: 'a@example.com', fullName: 'Analyst One' }, - templateName: 'Triage', - isReadOnly: false, - tags: ['phishing'], - }) - expect(JSON.parse(fetchMock.mock.calls[1][1].body)).toEqual({ ids: ['case-1'] }) - }) - - it('propagates a CrowdStrike error status', async () => { - fetchMock.mockResolvedValueOnce( - jsonResponse({ errors: [{ code: 403, message: 'access denied' }] }, 403) - ) - - const response = await POST( - requestFor({ operation: 'crowdstrike_query_host_groups', filter: 'name:"SOC"' }) - ) - const data = await response.json() - - expect(response.status).toBe(403) - expect(data).toEqual({ success: false, error: 'access denied' }) - }) - - it('fails an alert query whose 200 envelope carries only errors', async () => { - fetchMock.mockResolvedValueOnce( - jsonResponse({ - resources: [], - errors: [{ code: 403, id: null, message: 'insufficient scope' }], - }) - ) - - const response = await POST( - requestFor({ operation: 'crowdstrike_query_alerts', filter: 'status:"new"' }) - ) - const data = await response.json() - - expect(response.status).toBe(403) - expect(data).toEqual({ success: false, error: 'insufficient scope' }) - }) - - it('fails a case query whose 200 envelope carries only errors', async () => { - fetchMock.mockResolvedValueOnce( - jsonResponse({ - resources: [], - errors: [{ code: 500, id: null, message: 'case service unavailable' }], - }) - ) - - const response = await POST(requestFor({ operation: 'crowdstrike_query_cases' })) - const data = await response.json() - - expect(response.status).toBe(500) - expect(data).toEqual({ success: false, error: 'case service unavailable' }) - }) - - it('still reports a genuinely empty alert query as a success', async () => { - fetchMock.mockResolvedValueOnce(jsonResponse({ resources: [], errors: [] })) - - const response = await POST( - requestFor({ operation: 'crowdstrike_query_alerts', filter: 'status:"new"' }) - ) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.output.alertIds).toEqual([]) - expect(data.output.count).toBe(0) - }) - - it('rejects an indicator query that combines offset and after pagination', async () => { - const response = await POST( - requestFor({ operation: 'crowdstrike_query_indicators', offset: 0, after: 'cursor-1' }) - ) - - expect(response.status).toBe(400) - expect(fetchMock).toHaveBeenCalledTimes(0) - }) - - it('rejects a blank alert filter so Falcon never sees an empty FQL expression', async () => { - const response = await POST( - requestFor({ operation: 'crowdstrike_query_alerts', filter: ' ' }) - ) - - expect(response.status).toBe(400) - expect(fetchMock).toHaveBeenCalledTimes(0) - }) - - it('rejects a blank sensor filter instead of sending an empty FQL expression', async () => { - const response = await POST( - requestFor({ operation: 'crowdstrike_query_sensors', filter: ' ' }) - ) - - expect(response.status).toBe(400) - expect(fetchMock).toHaveBeenCalledTimes(0) - }) - - it('rejects a blank sensor sort instead of sending an empty sort expression', async () => { - const response = await POST(requestFor({ operation: 'crowdstrike_query_sensors', sort: '' })) - - expect(response.status).toBe(400) - expect(fetchMock).toHaveBeenCalledTimes(0) - }) - - it('forwards trimmed sensor filter and sort values', async () => { - fetchMock.mockResolvedValueOnce(jsonResponse({ resources: [] })) - - const response = await POST( - requestFor({ - operation: 'crowdstrike_query_sensors', - filter: ' hostname:"dc-01" ', - sort: ' hostname.asc ', - }) - ) - - expect(response.status).toBe(200) - - const queryUrl = new URL(fetchMock.mock.calls[1][0]) - expect(queryUrl.pathname).toBe('/identity-protection/queries/devices/v1') - expect(queryUrl.searchParams.get('filter')).toBe('hostname:"dc-01"') - expect(queryUrl.searchParams.get('sort')).toBe('hostname.asc') - }) - - it('fails an RTR session close whose 200 envelope reports an error', async () => { - fetchMock.mockResolvedValueOnce( - jsonResponse({ meta: {}, errors: [{ code: 404, message: 'session not found' }] }) - ) - - const response = await POST( - requestFor({ operation: 'crowdstrike_delete_rtr_session', sessionId: 'session-1' }) - ) - const data = await response.json() - - expect(response.status).toBe(404) - expect(data.success).toBe(false) - expect(data.error).toBe('session not found') - }) - - it('fails a sensor query whose 200 envelope carries only errors', async () => { - fetchMock.mockResolvedValueOnce( - jsonResponse({ - resources: [], - errors: [{ code: 403, message: 'access denied for Identity Protection' }], - }) - ) - - const response = await POST(requestFor({ operation: 'crowdstrike_query_sensors' })) - const data = await response.json() - - expect(response.status).toBe(403) - expect(data.success).toBe(false) - expect(data.error).toBe('access denied for Identity Protection') - }) - - it('fails a sensor detail lookup whose 200 envelope carries only errors', async () => { - fetchMock.mockResolvedValueOnce( - jsonResponse({ resources: [], errors: [{ code: 403, message: 'access denied' }] }) - ) - - const response = await POST( - requestFor({ - operation: 'crowdstrike_get_sensor_details', - ids: ['a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1'], - }) - ) - const data = await response.json() - - expect(response.status).toBe(403) - expect(data.success).toBe(false) - }) - - it('fails a sensor aggregate whose 200 envelope carries only errors', async () => { - fetchMock.mockResolvedValueOnce( - jsonResponse({ resources: [], errors: [{ code: 403, message: 'access denied' }] }) - ) - - const response = await POST( - requestFor({ - operation: 'crowdstrike_get_sensor_aggregates', - aggregateQuery: { field: 'status', name: 'by_status', type: 'terms' }, - }) - ) - const data = await response.json() - - expect(response.status).toBe(403) - expect(data.success).toBe(false) - }) - - it('surfaces partial sensor errors alongside the sensors that resolved', async () => { - fetchMock.mockResolvedValueOnce( - jsonResponse({ - resources: [{ device_id: 'a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1', hostname: 'dc-01' }], - errors: [ - { code: 404, id: 'b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2', message: 'sensor not found' }, - ], - }) - ) - - const response = await POST( - requestFor({ - operation: 'crowdstrike_get_sensor_details', - ids: ['a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1', 'b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2'], - }) - ) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.output.count).toBe(1) - expect(data.output.errors).toEqual([ - { code: 404, id: 'b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2', message: 'sensor not found' }, - ]) - }) - - it('preserves a scalar aggregate bucket label', async () => { - fetchMock.mockResolvedValueOnce( - jsonResponse({ - resources: [{ name: 'by_status', buckets: [{ label: 'contained', count: 3 }] }], - }) - ) - - const response = await POST( - requestFor({ - operation: 'crowdstrike_get_sensor_aggregates', - aggregateQuery: { field: 'status', name: 'by_status', type: 'terms' }, - }) - ) - const data = await response.json() - - expect(data.output.aggregates[0].buckets[0].label).toBe('contained') - }) - - it('rejects an indicator payload whose blank field would clear stored data', async () => { - const response = await POST( - requestFor({ - operation: 'crowdstrike_update_indicators', - indicators: [{ id: 'ioc-1', description: '' }], - }) - ) - - expect(response.status).toBe(400) - expect(fetchMock).toHaveBeenCalledTimes(0) - }) - - it('rejects a host action targeting more than the documented 100 hosts', async () => { - const response = await POST( - requestFor({ - operation: 'crowdstrike_perform_host_action', - actionName: 'contain', - deviceIds: Array.from({ length: 101 }, (_, index) => `aid-${index}`), - }) - ) - - expect(response.status).toBe(400) - expect(fetchMock).toHaveBeenCalledTimes(0) - }) - - it('accepts the documented detection suppression host actions', async () => { - fetchMock.mockResolvedValueOnce( - jsonResponse({ resources: [{ id: 'a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1' }] }) - ) - - const response = await POST( - requestFor({ - operation: 'crowdstrike_perform_host_action', - actionName: 'detection_suppress', - deviceIds: ['a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1'], - }) - ) - - expect(response.status).toBe(200) - expect(new URL(fetchMock.mock.calls[1][0]).searchParams.get('action_name')).toBe( - 'detection_suppress' - ) - }) - - describe('by-ids URL byte budget', () => { - /** Long enough that the contract maxima would generate a URL past any proxy limit. */ - function longIds(count: number, prefix: string): string[] { - return Array.from({ length: count }, (_, index) => - `${prefix}-${String(index).padStart(4, '0')}`.padEnd(64, 'x') - ) - } - - function idsFromUrl(rawUrl: string): string[] { - return new URL(rawUrl).searchParams.getAll('ids') - } - - /** The 8 KB request line + header cap common to proxies and load balancers. */ - const PROXY_REQUEST_LINE_LIMIT = 8192 - - it('keeps the budget under the request-line limit proxies enforce', () => { - expect(MAX_ID_URL_BYTES).toBeLessThanOrEqual(PROXY_REQUEST_LINE_LIMIT) - }) - - it('splits an oversized indicator lookup into batches that each stay under the URL budget', async () => { - const indicatorIds = longIds(300, 'ioc') - - fetchMock.mockImplementation((rawUrl: string) => - Promise.resolve( - jsonResponse({ - meta: { pagination: { limit: 1, offset: 0, total: 300 } }, - resources: idsFromUrl(rawUrl).map((id) => ({ id, type: 'sha256', value: id })), - }) - ) - ) - - const response = await POST( - requestFor({ operation: 'crowdstrike_get_indicator_details', indicatorIds }) - ) - const data = await response.json() - - const lookupUrls = fetchMock.mock.calls.slice(1).map((call) => String(call[0])) - expect(lookupUrls.length).toBeGreaterThan(1) - for (const url of lookupUrls) { - expect(url.length).toBeLessThanOrEqual(MAX_ID_URL_BYTES) - } - - expect(response.status).toBe(200) - expect(data.output.count).toBe(300) - expect(data.output.indicators.map((indicator: { id: string }) => indicator.id)).toEqual( - indicatorIds - ) - expect(lookupUrls.flatMap(idsFromUrl)).toEqual(indicatorIds) - }) - - it('merges the envelope errors every batch reported', async () => { - const indicatorIds = longIds(300, 'ioc') - - fetchMock.mockImplementation((rawUrl: string) => { - const ids = idsFromUrl(rawUrl) - return Promise.resolve( - jsonResponse({ - resources: ids.slice(1).map((id) => ({ id, type: 'sha256', value: id })), - errors: [{ code: 404, id: ids[0], message: 'Indicator not found' }], - }) - ) - }) - - const response = await POST( - requestFor({ operation: 'crowdstrike_get_indicator_details', indicatorIds }) - ) - const data = await response.json() - - const batchCount = fetchMock.mock.calls.length - 1 - expect(batchCount).toBeGreaterThan(1) - expect(data.output.errors).toHaveLength(batchCount) - }) - - it('surfaces an upstream failure raised by a later batch instead of swallowing it', async () => { - const indicatorIds = longIds(300, 'ioc') - let lookupCall = 0 - - fetchMock.mockImplementation((rawUrl: string) => { - lookupCall += 1 - if (lookupCall === 2) { - return Promise.resolve( - jsonResponse({ errors: [{ code: 429, message: 'Rate limit exceeded' }] }, 429) - ) - } - return Promise.resolve( - jsonResponse({ - resources: idsFromUrl(rawUrl).map((id) => ({ id, type: 'sha256', value: id })), - }) - ) - }) - - const response = await POST( - requestFor({ operation: 'crowdstrike_get_indicator_details', indicatorIds }) - ) - const data = await response.json() - - expect(response.status).toBe(429) - expect(data.success).toBe(false) - expect(data.error).toBe('Rate limit exceeded') - }) - - /** - * A batched delete has no rollback: every batch that answered `ok` really removed - * its indicators. Reporting only the failing batch's message would leave the - * caller unable to tell what is already gone, and a blind retry would re-target - * IDs that no longer exist. - */ - it('names the indicators earlier batches already deleted when a later batch fails', async () => { - const indicatorIds = longIds(300, 'ioc') - let deleteCall = 0 - const deletedByFirstBatch: string[] = [] - - fetchMock.mockImplementation((rawUrl: string) => { - deleteCall += 1 - if (deleteCall === 2) { - return Promise.resolve( - jsonResponse({ errors: [{ code: 429, message: 'Rate limit exceeded' }] }, 429) - ) - } - const ids = idsFromUrl(rawUrl) - deletedByFirstBatch.push(...ids) - return Promise.resolve(jsonResponse({ resources: ids })) - }) - - const response = await POST( - requestFor({ operation: 'crowdstrike_delete_indicators', indicatorIds }) - ) - const data = await response.json() - - expect(response.status).toBe(429) - expect(data.success).toBe(false) - expect(deletedByFirstBatch.length).toBeGreaterThan(0) - expect(data.error).toContain('Rate limit exceeded') - expect(data.error).toContain(`${deletedByFirstBatch.length} ID(s) were already deleted`) - expect(data.error).toContain(deletedByFirstBatch[0]) - }) - - /** - * A batch can answer 200 while reporting per-ID failures in `errors`. Recording - * the requested chunk would name indicators that are still live and tell the - * caller to drop them from the retry, so only the IDs Falcon echoed in - * `resources` count as committed. - */ - it('reports only the IDs Falcon confirmed, not every ID in a partly-failed batch', async () => { - const indicatorIds = longIds(300, 'ioc') - let deleteCall = 0 - let skipped = '' - const confirmed: string[] = [] - - fetchMock.mockImplementation((rawUrl: string) => { - deleteCall += 1 - if (deleteCall === 2) { - return Promise.resolve( - jsonResponse({ errors: [{ code: 429, message: 'Rate limit exceeded' }] }, 429) - ) - } - const ids = idsFromUrl(rawUrl) - skipped = ids[0] - confirmed.push(...ids.slice(1)) - return Promise.resolve( - jsonResponse({ - resources: ids.slice(1), - errors: [{ code: 404, id: ids[0], message: 'Indicator not found' }], - }) - ) - }) - - const response = await POST( - requestFor({ operation: 'crowdstrike_delete_indicators', indicatorIds }) - ) - const data = await response.json() - - expect(response.status).toBe(429) - expect(confirmed.length).toBeGreaterThan(0) - expect(data.error).toContain(`${confirmed.length} ID(s) were already deleted`) - expect(data.error).not.toContain(skipped) - }) - - /** - * A 2xx envelope carrying per-ID errors is a partial success, not a failure — - * `failedWithoutResources` only fails the operation when nothing came back at - * all. The batched path must report it exactly as a single request would: - * `deletedIds` names what Falcon confirmed and `errors` names what it refused, - * which is stricter reconciliation than the prose message the transport-failure - * path has to fall back on. A retry excludes `deletedIds`. - */ - it('reports a 2xx per-ID delete failure as partial success, matching an unbatched request', async () => { - const indicatorIds = longIds(300, 'ioc') - let deleteCall = 0 - let refused = '' - - fetchMock.mockImplementation((rawUrl: string) => { - deleteCall += 1 - const ids = idsFromUrl(rawUrl) - if (deleteCall === 2) { - refused = ids[0] - return Promise.resolve( - jsonResponse({ - resources: ids.slice(1), - errors: [{ code: 404, id: ids[0], message: 'Indicator not found' }], - }) - ) - } - return Promise.resolve(jsonResponse({ resources: ids })) - }) - - const response = await POST( - requestFor({ operation: 'crowdstrike_delete_indicators', indicatorIds }) - ) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(refused).not.toBe('') - expect(data.output.deletedIds).not.toContain(refused) - expect(data.output.count).toBe(indicatorIds.length - 1) - expect(data.output.errors).toContainEqual(expect.objectContaining({ id: refused, code: 404 })) - }) - - it('leaves a first-batch delete failure unannotated — nothing was committed', async () => { - const indicatorIds = longIds(300, 'ioc') - - fetchMock.mockImplementation(() => - Promise.resolve(jsonResponse({ errors: [{ code: 403, message: 'Access denied' }] }, 403)) - ) - - const response = await POST( - requestFor({ operation: 'crowdstrike_delete_indicators', indicatorIds }) - ) - const data = await response.json() - - expect(response.status).toBe(403) - expect(data.error).toBe('Access denied') - }) - - it('splits an oversized vulnerability lookup at the Spotlight cap', async () => { - const vulnerabilityIds = longIds(400, 'vuln') - - fetchMock.mockImplementation((rawUrl: string) => - Promise.resolve( - jsonResponse({ - resources: idsFromUrl(rawUrl).map((id) => ({ id })), - }) - ) - ) - - const response = await POST( - requestFor({ operation: 'crowdstrike_get_vulnerability_details', vulnerabilityIds }) - ) - const data = await response.json() - - const lookupUrls = fetchMock.mock.calls.slice(1).map((call) => String(call[0])) - expect(lookupUrls.length).toBeGreaterThan(1) - for (const url of lookupUrls) { - expect(url.length).toBeLessThanOrEqual(MAX_ID_URL_BYTES) - } - expect(data.output.count).toBe(400) - }) - - it('keeps a filter-only delete on a single request', async () => { - fetchMock.mockResolvedValueOnce(jsonResponse({ resources: ['ioc-1'] })) - - const response = await POST( - requestFor({ - operation: 'crowdstrike_delete_indicators', - filter: "source:'automation'", - comment: 'cleanup', - }) - ) - - expect(response.status).toBe(200) - expect(fetchMock).toHaveBeenCalledTimes(2) - }) - }) -}) diff --git a/apps/sim/app/api/tools/crowdstrike/query/operations.ts b/apps/sim/app/api/tools/crowdstrike/query/operations.ts deleted file mode 100644 index a1b2b1e7b67..00000000000 --- a/apps/sim/app/api/tools/crowdstrike/query/operations.ts +++ /dev/null @@ -1,795 +0,0 @@ -import { isRecordLike } from '@sim/utils/object' -import { truncate } from '@sim/utils/string' -import type { CrowdstrikeQueryBody } from '@/lib/api/contracts/tools/crowdstrike' -import { - buildUrl, - type CrowdStrikeCallResult, - callCrowdStrike, - getBoolean, - getCursorPagination, - getEnvelopeErrors, - getFalconErrorMessage, - getFirstRecordResource, - getNumber, - getPagination, - getRecordArray, - getRecordResources, - getResourcesArray, - getSpotlightPagination, - getString, - getStringResources, -} from '@/app/api/tools/crowdstrike/query/falcon' -import { - normalizeAffectedEntity, - normalizeAlert, - normalizeCase, - normalizeHostGroup, - normalizeIndicator, - normalizeVulnerability, -} from '@/app/api/tools/crowdstrike/query/normalize' -import type { CrowdStrikeActionParameter } from '@/tools/crowdstrike/types' - -type ExtendedOperation = Exclude< - CrowdstrikeQueryBody['operation'], - | 'crowdstrike_query_sensors' - | 'crowdstrike_get_sensor_details' - | 'crowdstrike_get_sensor_aggregates' -> - -type ExtendedBody = Extract - -export interface OperationFailure { - ok: false - status: number - error: string -} - -export interface OperationSuccess { - ok: true - output: Record -} - -export type OperationResult = OperationSuccess | OperationFailure - -/** - * CrowdStrike can answer 200 while the envelope carries only errors. Reporting - * that as HTTP 200 would read as a success, so fall back to the per-item error - * code the envelope supplies, and to 502 when it supplies none. - */ -export function failureStatus(result: CrowdStrikeCallResult): number { - if (!result.ok) { - return result.status - } - - const envelopeCode = getEnvelopeErrors(result.data)[0]?.code - if (envelopeCode != null && envelopeCode >= 400 && envelopeCode <= 599) { - return envelopeCode - } - - return 502 -} - -function fail(result: CrowdStrikeCallResult, fallback: string): OperationFailure { - return { - ok: false, - status: failureStatus(result), - error: getFalconErrorMessage(result.data, fallback), - } -} - -/** - * CrowdStrike answers 200 with a populated `errors` array when only some IDs - * fail. Treat that as an outright failure only when nothing came back at all. - */ -export function failedWithoutResources( - result: CrowdStrikeCallResult, - resourceCount: number -): boolean { - return resourceCount === 0 && getEnvelopeErrors(result.data).length > 0 -} - -/** - * Falcon's by-ids lookups carry every ID in the query string, so a request at the - * contract maxima (1000 indicator IDs at ~68 bytes each) would generate a ~68 KB - * URL. Proxies and load balancers commonly cap the request line plus headers at - * 8 KB, so batches are sized to keep each generated URL at or under half of that, - * leaving the rest of the budget for headers. - */ -export const MAX_ID_URL_BYTES = 4096 - -/** - * Batches by cumulative encoded length rather than by a fixed count: Falcon IDs - * range from 32-character AIDs to long composite alert IDs, so a count-based cap - * would either waste the budget or blow past it. A single ID longer than the - * budget still gets its own batch — truncating the list would silently drop it. - */ -export function chunkIdsByUrlBudget(ids: string[], budget: number): string[][] { - const chunks: string[][] = [] - let current: string[] = [] - let used = 0 - - for (const id of ids) { - const cost = `&ids=${encodeURIComponent(id)}`.length - if (current.length > 0 && used + cost > budget) { - chunks.push(current) - current = [] - used = 0 - } - current.push(id) - used += cost - } - - if (current.length > 0) { - chunks.push(current) - } - - return chunks -} - -/** - * Caps how much of the already-committed ID list is spelled out in a partial-failure - * message. A by-ids delete can carry 1000 IDs at ~68 bytes each, so the full list - * would bury the actual failure under ~68 KB of text. - */ -const MAX_COMMITTED_IDS_IN_MESSAGE = 400 - -/** - * Rewrites a failed batch's envelope so the reported error names the deletions the - * earlier batches already committed. - * - * Batches run sequentially and Falcon has no way to roll back a deletion it already - * performed. Short-circuiting on a later batch would therefore report a bare failure - * over work that already happened, and a blind retry would target IDs that no longer - * exist. Only the message survives to the caller ({@link fail} keeps `status` and the - * message, not `data`), so the committed list is written onto `errors[0].message`, - * which is the first thing {@link getFalconErrorMessage} reads. - * - * `committed` holds the IDs Falcon echoed in `resources`, never the IDs that were - * requested, so an ID that failed inside an otherwise-200 batch is not reported as - * deleted. - */ -function withCommittedIds( - result: CrowdStrikeCallResult, - committed: string[] -): CrowdStrikeCallResult { - if (committed.length === 0) return result - - const envelope = isRecordLike(result.data) ? result.data : {} - const existing = getRecordArray(envelope.errors) - const reason = getFalconErrorMessage(result.data, 'CrowdStrike rejected a later batch.') - const message = - `${reason} This request was split into batches and ${committed.length} ID(s) were already deleted ` + - `before the failing batch; they were not rolled back, so retry only the remainder. ` + - `Deleted: ${truncate(committed.join(', '), MAX_COMMITTED_IDS_IN_MESSAGE)}` - - return { - ...result, - data: { ...envelope, errors: [{ ...(existing[0] ?? {}), message }, ...existing.slice(1)] }, - } -} - -interface ByIdsRequestOptions { - method: 'GET' | 'DELETE' - path: string - ids: string[] | undefined - query?: Record -} - -/** - * Issues a by-ids lookup as however many requests it takes to stay under - * `MAX_ID_URL_BYTES`, then presents the batches as one `{ meta, resources, errors }` - * envelope so callers read the same shape a single request returns. - * - * Batches run sequentially: resource order matches the caller's ID order, the - * endpoint's rate limit only ever sees one request at a time, and a failing batch - * short-circuits with its own status instead of being merged away. A `DELETE` that - * fails partway also carries the IDs its earlier batches already removed — see - * {@link withCommittedIds}. `meta` comes - * from the first batch — pagination is meaningless for a lookup that names every - * ID it wants, and no by-ids operation here reads it. - */ -async function callCrowdStrikeByIds( - baseUrl: string, - accessToken: string, - options: ByIdsRequestOptions -): Promise { - const prefix = buildUrl(baseUrl, { - method: options.method, - path: options.path, - query: options.query, - }) - const chunks = chunkIdsByUrlBudget( - options.ids ?? [], - Math.max(MAX_ID_URL_BYTES - prefix.length, 1) - ) - - if (chunks.length <= 1) { - return callCrowdStrike(baseUrl, accessToken, { - method: options.method, - path: options.path, - query: options.query, - repeatedQuery: { ids: options.ids }, - }) - } - - const resources: unknown[] = [] - const errors: unknown[] = [] - const committed: string[] = [] - let meta: unknown - let status = 200 - - for (const [index, chunk] of chunks.entries()) { - const result = await callCrowdStrike(baseUrl, accessToken, { - method: options.method, - path: options.path, - query: options.query, - repeatedQuery: { ids: chunk }, - }) - - if (!result.ok) { - return options.method === 'DELETE' ? withCommittedIds(result, committed) : result - } - - /** - * Only the IDs Falcon echoed in `resources` were actually deleted. A batch can - * answer 200 while reporting per-ID failures in `errors`, so recording the - * requested chunk would name indicators that are still live and tell the - * caller to drop them from the retry. - */ - if (options.method === 'DELETE') { - committed.push(...getStringResources(result.data)) - } - - if (index === 0) { - status = result.status - meta = isRecordLike(result.data) ? result.data.meta : undefined - } - - resources.push(...getResourcesArray(result.data)) - errors.push(...getRecordArray(isRecordLike(result.data) ? result.data.errors : undefined)) - } - - return { ok: true, status, data: { meta, resources, errors } } -} - -function buildAlertActionParameters( - body: Extract -) { - const parameters: CrowdStrikeActionParameter[] = [] - - const push = (name: string, value: string | undefined) => { - if (value !== undefined) { - parameters.push({ name, value }) - } - } - - push('update_status', body.updateStatus) - push('assign_to_uuid', body.assignToUuid) - push('assign_to_user_id', body.assignToUserId) - push('assign_to_name', body.assignToName) - push('append_comment', body.appendComment) - push('add_tag', body.addTag) - push('remove_tag', body.removeTag) - push('remove_tags_by_prefix', body.removeTagsByPrefix) - - if (body.unassign === true) { - parameters.push({ name: 'unassign', value: '' }) - } - - if (body.showInUi !== undefined) { - parameters.push({ name: 'show_in_ui', value: String(body.showInUi) }) - } - - for (const parameter of body.actionParameters ?? []) { - parameters.push({ name: parameter.name, value: parameter.value }) - } - - return parameters -} - -/** - * CrowdStrike's host-group action endpoint selects the hosts to add or remove - * with an FQL `device_id` filter rather than an ID list. - */ -function buildDeviceIdFilter(deviceIds: string[]): string { - const values = deviceIds.map((id) => `'${id.replaceAll("'", "\\'")}'`).join(',') - return `(device_id:[${values}])` -} - -export async function executeCrowdStrikeOperation( - body: ExtendedBody, - baseUrl: string, - accessToken: string -): Promise { - switch (body.operation) { - case 'crowdstrike_query_alerts': { - const result = await callCrowdStrike(baseUrl, accessToken, { - method: 'GET', - path: '/alerts/queries/alerts/v2', - query: { - filter: body.filter, - include_hidden: body.includeHidden, - limit: body.limit, - offset: body.offset, - q: body.q, - sort: body.sort, - }, - }) - if (!result.ok) return fail(result, 'Failed to query CrowdStrike alerts') - - const alertIds = getStringResources(result.data) - if (failedWithoutResources(result, alertIds.length)) { - return fail(result, 'Failed to query CrowdStrike alerts') - } - - return { - ok: true, - output: { alertIds, count: alertIds.length, pagination: getPagination(result.data) }, - } - } - - case 'crowdstrike_get_alert_details': { - const result = await callCrowdStrike(baseUrl, accessToken, { - method: 'POST', - path: '/alerts/entities/alerts/v2', - query: { include_hidden: body.includeHidden }, - body: { composite_ids: body.compositeIds }, - }) - if (!result.ok) return fail(result, 'Failed to fetch CrowdStrike alert details') - - const alerts = getRecordResources(result.data).map(normalizeAlert) - if (failedWithoutResources(result, alerts.length)) { - return fail(result, 'Failed to fetch CrowdStrike alert details') - } - - return { - ok: true, - output: { alerts, count: alerts.length, errors: getEnvelopeErrors(result.data) }, - } - } - - case 'crowdstrike_update_alerts': { - const result = await callCrowdStrike(baseUrl, accessToken, { - method: 'PATCH', - path: '/alerts/entities/alerts/v3', - query: { include_hidden: body.includeHidden }, - body: { - action_parameters: buildAlertActionParameters(body), - composite_ids: body.compositeIds, - }, - }) - if (!result.ok) return fail(result, 'Failed to update CrowdStrike alerts') - - const errors = getEnvelopeErrors(result.data) - if (errors.length > 0) { - return fail(result, 'Failed to update CrowdStrike alerts') - } - - return { - ok: true, - output: { updatedIds: body.compositeIds, count: body.compositeIds.length, errors }, - } - } - - case 'crowdstrike_perform_host_action': { - const result = await callCrowdStrike(baseUrl, accessToken, { - method: 'POST', - path: '/devices/entities/devices-actions/v2', - query: { action_name: body.actionName }, - body: { ids: body.deviceIds }, - }) - if (!result.ok) return fail(result, 'Failed to perform CrowdStrike host action') - - const affected = getRecordResources(result.data).map(normalizeAffectedEntity) - if (failedWithoutResources(result, affected.length)) { - return fail(result, 'Failed to perform CrowdStrike host action') - } - - return { - ok: true, - output: { affected, count: affected.length, errors: getEnvelopeErrors(result.data) }, - } - } - - case 'crowdstrike_query_host_groups': { - const result = await callCrowdStrike(baseUrl, accessToken, { - method: 'GET', - path: '/devices/queries/host-groups/v1', - query: { - filter: body.filter, - limit: body.limit, - offset: body.offset, - sort: body.sort, - }, - }) - if (!result.ok) return fail(result, 'Failed to query CrowdStrike host groups') - - const hostGroupIds = getStringResources(result.data) - if (failedWithoutResources(result, hostGroupIds.length)) { - return fail(result, 'Failed to query CrowdStrike host groups') - } - - return { - ok: true, - output: { - hostGroupIds, - count: hostGroupIds.length, - pagination: getPagination(result.data), - }, - } - } - - case 'crowdstrike_get_host_group_details': { - const result = await callCrowdStrikeByIds(baseUrl, accessToken, { - method: 'GET', - path: '/devices/entities/host-groups/v1', - ids: body.hostGroupIds, - }) - if (!result.ok) return fail(result, 'Failed to fetch CrowdStrike host group details') - - const hostGroups = getRecordResources(result.data).map(normalizeHostGroup) - if (failedWithoutResources(result, hostGroups.length)) { - return fail(result, 'Failed to fetch CrowdStrike host group details') - } - - return { - ok: true, - output: { - hostGroups, - count: hostGroups.length, - errors: getEnvelopeErrors(result.data), - }, - } - } - - case 'crowdstrike_perform_host_group_action': { - const result = await callCrowdStrike(baseUrl, accessToken, { - method: 'POST', - path: '/devices/entities/host-group-actions/v1', - query: { action_name: body.actionName }, - body: { - action_parameters: [{ name: 'filter', value: buildDeviceIdFilter(body.deviceIds) }], - ids: [body.hostGroupId], - }, - }) - if (!result.ok) return fail(result, 'Failed to perform CrowdStrike host group action') - - const hostGroups = getRecordResources(result.data).map(normalizeHostGroup) - if (failedWithoutResources(result, hostGroups.length)) { - return fail(result, 'Failed to perform CrowdStrike host group action') - } - - return { - ok: true, - output: { - hostGroups, - count: hostGroups.length, - errors: getEnvelopeErrors(result.data), - }, - } - } - - case 'crowdstrike_query_indicators': { - const result = await callCrowdStrike(baseUrl, accessToken, { - method: 'GET', - path: '/iocs/queries/indicators/v1', - query: { - after: body.after, - filter: body.filter, - limit: body.limit, - offset: body.offset, - sort: body.sort, - }, - }) - if (!result.ok) return fail(result, 'Failed to query CrowdStrike indicators') - - const indicatorIds = getStringResources(result.data) - if (failedWithoutResources(result, indicatorIds.length)) { - return fail(result, 'Failed to query CrowdStrike indicators') - } - - return { - ok: true, - output: { - indicatorIds, - count: indicatorIds.length, - pagination: getCursorPagination(result.data), - }, - } - } - - case 'crowdstrike_get_indicator_details': { - const result = await callCrowdStrikeByIds(baseUrl, accessToken, { - method: 'GET', - path: '/iocs/entities/indicators/v1', - ids: body.indicatorIds, - }) - if (!result.ok) return fail(result, 'Failed to fetch CrowdStrike indicator details') - - const indicators = getRecordResources(result.data).map(normalizeIndicator) - if (failedWithoutResources(result, indicators.length)) { - return fail(result, 'Failed to fetch CrowdStrike indicator details') - } - - return { - ok: true, - output: { - indicators, - count: indicators.length, - errors: getEnvelopeErrors(result.data), - }, - } - } - - case 'crowdstrike_create_indicators': - case 'crowdstrike_update_indicators': { - const isCreate = body.operation === 'crowdstrike_create_indicators' - const result = await callCrowdStrike(baseUrl, accessToken, { - method: isCreate ? 'POST' : 'PATCH', - path: '/iocs/entities/indicators/v1', - query: { - ignore_warnings: body.ignoreWarnings, - retrodetects: body.retrodetects, - }, - body: { - comment: body.comment, - indicators: body.indicators, - }, - }) - if (!result.ok) { - return fail( - result, - isCreate - ? 'Failed to create CrowdStrike indicators' - : 'Failed to update CrowdStrike indicators' - ) - } - - const indicators = getRecordResources(result.data).map(normalizeIndicator) - if (failedWithoutResources(result, indicators.length)) { - return fail( - result, - isCreate - ? 'Failed to create CrowdStrike indicators' - : 'Failed to update CrowdStrike indicators' - ) - } - - return { - ok: true, - output: { - indicators, - count: indicators.length, - errors: getEnvelopeErrors(result.data), - }, - } - } - - case 'crowdstrike_delete_indicators': { - const result = await callCrowdStrikeByIds(baseUrl, accessToken, { - method: 'DELETE', - path: '/iocs/entities/indicators/v1', - query: { comment: body.comment, filter: body.filter }, - ids: body.filter ? undefined : body.indicatorIds, - }) - if (!result.ok) return fail(result, 'Failed to delete CrowdStrike indicators') - - const deletedIds = getStringResources(result.data) - if (failedWithoutResources(result, deletedIds.length)) { - return fail(result, 'Failed to delete CrowdStrike indicators') - } - - return { - ok: true, - output: { - deletedIds, - count: deletedIds.length, - errors: getEnvelopeErrors(result.data), - }, - } - } - - case 'crowdstrike_query_vulnerabilities': { - const result = await callCrowdStrike(baseUrl, accessToken, { - method: 'GET', - path: '/spotlight/queries/vulnerabilities/v1', - query: { - after: body.after, - filter: body.filter, - limit: body.limit, - sort: body.sort, - }, - }) - if (!result.ok) return fail(result, 'Failed to query CrowdStrike vulnerabilities') - - const vulnerabilityIds = getStringResources(result.data) - if (failedWithoutResources(result, vulnerabilityIds.length)) { - return fail(result, 'Failed to query CrowdStrike vulnerabilities') - } - - return { - ok: true, - output: { - vulnerabilityIds, - count: vulnerabilityIds.length, - pagination: getSpotlightPagination(result.data), - }, - } - } - - case 'crowdstrike_get_vulnerability_details': { - const result = await callCrowdStrikeByIds(baseUrl, accessToken, { - method: 'GET', - path: '/spotlight/entities/vulnerabilities/v2', - ids: body.vulnerabilityIds, - }) - if (!result.ok) return fail(result, 'Failed to fetch CrowdStrike vulnerability details') - - const vulnerabilities = getRecordResources(result.data).map(normalizeVulnerability) - if (failedWithoutResources(result, vulnerabilities.length)) { - return fail(result, 'Failed to fetch CrowdStrike vulnerability details') - } - - return { - ok: true, - output: { - vulnerabilities, - count: vulnerabilities.length, - errors: getEnvelopeErrors(result.data), - }, - } - } - - case 'crowdstrike_init_rtr_session': { - const result = await callCrowdStrike(baseUrl, accessToken, { - method: 'POST', - path: '/real-time-response/entities/sessions/v1', - body: { - device_id: body.deviceId, - origin: body.origin, - queue_offline: body.queueOffline, - }, - }) - if (!result.ok) return fail(result, 'Failed to initialize CrowdStrike RTR session') - - const session = getFirstRecordResource(result.data) - if (!session) return fail(result, 'CrowdStrike did not return an RTR session') - - return { - ok: true, - output: { - sessionId: getString(session.session_id), - deviceId: getString(session.device_id), - platform: getString(session.platform), - pwd: getString(session.pwd), - offlineQueued: getBoolean(session.offline_queued), - existingAidSessions: getNumber(session.existing_aid_sessions), - createdAt: getString(session.created_at), - errors: getEnvelopeErrors(result.data), - }, - } - } - - case 'crowdstrike_execute_rtr_command': { - const result = await callCrowdStrike(baseUrl, accessToken, { - method: 'POST', - path: '/real-time-response/entities/command/v1', - body: { - base_command: body.baseCommand, - command_string: body.commandString, - session_id: body.sessionId, - }, - }) - if (!result.ok) return fail(result, 'Failed to execute CrowdStrike RTR command') - - const command = getFirstRecordResource(result.data) - if (!command) return fail(result, 'CrowdStrike did not return an RTR command result') - - return { - ok: true, - output: { - cloudRequestId: getString(command.cloud_request_id), - sessionId: getString(command.session_id), - queuedCommandOffline: getBoolean(command.queued_command_offline), - errors: getEnvelopeErrors(result.data), - }, - } - } - - case 'crowdstrike_get_rtr_command_status': { - const result = await callCrowdStrike(baseUrl, accessToken, { - method: 'GET', - path: '/real-time-response/entities/command/v1', - query: { - cloud_request_id: body.cloudRequestId, - sequence_id: body.sequenceId ?? 0, - }, - }) - if (!result.ok) return fail(result, 'Failed to fetch CrowdStrike RTR command status') - - const status = getFirstRecordResource(result.data) - if (!status) return fail(result, 'CrowdStrike did not return an RTR command status') - - return { - ok: true, - output: { - complete: getBoolean(status.complete), - stdout: getString(status.stdout), - stderr: getString(status.stderr), - baseCommand: getString(status.base_command), - sessionId: getString(status.session_id), - taskId: getString(status.task_id), - sequenceId: getNumber(status.sequence_id), - errors: getEnvelopeErrors(result.data), - }, - } - } - - case 'crowdstrike_delete_rtr_session': { - const result = await callCrowdStrike(baseUrl, accessToken, { - method: 'DELETE', - path: '/real-time-response/entities/sessions/v1', - query: { session_id: body.sessionId }, - }) - if (!result.ok) return fail(result, 'Failed to delete CrowdStrike RTR session') - - const deleteErrors = getEnvelopeErrors(result.data) - if (deleteErrors.length > 0) { - return fail(result, 'Failed to delete CrowdStrike RTR session') - } - - return { - ok: true, - output: { - sessionId: body.sessionId, - deleted: true, - errors: deleteErrors, - }, - } - } - - case 'crowdstrike_query_cases': { - const result = await callCrowdStrike(baseUrl, accessToken, { - method: 'GET', - path: '/cases/queries/cases/v1', - query: { - filter: body.filter, - limit: body.limit, - offset: body.offset, - q: body.q, - sort: body.sort, - }, - }) - if (!result.ok) return fail(result, 'Failed to query CrowdStrike cases') - - const caseIds = getStringResources(result.data) - if (failedWithoutResources(result, caseIds.length)) { - return fail(result, 'Failed to query CrowdStrike cases') - } - - return { - ok: true, - output: { caseIds, count: caseIds.length, pagination: getPagination(result.data) }, - } - } - - case 'crowdstrike_get_case_details': { - const result = await callCrowdStrike(baseUrl, accessToken, { - method: 'POST', - path: '/cases/entities/cases/v2', - body: { ids: body.caseIds }, - }) - if (!result.ok) return fail(result, 'Failed to fetch CrowdStrike case details') - - const cases = getRecordResources(result.data).map(normalizeCase) - if (failedWithoutResources(result, cases.length)) { - return fail(result, 'Failed to fetch CrowdStrike case details') - } - - return { - ok: true, - output: { cases, count: cases.length, errors: getEnvelopeErrors(result.data) }, - } - } - } -} diff --git a/apps/sim/app/api/tools/crowdstrike/query/route.test.ts b/apps/sim/app/api/tools/crowdstrike/query/route.test.ts deleted file mode 100644 index d4657e8690d..00000000000 --- a/apps/sim/app/api/tools/crowdstrike/query/route.test.ts +++ /dev/null @@ -1,313 +0,0 @@ -/** - * @vitest-environment node - */ -import { createMockRequest, hybridAuthMockFns } from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { fetchMock } = vi.hoisted(() => ({ - fetchMock: vi.fn(), -})) - -import { POST } from '@/app/api/tools/crowdstrike/query/route' - -function jsonResponse(body: unknown, status = 200): Response { - return new Response(JSON.stringify(body), { - status, - headers: { 'Content-Type': 'application/json' }, - }) -} - -const sensorResource = { - agent_version: '6.1.0', - cid: 'cid-1', - device_id: 'sensor-1', - heartbeat_time: 1700, - hostname: 'host-1', - idp_policy_id: 'policy-1', - idp_policy_name: 'Default Policy', - kerberos_config: 'configured', - ldap_config: 'configured', - ldaps_config: 'configured', - local_ip: '10.0.0.1', - machine_domain: 'corp.local', - ntlm_config: 'configured', - os_version: 'Windows Server 2022', - rdp_to_dc_config: 'configured', - smb_to_dc_config: 'configured', - status: 'protected', - status_causes: ['healthy'], - ti_enabled: 'enabled', -} - -const normalizedSensor = { - agentVersion: '6.1.0', - cid: 'cid-1', - deviceId: 'sensor-1', - heartbeatTime: 1700, - hostname: 'host-1', - idpPolicyId: 'policy-1', - idpPolicyName: 'Default Policy', - ipAddress: '10.0.0.1', - kerberosConfig: 'configured', - ldapConfig: 'configured', - ldapsConfig: 'configured', - machineDomain: 'corp.local', - ntlmConfig: 'configured', - osVersion: 'Windows Server 2022', - rdpToDcConfig: 'configured', - smbToDcConfig: 'configured', - status: 'protected', - statusCauses: ['healthy'], - tiEnabled: 'enabled', -} - -describe('CrowdStrike query route', () => { - beforeEach(() => { - vi.clearAllMocks() - vi.stubGlobal('fetch', fetchMock) - - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-123', - authType: 'internal_jwt', - }) - }) - - it('surfaces a credential failure with the Falcon status, not a generic 500', async () => { - // getAccessToken runs before the operation dispatch, so a Falcon 401 used to - // fall through to the catch-all and reach the caller as a 500. - fetchMock.mockResolvedValueOnce( - jsonResponse({ errors: [{ code: 401, message: 'access denied, invalid bearer token' }] }, 401) - ) - - const response = await POST( - createMockRequest('POST', { - clientId: 'client-id', - clientSecret: 'wrong-secret', - cloud: 'us-1', - limit: 1, - operation: 'crowdstrike_query_sensors', - }) - ) - const data = await response.json() - - expect(response.status).toBe(401) - expect(data).toEqual({ success: false, error: 'access denied, invalid bearer token' }) - expect(fetchMock).toHaveBeenCalledTimes(1) - }) - - it('reports an unusable token response as a bad gateway rather than a 500', async () => { - fetchMock.mockResolvedValueOnce(jsonResponse({ nothing: true })) - - const response = await POST( - createMockRequest('POST', { - clientId: 'client-id', - clientSecret: 'client-secret', - cloud: 'us-1', - limit: 1, - operation: 'crowdstrike_query_sensors', - }) - ) - - expect(response.status).toBe(502) - }) - - it('hydrates sensor details after querying sensor ids', async () => { - fetchMock - .mockResolvedValueOnce(jsonResponse({ access_token: 'token-123' })) - .mockResolvedValueOnce( - jsonResponse({ - meta: { pagination: { expires_at: 111, limit: 1, offset: 0, total: 1 } }, - resources: ['sensor-1'], - }) - ) - .mockResolvedValueOnce( - jsonResponse({ - resources: [sensorResource], - }) - ) - - const request = createMockRequest('POST', { - clientId: 'client-id', - clientSecret: 'client-secret', - cloud: 'us-1', - limit: 1, - operation: 'crowdstrike_query_sensors', - }) - - const response = await POST(request) - const data = await response.json() - - expect(response.status).toBe(200) - expect(fetchMock).toHaveBeenCalledTimes(3) - expect(fetchMock.mock.calls[1]?.[0]).toBe( - 'https://api.crowdstrike.com/identity-protection/queries/devices/v1?limit=1' - ) - expect(fetchMock.mock.calls[2]?.[0]).toBe( - 'https://api.crowdstrike.com/identity-protection/entities/devices/GET/v1' - ) - expect(fetchMock.mock.calls[2]?.[1]).toMatchObject({ - body: JSON.stringify({ ids: ['sensor-1'] }), - method: 'POST', - }) - expect(data.output).toEqual({ - count: 1, - errors: [], - pagination: { - limit: 1, - offset: 0, - total: 1, - }, - sensors: [normalizedSensor], - }) - }) - - it('fetches sensor details directly from device ids', async () => { - fetchMock - .mockResolvedValueOnce(jsonResponse({ access_token: 'token-123' })) - .mockResolvedValueOnce( - jsonResponse({ - resources: [sensorResource], - }) - ) - - const request = createMockRequest('POST', { - clientId: 'client-id', - clientSecret: 'client-secret', - cloud: 'us-1', - ids: ['sensor-1'], - operation: 'crowdstrike_get_sensor_details', - }) - - const response = await POST(request) - const data = await response.json() - - expect(response.status).toBe(200) - expect(fetchMock).toHaveBeenCalledTimes(2) - expect(fetchMock.mock.calls[1]?.[0]).toBe( - 'https://api.crowdstrike.com/identity-protection/entities/devices/GET/v1' - ) - expect(fetchMock.mock.calls[1]?.[1]).toMatchObject({ - body: JSON.stringify({ ids: ['sensor-1'] }), - method: 'POST', - }) - expect(data.output).toEqual({ - count: 1, - errors: [], - pagination: null, - sensors: [normalizedSensor], - }) - }) - - it('normalizes sensor aggregate results', async () => { - fetchMock - .mockResolvedValueOnce(jsonResponse({ access_token: 'token-123' })) - .mockResolvedValueOnce( - jsonResponse({ - resources: [ - { - buckets: [ - { - count: 2, - key_as_string: 'protected', - sub_aggregates: [ - { - buckets: [ - { - count: 2, - key_as_string: 'corp.local', - value: 2, - value_as_string: '2', - }, - ], - doc_count_error_upper_bound: 0, - name: 'machine_domain_counts', - sum_other_doc_count: 0, - }, - ], - value: 2, - value_as_string: '2', - }, - ], - doc_count_error_upper_bound: 0, - name: 'status_counts', - sum_other_doc_count: 0, - }, - ], - }) - ) - - const aggregateQuery = { - field: 'status', - name: 'status_counts', - size: 10, - type: 'terms', - } - - const request = createMockRequest('POST', { - aggregateQuery, - clientId: 'client-id', - clientSecret: 'client-secret', - cloud: 'us-1', - operation: 'crowdstrike_get_sensor_aggregates', - }) - - const response = await POST(request) - const data = await response.json() - - expect(response.status).toBe(200) - expect(fetchMock).toHaveBeenCalledTimes(2) - expect(fetchMock.mock.calls[1]?.[0]).toBe( - 'https://api.crowdstrike.com/identity-protection/aggregates/devices/GET/v1' - ) - expect(fetchMock.mock.calls[1]?.[1]).toMatchObject({ - body: JSON.stringify(aggregateQuery), - method: 'POST', - }) - expect(data.output).toEqual({ - aggregates: [ - { - buckets: [ - { - count: 2, - from: null, - keyAsString: 'protected', - label: null, - stringFrom: null, - stringTo: null, - subAggregates: [ - { - buckets: [ - { - count: 2, - from: null, - keyAsString: 'corp.local', - label: null, - stringFrom: null, - stringTo: null, - subAggregates: [], - to: null, - value: 2, - valueAsString: '2', - }, - ], - docCountErrorUpperBound: 0, - name: 'machine_domain_counts', - sumOtherDocCount: 0, - }, - ], - to: null, - value: 2, - valueAsString: '2', - }, - ], - docCountErrorUpperBound: 0, - name: 'status_counts', - sumOtherDocCount: 0, - }, - ], - count: 1, - errors: [], - }) - }) -}) diff --git a/apps/sim/app/api/tools/crowdstrike/query/route.ts b/apps/sim/app/api/tools/crowdstrike/query/route.ts deleted file mode 100644 index e943aa77904..00000000000 --- a/apps/sim/app/api/tools/crowdstrike/query/route.ts +++ /dev/null @@ -1,331 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { crowdstrikeQueryContract } from '@/lib/api/contracts/tools/crowdstrike' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - CrowdStrikeAuthError, - type CrowdStrikeCallResult, - callCrowdStrike, - getAccessToken, - getCloudBaseUrl, - getEnvelopeErrors, - getFalconErrorMessage, - getNumber, - getPagination, - getRecordArray, - getRecordResources, - getString, - getStringArray, - getStringResources, - type JsonRecord, -} from '@/app/api/tools/crowdstrike/query/falcon' -import { - executeCrowdStrikeOperation, - failedWithoutResources, - failureStatus, -} from '@/app/api/tools/crowdstrike/query/operations' -import type { - CrowdStrikeQuerySensorsParams, - CrowdStrikeSensorAggregateBucket, - CrowdStrikeSensorAggregateResult, -} from '@/tools/crowdstrike/types' - -const logger = createLogger('CrowdStrikeAPI') - -function normalizeSensor(resource: JsonRecord) { - return { - agentVersion: getString(resource.agent_version), - cid: getString(resource.cid), - deviceId: getString(resource.device_id), - heartbeatTime: getNumber(resource.heartbeat_time), - hostname: getString(resource.hostname), - idpPolicyId: getString(resource.idp_policy_id), - idpPolicyName: getString(resource.idp_policy_name), - ipAddress: getString(resource.local_ip), - kerberosConfig: getString(resource.kerberos_config), - ldapConfig: getString(resource.ldap_config), - ldapsConfig: getString(resource.ldaps_config), - machineDomain: getString(resource.machine_domain), - ntlmConfig: getString(resource.ntlm_config), - osVersion: getString(resource.os_version), - rdpToDcConfig: getString(resource.rdp_to_dc_config), - smbToDcConfig: getString(resource.smb_to_dc_config), - status: getString(resource.status), - statusCauses: getStringArray(resource.status_causes), - tiEnabled: getString(resource.ti_enabled), - } -} - -function normalizeSensorsOutput(data: unknown, paginationData?: unknown) { - const sensors = getRecordResources(data).map(normalizeSensor) - - return { - count: sensors.length, - errors: getEnvelopeErrors(data), - pagination: paginationData == null ? null : getPagination(paginationData), - sensors, - } -} - -/** - * CrowdStrike answers 200 while the envelope carries only errors. Mirrors the - * shared operation executor so the Identity Protection branches cannot report a - * resource-less error envelope as a success. - */ -function envelopeFailureResponse( - result: CrowdStrikeCallResult, - resourceCount: number, - fallback: string -) { - if (!failedWithoutResources(result, resourceCount)) { - return null - } - - return NextResponse.json( - { success: false, error: getFalconErrorMessage(result.data, fallback) }, - { status: failureStatus(result) } - ) -} - -function normalizeAggregationResult(resource: JsonRecord): CrowdStrikeSensorAggregateResult { - return { - buckets: getRecordArray(resource.buckets).map(normalizeAggregationBucket), - docCountErrorUpperBound: getNumber(resource.doc_count_error_upper_bound), - name: getString(resource.name), - sumOtherDocCount: getNumber(resource.sum_other_doc_count), - } -} - -function normalizeAggregationBucket(resource: JsonRecord): CrowdStrikeSensorAggregateBucket { - return { - count: getNumber(resource.count), - from: getNumber(resource.from), - keyAsString: getString(resource.key_as_string), - label: resource.label ?? null, - stringFrom: getString(resource.string_from), - stringTo: getString(resource.string_to), - subAggregates: getRecordArray(resource.sub_aggregates).map(normalizeAggregationResult), - to: getNumber(resource.to), - value: getNumber(resource.value), - valueAsString: getString(resource.value_as_string), - } -} - -function normalizeAggregatesOutput(data: unknown) { - const aggregates = getRecordResources(data).map(normalizeAggregationResult) - - return { - aggregates, - count: aggregates.length, - errors: getEnvelopeErrors(data), - } -} - -function sensorQuery(params: CrowdStrikeQuerySensorsParams) { - return { - filter: params.filter, - limit: params.limit, - offset: params.offset, - sort: params.sort, - } -} - -/** - * Special route: this proxies workflow tool calls to CrowdStrike Falcon with the - * caller's own API credentials, so it authenticates through `checkInternalAuth` - * rather than an application use case and uses raw `withRouteHandler`. - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json( - { success: false, error: authResult.error || 'Unauthorized' }, - { status: 401 } - ) - } - - try { - const parsed = await parseRequest( - crowdstrikeQueryContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ), - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - const baseUrl = getCloudBaseUrl(params.cloud) - const accessToken = await getAccessToken(params) - - logger.info('CrowdStrike request', { - cloud: params.cloud, - operation: params.operation, - }) - - if (params.operation === 'crowdstrike_query_sensors') { - const queryResponse = await callCrowdStrike(baseUrl, accessToken, { - method: 'GET', - path: '/identity-protection/queries/devices/v1', - query: sensorQuery(params), - }) - - if (!queryResponse.ok) { - return NextResponse.json( - { - success: false, - error: getFalconErrorMessage(queryResponse.data, 'CrowdStrike request failed'), - }, - { status: queryResponse.status } - ) - } - - const ids = getStringResources(queryResponse.data) - const queryFailure = envelopeFailureResponse( - queryResponse, - ids.length, - 'Failed to query CrowdStrike sensors' - ) - if (queryFailure) return queryFailure - - if (ids.length === 0) { - return NextResponse.json({ - success: true, - output: normalizeSensorsOutput({ resources: [] }, queryResponse.data), - }) - } - - const detailResponse = await callCrowdStrike(baseUrl, accessToken, { - method: 'POST', - path: '/identity-protection/entities/devices/GET/v1', - body: { ids }, - }) - - if (!detailResponse.ok) { - return NextResponse.json( - { - success: false, - error: getFalconErrorMessage( - detailResponse.data, - 'Failed to fetch CrowdStrike sensor details' - ), - }, - { status: detailResponse.status } - ) - } - - const detailFailure = envelopeFailureResponse( - detailResponse, - getRecordResources(detailResponse.data).length, - 'Failed to fetch CrowdStrike sensor details' - ) - if (detailFailure) return detailFailure - - return NextResponse.json({ - success: true, - output: normalizeSensorsOutput(detailResponse.data, queryResponse.data), - }) - } - - if (params.operation === 'crowdstrike_get_sensor_details') { - const detailResponse = await callCrowdStrike(baseUrl, accessToken, { - method: 'POST', - path: '/identity-protection/entities/devices/GET/v1', - body: { ids: params.ids }, - }) - - if (!detailResponse.ok) { - return NextResponse.json( - { - success: false, - error: getFalconErrorMessage( - detailResponse.data, - 'Failed to fetch CrowdStrike sensor details' - ), - }, - { status: detailResponse.status } - ) - } - - const detailsFailure = envelopeFailureResponse( - detailResponse, - getRecordResources(detailResponse.data).length, - 'Failed to fetch CrowdStrike sensor details' - ) - if (detailsFailure) return detailsFailure - - return NextResponse.json({ - success: true, - output: normalizeSensorsOutput(detailResponse.data), - }) - } - - if (params.operation === 'crowdstrike_get_sensor_aggregates') { - const aggregateResponse = await callCrowdStrike(baseUrl, accessToken, { - method: 'POST', - path: '/identity-protection/aggregates/devices/GET/v1', - body: params.aggregateQuery, - }) - - if (!aggregateResponse.ok) { - return NextResponse.json( - { - success: false, - error: getFalconErrorMessage( - aggregateResponse.data, - 'Failed to fetch CrowdStrike sensor aggregates' - ), - }, - { status: aggregateResponse.status } - ) - } - - const aggregateFailure = envelopeFailureResponse( - aggregateResponse, - getRecordResources(aggregateResponse.data).length, - 'Failed to fetch CrowdStrike sensor aggregates' - ) - if (aggregateFailure) return aggregateFailure - - return NextResponse.json({ - success: true, - output: normalizeAggregatesOutput(aggregateResponse.data), - }) - } - - const result = await executeCrowdStrikeOperation(params, baseUrl, accessToken) - if (!result.ok) { - return NextResponse.json( - { success: false, error: result.error }, - { status: result.status || 502 } - ) - } - - return NextResponse.json({ success: true, output: result.output }) - } catch (error) { - const message = toError(error).message - - /** - * The token exchange runs before the operation dispatch, so without this a - * bad client ID or secret (Falcon 401) reaches the caller as a 500. - */ - if (error instanceof CrowdStrikeAuthError) { - logger.warn('CrowdStrike authentication failed', { error: message, status: error.status }) - return NextResponse.json({ success: false, error: message }, { status: error.status }) - } - - logger.error('CrowdStrike request failed', { error: message }) - return NextResponse.json({ success: false, error: message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/cursor/download-artifact/route.ts b/apps/sim/app/api/tools/cursor/download-artifact/route.ts deleted file mode 100644 index b32e78ddd15..00000000000 --- a/apps/sim/app/api/tools/cursor/download-artifact/route.ts +++ /dev/null @@ -1,159 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { cursorDownloadArtifactContract } from '@/lib/api/contracts/tools/cursor' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('CursorDownloadArtifactAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn( - `[${requestId}] Unauthorized Cursor download artifact attempt: ${authResult.error}` - ) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - logger.info( - `[${requestId}] Authenticated Cursor download artifact request via ${authResult.authType}`, - { - userId: authResult.userId, - } - ) - - const parsed = await parseRequest( - cursorDownloadArtifactContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ), - } - ) - if (!parsed.success) return parsed.response - const { apiKey, agentId, path } = parsed.data.body - - const authHeader = `Basic ${Buffer.from(`${apiKey}:`).toString('base64')}` - - logger.info(`[${requestId}] Requesting presigned URL for artifact`, { agentId, path }) - - const artifactResponse = await fetch( - `https://api.cursor.com/v0/agents/${encodeURIComponent(agentId)}/artifacts/download?path=${encodeURIComponent(path)}`, - { - method: 'GET', - headers: { - Authorization: authHeader, - }, - } - ) - - if (!artifactResponse.ok) { - const errorText = await artifactResponse.text().catch(() => '') - logger.error(`[${requestId}] Failed to get artifact presigned URL`, { - status: artifactResponse.status, - error: errorText, - }) - return NextResponse.json( - { - success: false, - error: errorText || `Failed to get artifact URL (${artifactResponse.status})`, - }, - { status: artifactResponse.status } - ) - } - - const artifactData = await artifactResponse.json() - const downloadUrl = artifactData.url || artifactData.downloadUrl || artifactData.presignedUrl - - if (!downloadUrl) { - logger.error(`[${requestId}] No download URL in artifact response`, { artifactData }) - return NextResponse.json( - { success: false, error: 'No download URL returned for artifact' }, - { status: 400 } - ) - } - - const urlValidation = await validateUrlWithDNS(downloadUrl, 'downloadUrl') - if (!urlValidation.isValid) { - return NextResponse.json({ success: false, error: urlValidation.error }, { status: 400 }) - } - - logger.info(`[${requestId}] Downloading artifact from presigned URL`, { agentId, path }) - - const downloadResponse = await secureFetchWithPinnedIP( - downloadUrl, - urlValidation.resolvedIP!, - {} - ) - - if (!downloadResponse.ok) { - logger.error(`[${requestId}] Failed to download artifact content`, { - status: downloadResponse.status, - statusText: downloadResponse.statusText, - }) - return NextResponse.json( - { - success: false, - error: `Failed to download artifact content (${downloadResponse.status}: ${downloadResponse.statusText})`, - }, - { status: downloadResponse.status } - ) - } - - const contentType = downloadResponse.headers.get('content-type') || 'application/octet-stream' - const arrayBuffer = await downloadResponse.arrayBuffer() - const fileBuffer = Buffer.from(arrayBuffer) - - const fileName = path.split('/').pop() || 'artifact' - - logger.info(`[${requestId}] Artifact downloaded successfully`, { - agentId, - path, - name: fileName, - size: fileBuffer.length, - mimeType: contentType, - }) - - return NextResponse.json({ - success: true, - output: { - file: { - name: fileName, - mimeType: contentType, - data: fileBuffer.toString('base64'), - size: fileBuffer.length, - }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error downloading Cursor artifact:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error occurred') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/daytona/upload/route.ts b/apps/sim/app/api/tools/daytona/upload/route.ts deleted file mode 100644 index 071def4c217..00000000000 --- a/apps/sim/app/api/tools/daytona/upload/route.ts +++ /dev/null @@ -1,165 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { daytonaUploadFileContract } from '@/lib/api/contracts/tools/daytona' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { daytonaToolboxUrl, extractDaytonaError } from '@/tools/daytona/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('DaytonaUploadAPI') - -const MAX_UPLOAD_SIZE_BYTES = 100 * 1024 * 1024 - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Daytona upload attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated Daytona upload request via ${authResult.authType}`) - - const parsed = await parseRequest(daytonaUploadFileContract, request, {}) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - let fileBuffer: Buffer - let fileName: string - - if (params.file) { - const userFiles = processFilesToUserFiles([params.file as RawFileInput], requestId, logger) - - if (userFiles.length === 0) { - return NextResponse.json({ success: false, error: 'Invalid file input' }, { status: 400 }) - } - - const userFile = userFiles[0] - const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) - if (denied) return denied - - if (userFile.size > MAX_UPLOAD_SIZE_BYTES) { - const sizeMB = (userFile.size / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { success: false, error: `File size (${sizeMB}MB) exceeds upload limit of 100MB` }, - { status: 400 } - ) - } - - logger.info(`[${requestId}] Downloading file: ${userFile.name} (${userFile.size} bytes)`) - try { - const servable = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_UPLOAD_SIZE_BYTES, - }) - fileBuffer = servable.buffer - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - if (isPayloadSizeLimitError(error)) { - return NextResponse.json( - { success: false, error: 'File exceeds upload limit of 100MB' }, - { status: 400 } - ) - } - logger.error(`[${requestId}] Failed to download file from storage:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Failed to download file') }, - { status: 500 } - ) - } - fileName = params.fileName || userFile.name - } else if (params.fileContent) { - logger.info(`[${requestId}] Using legacy base64 content input`) - const estimatedSize = Math.floor((params.fileContent.length * 3) / 4) - if (estimatedSize > MAX_UPLOAD_SIZE_BYTES) { - const sizeMB = (estimatedSize / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { success: false, error: `File size (${sizeMB}MB) exceeds upload limit of 100MB` }, - { status: 400 } - ) - } - fileBuffer = Buffer.from(params.fileContent, 'base64') - fileName = params.fileName || 'file' - } else { - return NextResponse.json({ success: false, error: 'File is required' }, { status: 400 }) - } - - if (fileBuffer.length > MAX_UPLOAD_SIZE_BYTES) { - const sizeMB = (fileBuffer.length / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { success: false, error: `File size (${sizeMB}MB) exceeds upload limit of 100MB` }, - { status: 400 } - ) - } - - const requestedPath = params.destinationPath.trim() - if (!requestedPath) { - return NextResponse.json( - { success: false, error: 'Destination path is required' }, - { status: 400 } - ) - } - const destinationPath = requestedPath.endsWith('/') - ? `${requestedPath}${fileName}` - : requestedPath - - logger.info( - `[${requestId}] Uploading to Daytona sandbox ${params.sandboxId}: ${destinationPath} (${fileBuffer.length} bytes)` - ) - - const formData = new FormData() - formData.append( - 'file', - new Blob([new Uint8Array(fileBuffer)], { type: 'application/octet-stream' }), - fileName - ) - - const uploadUrl = daytonaToolboxUrl( - params.sandboxId, - `/files/upload-v2?path=${encodeURIComponent(destinationPath)}` - ) - const response = await fetch(uploadUrl, { - method: 'POST', - headers: { - Authorization: `Bearer ${params.apiKey}`, - }, - body: formData, - }) - - if (!response.ok) { - const errorMessage = await extractDaytonaError(response, 'Failed to upload file') - logger.error(`[${requestId}] Daytona API error:`, { status: response.status, errorMessage }) - return NextResponse.json({ success: false, error: errorMessage }, { status: response.status }) - } - - logger.info(`[${requestId}] File uploaded successfully: ${destinationPath}`) - - return NextResponse.json({ - success: true, - uploadedPath: destinationPath, - name: fileName, - size: fileBuffer.length, - }) - } catch (error) { - logger.error(`[${requestId}] Unexpected error:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/deployments/deploy/route.ts b/apps/sim/app/api/tools/deployments/deploy/route.ts deleted file mode 100644 index 0795475b549..00000000000 --- a/apps/sim/app/api/tools/deployments/deploy/route.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { createLogger } from '@sim/logger' -import { assertWorkflowMutable, WorkflowLockedError } from '@sim/platform-authz/workflow' -import { type NextRequest, NextResponse } from 'next/server' -import { deploymentsDeployContract } from '@/lib/api/contracts/tools/deployments' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { statusForOrchestrationError } from '@/lib/core/orchestration/types' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { performFullDeploy } from '@/lib/workflows/orchestration' -import { - authenticateDeploymentToolRequest, - authorizeDeploymentWorkflow, - deploymentToolError, -} from '@/app/api/tools/deployments/utils' - -const logger = createLogger('DeploymentsDeployAPI') - -export const dynamic = 'force-dynamic' -export const runtime = 'nodejs' -export const maxDuration = 120 - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const auth = await authenticateDeploymentToolRequest(request, requestId) - if (!auth.ok) return auth.response - - const parsed = await parseRequest( - deploymentsDeployContract, - request, - {}, - { - validationErrorResponse: (error) => - deploymentToolError(getValidationErrorMessage(error, 'Invalid request data'), 400), - } - ) - if (!parsed.success) return parsed.response - - const { workflowId, workspaceId, name, description } = parsed.data.body - - const access = await authorizeDeploymentWorkflow(auth.userId, workflowId, workspaceId, 'admin') - if (!access.ok) return access.response - - await assertWorkflowMutable(workflowId) - - logger.info(`[${requestId}] Deploying workflow ${workflowId} via deployments tool`, { - userId: auth.userId, - }) - - const result = await performFullDeploy({ - workflowId, - userId: auth.userId, - versionName: name, - versionDescription: description ?? undefined, - requestId, - }) - - if (!result.success) { - return deploymentToolError( - result.error || 'Failed to deploy workflow', - statusForOrchestrationError(result.errorCode) - ) - } - - return NextResponse.json({ - success: true, - output: { - workflowId, - isDeployed: Boolean(result.activeDeployment), - deployedAt: result.deployedAt?.toISOString() ?? null, - version: result.version, - activeDeployment: result.activeDeployment, - latestDeploymentAttempt: result.latestDeploymentAttempt, - warnings: result.warnings ?? [], - }, - }) - } catch (error: unknown) { - if (error instanceof WorkflowLockedError) { - return deploymentToolError(error.message, error.status) - } - logger.error(`[${requestId}] Deployment tool deploy error`, { error }) - return deploymentToolError('Failed to deploy workflow', 500) - } -}) diff --git a/apps/sim/app/api/tools/deployments/promote/route.ts b/apps/sim/app/api/tools/deployments/promote/route.ts deleted file mode 100644 index 523a5630a32..00000000000 --- a/apps/sim/app/api/tools/deployments/promote/route.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { createLogger } from '@sim/logger' -import { assertWorkflowMutable, WorkflowLockedError } from '@sim/platform-authz/workflow' -import { type NextRequest, NextResponse } from 'next/server' -import { deploymentsPromoteContract } from '@/lib/api/contracts/tools/deployments' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { statusForOrchestrationError } from '@/lib/core/orchestration/types' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { performActivateVersion } from '@/lib/workflows/orchestration' -import { - authenticateDeploymentToolRequest, - authorizeDeploymentWorkflow, - deploymentToolError, -} from '@/app/api/tools/deployments/utils' - -const logger = createLogger('DeploymentsPromoteAPI') - -export const dynamic = 'force-dynamic' -export const runtime = 'nodejs' -export const maxDuration = 120 - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const auth = await authenticateDeploymentToolRequest(request, requestId) - if (!auth.ok) return auth.response - - const parsed = await parseRequest( - deploymentsPromoteContract, - request, - {}, - { - validationErrorResponse: (error) => - deploymentToolError(getValidationErrorMessage(error, 'Invalid request data'), 400), - } - ) - if (!parsed.success) return parsed.response - - const { workflowId, workspaceId, version } = parsed.data.body - - const access = await authorizeDeploymentWorkflow(auth.userId, workflowId, workspaceId, 'admin') - if (!access.ok) return access.response - - await assertWorkflowMutable(workflowId) - - logger.info( - `[${requestId}] Promoting workflow ${workflowId} to version ${version} via deployments tool`, - { userId: auth.userId } - ) - - const result = await performActivateVersion({ - workflowId, - version, - userId: auth.userId, - requestId, - }) - - if (!result.success) { - return deploymentToolError( - result.error || 'Failed to promote deployment version', - statusForOrchestrationError(result.errorCode) - ) - } - - return NextResponse.json({ - success: true, - output: { - workflowId, - isDeployed: Boolean(result.activeDeployment), - deployedAt: result.deployedAt?.toISOString() ?? null, - version, - activeDeployment: result.activeDeployment, - latestDeploymentAttempt: result.latestDeploymentAttempt, - warnings: result.warnings ?? [], - }, - }) - } catch (error: unknown) { - if (error instanceof WorkflowLockedError) { - return deploymentToolError(error.message, error.status) - } - logger.error(`[${requestId}] Deployment tool promote error`, { error }) - return deploymentToolError('Failed to promote deployment version', 500) - } -}) diff --git a/apps/sim/app/api/tools/deployments/routes.test.ts b/apps/sim/app/api/tools/deployments/routes.test.ts deleted file mode 100644 index f0313a779b3..00000000000 --- a/apps/sim/app/api/tools/deployments/routes.test.ts +++ /dev/null @@ -1,376 +0,0 @@ -/** - * @vitest-environment node - * - * Tests for the deployment tool routes under /api/tools/deployments — verifies - * session/internal auth, workspace permission enforcement, and the mapping of - * orchestration results to tool responses. - */ - -import { WorkflowLockedError } from '@sim/platform-authz/workflow' -import { createMockRequest, hybridAuthMockFns, workflowAuthzMockFns } from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { - mockEnforceUserRateLimit, - mockPerformFullDeploy, - mockPerformFullUndeploy, - mockPerformActivateVersion, - mockListWorkflowVersions, - mockGetWorkflowDeploymentVersion, -} = vi.hoisted(() => ({ - mockEnforceUserRateLimit: vi.fn(), - mockPerformFullDeploy: vi.fn(), - mockPerformFullUndeploy: vi.fn(), - mockPerformActivateVersion: vi.fn(), - mockListWorkflowVersions: vi.fn(), - mockGetWorkflowDeploymentVersion: vi.fn(), -})) - -vi.mock('@/lib/core/rate-limiter', () => ({ - enforceUserRateLimit: mockEnforceUserRateLimit, -})) - -vi.mock('@/lib/workflows/orchestration', () => ({ - performFullDeploy: mockPerformFullDeploy, - performFullUndeploy: mockPerformFullUndeploy, - performActivateVersion: mockPerformActivateVersion, -})) - -vi.mock('@/lib/workflows/persistence/utils', () => ({ - listWorkflowVersions: mockListWorkflowVersions, - getWorkflowDeploymentVersion: mockGetWorkflowDeploymentVersion, -})) - -import { POST as deployPost } from '@/app/api/tools/deployments/deploy/route' -import { POST as promotePost } from '@/app/api/tools/deployments/promote/route' -import { POST as undeployPost } from '@/app/api/tools/deployments/undeploy/route' -import { GET as getVersionGet } from '@/app/api/tools/deployments/version/route' -import { GET as listVersionsGet } from '@/app/api/tools/deployments/versions/route' - -const WORKFLOW_ID = 'wf-1' -const WORKFLOW_RECORD = { - id: WORKFLOW_ID, - name: 'My Workflow', - workspaceId: 'ws-1', - isDeployed: true, -} - -function authorized() { - return { allowed: true, status: 200, workflow: WORKFLOW_RECORD, workspacePermission: 'admin' } -} - -function makePost(path: string, body: unknown) { - return createMockRequest('POST', body, {}, `http://localhost:3000/api/tools/deployments/${path}`) -} - -function makeGet(path: string, query: string) { - return createMockRequest( - 'GET', - undefined, - {}, - `http://localhost:3000/api/tools/deployments/${path}?${query}` - ) -} - -beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - mockEnforceUserRateLimit.mockResolvedValue(null) - workflowAuthzMockFns.mockAuthorizeWorkflowByWorkspacePermission.mockResolvedValue(authorized()) - workflowAuthzMockFns.mockAssertWorkflowMutable.mockResolvedValue(undefined) -}) - -describe('POST /api/tools/deployments/deploy', () => { - beforeEach(() => { - mockPerformFullDeploy.mockResolvedValue({ - success: true, - deployedAt: new Date('2026-06-12T00:00:00Z'), - version: 4, - activeDeployment: { - deploymentVersionId: 'dv-4', - version: 4, - deployedAt: '2026-06-12T00:00:00.000Z', - }, - }) - }) - - it('rejects unauthenticated requests', async () => { - hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ - success: false, - error: 'Unauthorized', - }) - - const response = await deployPost( - makePost('deploy', { workflowId: WORKFLOW_ID, workspaceId: 'ws-1' }) - ) - - expect(response.status).toBe(401) - expect(mockPerformFullDeploy).not.toHaveBeenCalled() - }) - - it('requires admin permission on the workflow workspace', async () => { - workflowAuthzMockFns.mockAuthorizeWorkflowByWorkspacePermission.mockResolvedValue({ - allowed: false, - status: 403, - message: 'Access denied', - workflow: WORKFLOW_RECORD, - workspacePermission: 'write', - }) - - const response = await deployPost( - makePost('deploy', { workflowId: WORKFLOW_ID, workspaceId: 'ws-1' }) - ) - - expect(response.status).toBe(403) - expect(workflowAuthzMockFns.mockAuthorizeWorkflowByWorkspacePermission).toHaveBeenCalledWith({ - workflowId: WORKFLOW_ID, - userId: 'user-1', - action: 'admin', - }) - expect(mockPerformFullDeploy).not.toHaveBeenCalled() - }) - - it('deploys and returns the new version', async () => { - const response = await deployPost( - makePost('deploy', { - workflowId: WORKFLOW_ID, - workspaceId: 'ws-1', - name: 'Release 4', - description: 'Fixes the agent prompt', - }) - ) - - expect(response.status).toBe(200) - expect(mockPerformFullDeploy).toHaveBeenCalledWith( - expect.objectContaining({ - workflowId: WORKFLOW_ID, - userId: 'user-1', - versionName: 'Release 4', - versionDescription: 'Fixes the agent prompt', - }) - ) - - const body = await response.json() - expect(body).toEqual({ - success: true, - output: { - workflowId: WORKFLOW_ID, - isDeployed: true, - deployedAt: '2026-06-12T00:00:00.000Z', - version: 4, - activeDeployment: { - deploymentVersionId: 'dv-4', - version: 4, - deployedAt: '2026-06-12T00:00:00.000Z', - }, - warnings: [], - }, - }) - }) - - it('returns 423 when the workflow is locked', async () => { - workflowAuthzMockFns.mockAssertWorkflowMutable.mockRejectedValue(new WorkflowLockedError()) - - const response = await deployPost( - makePost('deploy', { workflowId: WORKFLOW_ID, workspaceId: 'ws-1' }) - ) - - expect(response.status).toBe(423) - expect(mockPerformFullDeploy).not.toHaveBeenCalled() - }) - - it('rejects a request without a workflowId', async () => { - const response = await deployPost(makePost('deploy', { workspaceId: 'ws-1' })) - - expect(response.status).toBe(400) - expect(mockPerformFullDeploy).not.toHaveBeenCalled() - }) - - it('returns 404 when the workflow belongs to a different workspace', async () => { - const response = await deployPost( - makePost('deploy', { workflowId: WORKFLOW_ID, workspaceId: 'ws-other' }) - ) - - expect(response.status).toBe(404) - const body = await response.json() - expect(body.error).toBe('Workflow not found in this workspace') - expect(mockPerformFullDeploy).not.toHaveBeenCalled() - }) -}) - -describe('POST /api/tools/deployments/undeploy', () => { - beforeEach(() => { - mockPerformFullUndeploy.mockResolvedValue({ success: true }) - }) - - it('returns 400 when the workflow is not deployed', async () => { - workflowAuthzMockFns.mockAuthorizeWorkflowByWorkspacePermission.mockResolvedValue({ - ...authorized(), - workflow: { ...WORKFLOW_RECORD, isDeployed: false }, - }) - - const response = await undeployPost( - makePost('undeploy', { workflowId: WORKFLOW_ID, workspaceId: 'ws-1' }) - ) - - expect(response.status).toBe(400) - expect(mockPerformFullUndeploy).not.toHaveBeenCalled() - }) - - it('undeploys a deployed workflow', async () => { - const response = await undeployPost( - makePost('undeploy', { workflowId: WORKFLOW_ID, workspaceId: 'ws-1' }) - ) - - expect(response.status).toBe(200) - expect(mockPerformFullUndeploy).toHaveBeenCalledWith( - expect.objectContaining({ workflowId: WORKFLOW_ID, userId: 'user-1' }) - ) - - const body = await response.json() - expect(body.output).toEqual({ - workflowId: WORKFLOW_ID, - isDeployed: false, - deployedAt: null, - warnings: [], - }) - }) -}) - -describe('POST /api/tools/deployments/promote', () => { - beforeEach(() => { - mockPerformActivateVersion.mockResolvedValue({ - success: true, - deployedAt: new Date('2026-06-12T00:00:00Z'), - activeDeployment: { - deploymentVersionId: 'dv-3', - version: 3, - deployedAt: '2026-06-12T00:00:00.000Z', - }, - }) - }) - - it('promotes the given version to live', async () => { - const response = await promotePost( - makePost('promote', { workflowId: WORKFLOW_ID, workspaceId: 'ws-1', version: 3 }) - ) - - expect(response.status).toBe(200) - expect(mockPerformActivateVersion).toHaveBeenCalledWith( - expect.objectContaining({ workflowId: WORKFLOW_ID, version: 3, userId: 'user-1' }) - ) - - const body = await response.json() - expect(body.output).toEqual({ - workflowId: WORKFLOW_ID, - isDeployed: true, - deployedAt: '2026-06-12T00:00:00.000Z', - version: 3, - activeDeployment: { - deploymentVersionId: 'dv-3', - version: 3, - deployedAt: '2026-06-12T00:00:00.000Z', - }, - warnings: [], - }) - }) - - it('rejects a missing version', async () => { - const response = await promotePost( - makePost('promote', { workflowId: WORKFLOW_ID, workspaceId: 'ws-1' }) - ) - - expect(response.status).toBe(400) - expect(mockPerformActivateVersion).not.toHaveBeenCalled() - }) - - it('maps a missing target version to 404', async () => { - mockPerformActivateVersion.mockResolvedValue({ - success: false, - error: 'Deployment version not found', - errorCode: 'not_found', - }) - - const response = await promotePost( - makePost('promote', { workflowId: WORKFLOW_ID, workspaceId: 'ws-1', version: 99 }) - ) - - expect(response.status).toBe(404) - }) -}) - -describe('GET /api/tools/deployments/versions', () => { - it('lists deployment versions with read permission', async () => { - const versions = [ - { - id: 'v-2', - version: 2, - name: null, - description: null, - isActive: true, - createdAt: '2026-06-12T00:00:00.000Z', - createdBy: 'user-1', - deployedByName: 'Waleed', - }, - ] - mockListWorkflowVersions.mockResolvedValue({ versions }) - - const response = await listVersionsGet( - makeGet('versions', `workflowId=${WORKFLOW_ID}&workspaceId=ws-1`) - ) - - expect(response.status).toBe(200) - expect(workflowAuthzMockFns.mockAuthorizeWorkflowByWorkspacePermission).toHaveBeenCalledWith({ - workflowId: WORKFLOW_ID, - userId: 'user-1', - action: 'read', - }) - - const body = await response.json() - expect(body.output).toEqual({ workflowId: WORKFLOW_ID, versions }) - }) -}) - -describe('GET /api/tools/deployments/version', () => { - it('returns version metadata and the deployed state', async () => { - mockGetWorkflowDeploymentVersion.mockResolvedValue({ - id: 'v-3', - version: 3, - name: 'Release 3', - description: null, - isActive: false, - createdAt: '2026-06-12T00:00:00.000Z', - state: { blocks: {}, edges: [] }, - }) - - const response = await getVersionGet( - makeGet('version', `workflowId=${WORKFLOW_ID}&workspaceId=ws-1&version=3`) - ) - - expect(response.status).toBe(200) - const body = await response.json() - expect(body.output).toEqual({ - workflowId: WORKFLOW_ID, - version: 3, - name: 'Release 3', - description: null, - isActive: false, - createdAt: '2026-06-12T00:00:00.000Z', - deployedState: { blocks: {}, edges: [] }, - }) - }) - - it('returns 404 when the version does not exist', async () => { - mockGetWorkflowDeploymentVersion.mockResolvedValue(null) - - const response = await getVersionGet( - makeGet('version', `workflowId=${WORKFLOW_ID}&workspaceId=ws-1&version=9`) - ) - - expect(response.status).toBe(404) - }) -}) diff --git a/apps/sim/app/api/tools/deployments/undeploy/route.ts b/apps/sim/app/api/tools/deployments/undeploy/route.ts deleted file mode 100644 index 942cf9f4895..00000000000 --- a/apps/sim/app/api/tools/deployments/undeploy/route.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { createLogger } from '@sim/logger' -import { assertWorkflowMutable, WorkflowLockedError } from '@sim/platform-authz/workflow' -import { type NextRequest, NextResponse } from 'next/server' -import { deploymentsUndeployContract } from '@/lib/api/contracts/tools/deployments' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { performFullUndeploy } from '@/lib/workflows/orchestration' -import { - authenticateDeploymentToolRequest, - authorizeDeploymentWorkflow, - deploymentToolError, -} from '@/app/api/tools/deployments/utils' - -const logger = createLogger('DeploymentsUndeployAPI') - -export const dynamic = 'force-dynamic' -export const runtime = 'nodejs' -export const maxDuration = 120 - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const auth = await authenticateDeploymentToolRequest(request, requestId) - if (!auth.ok) return auth.response - - const parsed = await parseRequest( - deploymentsUndeployContract, - request, - {}, - { - validationErrorResponse: (error) => - deploymentToolError(getValidationErrorMessage(error, 'Invalid request data'), 400), - } - ) - if (!parsed.success) return parsed.response - - const { workflowId, workspaceId } = parsed.data.body - - const access = await authorizeDeploymentWorkflow(auth.userId, workflowId, workspaceId, 'admin') - if (!access.ok) return access.response - - if (!access.workflow.isDeployed) { - return deploymentToolError('Workflow is not deployed', 400) - } - - await assertWorkflowMutable(workflowId) - - logger.info(`[${requestId}] Undeploying workflow ${workflowId} via deployments tool`, { - userId: auth.userId, - }) - - const result = await performFullUndeploy({ - workflowId, - userId: auth.userId, - requestId, - }) - - if (!result.success) { - return deploymentToolError(result.error || 'Failed to undeploy workflow', 500) - } - - return NextResponse.json({ - success: true, - output: { - workflowId, - isDeployed: false, - deployedAt: null, - warnings: result.warnings ?? [], - }, - }) - } catch (error: unknown) { - if (error instanceof WorkflowLockedError) { - return deploymentToolError(error.message, error.status) - } - logger.error(`[${requestId}] Deployment tool undeploy error`, { error }) - return deploymentToolError('Failed to undeploy workflow', 500) - } -}) diff --git a/apps/sim/app/api/tools/deployments/utils.ts b/apps/sim/app/api/tools/deployments/utils.ts deleted file mode 100644 index 2812b86d39a..00000000000 --- a/apps/sim/app/api/tools/deployments/utils.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { createLogger } from '@sim/logger' -import { - authorizeWorkflowByWorkspacePermission, - type WorkflowWorkspaceAuthorizationResult, -} from '@sim/platform-authz/workflow' -import { type NextRequest, NextResponse } from 'next/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { enforceUserRateLimit } from '@/lib/core/rate-limiter' - -const logger = createLogger('DeploymentToolsAPI') - -export type AuthorizedDeploymentWorkflow = NonNullable< - WorkflowWorkspaceAuthorizationResult['workflow'] -> - -/** Standard error body for deployment tool routes, matching the generic tool response shape. */ -export function deploymentToolError(error: string, status: number): NextResponse { - return NextResponse.json({ success: false, error }, { status }) -} - -/** - * Authenticates a deployment tool request via session or internal token (API - * keys are rejected) and applies per-user rate limiting. Runs before request - * parsing, so it must not read the body. - */ -export async function authenticateDeploymentToolRequest( - request: NextRequest, - requestId: string -): Promise<{ ok: true; userId: string } | { ok: false; response: NextResponse }> { - const auth = await checkSessionOrInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized deployment tool request`, { error: auth.error }) - return { - ok: false, - response: deploymentToolError(auth.error || 'Authentication required', 401), - } - } - - const rateLimited = await enforceUserRateLimit('deployment-tools', auth.userId) - if (rateLimited) return { ok: false, response: rateLimited } - - return { ok: true, userId: auth.userId } -} - -/** - * Verifies the user holds the required workspace permission on the target - * workflow and that the workflow belongs to the calling workspace. Deployment - * mutations require `admin`, reads require `read`, matching the UI deploy - * routes. The workspace binding keeps workflow-driven executions (schedules, - * webhooks) from reaching into other workspaces the actor administers. - */ -export async function authorizeDeploymentWorkflow( - userId: string, - workflowId: string, - workspaceId: string, - action: 'read' | 'admin' -): Promise< - { ok: true; workflow: AuthorizedDeploymentWorkflow } | { ok: false; response: NextResponse } -> { - const authorization = await authorizeWorkflowByWorkspacePermission({ - workflowId, - userId, - action, - }) - - if (!authorization.allowed || !authorization.workflow) { - return { - ok: false, - response: deploymentToolError(authorization.message || 'Access denied', authorization.status), - } - } - - if (authorization.workflow.workspaceId !== workspaceId) { - return { - ok: false, - response: deploymentToolError('Workflow not found in this workspace', 404), - } - } - - return { ok: true, workflow: authorization.workflow } -} diff --git a/apps/sim/app/api/tools/deployments/version/route.ts b/apps/sim/app/api/tools/deployments/version/route.ts deleted file mode 100644 index 86e56cbaabc..00000000000 --- a/apps/sim/app/api/tools/deployments/version/route.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { deploymentsGetVersionContract } from '@/lib/api/contracts/tools/deployments' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getWorkflowDeploymentVersion } from '@/lib/workflows/persistence/utils' -import { - authenticateDeploymentToolRequest, - authorizeDeploymentWorkflow, - deploymentToolError, -} from '@/app/api/tools/deployments/utils' - -const logger = createLogger('DeploymentsGetVersionAPI') - -export const dynamic = 'force-dynamic' -export const runtime = 'nodejs' - -export const GET = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const auth = await authenticateDeploymentToolRequest(request, requestId) - if (!auth.ok) return auth.response - - const parsed = await parseRequest( - deploymentsGetVersionContract, - request, - {}, - { - validationErrorResponse: (error) => - deploymentToolError(getValidationErrorMessage(error, 'Invalid request data'), 400), - } - ) - if (!parsed.success) return parsed.response - - const { workflowId, workspaceId, version } = parsed.data.query - - const access = await authorizeDeploymentWorkflow(auth.userId, workflowId, workspaceId, 'read') - if (!access.ok) return access.response - - const row = await getWorkflowDeploymentVersion(workflowId, version) - if (!row) { - return deploymentToolError('Deployment version not found', 404) - } - - return NextResponse.json({ - success: true, - output: { - workflowId, - version: row.version, - name: row.name, - description: row.description, - isActive: row.isActive, - createdAt: row.createdAt, - deployedState: row.state, - }, - }) - } catch (error: unknown) { - logger.error(`[${requestId}] Deployment tool get version error`, { error }) - return deploymentToolError('Failed to get deployment version', 500) - } -}) diff --git a/apps/sim/app/api/tools/deployments/versions/route.ts b/apps/sim/app/api/tools/deployments/versions/route.ts deleted file mode 100644 index 42abd9eb9ea..00000000000 --- a/apps/sim/app/api/tools/deployments/versions/route.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { deploymentsListVersionsContract } from '@/lib/api/contracts/tools/deployments' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { listWorkflowVersions } from '@/lib/workflows/persistence/utils' -import { - authenticateDeploymentToolRequest, - authorizeDeploymentWorkflow, - deploymentToolError, -} from '@/app/api/tools/deployments/utils' - -const logger = createLogger('DeploymentsListVersionsAPI') - -export const dynamic = 'force-dynamic' -export const runtime = 'nodejs' - -export const GET = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const auth = await authenticateDeploymentToolRequest(request, requestId) - if (!auth.ok) return auth.response - - const parsed = await parseRequest( - deploymentsListVersionsContract, - request, - {}, - { - validationErrorResponse: (error) => - deploymentToolError(getValidationErrorMessage(error, 'Invalid request data'), 400), - } - ) - if (!parsed.success) return parsed.response - - const { workflowId, workspaceId } = parsed.data.query - - const access = await authorizeDeploymentWorkflow(auth.userId, workflowId, workspaceId, 'read') - if (!access.ok) return access.response - - const { versions } = await listWorkflowVersions(workflowId) - - return NextResponse.json({ - success: true, - output: { workflowId, versions }, - }) - } catch (error: unknown) { - logger.error(`[${requestId}] Deployment tool list versions error`, { error }) - return deploymentToolError('Failed to list deployment versions', 500) - } -}) diff --git a/apps/sim/app/api/tools/discord/channels/route.ts b/apps/sim/app/api/tools/discord/channels/route.ts deleted file mode 100644 index 170dc658ad7..00000000000 --- a/apps/sim/app/api/tools/discord/channels/route.ts +++ /dev/null @@ -1,141 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { discordChannelsContract } from '@/lib/api/contracts/tools/communication/discord' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateNumericId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -interface DiscordChannel { - id: string - name: string - type: number - guild_id?: string -} - -export const dynamic = 'force-dynamic' - -const logger = createLogger('DiscordChannelsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(discordChannelsContract, request, {}) - if (!parsed.success) return parsed.response - const { botToken, serverId, channelId } = parsed.data.body - - const serverIdValidation = validateNumericId(serverId, 'serverId') - if (!serverIdValidation.isValid) { - logger.error(`Invalid server ID: ${serverIdValidation.error}`) - return NextResponse.json({ error: serverIdValidation.error }, { status: 400 }) - } - - if (channelId) { - const channelIdValidation = validateNumericId(channelId, 'channelId') - if (!channelIdValidation.isValid) { - logger.error(`Invalid channel ID: ${channelIdValidation.error}`) - return NextResponse.json({ error: channelIdValidation.error }, { status: 400 }) - } - - logger.info(`Fetching single Discord channel: ${channelId}`) - - const response = await fetch(`https://discord.com/api/v10/channels/${channelId}`, { - method: 'GET', - headers: { - Authorization: `Bot ${botToken}`, - 'Content-Type': 'application/json', - }, - }) - - if (!response.ok) { - logger.error('Discord API error fetching channel:', { - status: response.status, - statusText: response.statusText, - }) - - let errorMessage - try { - const errorData = await response.json() - logger.error('Error details:', errorData) - errorMessage = errorData.message || `Failed to fetch channel (${response.status})` - } catch (_e) { - errorMessage = `Failed to fetch channel: ${response.status} ${response.statusText}` - } - return NextResponse.json({ error: errorMessage }, { status: response.status }) - } - - const channel = (await response.json()) as DiscordChannel - - if (channel.guild_id !== serverId) { - logger.error('Channel does not belong to the specified server') - return NextResponse.json( - { error: 'Channel not found in specified server' }, - { status: 404 } - ) - } - - if (channel.type !== 0) { - logger.warn('Requested channel is not a text channel') - return NextResponse.json({ error: 'Channel is not a text channel' }, { status: 400 }) - } - - logger.info(`Successfully fetched channel: ${channel.name}`) - - return NextResponse.json({ - channel: { - id: channel.id, - name: channel.name, - type: channel.type, - }, - }) - } - - logger.info(`Fetching all Discord channels for server: ${serverId}`) - - const response = await fetch(`https://discord.com/api/v10/guilds/${serverId}/channels`, { - method: 'GET', - headers: { - Authorization: `Bot ${botToken}`, - 'Content-Type': 'application/json', - }, - }) - - if (!response.ok) { - logger.warn( - 'Discord API returned non-OK for channels; returning empty list to avoid UX break', - { - status: response.status, - statusText: response.statusText, - } - ) - return NextResponse.json({ channels: [] }) - } - - const channels = (await response.json()) as DiscordChannel[] - - const textChannels = channels.filter((channel: DiscordChannel) => channel.type === 0) - - logger.info(`Successfully fetched ${textChannels.length} text channels`) - - return NextResponse.json({ - channels: textChannels.map((channel: DiscordChannel) => ({ - id: channel.id, - name: channel.name, - type: channel.type, - })), - }) - } catch (error) { - logger.error('Error processing request:', error) - return NextResponse.json( - { - error: 'Failed to retrieve Discord channels', - details: (error as Error).message, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/discord/send-message/route.ts b/apps/sim/app/api/tools/discord/send-message/route.ts deleted file mode 100644 index bc9d30d526a..00000000000 --- a/apps/sim/app/api/tools/discord/send-message/route.ts +++ /dev/null @@ -1,226 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { discordSendMessageContract } from '@/lib/api/contracts/tools/communication/discord' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateNumericId } from '@/lib/core/security/input-validation' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('DiscordSendMessageAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Discord send attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - const userId = authResult.userId - logger.info(`[${requestId}] Authenticated Discord send request via ${authResult.authType}`, { - userId, - }) - - const parsed = await parseRequest(discordSendMessageContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const channelIdValidation = validateNumericId(validatedData.channelId, 'channelId') - if (!channelIdValidation.isValid) { - logger.warn(`[${requestId}] Invalid channelId format`, { - error: channelIdValidation.error, - }) - return NextResponse.json( - { success: false, error: channelIdValidation.error }, - { status: 400 } - ) - } - - logger.info(`[${requestId}] Sending Discord message`, { - channelId: validatedData.channelId, - hasFiles: !!(validatedData.files && validatedData.files.length > 0), - fileCount: validatedData.files?.length || 0, - }) - - const discordApiUrl = `https://discord.com/api/v10/channels/${validatedData.channelId}/messages` - - if (!validatedData.files || validatedData.files.length === 0) { - logger.info(`[${requestId}] No files, using JSON POST`) - - const response = await fetch(discordApiUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bot ${validatedData.botToken}`, - }, - body: JSON.stringify({ - content: validatedData.content || '', - }), - }) - - if (!response.ok) { - const errorData = await response.json().catch(() => ({})) - logger.error(`[${requestId}] Discord API error:`, errorData) - return NextResponse.json( - { - success: false, - error: errorData.message || 'Failed to send message', - }, - { status: response.status } - ) - } - - const data = await response.json() - logger.info(`[${requestId}] Message sent successfully`) - return NextResponse.json({ - success: true, - output: { - message: data.content, - data: data, - }, - }) - } - - logger.info(`[${requestId}] Processing ${validatedData.files.length} file(s)`) - - const userFiles = processFilesToUserFiles(validatedData.files, requestId, logger) - const filesOutput: Array<{ - name: string - mimeType: string - data: string - size: number - }> = [] - - if (userFiles.length === 0) { - logger.warn(`[${requestId}] No valid files to upload, falling back to text-only`) - const response = await fetch(discordApiUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bot ${validatedData.botToken}`, - }, - body: JSON.stringify({ - content: validatedData.content || '', - }), - }) - - const data = await response.json() - return NextResponse.json({ - success: true, - output: { - message: data.content, - data: data, - }, - }) - } - - const formData = new FormData() - - const payload = { - content: validatedData.content || '', - } - formData.append('payload_json', JSON.stringify(payload)) - - const accessResults = await Promise.all( - userFiles.map((file) => assertToolFileAccess(file.key, userId, requestId, logger)) - ) - const denied = accessResults.find((r) => r !== null) - if (denied) return denied - - let resolved: Array<{ buffer: Buffer; contentType: string }> - try { - resolved = await downloadServableFilesWithinBudget(userFiles, requestId, logger, { - totalMaxBytes: MAX_BUFFERED_TRANSFER_BYTES, - label: 'Total attachment size', - }) - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - logger.error(`[${requestId}] Failed to download an attachment:`, error) - return NextResponse.json( - { - success: false, - error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`, - }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } - - for (let i = 0; i < userFiles.length; i++) { - const userFile = userFiles[i] - const buffer = resolved[i].buffer - const mimeType = resolved[i].contentType || userFile.type || 'application/octet-stream' - logger.info(`[${requestId}] Added file ${i}: ${userFile.name} (${buffer.length} bytes)`) - filesOutput.push({ - name: userFile.name, - mimeType, - data: buffer.toString('base64'), - size: buffer.length, - }) - const blob = new Blob([new Uint8Array(buffer)], { type: mimeType }) - formData.append(`files[${i}]`, blob, userFile.name) - } - - logger.info(`[${requestId}] Sending multipart request with ${userFiles.length} file(s)`) - const response = await fetch(discordApiUrl, { - method: 'POST', - headers: { - Authorization: `Bot ${validatedData.botToken}`, - }, - body: formData, - }) - - if (!response.ok) { - const errorData = await response.json().catch(() => ({})) - logger.error(`[${requestId}] Discord API error:`, errorData) - return NextResponse.json( - { - success: false, - error: errorData.message || 'Failed to send message with files', - }, - { status: response.status } - ) - } - - const data = await response.json() - logger.info(`[${requestId}] Message with files sent successfully`) - - return NextResponse.json({ - success: true, - output: { - message: data.content, - data: data, - fileCount: userFiles.length, - files: filesOutput, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error sending Discord message:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/discord/servers/route.ts b/apps/sim/app/api/tools/discord/servers/route.ts deleted file mode 100644 index 6cd82e79028..00000000000 --- a/apps/sim/app/api/tools/discord/servers/route.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { discordServersContract } from '@/lib/api/contracts/tools/communication/discord' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateNumericId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -interface DiscordServer { - id: string - name: string - icon: string | null -} - -export const dynamic = 'force-dynamic' - -const logger = createLogger('DiscordServersAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(discordServersContract, request, {}) - if (!parsed.success) return parsed.response - const { botToken, serverId } = parsed.data.body - - if (serverId) { - const serverIdValidation = validateNumericId(serverId, 'serverId') - if (!serverIdValidation.isValid) { - logger.error(`Invalid server ID: ${serverIdValidation.error}`) - return NextResponse.json({ error: serverIdValidation.error }, { status: 400 }) - } - - logger.info(`Fetching single Discord server: ${serverId}`) - - const response = await fetch(`https://discord.com/api/v10/guilds/${serverId}`, { - method: 'GET', - headers: { - Authorization: `Bot ${botToken}`, - 'Content-Type': 'application/json', - }, - }) - - if (!response.ok) { - logger.error('Discord API error fetching server:', { - status: response.status, - statusText: response.statusText, - }) - - let errorMessage - try { - const errorData = await response.json() - logger.error('Error details:', errorData) - errorMessage = errorData.message || `Failed to fetch server (${response.status})` - } catch (_e) { - errorMessage = `Failed to fetch server: ${response.status} ${response.statusText}` - } - return NextResponse.json({ error: errorMessage }, { status: response.status }) - } - - const server = (await response.json()) as DiscordServer - logger.info(`Successfully fetched server: ${server.name}`) - - return NextResponse.json({ - server: { - id: server.id, - name: server.name, - icon: server.icon - ? `https://cdn.discordapp.com/icons/${server.id}/${server.icon}.png` - : null, - }, - }) - } - - logger.info( - 'Skipping guild listing: bot token cannot list /users/@me/guilds; returning empty list' - ) - return NextResponse.json({ servers: [] }) - } catch (error) { - logger.error('Error processing request:', error) - return NextResponse.json( - { - error: 'Failed to retrieve Discord servers', - details: (error as Error).message, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/docusign/route.ts b/apps/sim/app/api/tools/docusign/route.ts deleted file mode 100644 index 0f21e39b413..00000000000 --- a/apps/sim/app/api/tools/docusign/route.ts +++ /dev/null @@ -1,646 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { docusignToolContract } from '@/lib/api/contracts/tools/docusign' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - assertKnownSizeWithinLimit, - DEFAULT_MAX_ERROR_BODY_BYTES, - isPayloadSizeLimitError, - readResponseJsonWithLimit, - readResponseTextWithLimit, - readResponseToBufferWithLimit, -} from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getDocusignOAuthUrl } from '@/lib/oauth/docusign' -import { uploadCopilotFile } from '@/lib/uploads/contexts/copilot' -import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' -import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' -import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' - -const logger = createLogger('DocuSignAPI') -const MAX_DOCUSIGN_DOCUMENT_BYTES = 25 * 1024 * 1024 -const MAX_LEGACY_INLINE_DOCUMENT_BYTES = 7 * 1024 * 1024 -const MAX_DOCUSIGN_JSON_BYTES = 2 * 1024 * 1024 -const DOCUSIGN_FETCH_TIMEOUT_MS = 30_000 - -interface DocuSignAccountInfo { - accountId: string - baseUri: string -} - -async function readDocusignJson( - response: Response, - label: string -): Promise> { - return readResponseJsonWithLimit>(response, { - maxBytes: MAX_DOCUSIGN_JSON_BYTES, - label, - }) -} - -function docusignError(data: Record, fallback: string): string { - return ( - (typeof data.message === 'string' && data.message) || - (typeof data.errorCode === 'string' && data.errorCode) || - fallback - ) -} - -async function fetchDocusign( - input: string, - init: RequestInit = {}, - parentSignal?: AbortSignal -): Promise { - const controller = new AbortController() - const timeout = setTimeout(() => { - controller.abort(new Error('DocuSign request timed out')) - }, DOCUSIGN_FETCH_TIMEOUT_MS) - const abort = () => controller.abort(parentSignal?.reason ?? new Error('Request aborted')) - parentSignal?.addEventListener('abort', abort, { once: true }) - - try { - return await fetch(input, { ...init, signal: controller.signal }) - } finally { - clearTimeout(timeout) - parentSignal?.removeEventListener('abort', abort) - } -} - -/** - * Resolves the user's DocuSign account info from their access token - * by calling the DocuSign userinfo endpoint. - */ -async function resolveAccount( - accessToken: string, - signal?: AbortSignal -): Promise { - const response = await fetchDocusign( - getDocusignOAuthUrl('/oauth/userinfo'), - { - headers: { Authorization: `Bearer ${accessToken}` }, - }, - signal - ) - - if (!response.ok) { - const errorText = await readResponseTextWithLimit(response, { - maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, - label: 'DocuSign account error response', - }).catch(() => '') - logger.error('Failed to resolve DocuSign account', { - status: response.status, - error: errorText, - }) - throw new Error(`Failed to resolve DocuSign account: ${response.status}`) - } - - const data = await readDocusignJson(response, 'DocuSign account response') - const accounts = Array.isArray(data.accounts) - ? (data.accounts as Array<{ - is_default?: boolean - base_uri?: string - account_id?: string - }>) - : [] - - const defaultAccount = accounts.find((account) => account.is_default) ?? accounts[0] - if (!defaultAccount) { - throw new Error('No DocuSign accounts found for this user') - } - - const baseUri = defaultAccount.base_uri - if (!baseUri) { - throw new Error('DocuSign account is missing base_uri') - } - const accountId = defaultAccount.account_id - if (!accountId) { - throw new Error('DocuSign account is missing account_id') - } - - return { - accountId, - baseUri, - } -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest( - docusignToolContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Invalid request data') }, - { status: 400 } - ), - } - ) - if (!parsed.success) return parsed.response - - const { accessToken, operation, ...params } = parsed.data.body - - try { - const account = await resolveAccount(accessToken, request.signal) - const apiBase = `${account.baseUri}/restapi/v2.1/accounts/${account.accountId}` - const headers: Record = { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json', - } - - switch (operation) { - case 'send_envelope': - return await handleSendEnvelope(apiBase, headers, params, authResult.userId, request.signal) - case 'create_from_template': - return await handleCreateFromTemplate(apiBase, headers, params, request.signal) - case 'get_envelope': - return await handleGetEnvelope(apiBase, headers, params, request.signal) - case 'list_envelopes': - return await handleListEnvelopes(apiBase, headers, params, request.signal) - case 'void_envelope': - return await handleVoidEnvelope(apiBase, headers, params, request.signal) - case 'download_document': - return await handleDownloadDocument( - apiBase, - headers, - params, - authResult.userId, - request.signal - ) - case 'list_templates': - return await handleListTemplates(apiBase, headers, params, request.signal) - case 'list_recipients': - return await handleListRecipients(apiBase, headers, params, request.signal) - default: - return NextResponse.json( - { success: false, error: `Unknown operation: ${operation}` }, - { status: 400 } - ) - } - } catch (error) { - logger.error('DocuSign API error', { operation, error }) - const message = getErrorMessage(error, 'Internal server error') - return NextResponse.json( - { success: false, error: message }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } -}) - -async function handleSendEnvelope( - apiBase: string, - headers: Record, - params: Record, - userId: string, - signal?: AbortSignal -) { - const { signerEmail, signerName, emailSubject, emailBody, ccEmail, ccName, file, status } = params - - if (!signerEmail || !signerName || !emailSubject) { - return NextResponse.json( - { success: false, error: 'signerEmail, signerName, and emailSubject are required' }, - { status: 400 } - ) - } - - let documentBase64 = '' - let documentName = 'document.pdf' - - if (file) { - try { - const parsed = FileInputSchema.parse(file) - const userFiles = processFilesToUserFiles([parsed as RawFileInput], 'docusign-send', logger) - if (userFiles.length > 0) { - const userFile = userFiles[0] - const denied = await assertToolFileAccess(userFile.key, userId, 'docusign-send', logger) - if (denied) return denied - if (userFile.size > MAX_DOCUSIGN_DOCUMENT_BYTES) { - return NextResponse.json( - { success: false, error: 'Document is too large to send through DocuSign' }, - { status: 413 } - ) - } - const { buffer } = await downloadServableFileFromStorage( - userFile, - 'docusign-send', - logger, - { - maxBytes: MAX_DOCUSIGN_DOCUMENT_BYTES, - } - ) - assertKnownSizeWithinLimit(buffer.length, MAX_DOCUSIGN_DOCUMENT_BYTES, 'DocuSign document') - documentBase64 = buffer.toString('base64') - documentName = userFile.name - } - } catch (fileError) { - const notReady = docNotReadyResponse(fileError) - if (notReady) return notReady - logger.error('Failed to process file for DocuSign envelope', { fileError }) - return NextResponse.json( - { - success: false, - error: isPayloadSizeLimitError(fileError) - ? getErrorMessage(fileError, 'Document is too large to send through DocuSign') - : 'Failed to process uploaded file', - }, - { status: isPayloadSizeLimitError(fileError) ? 413 : 400 } - ) - } - } - - const envelopeBody: Record = { - emailSubject, - status: (status as string) || 'sent', - recipients: { - signers: [ - { - email: signerEmail, - name: signerName, - recipientId: '1', - routingOrder: '1', - tabs: { - signHereTabs: [ - { - anchorString: '/sig1/', - anchorUnits: 'pixels', - anchorXOffset: '0', - anchorYOffset: '0', - }, - ], - dateSignedTabs: [ - { - anchorString: '/date1/', - anchorUnits: 'pixels', - anchorXOffset: '0', - anchorYOffset: '0', - }, - ], - }, - }, - ], - carbonCopies: ccEmail - ? [ - { - email: ccEmail, - name: ccName || (ccEmail as string), - recipientId: '2', - routingOrder: '2', - }, - ] - : [], - }, - } - - if (emailBody) { - envelopeBody.emailBlurb = emailBody - } - - if (documentBase64) { - envelopeBody.documents = [ - { - documentBase64, - name: documentName, - fileExtension: documentName.split('.').pop() || 'pdf', - documentId: '1', - }, - ] - } else if (((status as string) || 'sent') === 'sent') { - return NextResponse.json( - { success: false, error: 'A document file is required to send an envelope' }, - { status: 400 } - ) - } - - const response = await fetchDocusign( - `${apiBase}/envelopes`, - { - method: 'POST', - headers, - body: JSON.stringify(envelopeBody), - }, - signal - ) - - const data = await readDocusignJson(response, 'DocuSign send envelope response') - if (!response.ok) { - logger.error('DocuSign send envelope failed', { data, status: response.status }) - return NextResponse.json( - { success: false, error: docusignError(data, 'Failed to send envelope') }, - { status: response.status } - ) - } - - return NextResponse.json(data) -} - -async function handleCreateFromTemplate( - apiBase: string, - headers: Record, - params: Record, - signal?: AbortSignal -) { - const { templateId, emailSubject, emailBody, templateRoles, status } = params - - if (!templateId) { - return NextResponse.json({ success: false, error: 'templateId is required' }, { status: 400 }) - } - - let parsedRoles: unknown[] = [] - if (templateRoles) { - if (typeof templateRoles === 'string') { - try { - parsedRoles = JSON.parse(templateRoles) - } catch { - return NextResponse.json( - { success: false, error: 'Invalid JSON for templateRoles' }, - { status: 400 } - ) - } - } else if (Array.isArray(templateRoles)) { - parsedRoles = templateRoles - } - } - - const envelopeBody: Record = { - templateId, - status: (status as string) || 'sent', - templateRoles: parsedRoles, - } - - if (emailSubject) envelopeBody.emailSubject = emailSubject - if (emailBody) envelopeBody.emailBlurb = emailBody - - const response = await fetchDocusign( - `${apiBase}/envelopes`, - { - method: 'POST', - headers, - body: JSON.stringify(envelopeBody), - }, - signal - ) - - const data = await readDocusignJson(response, 'DocuSign create from template response') - if (!response.ok) { - logger.error('DocuSign create from template failed', { data, status: response.status }) - return NextResponse.json( - { - success: false, - error: docusignError(data, 'Failed to create envelope from template'), - }, - { status: response.status } - ) - } - - return NextResponse.json(data) -} - -async function handleGetEnvelope( - apiBase: string, - headers: Record, - params: Record, - signal?: AbortSignal -) { - const { envelopeId } = params - if (!envelopeId) { - return NextResponse.json({ success: false, error: 'envelopeId is required' }, { status: 400 }) - } - - const response = await fetchDocusign( - `${apiBase}/envelopes/${(envelopeId as string).trim()}?include=recipients,documents`, - { headers }, - signal - ) - const data = await readDocusignJson(response, 'DocuSign envelope response') - - if (!response.ok) { - return NextResponse.json( - { success: false, error: docusignError(data, 'Failed to get envelope') }, - { status: response.status } - ) - } - - return NextResponse.json(data) -} - -async function handleListEnvelopes( - apiBase: string, - headers: Record, - params: Record, - signal?: AbortSignal -) { - const queryParams = new URLSearchParams() - - const fromDate = params.fromDate as string | undefined - if (fromDate) { - queryParams.append('from_date', fromDate) - } else { - const thirtyDaysAgo = new Date() - thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30) - queryParams.append('from_date', thirtyDaysAgo.toISOString()) - } - - if (params.toDate) queryParams.append('to_date', params.toDate as string) - if (params.envelopeStatus) queryParams.append('status', params.envelopeStatus as string) - if (params.searchText) queryParams.append('search_text', params.searchText as string) - if (params.count) queryParams.append('count', params.count as string) - - const response = await fetchDocusign(`${apiBase}/envelopes?${queryParams}`, { headers }, signal) - const data = await readDocusignJson(response, 'DocuSign envelope list response') - - if (!response.ok) { - return NextResponse.json( - { success: false, error: docusignError(data, 'Failed to list envelopes') }, - { status: response.status } - ) - } - - return NextResponse.json(data) -} - -async function handleVoidEnvelope( - apiBase: string, - headers: Record, - params: Record, - signal?: AbortSignal -) { - const { envelopeId, voidedReason } = params - if (!envelopeId) { - return NextResponse.json({ success: false, error: 'envelopeId is required' }, { status: 400 }) - } - if (!voidedReason) { - return NextResponse.json({ success: false, error: 'voidedReason is required' }, { status: 400 }) - } - - const response = await fetchDocusign( - `${apiBase}/envelopes/${(envelopeId as string).trim()}`, - { - method: 'PUT', - headers, - body: JSON.stringify({ status: 'voided', voidedReason }), - }, - signal - ) - - const data = await readDocusignJson(response, 'DocuSign void envelope response') - if (!response.ok) { - return NextResponse.json( - { success: false, error: docusignError(data, 'Failed to void envelope') }, - { status: response.status } - ) - } - - return NextResponse.json({ envelopeId, status: 'voided' }) -} - -async function handleDownloadDocument( - apiBase: string, - headers: Record, - params: Record, - userId: string, - signal?: AbortSignal -) { - const { envelopeId, documentId } = params - if (!envelopeId) { - return NextResponse.json({ success: false, error: 'envelopeId is required' }, { status: 400 }) - } - - const docId = (documentId as string) || 'combined' - - const response = await fetchDocusign( - `${apiBase}/envelopes/${(envelopeId as string).trim()}/documents/${docId}`, - { - headers: { Authorization: headers.Authorization }, - }, - signal - ) - - if (!response.ok) { - let errorText = '' - try { - errorText = await readResponseTextWithLimit(response, { - maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, - label: 'DocuSign document error response', - }) - } catch { - // ignore - } - return NextResponse.json( - { success: false, error: `Failed to download document: ${response.status} ${errorText}` }, - { status: response.status } - ) - } - - const contentType = response.headers.get('content-type') || 'application/pdf' - const contentDisposition = response.headers.get('content-disposition') || '' - let fileName = `document-${docId}.pdf` - - const filenameMatch = contentDisposition.match(/filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/) - if (filenameMatch) { - fileName = filenameMatch[1].replace(/['"]/g, '') - } - - const buffer = await readResponseToBufferWithLimit(response, { - maxBytes: MAX_DOCUSIGN_DOCUMENT_BYTES, - label: 'DocuSign document download', - }) - - const workspaceId = typeof params.workspaceId === 'string' ? params.workspaceId : undefined - const workflowId = typeof params.workflowId === 'string' ? params.workflowId : undefined - const executionId = typeof params.executionId === 'string' ? params.executionId : undefined - const legacyInlineContent = - buffer.length <= MAX_LEGACY_INLINE_DOCUMENT_BYTES - ? { base64Content: buffer.toString('base64') } - : {} - - if (workspaceId && workflowId && executionId) { - const file = await uploadExecutionFile( - { workspaceId, workflowId, executionId }, - buffer, - fileName, - contentType, - userId - ) - return NextResponse.json({ - file, - mimeType: contentType, - fileName, - ...legacyInlineContent, - }) - } - - const file = await uploadCopilotFile({ - buffer, - fileName, - contentType, - userId, - }) - - return NextResponse.json({ file, mimeType: contentType, fileName, ...legacyInlineContent }) -} - -async function handleListTemplates( - apiBase: string, - headers: Record, - params: Record, - signal?: AbortSignal -) { - const queryParams = new URLSearchParams() - if (params.searchText) queryParams.append('search_text', params.searchText as string) - if (params.count) queryParams.append('count', params.count as string) - - const queryString = queryParams.toString() - const url = queryString ? `${apiBase}/templates?${queryString}` : `${apiBase}/templates` - - const response = await fetchDocusign(url, { headers }, signal) - const data = await readDocusignJson(response, 'DocuSign template list response') - - if (!response.ok) { - return NextResponse.json( - { success: false, error: docusignError(data, 'Failed to list templates') }, - { status: response.status } - ) - } - - return NextResponse.json(data) -} - -async function handleListRecipients( - apiBase: string, - headers: Record, - params: Record, - signal?: AbortSignal -) { - const { envelopeId } = params - if (!envelopeId) { - return NextResponse.json({ success: false, error: 'envelopeId is required' }, { status: 400 }) - } - - const response = await fetchDocusign( - `${apiBase}/envelopes/${(envelopeId as string).trim()}/recipients`, - { - headers, - }, - signal - ) - const data = await readDocusignJson(response, 'DocuSign recipients response') - - if (!response.ok) { - return NextResponse.json( - { success: false, error: docusignError(data, 'Failed to list recipients') }, - { status: response.status } - ) - } - - return NextResponse.json(data) -} diff --git a/apps/sim/app/api/tools/dropbox/upload/route.ts b/apps/sim/app/api/tools/dropbox/upload/route.ts deleted file mode 100644 index af04b69db16..00000000000 --- a/apps/sim/app/api/tools/dropbox/upload/route.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { dropboxUploadContract } from '@/lib/api/contracts/storage-transfer' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { httpHeaderSafeJson } from '@/lib/core/utils/validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('DropboxUploadAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Dropbox upload attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated Dropbox upload request via ${authResult.authType}`) - - const parsed = await parseRequest(dropboxUploadContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - let fileBuffer: Buffer - let fileName: string - - // Prefer UserFile input, fall back to legacy base64 string - if (validatedData.file) { - // Process UserFile input - const userFiles = processFilesToUserFiles( - [validatedData.file as RawFileInput], - requestId, - logger - ) - - if (userFiles.length === 0) { - return NextResponse.json({ success: false, error: 'Invalid file input' }, { status: 400 }) - } - - const userFile = userFiles[0] - logger.info(`[${requestId}] Downloading file: ${userFile.name} (${userFile.size} bytes)`) - - const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) - if (denied) return denied - try { - const result = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - }) - fileBuffer = result.buffer - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Failed to download file') }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } - fileName = userFile.name - } else if (validatedData.fileContent) { - // Legacy: base64 string input (backwards compatibility) - logger.info(`[${requestId}] Using legacy base64 content input`) - fileBuffer = Buffer.from(validatedData.fileContent, 'base64') - fileName = validatedData.fileName || 'file' - } else { - return NextResponse.json({ success: false, error: 'File is required' }, { status: 400 }) - } - - // Determine final path - let finalPath = validatedData.path - if (finalPath.endsWith('/')) { - finalPath = `${finalPath}${fileName}` - } - - logger.info(`[${requestId}] Uploading to Dropbox: ${finalPath} (${fileBuffer.length} bytes)`) - - const dropboxApiArg = { - path: finalPath, - mode: validatedData.mode || 'add', - autorename: validatedData.autorename ?? false, - mute: validatedData.mute ?? false, - } - - const response = await fetch('https://content.dropboxapi.com/2/files/upload', { - method: 'POST', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': 'application/octet-stream', - 'Dropbox-API-Arg': httpHeaderSafeJson(dropboxApiArg), - }, - body: new Uint8Array(fileBuffer), - }) - - const data = await response.json() - - if (!response.ok) { - const errorMessage = data.error_summary || data.error?.message || 'Failed to upload file' - logger.error(`[${requestId}] Dropbox API error:`, { status: response.status, data }) - return NextResponse.json({ success: false, error: errorMessage }, { status: response.status }) - } - - logger.info(`[${requestId}] File uploaded successfully to ${data.path_display}`) - - return NextResponse.json({ - success: true, - output: { - file: data, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Unexpected error:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/dynamodb/delete/route.ts b/apps/sim/app/api/tools/dynamodb/delete/route.ts deleted file mode 100644 index e3097142d7c..00000000000 --- a/apps/sim/app/api/tools/dynamodb/delete/route.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsDynamodbDeleteContract } from '@/lib/api/contracts/tools/aws/dynamodb-delete' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createDynamoDBClient, deleteItem } from '@/app/api/tools/dynamodb/utils' - -const logger = createLogger('DynamoDBDeleteAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsDynamodbDeleteContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`Deleting item from table '${validatedData.tableName}'`) - - const client = createDynamoDBClient({ - region: validatedData.region, - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }) - - try { - await deleteItem(client, validatedData.tableName, validatedData.key, { - conditionExpression: validatedData.conditionExpression, - expressionAttributeNames: validatedData.expressionAttributeNames, - expressionAttributeValues: validatedData.expressionAttributeValues, - }) - - logger.info(`Delete completed for table '${validatedData.tableName}'`) - - return NextResponse.json({ - message: 'Item deleted successfully', - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = toError(error).message || 'DynamoDB delete failed' - logger.error('DynamoDB delete failed:', error) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/dynamodb/get/route.ts b/apps/sim/app/api/tools/dynamodb/get/route.ts deleted file mode 100644 index a66bb21e090..00000000000 --- a/apps/sim/app/api/tools/dynamodb/get/route.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsDynamodbGetContract } from '@/lib/api/contracts/tools/aws/dynamodb-get' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createDynamoDBClient, getItem } from '@/app/api/tools/dynamodb/utils' - -const logger = createLogger('DynamoDBGetAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsDynamodbGetContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`Getting item from table '${validatedData.tableName}'`) - - const client = createDynamoDBClient({ - region: validatedData.region, - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }) - - try { - const result = await getItem( - client, - validatedData.tableName, - validatedData.key, - validatedData.consistentRead - ) - - logger.info(`Get item completed for table '${validatedData.tableName}'`) - - return NextResponse.json({ - message: result.item ? 'Item retrieved successfully' : 'Item not found', - item: result.item, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = toError(error).message || 'DynamoDB get failed' - logger.error('DynamoDB get failed:', error) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/dynamodb/introspect/route.ts b/apps/sim/app/api/tools/dynamodb/introspect/route.ts deleted file mode 100644 index c0e5dd3e2e0..00000000000 --- a/apps/sim/app/api/tools/dynamodb/introspect/route.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsDynamodbIntrospectContract } from '@/lib/api/contracts/tools/aws/dynamodb-introspect' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createRawDynamoDBClient, describeTable, listTables } from '@/app/api/tools/dynamodb/utils' - -const logger = createLogger('DynamoDBIntrospectAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsDynamodbIntrospectContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Introspecting DynamoDB in region ${params.region}`) - - const client = createRawDynamoDBClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const { tables } = await listTables(client) - - if (params.tableName) { - logger.info(`Describing table: ${params.tableName}`) - const { tableDetails } = await describeTable(client, params.tableName) - - logger.info(`Table description completed for '${params.tableName}'`) - - return NextResponse.json({ - message: `Table '${params.tableName}' described successfully.`, - tables, - tableDetails, - }) - } - - logger.info(`Listed ${tables.length} tables`) - - return NextResponse.json({ - message: `Found ${tables.length} table(s) in region '${params.region}'.`, - tables, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = toError(error).message || 'Unknown error occurred' - logger.error('DynamoDB introspection failed:', error) - - return NextResponse.json( - { error: `DynamoDB introspection failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/dynamodb/put/route.ts b/apps/sim/app/api/tools/dynamodb/put/route.ts deleted file mode 100644 index 5eabcc20d3f..00000000000 --- a/apps/sim/app/api/tools/dynamodb/put/route.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsDynamodbPutContract } from '@/lib/api/contracts/tools/aws/dynamodb-put' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createDynamoDBClient, putItem } from '@/app/api/tools/dynamodb/utils' - -const logger = createLogger('DynamoDBPutAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsDynamodbPutContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`Putting item into table '${validatedData.tableName}'`) - - const client = createDynamoDBClient({ - region: validatedData.region, - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }) - - try { - await putItem(client, validatedData.tableName, validatedData.item, { - conditionExpression: validatedData.conditionExpression, - expressionAttributeNames: validatedData.expressionAttributeNames, - expressionAttributeValues: validatedData.expressionAttributeValues, - }) - - logger.info(`Put item completed for table '${validatedData.tableName}'`) - - return NextResponse.json({ - message: 'Item created successfully', - item: validatedData.item, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = toError(error).message || 'DynamoDB put failed' - logger.error('DynamoDB put failed:', error) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/dynamodb/query/route.ts b/apps/sim/app/api/tools/dynamodb/query/route.ts deleted file mode 100644 index 62beffffe38..00000000000 --- a/apps/sim/app/api/tools/dynamodb/query/route.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsDynamodbQueryContract } from '@/lib/api/contracts/tools/aws/dynamodb-query' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createDynamoDBClient, queryItems } from '@/app/api/tools/dynamodb/utils' - -const logger = createLogger('DynamoDBQueryAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsDynamodbQueryContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`Querying table '${validatedData.tableName}'`) - - const client = createDynamoDBClient({ - region: validatedData.region, - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }) - - try { - const result = await queryItems( - client, - validatedData.tableName, - validatedData.keyConditionExpression, - { - filterExpression: validatedData.filterExpression, - expressionAttributeNames: validatedData.expressionAttributeNames, - expressionAttributeValues: validatedData.expressionAttributeValues, - indexName: validatedData.indexName, - limit: validatedData.limit, - exclusiveStartKey: validatedData.exclusiveStartKey, - scanIndexForward: validatedData.scanIndexForward, - } - ) - - logger.info( - `Query completed for table '${validatedData.tableName}', returned ${result.count} items` - ) - - return NextResponse.json({ - message: `Query returned ${result.count} items`, - items: result.items, - count: result.count, - ...(result.lastEvaluatedKey && { lastEvaluatedKey: result.lastEvaluatedKey }), - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = toError(error).message || 'DynamoDB query failed' - logger.error('DynamoDB query failed:', error) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/dynamodb/scan/route.ts b/apps/sim/app/api/tools/dynamodb/scan/route.ts deleted file mode 100644 index 8d9ffe1d68e..00000000000 --- a/apps/sim/app/api/tools/dynamodb/scan/route.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsDynamodbScanContract } from '@/lib/api/contracts/tools/aws/dynamodb-scan' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createDynamoDBClient, scanItems } from '@/app/api/tools/dynamodb/utils' - -const logger = createLogger('DynamoDBScanAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsDynamodbScanContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`Scanning table '${validatedData.tableName}'`) - - const client = createDynamoDBClient({ - region: validatedData.region, - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }) - - try { - const result = await scanItems(client, validatedData.tableName, { - filterExpression: validatedData.filterExpression, - projectionExpression: validatedData.projectionExpression, - expressionAttributeNames: validatedData.expressionAttributeNames, - expressionAttributeValues: validatedData.expressionAttributeValues, - limit: validatedData.limit, - exclusiveStartKey: validatedData.exclusiveStartKey, - }) - - logger.info( - `Scan completed for table '${validatedData.tableName}', returned ${result.count} items` - ) - - return NextResponse.json({ - message: `Scan returned ${result.count} items`, - items: result.items, - count: result.count, - ...(result.lastEvaluatedKey && { lastEvaluatedKey: result.lastEvaluatedKey }), - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = toError(error).message || 'DynamoDB scan failed' - logger.error('DynamoDB scan failed:', error) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/dynamodb/update/route.ts b/apps/sim/app/api/tools/dynamodb/update/route.ts deleted file mode 100644 index 02e1ee33687..00000000000 --- a/apps/sim/app/api/tools/dynamodb/update/route.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsDynamodbUpdateContract } from '@/lib/api/contracts/tools/aws/dynamodb-update' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createDynamoDBClient, updateItem } from '@/app/api/tools/dynamodb/utils' - -const logger = createLogger('DynamoDBUpdateAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(awsDynamodbUpdateContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`Updating item in table '${validatedData.tableName}'`) - - const client = createDynamoDBClient({ - region: validatedData.region, - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }) - - try { - const result = await updateItem( - client, - validatedData.tableName, - validatedData.key, - validatedData.updateExpression, - { - expressionAttributeNames: validatedData.expressionAttributeNames, - expressionAttributeValues: validatedData.expressionAttributeValues, - conditionExpression: validatedData.conditionExpression, - } - ) - - logger.info(`Update completed for table '${validatedData.tableName}'`) - - return NextResponse.json({ - message: 'Item updated successfully', - item: result.attributes, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = toError(error).message || 'DynamoDB update failed' - logger.error('DynamoDB update failed:', error) - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/dynamodb/utils.ts b/apps/sim/app/api/tools/dynamodb/utils.ts deleted file mode 100644 index e9cf2cb1b54..00000000000 --- a/apps/sim/app/api/tools/dynamodb/utils.ts +++ /dev/null @@ -1,308 +0,0 @@ -import { DescribeTableCommand, DynamoDBClient, ListTablesCommand } from '@aws-sdk/client-dynamodb' -import { - DeleteCommand, - DynamoDBDocumentClient, - GetCommand, - PutCommand, - QueryCommand, - ScanCommand, - UpdateCommand, -} from '@aws-sdk/lib-dynamodb' -import type { DynamoDBConnectionConfig, DynamoDBTableSchema } from '@/tools/dynamodb/types' - -export function createDynamoDBClient(config: DynamoDBConnectionConfig): DynamoDBDocumentClient { - const client = new DynamoDBClient({ - region: config.region, - credentials: { - accessKeyId: config.accessKeyId, - secretAccessKey: config.secretAccessKey, - }, - }) - - return DynamoDBDocumentClient.from(client, { - marshallOptions: { - removeUndefinedValues: true, - convertEmptyValues: false, - }, - unmarshallOptions: { - wrapNumbers: false, - }, - }) -} - -export async function getItem( - client: DynamoDBDocumentClient, - tableName: string, - key: Record, - consistentRead?: boolean -): Promise<{ item: Record | null }> { - const command = new GetCommand({ - TableName: tableName, - Key: key, - ConsistentRead: consistentRead, - }) - - const response = await client.send(command) - return { - item: (response.Item as Record) || null, - } -} - -export async function putItem( - client: DynamoDBDocumentClient, - tableName: string, - item: Record, - options?: { - conditionExpression?: string - expressionAttributeNames?: Record - expressionAttributeValues?: Record - } -): Promise<{ success: boolean }> { - const command = new PutCommand({ - TableName: tableName, - Item: item, - ...(options?.conditionExpression && { ConditionExpression: options.conditionExpression }), - ...(options?.expressionAttributeNames && { - ExpressionAttributeNames: options.expressionAttributeNames, - }), - ...(options?.expressionAttributeValues && { - ExpressionAttributeValues: options.expressionAttributeValues, - }), - }) - - await client.send(command) - return { success: true } -} - -export async function queryItems( - client: DynamoDBDocumentClient, - tableName: string, - keyConditionExpression: string, - options?: { - filterExpression?: string - expressionAttributeNames?: Record - expressionAttributeValues?: Record - indexName?: string - limit?: number - exclusiveStartKey?: Record - scanIndexForward?: boolean - } -): Promise<{ - items: Record[] - count: number - lastEvaluatedKey?: Record -}> { - const command = new QueryCommand({ - TableName: tableName, - KeyConditionExpression: keyConditionExpression, - ...(options?.filterExpression && { FilterExpression: options.filterExpression }), - ...(options?.expressionAttributeNames && { - ExpressionAttributeNames: options.expressionAttributeNames, - }), - ...(options?.expressionAttributeValues && { - ExpressionAttributeValues: options.expressionAttributeValues, - }), - ...(options?.indexName && { IndexName: options.indexName }), - ...(options?.limit && { Limit: options.limit }), - ...(options?.exclusiveStartKey && { ExclusiveStartKey: options.exclusiveStartKey }), - ...(options?.scanIndexForward !== undefined && { ScanIndexForward: options.scanIndexForward }), - }) - - const response = await client.send(command) - return { - items: (response.Items as Record[]) || [], - count: response.Count || 0, - lastEvaluatedKey: response.LastEvaluatedKey as Record | undefined, - } -} - -export async function scanItems( - client: DynamoDBDocumentClient, - tableName: string, - options?: { - filterExpression?: string - projectionExpression?: string - expressionAttributeNames?: Record - expressionAttributeValues?: Record - limit?: number - exclusiveStartKey?: Record - } -): Promise<{ - items: Record[] - count: number - lastEvaluatedKey?: Record -}> { - const command = new ScanCommand({ - TableName: tableName, - ...(options?.filterExpression && { FilterExpression: options.filterExpression }), - ...(options?.projectionExpression && { ProjectionExpression: options.projectionExpression }), - ...(options?.expressionAttributeNames && { - ExpressionAttributeNames: options.expressionAttributeNames, - }), - ...(options?.expressionAttributeValues && { - ExpressionAttributeValues: options.expressionAttributeValues, - }), - ...(options?.limit && { Limit: options.limit }), - ...(options?.exclusiveStartKey && { ExclusiveStartKey: options.exclusiveStartKey }), - }) - - const response = await client.send(command) - return { - items: (response.Items as Record[]) || [], - count: response.Count || 0, - lastEvaluatedKey: response.LastEvaluatedKey as Record | undefined, - } -} - -export async function updateItem( - client: DynamoDBDocumentClient, - tableName: string, - key: Record, - updateExpression: string, - options?: { - expressionAttributeNames?: Record - expressionAttributeValues?: Record - conditionExpression?: string - } -): Promise<{ attributes: Record | null }> { - const command = new UpdateCommand({ - TableName: tableName, - Key: key, - UpdateExpression: updateExpression, - ...(options?.expressionAttributeNames && { - ExpressionAttributeNames: options.expressionAttributeNames, - }), - ...(options?.expressionAttributeValues && { - ExpressionAttributeValues: options.expressionAttributeValues, - }), - ...(options?.conditionExpression && { ConditionExpression: options.conditionExpression }), - ReturnValues: 'ALL_NEW', - }) - - const response = await client.send(command) - return { - attributes: (response.Attributes as Record) || null, - } -} - -export async function deleteItem( - client: DynamoDBDocumentClient, - tableName: string, - key: Record, - options?: { - conditionExpression?: string - expressionAttributeNames?: Record - expressionAttributeValues?: Record - } -): Promise<{ success: boolean }> { - const command = new DeleteCommand({ - TableName: tableName, - Key: key, - ...(options?.conditionExpression && { ConditionExpression: options.conditionExpression }), - ...(options?.expressionAttributeNames && { - ExpressionAttributeNames: options.expressionAttributeNames, - }), - ...(options?.expressionAttributeValues && { - ExpressionAttributeValues: options.expressionAttributeValues, - }), - }) - - await client.send(command) - return { success: true } -} - -/** - * Creates a raw DynamoDB client for operations that don't require DocumentClient - */ -export function createRawDynamoDBClient(config: DynamoDBConnectionConfig): DynamoDBClient { - return new DynamoDBClient({ - region: config.region, - credentials: { - accessKeyId: config.accessKeyId, - secretAccessKey: config.secretAccessKey, - }, - }) -} - -/** - * Lists all DynamoDB tables in the configured region - */ -export async function listTables(client: DynamoDBClient): Promise<{ tables: string[] }> { - const tables: string[] = [] - let exclusiveStartTableName: string | undefined - - do { - const command = new ListTablesCommand({ - ExclusiveStartTableName: exclusiveStartTableName, - }) - - const response = await client.send(command) - if (response.TableNames) { - tables.push(...response.TableNames) - } - exclusiveStartTableName = response.LastEvaluatedTableName - } while (exclusiveStartTableName) - - return { tables } -} - -/** - * Describes a specific DynamoDB table and returns its schema information - */ -export async function describeTable( - client: DynamoDBClient, - tableName: string -): Promise<{ tableDetails: DynamoDBTableSchema }> { - const command = new DescribeTableCommand({ - TableName: tableName, - }) - - const response = await client.send(command) - const table = response.Table - - if (!table) { - throw new Error(`Table '${tableName}' not found`) - } - - const tableDetails: DynamoDBTableSchema = { - tableName: table.TableName || tableName, - tableStatus: table.TableStatus || 'UNKNOWN', - keySchema: - table.KeySchema?.map((key) => ({ - attributeName: key.AttributeName || '', - keyType: (key.KeyType as 'HASH' | 'RANGE') || 'HASH', - })) || [], - attributeDefinitions: - table.AttributeDefinitions?.map((attr) => ({ - attributeName: attr.AttributeName || '', - attributeType: (attr.AttributeType as 'S' | 'N' | 'B') || 'S', - })) || [], - globalSecondaryIndexes: - table.GlobalSecondaryIndexes?.map((gsi) => ({ - indexName: gsi.IndexName || '', - keySchema: - gsi.KeySchema?.map((key) => ({ - attributeName: key.AttributeName || '', - keyType: (key.KeyType as 'HASH' | 'RANGE') || 'HASH', - })) || [], - projectionType: gsi.Projection?.ProjectionType || 'ALL', - indexStatus: gsi.IndexStatus || 'UNKNOWN', - })) || [], - localSecondaryIndexes: - table.LocalSecondaryIndexes?.map((lsi) => ({ - indexName: lsi.IndexName || '', - keySchema: - lsi.KeySchema?.map((key) => ({ - attributeName: key.AttributeName || '', - keyType: (key.KeyType as 'HASH' | 'RANGE') || 'HASH', - })) || [], - projectionType: lsi.Projection?.ProjectionType || 'ALL', - indexStatus: 'ACTIVE', - })) || [], - itemCount: Number(table.ItemCount) || 0, - tableSizeBytes: Number(table.TableSizeBytes) || 0, - billingMode: table.BillingModeSummary?.BillingMode || 'PROVISIONED', - } - - return { tableDetails } -} diff --git a/apps/sim/app/api/tools/elevenlabs/audio/route.test.ts b/apps/sim/app/api/tools/elevenlabs/audio/route.test.ts deleted file mode 100644 index b8226fb29c5..00000000000 --- a/apps/sim/app/api/tools/elevenlabs/audio/route.test.ts +++ /dev/null @@ -1,135 +0,0 @@ -/** - * @vitest-environment node - */ -import { createMockRequest, hybridAuthMockFns } from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' -import { PRIVATE_MODEL_INPUT_PROVENANCE_HEADER } from '@/lib/execution/model-input-provenance' -import { - RESOLVED_SECRET_PROVENANCE_FIELD, - RESOLVED_SECRET_PROVENANCE_METADATA_V1, -} from '@/lib/execution/private-tool-metadata' - -const { mockDownloadFileFromStorage, mockIsModelSafeWorkspaceFileKey, mockUploadFile } = vi.hoisted( - () => ({ - mockDownloadFileFromStorage: vi.fn(), - mockIsModelSafeWorkspaceFileKey: vi.fn(), - mockUploadFile: vi.fn(), - }) -) - -vi.mock('@/app/api/files/authorization', () => ({ - assertToolFileAccess: vi.fn().mockResolvedValue(null), -})) -vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ - downloadFileFromStorage: mockDownloadFileFromStorage, -})) -vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({ - isModelSafeWorkspaceFileKey: mockIsModelSafeWorkspaceFileKey, - MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE: - 'File cannot be sent to a model because its secret provenance is unavailable', -})) -vi.mock('@/lib/uploads', () => ({ - StorageService: { uploadFile: mockUploadFile }, -})) -vi.mock('@/lib/core/utils/urls', () => ({ - getBaseUrl: vi.fn(() => 'http://localhost:3000'), -})) - -import { POST } from '@/app/api/tools/elevenlabs/audio/route' - -const audioFile = { - id: 'file-1', - name: 'audio.wav', - size: 5, - type: 'audio/wav', - key: 'workspace/workspace-1/audio.wav', -} - -describe('POST /api/tools/elevenlabs/audio', () => { - beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - mockIsModelSafeWorkspaceFileKey.mockResolvedValue(true) - mockDownloadFileFromStorage.mockResolvedValue(Buffer.from('audio')) - mockUploadFile.mockResolvedValue({ path: '/api/files/serve/generated.mp3', size: 3 }) - vi.stubGlobal( - 'fetch', - vi.fn().mockResolvedValue( - new Response(new Uint8Array([1, 2, 3]), { - status: 200, - headers: { 'content-type': 'audio/mpeg' }, - }) - ) - ) - }) - - it('keeps headerless sound-effect calls compatible', async () => { - const response = await POST( - createMockRequest('POST', { - operation: 'sound_effects', - apiKey: 'test-api-key', - text: 'A soft chime', - }) - ) - - expect(response.status).toBe(200) - expect(fetch).toHaveBeenCalledOnce() - expect(mockIsModelSafeWorkspaceFileKey).not.toHaveBeenCalled() - }) - - it('rejects an incomplete private provenance envelope before reading audio bytes', async () => { - const response = await POST( - createMockRequest( - 'POST', - { - operation: 'audio_isolation', - apiKey: 'test-api-key', - audioFile, - [RESOLVED_SECRET_PROVENANCE_FIELD]: { - version: 1, - complete: false, - entries: [], - }, - }, - { [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1 } - ) - ) - - expect(response.status).toBe(400) - expect(await response.json()).toEqual({ error: 'Model input provenance is unavailable' }) - expect(mockDownloadFileFromStorage).not.toHaveBeenCalled() - expect(fetch).not.toHaveBeenCalled() - }) - - it('rejects tracked unsafe audio before reading or sending it', async () => { - mockIsModelSafeWorkspaceFileKey.mockResolvedValueOnce(false) - - const response = await POST( - createMockRequest( - 'POST', - { - operation: 'audio_isolation', - apiKey: 'test-api-key', - audioFile, - [RESOLVED_SECRET_PROVENANCE_FIELD]: { - version: 1, - complete: true, - entries: [], - }, - }, - { [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1 } - ) - ) - - expect(response.status).toBe(400) - expect(await response.json()).toEqual({ - error: 'File cannot be sent to a model because its secret provenance is unavailable', - }) - expect(mockDownloadFileFromStorage).not.toHaveBeenCalled() - expect(fetch).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/app/api/tools/elevenlabs/audio/route.ts b/apps/sim/app/api/tools/elevenlabs/audio/route.ts deleted file mode 100644 index 871868037ad..00000000000 --- a/apps/sim/app/api/tools/elevenlabs/audio/route.ts +++ /dev/null @@ -1,240 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import type { NextRequest } from 'next/server' -import { NextResponse } from 'next/server' -import { elevenLabsAudioToolContract } from '@/lib/api/contracts/tools/media/elevenlabs' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' -import { validateAlphanumericId } from '@/lib/core/security/input-validation' -import { - isPayloadSizeLimitError, - readResponseToBufferWithLimit, -} from '@/lib/core/utils/stream-limits' -import { getBaseUrl } from '@/lib/core/utils/urls' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' -import { StorageService } from '@/lib/uploads' -import { - isModelSafeWorkspaceFileKey, - MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, -} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import { getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { assertToolFileAccess } from '@/app/api/files/authorization' - -const logger = createLogger('ElevenLabsAudioAPI') -const MAX_AUDIO_BYTES = 25 * 1024 * 1024 -const BASE_URL = 'https://api.elevenlabs.io/v1' - -type AudioOperation = 'sound_effects' | 'speech_to_speech' | 'audio_isolation' - -interface SourceAudio { - buffer: Buffer - fileName: string - mimeType: string -} - -/** Builds the upstream ElevenLabs request for an audio-producing operation. */ -function buildElevenLabsRequest( - operation: AudioOperation, - body: { - apiKey: string - voiceId?: string - text?: string - modelId?: string - durationSeconds?: number - promptInfluence?: number - loop?: boolean - removeBackgroundNoise?: boolean - }, - source: SourceAudio | null -): { url: string; init: RequestInit } { - const headers: Record = { 'xi-api-key': body.apiKey, Accept: 'audio/mpeg' } - const signal = AbortSignal.timeout(DEFAULT_EXECUTION_TIMEOUT_MS) - - if (operation === 'sound_effects') { - const payload: Record = { text: body.text } - if (body.modelId) payload.model_id = body.modelId - if (body.durationSeconds !== undefined) payload.duration_seconds = body.durationSeconds - if (body.promptInfluence !== undefined) payload.prompt_influence = body.promptInfluence - if (body.loop !== undefined) payload.loop = body.loop - return { - url: `${BASE_URL}/sound-generation`, - init: { - method: 'POST', - headers: { ...headers, 'Content-Type': 'application/json' }, - body: JSON.stringify(payload), - signal, - }, - } - } - - const formData = new FormData() - const file = source as SourceAudio - formData.append( - 'audio', - new Blob([new Uint8Array(file.buffer)], { type: file.mimeType }), - file.fileName - ) - - if (operation === 'speech_to_speech') { - if (body.modelId) formData.append('model_id', body.modelId) - if (body.removeBackgroundNoise !== undefined) { - formData.append('remove_background_noise', String(body.removeBackgroundNoise)) - } - return { - url: `${BASE_URL}/speech-to-speech/${body.voiceId}`, - init: { method: 'POST', headers, body: formData, signal }, - } - } - - return { - url: `${BASE_URL}/audio-isolation`, - init: { method: 'POST', headers, body: formData, signal }, - } -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId() - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const userId = authResult.userId - - const parsed = await parseRequest( - elevenLabsAudioToolContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { error: getValidationErrorMessage(error, 'Missing required parameters') }, - { status: 400 } - ), - } - ) - if (!parsed.success) return parsed.response - - const body = parsed.data.body - const operation = body.operation as AudioOperation - if (operation === 'speech_to_speech' || operation === 'audio_isolation') { - const modelInputProvenance = validateOpaqueModelInputProvenance({ - headers: request.headers, - payload: body, - isInternalRequest: true, - }) - if (!modelInputProvenance.success) { - return NextResponse.json( - { error: modelInputProvenance.error }, - { status: modelInputProvenance.status } - ) - } - } - - if (operation === 'sound_effects' && !body.text) { - return NextResponse.json({ error: 'text is required' }, { status: 400 }) - } - - let source: SourceAudio | null = null - if (operation === 'speech_to_speech' || operation === 'audio_isolation') { - if (!body.audioFile) { - return NextResponse.json({ error: 'audioFile is required' }, { status: 400 }) - } - const file = body.audioFile - const denied = await assertToolFileAccess(file.key, userId, requestId, logger) - if (denied) return denied - if (!(await isModelSafeWorkspaceFileKey(file.key))) { - return NextResponse.json( - { error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE }, - { status: 400 } - ) - } - const buffer = await downloadFileFromStorage(file, requestId, logger, { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - }) - const ext = file.name.split('.').pop()?.toLowerCase() || '' - source = { - buffer, - fileName: file.name, - mimeType: file.type || getMimeTypeFromExtension(ext), - } - } - - if (operation === 'speech_to_speech') { - if (!body.voiceId) { - return NextResponse.json({ error: 'voiceId is required' }, { status: 400 }) - } - const voiceIdValidation = validateAlphanumericId(body.voiceId, 'voiceId', 255) - if (!voiceIdValidation.isValid) { - return NextResponse.json({ error: voiceIdValidation.error }, { status: 400 }) - } - } - - const { url, init } = buildElevenLabsRequest(operation, body, source) - const response = await fetch(url, init) - - if (!response.ok) { - await response.text().catch(() => '') - logger.error(`[${requestId}] ElevenLabs request failed`, { - operation, - status: response.status, - statusText: response.statusText, - }) - return NextResponse.json( - { error: `ElevenLabs request failed: ${response.status} ${response.statusText}` }, - { status: response.status } - ) - } - - const outputBuffer = await readResponseToBufferWithLimit(response, { - maxBytes: MAX_AUDIO_BYTES, - label: `ElevenLabs ${operation} response`, - signal: request.signal, - }) - - if (outputBuffer.length === 0) { - return NextResponse.json({ error: 'Empty audio received' }, { status: 422 }) - } - - const fileName = `elevenlabs-${operation}-${Date.now()}.mp3` - const executionContext = - body.workspaceId && body.workflowId && body.executionId - ? { - workspaceId: body.workspaceId, - workflowId: body.workflowId, - executionId: body.executionId, - } - : null - - if (executionContext) { - const { uploadExecutionFile } = await import('@/lib/uploads/contexts/execution') - const userFile = await uploadExecutionFile( - executionContext, - outputBuffer, - fileName, - 'audio/mpeg', - userId - ) - return NextResponse.json({ audioFile: userFile, audioUrl: userFile.url }) - } - - const fileInfo = await StorageService.uploadFile({ - file: outputBuffer, - fileName, - contentType: 'audio/mpeg', - context: 'copilot', - }) - return NextResponse.json({ audioUrl: `${getBaseUrl()}${fileInfo.path}`, size: fileInfo.size }) - } catch (error) { - logger.error(`[${requestId}] ElevenLabs audio proxy error:`, error) - return NextResponse.json( - { error: `Internal Server Error: ${getErrorMessage(error, 'Unknown error')}` }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/embeddings/route.test.ts b/apps/sim/app/api/tools/embeddings/route.test.ts deleted file mode 100644 index b6586902869..00000000000 --- a/apps/sim/app/api/tools/embeddings/route.test.ts +++ /dev/null @@ -1,305 +0,0 @@ -/** - * @vitest-environment node - */ -import { createMockRequest, hybridAuthMockFns } from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { - MockEmbeddingOutputLimitError, - mockEmbed, - mockEmbedOpenRouter, - mockGetOpenRouterEmbeddingModelMetadata, -} = vi.hoisted(() => { - class MockEmbeddingOutputLimitError extends Error { - constructor(message: string) { - super(message) - this.name = 'EmbeddingOutputLimitError' - } - } - - return { - MockEmbeddingOutputLimitError, - mockEmbed: vi.fn(), - mockEmbedOpenRouter: vi.fn(), - mockGetOpenRouterEmbeddingModelMetadata: vi.fn(), - } -}) - -vi.mock('@/lib/embeddings/openrouter-model-catalog.server', () => ({ - getOpenRouterEmbeddingModelMetadata: mockGetOpenRouterEmbeddingModelMetadata, - OpenRouterEmbeddingModelNotFoundError: class OpenRouterEmbeddingModelNotFoundError extends Error { - constructor(model: string) { - super(`Unsupported OpenRouter embedding model: ${model}`) - this.name = 'OpenRouterEmbeddingModelNotFoundError' - } - }, -})) - -vi.mock('@/lib/embeddings', async () => { - const catalog = await import('@/lib/embeddings/catalog') - return { - embed: mockEmbed, - embedOpenRouter: mockEmbedOpenRouter, - EmbeddingOutputLimitError: MockEmbeddingOutputLimitError, - DEFAULT_OPENROUTER_EMBEDDING_MODEL: 'openrouter/openai/text-embedding-3-small', - findEmbeddingModelInfo: catalog.findEmbeddingModelInfo, - getModelsForProvider: catalog.getModelsForProvider, - resolveDimensions: catalog.resolveDimensions, - } -}) - -import { OpenRouterEmbeddingModelNotFoundError } from '@/lib/embeddings/openrouter-model-catalog.server' -import { POST } from '@/app/api/tools/embeddings/route' - -const baseBody = { - provider: 'openai', - model: 'text-embedding-3-small', - input: 'hello world', - apiKey: 'sk-test', -} - -function post(body: Record) { - return POST(createMockRequest('POST', body) as never, undefined as never) -} - -describe('POST /api/tools/embeddings', () => { - beforeEach(() => { - vi.clearAllMocks() - mockGetOpenRouterEmbeddingModelMetadata.mockResolvedValue({ - id: 'openrouter/qwen/qwen3-embedding-8b', - maxInputTokens: 32768, - }) - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - mockEmbed.mockResolvedValue({ - embeddings: [[0.1, 0.2]], - totalTokens: 3, - isBYOK: true, - modelName: 'text-embedding-3-small', - pricingId: 'text-embedding-3-small', - dimensions: 1536, - }) - mockEmbedOpenRouter.mockResolvedValue({ - embeddings: [[0.1, 0.2]], - totalTokens: 3, - billableTokens: 0, - isBYOK: true, - modelName: 'openrouter/qwen/qwen3-embedding-8b', - pricingId: 'openrouter/qwen/qwen3-embedding-8b', - dimensions: 2, - }) - }) - - it('rejects an unauthenticated caller', async () => { - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ success: false }) - const response = await post(baseBody) - expect(response.status).toBe(401) - expect(mockEmbed).not.toHaveBeenCalled() - }) - - it('embeds and returns the contract shape', async () => { - const response = await post(baseBody) - expect(response.status).toBe(200) - const json = await response.json() - expect(json).toMatchObject({ - success: true, - embeddings: [[0.1, 0.2]], - model: 'text-embedding-3-small', - provider: 'openai', - dimensions: 1536, - usage: { prompt_tokens: 3, total_tokens: 3 }, - __embeddingTokens: 3, - }) - }) - - it('rejects an unknown model', async () => { - const response = await post({ ...baseBody, model: 'not-a-real-model' }) - expect(response.status).toBe(400) - expect((await response.json()).error).toContain('Unsupported embedding model') - expect(mockEmbed).not.toHaveBeenCalled() - }) - - it('rejects a model that belongs to another provider', async () => { - const response = await post({ ...baseBody, provider: 'cohere', model: 'gemini-embedding-001' }) - expect(response.status).toBe(400) - expect((await response.json()).error).toContain('belongs to gemini, not cohere') - expect(mockEmbed).not.toHaveBeenCalled() - }) - - /** - * Regression: an unsupported `dimensions` used to escape as the generic 502 - * from the embed() catch, reporting a client input error as an upstream - * failure. - */ - it('rejects an unsupported dimension with 400, not 502', async () => { - const response = await post({ ...baseBody, dimensions: 777 }) - expect(response.status).toBe(400) - expect((await response.json()).error).toContain('does not support 777-dimensional output') - expect(mockEmbed).not.toHaveBeenCalled() - }) - - it('rejects any dimension for a model without Matryoshka support', async () => { - const response = await post({ - ...baseBody, - provider: 'mistral', - model: 'mistral-embed', - dimensions: 999, - }) - expect(response.status).toBe(400) - expect(mockEmbed).not.toHaveBeenCalled() - }) - - it('accepts a supported dimension', async () => { - const response = await post({ ...baseBody, dimensions: 512 }) - expect(response.status).toBe(200) - expect(mockEmbed).toHaveBeenCalledWith( - ['hello world'], - expect.objectContaining({ dimensions: 512 }) - ) - }) - - it('routes OpenRouter through its transport with an explicit key', async () => { - const response = await post({ - provider: 'openrouter', - model: 'openrouter/qwen/qwen3-embedding-8b', - input: 'hello world', - apiKey: 'or-test', - }) - - expect(response.status).toBe(200) - expect(mockEmbedOpenRouter).toHaveBeenCalledWith( - ['hello world'], - expect.objectContaining({ - apiKey: 'or-test', - maxInputTokens: 32768, - model: 'openrouter/qwen/qwen3-embedding-8b', - }) - ) - expect(mockEmbed).not.toHaveBeenCalled() - expect((await response.json()).provider).toBe('openrouter') - }) - - it('rejects OpenRouter without an explicit key', async () => { - const response = await post({ - provider: 'openrouter', - model: 'openrouter/openai/text-embedding-3-small', - input: 'hello world', - }) - - expect(response.status).toBe(400) - expect((await response.json()).error).toContain('apiKey') - expect(mockEmbed).not.toHaveBeenCalled() - expect(mockEmbedOpenRouter).not.toHaveBeenCalled() - }) - - it('rejects an invalid OpenRouter model id', async () => { - const response = await post({ - provider: 'openrouter', - model: 'openrouter/not-qualified', - input: 'hello world', - apiKey: 'or-test', - }) - - expect(response.status).toBe(400) - expect((await response.json()).error).toContain('Invalid OpenRouter embedding model') - expect(mockEmbedOpenRouter).not.toHaveBeenCalled() - }) - - it('rejects a qualified model that is absent from OpenRouter', async () => { - mockGetOpenRouterEmbeddingModelMetadata.mockRejectedValue( - new OpenRouterEmbeddingModelNotFoundError('openrouter/example/missing') - ) - - const response = await post({ - provider: 'openrouter', - model: 'openrouter/example/missing', - input: 'hello world', - apiKey: 'or-test', - }) - - expect(response.status).toBe(400) - expect((await response.json()).error).toContain('Unsupported OpenRouter embedding model') - expect(mockEmbedOpenRouter).not.toHaveBeenCalled() - }) - - it('keeps API keys required for non-OpenRouter providers', async () => { - const response = await post({ - provider: 'openai', - model: 'text-embedding-3-small', - input: 'hello world', - }) - - expect(response.status).toBe(400) - expect((await response.json()).error).toContain('apiKey') - expect(mockEmbed).not.toHaveBeenCalled() - }) - - it('surfaces a provider failure as 502', async () => { - mockEmbed.mockRejectedValue(new Error('Embedding API failed: 429 Too Many Requests')) - const response = await post(baseBody) - expect(response.status).toBe(502) - expect((await response.json()).error).toContain('429') - }) - - it('returns 413 when the requested embedding output exceeds the safe aggregate limit', async () => { - mockEmbed.mockRejectedValue( - new MockEmbeddingOutputLimitError('Embedding output exceeds the safe aggregate limit') - ) - const response = await post(baseBody) - expect(response.status).toBe(413) - expect((await response.json()).error).toContain('safe aggregate limit') - }) - - it('splits a JSON-array input into separate texts', async () => { - await post({ ...baseBody, input: '["alpha","beta"]' }) - expect(mockEmbed).toHaveBeenCalledWith(['alpha', 'beta'], expect.anything()) - }) - - /** - * The contract bounds the array arm, but a JSON-encoded array reaches the - * route as a plain string and is only expanded after validation — so the - * bounds have to be re-applied to the normalized list or they hold for a - * native array body only. - */ - describe('JSON-encoded array bounds', () => { - it('rejects a JSON array that exceeds the input count limit', async () => { - const many = JSON.stringify(Array.from({ length: 1001 }, (_, i) => `t${i}`)) - const response = await post({ ...baseBody, input: many }) - - expect(response.status).toBe(400) - expect((await response.json()).error).toContain('cannot exceed 1000 texts') - expect(mockEmbed).not.toHaveBeenCalled() - }) - - it('rejects an empty JSON array instead of reporting success with no vectors', async () => { - const response = await post({ ...baseBody, input: '[]' }) - - expect(response.status).toBe(400) - expect((await response.json()).error).toContain('at least one text') - expect(mockEmbed).not.toHaveBeenCalled() - }) - - it('rejects a JSON array containing a blank entry', async () => { - const response = await post({ ...baseBody, input: '["ok"," "]' }) - - expect(response.status).toBe(400) - expect((await response.json()).error).toContain('entries cannot be empty') - expect(mockEmbed).not.toHaveBeenCalled() - }) - - it('accepts a JSON array within the bounds', async () => { - const response = await post({ ...baseBody, input: '["alpha","beta"]' }) - - expect(response.status).toBe(200) - expect(mockEmbed).toHaveBeenCalledWith(['alpha', 'beta'], expect.anything()) - }) - }) - - it('embeds a non-JSON string as a single text', async () => { - await post({ ...baseBody, input: 'just a sentence' }) - expect(mockEmbed).toHaveBeenCalledWith(['just a sentence'], expect.anything()) - }) -}) diff --git a/apps/sim/app/api/tools/embeddings/route.ts b/apps/sim/app/api/tools/embeddings/route.ts deleted file mode 100644 index 23a306b0157..00000000000 --- a/apps/sim/app/api/tools/embeddings/route.ts +++ /dev/null @@ -1,238 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { - embeddingsToolContract, - MAX_EMBEDDING_INPUTS, - MAX_EMBEDDING_TOTAL_CHARS, -} from '@/lib/api/contracts/tools/embeddings' -import { getValidationErrorMessage, parseRequest, validationErrorResponse } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - DEFAULT_MODEL_BY_PROVIDER, - DEFAULT_OPENROUTER_EMBEDDING_MODEL, - EmbeddingOutputLimitError, - embed, - embedOpenRouter, - findEmbeddingModelInfo, - resolveDimensions, -} from '@/lib/embeddings' -import { - getOpenRouterEmbeddingModelMetadata, - type OpenRouterEmbeddingModelMetadata, - OpenRouterEmbeddingModelNotFoundError, -} from '@/lib/embeddings/openrouter-model-catalog.server' -import { normalizeOpenRouterEmbeddingModelId } from '@/lib/embeddings/openrouter-models' - -const logger = createLogger('EmbeddingsToolAPI') - -export const dynamic = 'force-dynamic' - -/** - * Accepts a single string, an array, or a JSON-encoded array from a reference - * expression. Probes for the opening bracket with a regex rather than `trim()`, - * which would copy the whole payload just to read one character. - */ -function normalizeInput(input: string | string[]): string[] { - if (Array.isArray(input)) return input - if (/^\s*\[/.test(input)) { - try { - const parsed = JSON.parse(input) - if (Array.isArray(parsed) && parsed.every((entry) => typeof entry === 'string')) { - return parsed - } - } catch { - // Not JSON — fall through and embed the raw string - } - } - return [input] -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest( - embeddingsToolContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn('Invalid embeddings request', { issues: error.issues }) - return validationErrorResponse( - error, - getValidationErrorMessage(error, 'Invalid request data') - ) - }, - } - ) - if (!parsed.success) return parsed.response - - const { provider, apiKey, model, input, taskType, dimensions } = parsed.data.body - const texts = normalizeInput(input) - - /** - * The contract bounds the array arm, but a JSON-encoded array arrives as a - * plain string and is only expanded here, after validation. Re-checking the - * normalized list is what makes the bounds hold for the reference-expression - * path too, rather than only for a native array body. - */ - if (texts.length === 0) { - return NextResponse.json( - { success: false, error: 'input must contain at least one text' }, - { status: 400 } - ) - } - if (texts.length > MAX_EMBEDDING_INPUTS) { - return NextResponse.json( - { - success: false, - error: `input cannot exceed ${MAX_EMBEDDING_INPUTS} texts, received ${texts.length}`, - }, - { status: 400 } - ) - } - /** - * Summing lengths is cheap and runs before the per-entry whitespace scan, so - * an oversized body is rejected without walking every character. - */ - const totalChars = texts.reduce((sum, text) => sum + text.length, 0) - if (totalChars > MAX_EMBEDDING_TOTAL_CHARS) { - return NextResponse.json( - { - success: false, - error: `Input is too large: ${totalChars} characters exceeds the ${MAX_EMBEDDING_TOTAL_CHARS} limit`, - }, - { status: 400 } - ) - } - - if (texts.some((text) => !/\S/.test(text))) { - return NextResponse.json( - { success: false, error: 'input entries cannot be empty' }, - { status: 400 } - ) - } - - let resolvedModel: string - let openRouterModelMetadata: OpenRouterEmbeddingModelMetadata | undefined - if (provider === 'openrouter') { - try { - resolvedModel = normalizeOpenRouterEmbeddingModelId( - model || DEFAULT_OPENROUTER_EMBEDDING_MODEL - ) - } catch (error) { - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Invalid OpenRouter embedding model') }, - { status: 400 } - ) - } - try { - openRouterModelMetadata = await getOpenRouterEmbeddingModelMetadata(resolvedModel) - } catch (error) { - const modelError = error instanceof OpenRouterEmbeddingModelNotFoundError - return NextResponse.json( - { - success: false, - error: getErrorMessage( - error, - modelError - ? 'Unsupported OpenRouter embedding model' - : 'Failed to load OpenRouter embedding model metadata' - ), - }, - { status: modelError ? 400 : 502 } - ) - } - } else { - resolvedModel = model || DEFAULT_MODEL_BY_PROVIDER[provider] - } - - if (provider !== 'openrouter') { - const info = findEmbeddingModelInfo(resolvedModel) - if (!info) { - return NextResponse.json( - { success: false, error: `Unsupported embedding model: ${resolvedModel}` }, - { status: 400 } - ) - } - if (info.provider !== provider) { - return NextResponse.json( - { - success: false, - error: `Model ${resolvedModel} belongs to ${info.provider}, not ${provider}`, - }, - { status: 400 } - ) - } - - /** - * Resolved here as well as inside `embed()` so an unsupported `dimensions` - * is reported as the client error it is. The block's dropdown constrains the - * field, but a reference expression can put any value on the wire. - */ - try { - resolveDimensions(info, dimensions) - } catch (error) { - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Invalid dimensions') }, - { status: 400 } - ) - } - } - - logger.info(`Embedding ${texts.length} input(s) with ${provider}/${resolvedModel}`) - - try { - let result - if (provider === 'openrouter') { - if (!openRouterModelMetadata) { - throw new Error('OpenRouter embedding model metadata was not resolved') - } - result = await embedOpenRouter(texts, { - model: resolvedModel, - dimensions, - apiKey, - maxInputTokens: openRouterModelMetadata.maxInputTokens, - projectInputs: null, - }) - } else { - result = await embed(texts, { - model: resolvedModel, - taskType, - dimensions, - apiKey, - /** - * Callers reach this route through a tool whose `request.modelInput` - * already projected `input` at the HTTP hop, so projecting again here - * would run the substitution over already-projected content. - */ - projectInputs: null, - }) - } - - return NextResponse.json({ - success: true, - embeddings: result.embeddings, - model: result.modelName, - provider, - dimensions: result.dimensions, - usage: { - prompt_tokens: result.totalTokens, - total_tokens: result.totalTokens, - }, - __embeddingTokens: result.totalTokens, - }) - } catch (error) { - const message = getErrorMessage(error, 'Embedding generation failed') - if (error instanceof EmbeddingOutputLimitError) { - logger.warn('Embedding output exceeds safe limit', { error: message }) - return NextResponse.json({ success: false, error: message }, { status: 413 }) - } - logger.error('Embedding generation failed', { error: message }) - return NextResponse.json({ success: false, error: message }, { status: 502 }) - } -}) diff --git a/apps/sim/app/api/tools/enrichment/run/route.ts b/apps/sim/app/api/tools/enrichment/run/route.ts deleted file mode 100644 index 586dc7f735e..00000000000 --- a/apps/sim/app/api/tools/enrichment/run/route.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { createLogger } from '@sim/logger' -import type { NextRequest } from 'next/server' -import { NextResponse } from 'next/server' -import { runEnrichmentContract } from '@/lib/api/contracts/tools/enrichment' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getEnrichment } from '@/enrichments/registry' -import { runEnrichment } from '@/enrichments/run' - -const logger = createLogger('EnrichmentRunAPI') - -/** - * POST /api/tools/enrichment/run - * - * Runs a registry enrichment's provider cascade and returns its outputs. Backs - * the Enrichment workflow block; called server-to-server by the executor, so it - * authenticates with the internal token. The cascade injects the workspace's - * BYOK / hosted key via `executeTool` using `workspaceId`. - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest( - runEnrichmentContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { error: getValidationErrorMessage(error, 'Invalid request') }, - { - status: 400, - } - ), - } - ) - if (!parsed.success) return parsed.response - - const { enrichmentId, inputs, workspaceId } = parsed.data.body - const enrichment = getEnrichment(enrichmentId) - if (!enrichment) { - return NextResponse.json({ error: `Unknown enrichment "${enrichmentId}"` }, { status: 400 }) - } - - const { result, cost, error, provider } = await runEnrichment(enrichment, inputs, { - workspaceId, - signal: request.signal, - }) - - logger.info('Enrichment block run', { - enrichmentId, - matched: Object.keys(result).length > 0, - provider, - }) - return NextResponse.json({ - matched: Object.keys(result).length > 0, - result, - cost, - error, - provider, - }) -}) diff --git a/apps/sim/app/api/tools/extend/parse/route.ts b/apps/sim/app/api/tools/extend/parse/route.ts deleted file mode 100644 index 0fee26a182f..00000000000 --- a/apps/sim/app/api/tools/extend/parse/route.ts +++ /dev/null @@ -1,200 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { extendParseContract } from '@/lib/api/contracts/tools/media/document-parse' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' -import { resolveFileInputToUrl } from '@/lib/uploads/utils/file-utils.server' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('ExtendParseAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Extend parse attempt`, { - error: authResult.error || 'Missing userId', - }) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Unauthorized', - }, - { status: 401 } - ) - } - - const userId = authResult.userId - - const parsed = await parseRequest( - extendParseContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - - const validatedData = parsed.data.body - const modelInputProvenance = validateOpaqueModelInputProvenance({ - headers: request.headers, - payload: validatedData, - isInternalRequest: true, - }) - if (!modelInputProvenance.success) { - return NextResponse.json( - { success: false, error: modelInputProvenance.error }, - { status: modelInputProvenance.status } - ) - } - - logger.info(`[${requestId}] Extend parse request`, { - hasInlineFile: Boolean(validatedData.file), - hasFilePath: Boolean(validatedData.filePath), - userId, - }) - - const resolution = await resolveFileInputToUrl({ - file: validatedData.file, - filePath: validatedData.filePath, - userId, - requestId, - logger, - modelEgress: true, - }) - - if (resolution.error) { - return NextResponse.json( - { success: false, error: resolution.error.message }, - { status: resolution.error.status } - ) - } - - const fileUrl = resolution.fileUrl - if (!fileUrl) { - return NextResponse.json({ success: false, error: 'File input is required' }, { status: 400 }) - } - - const extendBody: Record = { - file: { fileUrl }, - } - - const config: Record = {} - - if (validatedData.outputFormat) { - config.target = validatedData.outputFormat - } - - if (validatedData.chunking) { - config.chunkingStrategy = { type: validatedData.chunking } - } - - if (validatedData.engine) { - config.engine = validatedData.engine - } - - if (Object.keys(config).length > 0) { - extendBody.config = config - } - - const extendEndpoint = 'https://api.extend.ai/parse' - const extendValidation = await validateUrlWithDNS(extendEndpoint, 'Extend API URL') - if (!extendValidation.isValid) { - logger.error(`[${requestId}] Extend API URL validation failed`, { - error: extendValidation.error, - }) - return NextResponse.json( - { - success: false, - error: 'Failed to reach Extend API', - }, - { status: 502 } - ) - } - - const extendResponse = await secureFetchWithPinnedIP( - extendEndpoint, - extendValidation.resolvedIP!, - { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json', - Authorization: `Bearer ${validatedData.apiKey}`, - 'x-extend-api-version': '2025-04-21', - }, - body: JSON.stringify(extendBody), - } - ) - - if (!extendResponse.ok) { - const errorText = await extendResponse.text() - logger.error(`[${requestId}] Extend API error:`, errorText) - let clientError = `Extend API error: ${extendResponse.statusText || extendResponse.status}` - try { - const parsedError = JSON.parse(errorText) - if (parsedError?.message || parsedError?.error) { - clientError = (parsedError.message ?? parsedError.error) as string - } - } catch { - // errorText is not JSON; keep generic message - } - return NextResponse.json( - { - success: false, - error: clientError, - }, - { status: extendResponse.status } - ) - } - - const extendData = (await extendResponse.json()) as Record - - logger.info(`[${requestId}] Extend parse successful`) - - return NextResponse.json({ - success: true, - output: { - id: extendData.id ?? null, - status: extendData.status ?? 'PROCESSED', - chunks: extendData.chunks ?? [], - blocks: extendData.blocks ?? [], - pageCount: extendData.pageCount ?? extendData.page_count ?? null, - creditsUsed: extendData.creditsUsed ?? extendData.credits_used ?? null, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error in Extend parse:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/file/manage/route.test.ts b/apps/sim/app/api/tools/file/manage/route.test.ts deleted file mode 100644 index 1f4c65d5f83..00000000000 --- a/apps/sim/app/api/tools/file/manage/route.test.ts +++ /dev/null @@ -1,832 +0,0 @@ -/** - * @vitest-environment node - */ -import { createMockRequest, hybridAuthMockFns } from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' -import { MAX_FOLDER_PATH_SEGMENTS } from '@/lib/folders/paths' - -const { - mockAssertActiveWorkspaceAccess, - mockAssertToolFileAccess, - mockDownloadServableFileFromStorage, - mockDownloadFileFromStorage, - mockDecompressArchiveBufferToWorkspaceFiles, - mockEnsureWorkspaceFileFolderPath, - mockFetchWorkspaceFileBuffer, - mockGetBoundWorkspaceFileSecretProvenance, - mockLoadActiveWorkspaceContext, - mockLoadActiveWorkspaceFileContext, - mockMoveWorkspaceFileItems, - mockResolveEffectiveWorkspacePermission, - mockGetFileMetadataByKey, - mockGetWorkspaceFile, - mockResolveWorkspaceFileReference, - mockUpdateWorkspaceFileContent, - mockUploadWorkspaceFile, -} = vi.hoisted(() => ({ - mockAssertActiveWorkspaceAccess: vi.fn(), - mockAssertToolFileAccess: vi.fn(), - mockDownloadServableFileFromStorage: vi.fn(), - mockDownloadFileFromStorage: vi.fn(), - mockDecompressArchiveBufferToWorkspaceFiles: vi.fn(), - mockEnsureWorkspaceFileFolderPath: vi.fn(), - mockFetchWorkspaceFileBuffer: vi.fn(), - mockGetBoundWorkspaceFileSecretProvenance: vi.fn(), - mockLoadActiveWorkspaceContext: vi.fn(), - mockLoadActiveWorkspaceFileContext: vi.fn(), - mockMoveWorkspaceFileItems: vi.fn(), - mockResolveEffectiveWorkspacePermission: vi.fn(), - mockGetFileMetadataByKey: vi.fn(), - mockGetWorkspaceFile: vi.fn(), - mockResolveWorkspaceFileReference: vi.fn(), - mockUpdateWorkspaceFileContent: vi.fn(), - mockUploadWorkspaceFile: vi.fn(), -})) - -vi.mock('@/lib/uploads/archive', async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - decompressArchiveBufferToWorkspaceFiles: (...args: unknown[]) => - mockDecompressArchiveBufferToWorkspaceFiles(...args), - } -}) - -vi.mock('@/lib/file-parsers', () => ({ - isSupportedFileType: vi.fn(() => false), - parseBuffer: vi.fn(), -})) - -vi.mock('@sim/audit', () => ({ - AuditAction: { FILE_UPLOADED: 'file_uploaded', FILE_UPDATED: 'file_updated' }, - AuditResourceType: { FILE: 'file' }, - recordAudit: vi.fn(), -})) - -vi.mock('@/lib/realtime/notify', () => ({ - notifyWorkspaceFilesChanged: vi.fn(async () => undefined), -})) - -vi.mock('@/lib/public-shares/share-manager', () => ({ - getShareForResource: vi.fn().mockResolvedValue(null), - getSharesForResources: vi.fn().mockResolvedValue(new Map()), - ShareValidationError: class ShareValidationError extends Error {}, -})) - -vi.mock('@sim/platform-authz/workspace', () => ({ - permissionSatisfies: (permission: string | null, required: string) => - permission === 'admin' || - permission === required || - (permission === 'write' && required === 'read'), - resolveEffectiveWorkspacePermission: (...args: unknown[]) => - mockResolveEffectiveWorkspacePermission(...args), -})) - -vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ - fetchWorkspaceFileBuffer: (...args: unknown[]) => mockFetchWorkspaceFileBuffer(...args), - getWorkspaceFile: (...args: unknown[]) => mockGetWorkspaceFile(...args), - loadActiveWorkspaceContext: (...args: unknown[]) => mockLoadActiveWorkspaceContext(...args), - loadActiveWorkspaceFileContext: (...args: unknown[]) => - mockLoadActiveWorkspaceFileContext(...args), - resolveWorkspaceFileReference: (...args: unknown[]) => mockResolveWorkspaceFileReference(...args), - updateWorkspaceFileContent: (...args: unknown[]) => mockUpdateWorkspaceFileContent(...args), - uploadWorkspaceFile: (...args: unknown[]) => mockUploadWorkspaceFile(...args), -})) - -vi.mock('@/lib/uploads/contexts/workspace', () => ({ - FileConflictError: class FileConflictError extends Error {}, - ContentVersionConflictError: class ContentVersionConflictError extends Error {}, - fetchWorkspaceFileBuffer: (...args: unknown[]) => mockFetchWorkspaceFileBuffer(...args), - getWorkspaceFile: (...args: unknown[]) => mockGetWorkspaceFile(...args), - loadActiveWorkspaceContext: (...args: unknown[]) => mockLoadActiveWorkspaceContext(...args), - updateWorkspaceFileContent: (...args: unknown[]) => mockUpdateWorkspaceFileContent(...args), - uploadWorkspaceFile: (...args: unknown[]) => mockUploadWorkspaceFile(...args), -})) - -vi.mock('@/lib/workspace-files/application/workspace-file-folders', () => ({ - ensureWorkspaceFileFolderPathOperation: { - execute: (...args: unknown[]) => mockEnsureWorkspaceFileFolderPath(...args), - }, -})) - -vi.mock('@/lib/workspace-files/application/move-workspace-file-items', () => ({ - moveWorkspaceFileItemsOperation: { - execute: (...args: unknown[]) => mockMoveWorkspaceFileItems(...args), - }, -})) - -vi.mock('@/lib/core/config/redis', () => ({ - acquireLock: vi.fn(async () => true), - releaseLock: vi.fn(async () => undefined), -})) - -vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({ - EXACT_EMPTY_WORKSPACE_FILE_SECRET_PROVENANCE: { status: 'exact', entries: [] }, - getBoundWorkspaceFileSecretProvenance: (...args: unknown[]) => - mockGetBoundWorkspaceFileSecretProvenance(...args), - mergeWorkspaceFileSecretProvenance: ( - ...provenances: Array< - | { status: 'exact'; entries: Array<{ name: string; encryptedValue: string }> } - | { - status: 'unknown' - } - > - ) => - provenances.some((provenance) => provenance.status === 'unknown') - ? { status: 'unknown' } - : { - status: 'exact', - entries: provenances.flatMap((provenance) => - provenance.status === 'exact' ? provenance.entries : [] - ), - }, -})) - -vi.mock('@/lib/uploads/server/metadata', () => ({ - getFileMetadataByKey: (...args: unknown[]) => mockGetFileMetadataByKey(...args), -})) - -vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ - downloadFileFromStorage: (...args: unknown[]) => mockDownloadFileFromStorage(...args), - downloadServableFileFromStorage: (...args: unknown[]) => - mockDownloadServableFileFromStorage(...args), -})) - -vi.mock('@/lib/workspaces/permissions/utils', () => ({ - assertActiveWorkspaceAccess: (...args: unknown[]) => mockAssertActiveWorkspaceAccess(...args), - getUserEntityPermissions: vi.fn(), - isWorkspaceAccessDeniedError: vi.fn(() => false), -})) - -vi.mock('@/app/api/files/authorization', () => ({ - assertToolFileAccess: (...args: unknown[]) => mockAssertToolFileAccess(...args), -})) - -import { POST } from '@/app/api/tools/file/manage/route' - -const PRIVATE_REQUEST_HEADER = { - 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1', -} -const PRIVATE_SECRET_PROVENANCE_HEADER = { - 'x-sim-private-secret-provenance': 'private-secret-provenance-bundle-v1', -} -const CONTENT_UPDATED_AT = new Date('2026-08-04T00:00:00.000Z') - -function workspaceFile(id: string, ownerUserId = 'user-1') { - return { - id, - workspaceId: 'workspace-1', - name: `${id}.txt`, - key: `workspace/workspace-1/${id}.txt`, - path: `/api/files/serve/${id}`, - size: id.length, - type: 'text/plain', - uploadedBy: ownerUserId, - uploadedAt: CONTENT_UPDATED_AT, - updatedAt: CONTENT_UPDATED_AT, - contentUpdatedAt: CONTENT_UPDATED_AT, - } -} - -describe('POST /api/tools/file/manage content provenance', () => { - beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - mockAssertActiveWorkspaceAccess.mockResolvedValue(undefined) - mockResolveEffectiveWorkspacePermission.mockResolvedValue('write') - mockGetWorkspaceFile.mockImplementation(async (_workspaceId: string, fileId: string) => - workspaceFile(fileId) - ) - mockLoadActiveWorkspaceContext.mockResolvedValue({ - workspaceId: 'workspace-1', - workspaceOrganizationId: null, - allowPersonalApiKeys: true, - billedAccountUserId: 'user-1', - }) - mockLoadActiveWorkspaceFileContext.mockImplementation(async (fileId: string) => ({ - fileId, - workspaceId: 'workspace-1', - workspaceOrganizationId: null, - allowPersonalApiKeys: true, - billedAccountUserId: 'user-1', - })) - mockAssertToolFileAccess.mockResolvedValue(undefined) - mockEnsureWorkspaceFileFolderPath.mockImplementation( - async ({ input }: { input: { pathSegments: string[] } }) => ({ - folderId: input.pathSegments.length === 0 ? null : 'folder-1', - createdFolderIds: [], - }) - ) - mockDownloadServableFileFromStorage.mockImplementation(async (file: { name: string }) => ({ - buffer: Buffer.from(`content:${file.name}`), - })) - mockFetchWorkspaceFileBuffer.mockResolvedValue(Buffer.from('before')) - mockUpdateWorkspaceFileContent.mockResolvedValue({ file: workspaceFile('file-1') }) - mockMoveWorkspaceFileItems.mockResolvedValue({ moved: 1 }) - mockUploadWorkspaceFile.mockResolvedValue({ - id: 'new-file', - name: 'new.txt', - key: 'workspace/workspace-1/new.txt', - url: '/api/files/serve/new-file', - }) - }) - - it('returns a scoped, deduplicated union of exact canonical file provenance', async () => { - mockGetBoundWorkspaceFileSecretProvenance.mockImplementation( - async (_workspaceId: string, identity: { fileId: string }) => - identity.fileId === 'file-1' - ? { - status: 'exact', - entries: [ - { name: 'TOKEN', encryptedValue: 'encrypted-token' }, - { name: 'ALPHA', encryptedValue: 'encrypted-alpha' }, - ], - } - : { - status: 'exact', - entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], - } - ) - - const response = await POST( - createMockRequest( - 'POST', - { operation: 'content', workspaceId: 'workspace-1', fileId: ['file-1', 'file-2'] }, - PRIVATE_REQUEST_HEADER - ) - ) - - expect(response.status).toBe(200) - expect(response.headers.get('x-sim-private-tool-metadata')).toBe( - 'resolved-secret-provenance-v1' - ) - await expect(response.json()).resolves.toEqual({ - success: true, - data: { contents: ['content:file-1.txt', 'content:file-2.txt'] }, - __resolvedSecretTraceProvenance: { - version: 1, - complete: true, - entries: [ - { name: 'ALPHA', encryptedValue: 'encrypted-alpha' }, - { name: 'TOKEN', encryptedValue: 'encrypted-token' }, - ], - scope: { userId: 'user-1', workspaceId: 'workspace-1' }, - }, - }) - }) - - it('stores exact causal provenance from a different user in the actor workspace', async () => { - const response = await POST( - createMockRequest( - 'POST', - { - operation: 'write', - workspaceId: 'workspace-1', - fileName: 'new.txt', - content: 'secret-value', - __privateSecretProvenance: { - version: 1, - complete: true, - selections: [ - { - key: 'content', - provenance: { - version: 1, - complete: true, - entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], - scope: { userId: 'workflow-owner', workspaceId: 'workspace-1' }, - }, - }, - ], - }, - }, - PRIVATE_SECRET_PROVENANCE_HEADER - ) - ) - - expect(response.status).toBe(200) - expect(mockUploadWorkspaceFile).toHaveBeenCalledWith( - 'workspace-1', - 'user-1', - Buffer.from('secret-value'), - 'new.txt', - 'text/plain', - { - exactName: false, - folderId: null, - folderPath: undefined, - secretProvenance: { - status: 'exact', - entries: [ - { - name: 'TOKEN', - encryptedValue: 'encrypted-token', - sourceUserId: 'workflow-owner', - sourceWorkspaceId: 'workspace-1', - }, - ], - }, - } - ) - }) - - it('rejects file-write provenance from another workspace', async () => { - const response = await POST( - createMockRequest( - 'POST', - { - operation: 'write', - workspaceId: 'workspace-1', - fileName: 'new.txt', - content: 'secret-value', - __privateSecretProvenance: { - version: 1, - complete: true, - selections: [ - { - key: 'content', - provenance: { - version: 1, - complete: true, - entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], - scope: { userId: 'workflow-owner', workspaceId: 'workspace-2' }, - }, - }, - ], - }, - }, - PRIVATE_SECRET_PROVENANCE_HEADER - ) - ) - - expect(response.status).toBe(400) - expect(mockUploadWorkspaceFile).not.toHaveBeenCalled() - }) - - it('preserves existing file-path behavior when a filename was resolved from a secret', async () => { - const response = await POST( - createMockRequest( - 'POST', - { - operation: 'write', - workspaceId: 'workspace-1', - fileName: 'Reports & Plans/2026/secret-value.txt', - content: 'ordinary text', - __privateSecretProvenance: { - version: 1, - complete: true, - selections: [ - { - key: 'content', - provenance: { - version: 1, - complete: true, - entries: [], - scope: { userId: 'user-1', workspaceId: 'workspace-1' }, - }, - }, - ], - }, - }, - PRIVATE_SECRET_PROVENANCE_HEADER - ) - ) - - expect(response.status).toBe(200) - expect(mockEnsureWorkspaceFileFolderPath).toHaveBeenCalledWith( - expect.objectContaining({ - principal: expect.objectContaining({ kind: 'delegated', subjectUserId: 'user-1' }), - input: { workspaceId: 'workspace-1', pathSegments: ['Reports & Plans', '2026'] }, - }) - ) - expect(mockUploadWorkspaceFile).toHaveBeenCalledWith( - 'workspace-1', - 'user-1', - Buffer.from('ordinary text'), - 'secret-value.txt', - 'text/plain', - { - exactName: false, - folderId: 'folder-1', - folderPath: undefined, - secretProvenance: { status: 'exact', entries: [] }, - } - ) - }) - - it('keeps a headerless file write on the legacy untracked path', async () => { - const response = await POST( - createMockRequest('POST', { - operation: 'write', - workspaceId: 'workspace-1', - fileName: 'new.txt', - content: 'ordinary text', - }) - ) - expect(response.status).toBe(200) - expect(mockUploadWorkspaceFile).toHaveBeenCalledWith( - 'workspace-1', - 'user-1', - Buffer.from('ordinary text'), - 'new.txt', - 'text/plain', - { - exactName: false, - folderId: null, - folderPath: undefined, - secretProvenance: { status: 'exact', entries: [] }, - } - ) - }) - - it.each([ - ['Reports & Plans/2026', '/Reports%20%26%20Plans/2026'], - ['', '/'], - ])('moves files to the canonical folder path for %j', async (targetFolder, expectedPath) => { - const response = await POST( - createMockRequest('POST', { - operation: 'move', - workspaceId: 'workspace-1', - fileId: 'file-1', - targetFolder, - }) - ) - - expect(response.status).toBe(200) - expect(mockMoveWorkspaceFileItems).toHaveBeenCalledWith( - expect.objectContaining({ - input: { - workspaceId: 'workspace-1', - fileIds: ['file-1'], - targetFolderPath: expectedPath, - }, - }) - ) - }) - - it('returns 400 before moving when the target folder path exceeds canonical limits', async () => { - const response = await POST( - createMockRequest('POST', { - operation: 'move', - workspaceId: 'workspace-1', - fileId: 'file-1', - targetFolder: Array.from( - { length: MAX_FOLDER_PATH_SEGMENTS + 1 }, - (_, index) => `folder-${index}` - ).join('/'), - }) - ) - - expect(response.status).toBe(400) - await expect(response.json()).resolves.toMatchObject({ - success: false, - error: `Folder paths cannot exceed ${MAX_FOLDER_PATH_SEGMENTS} segments`, - }) - expect(mockMoveWorkspaceFileItems).not.toHaveBeenCalled() - }) - - it('persists an authenticated file write with unavailable lineage as unknown', async () => { - const response = await POST( - createMockRequest( - 'POST', - { - operation: 'write', - workspaceId: 'workspace-1', - fileName: 'new.txt', - content: 'possibly secret', - __privateSecretProvenance: { - version: 1, - complete: false, - selections: [], - }, - }, - PRIVATE_SECRET_PROVENANCE_HEADER - ) - ) - - expect(response.status).toBe(200) - expect(mockUploadWorkspaceFile).toHaveBeenCalledWith( - 'workspace-1', - 'user-1', - Buffer.from('possibly secret'), - 'new.txt', - 'text/plain', - { - exactName: false, - folderId: null, - folderPath: undefined, - secretProvenance: { status: 'unknown' }, - } - ) - }) - - it('atomically binds append provenance to the exact predecessor version', async () => { - const existing = workspaceFile('file-1') - mockResolveWorkspaceFileReference.mockResolvedValue(existing) - mockGetBoundWorkspaceFileSecretProvenance.mockResolvedValue({ - status: 'exact', - entries: [ - { - name: 'OLD', - encryptedValue: 'encrypted-old', - sourceUserId: 'user-1', - sourceWorkspaceId: 'workspace-1', - }, - ], - }) - - const response = await POST( - createMockRequest( - 'POST', - { - operation: 'append', - workspaceId: 'workspace-1', - fileName: 'file-1.txt', - content: 'secret-value', - __privateSecretProvenance: { - version: 1, - complete: true, - selections: [ - { - key: 'content', - provenance: { - version: 1, - complete: true, - entries: [{ name: 'NEW', encryptedValue: 'encrypted-new' }], - scope: { userId: 'user-1', workspaceId: 'workspace-1' }, - }, - }, - ], - }, - }, - PRIVATE_SECRET_PROVENANCE_HEADER - ) - ) - - expect(response.status).toBe(200) - expect(mockUpdateWorkspaceFileContent).toHaveBeenCalledWith( - 'workspace-1', - 'file-1', - 'user-1', - Buffer.from('beforesecret-value'), - undefined, - { - expectedUpdatedAt: CONTENT_UPDATED_AT, - secretProvenancePolicy: { - mode: 'replace', - provenance: { - status: 'exact', - entries: [ - { - name: 'OLD', - encryptedValue: 'encrypted-old', - sourceUserId: 'user-1', - sourceWorkspaceId: 'workspace-1', - }, - { - name: 'NEW', - encryptedValue: 'encrypted-new', - sourceUserId: 'user-1', - sourceWorkspaceId: 'workspace-1', - }, - ], - }, - }, - } - ) - }) - - it('preserves the prior classification for a legacy headerless append', async () => { - const existing = workspaceFile('file-1') - mockResolveWorkspaceFileReference.mockResolvedValue(existing) - mockGetBoundWorkspaceFileSecretProvenance.mockResolvedValue({ - status: 'exact', - entries: [{ name: 'OLD', encryptedValue: 'encrypted-old' }], - }) - - const response = await POST( - createMockRequest('POST', { - operation: 'append', - workspaceId: 'workspace-1', - fileName: 'file-1.txt', - content: 'ordinary text', - }) - ) - - expect(response.status).toBe(200) - expect(mockUpdateWorkspaceFileContent).toHaveBeenCalledWith( - 'workspace-1', - 'file-1', - 'user-1', - Buffer.from('beforeordinary text'), - undefined, - { - expectedUpdatedAt: CONTENT_UPDATED_AT, - secretProvenancePolicy: { mode: 'preserve' }, - } - ) - }) - - it('carries the union of source provenance into a compressed archive', async () => { - mockGetWorkspaceFile.mockResolvedValue(workspaceFile('file-1')) - mockGetBoundWorkspaceFileSecretProvenance.mockResolvedValue({ - status: 'exact', - entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], - }) - - const response = await POST( - createMockRequest('POST', { - operation: 'compress', - workspaceId: 'workspace-1', - fileId: 'file-1', - archiveName: 'bundle', - }) - ) - - expect(response.status).toBe(200) - expect(Buffer.isBuffer(mockUploadWorkspaceFile.mock.calls[0]?.[2])).toBe(true) - expect(mockUploadWorkspaceFile).toHaveBeenCalledWith( - 'workspace-1', - 'user-1', - expect.anything(), - 'bundle.zip', - 'application/zip', - expect.objectContaining({ - folderId: null, - secretProvenance: { - status: 'exact', - entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], - }, - }) - ) - }) - - it('passes secret-bearing archive provenance to the decompressor', async () => { - const archiveBuffer = Buffer.from('archive-bytes') - mockDownloadFileFromStorage.mockResolvedValue(archiveBuffer) - mockGetWorkspaceFile.mockResolvedValue({ - ...workspaceFile('archive'), - name: 'archive.zip', - type: 'application/zip', - }) - mockGetBoundWorkspaceFileSecretProvenance.mockResolvedValue({ - status: 'exact', - entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], - }) - mockDecompressArchiveBufferToWorkspaceFiles.mockResolvedValue({ - extracted: [ - { - id: 'new-file', - name: 'child.txt', - key: 'workspace/workspace-1/child.txt', - url: '/api/files/serve/new-file', - size: 12, - type: 'text/plain', - context: 'workspace', - }, - ], - skipped: 0, - skippedUnsafePaths: [], - }) - - const response = await POST( - createMockRequest('POST', { - operation: 'decompress', - workspaceId: 'workspace-1', - fileId: 'archive', - }) - ) - - expect(response.status).toBe(200) - expect(mockDownloadFileFromStorage).toHaveBeenCalledTimes(1) - expect(mockDecompressArchiveBufferToWorkspaceFiles).toHaveBeenCalledWith( - archiveBuffer, - expect.objectContaining({ - workspaceId: 'workspace-1', - principal: expect.objectContaining({ kind: 'delegated', subjectUserId: 'user-1' }), - secretProvenance: { - status: 'exact', - entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], - }, - }) - ) - }) - - it('omits source scope when canonical files have different owners', async () => { - mockGetWorkspaceFile.mockImplementation(async (_workspaceId: string, fileId: string) => - workspaceFile(fileId, fileId === 'file-1' ? 'user-1' : 'user-2') - ) - mockGetBoundWorkspaceFileSecretProvenance.mockResolvedValue({ - status: 'exact', - entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], - }) - - const response = await POST( - createMockRequest( - 'POST', - { operation: 'content', workspaceId: 'workspace-1', fileId: ['file-1', 'file-2'] }, - PRIVATE_REQUEST_HEADER - ) - ) - const body = await response.json() - - expect(body.__resolvedSecretTraceProvenance).toEqual({ - version: 1, - complete: true, - entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], - }) - }) - - it('returns incomplete provenance for an input that cannot bind to a canonical file row', async () => { - mockGetFileMetadataByKey.mockResolvedValue(null) - - const response = await POST( - createMockRequest( - 'POST', - { - operation: 'content', - workspaceId: 'workspace-1', - fileInput: { - key: 'workspace/workspace-1/unbound.txt', - name: 'unbound.txt', - type: 'text/plain', - size: 7, - }, - }, - PRIVATE_REQUEST_HEADER - ) - ) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toMatchObject({ - success: true, - __resolvedSecretTraceProvenance: { version: 1, complete: false, entries: [] }, - }) - expect(mockGetBoundWorkspaceFileSecretProvenance).not.toHaveBeenCalled() - }) - - it('keeps a normal not-found error while returning a valid private envelope', async () => { - mockGetWorkspaceFile.mockResolvedValue(null) - - const response = await POST( - createMockRequest( - 'POST', - { operation: 'content', workspaceId: 'workspace-1', fileId: 'missing-file' }, - PRIVATE_REQUEST_HEADER - ) - ) - - expect(response.status).toBe(404) - expect(response.headers.get('x-sim-private-tool-metadata')).toBe( - 'resolved-secret-provenance-v1' - ) - await expect(response.json()).resolves.toEqual({ - success: false, - error: 'File not found: "missing-file"', - __resolvedSecretTraceProvenance: { version: 1, complete: true, entries: [] }, - }) - }) - - it('does not add private transport fields when provenance was not requested', async () => { - mockGetWorkspaceFile.mockResolvedValue(workspaceFile('file-1')) - - const response = await POST( - createMockRequest('POST', { - operation: 'content', - workspaceId: 'workspace-1', - fileId: 'file-1', - }) - ) - - expect(response.headers.get('x-sim-private-tool-metadata')).toBeNull() - const body = await response.json() - expect(body).not.toHaveProperty('__resolvedSecretTraceProvenance') - expect(mockGetBoundWorkspaceFileSecretProvenance).not.toHaveBeenCalled() - }) - - it('never uses query.userId as the authorization identity', async () => { - mockGetWorkspaceFile.mockResolvedValue(workspaceFile('file-1')) - - const response = await POST( - createMockRequest( - 'POST', - { operation: 'get', workspaceId: 'workspace-1', fileId: 'file-1' }, - {}, - 'http://localhost:3000/api/tools/file/manage?userId=attacker' - ) - ) - - expect(response.status).toBe(200) - expect(mockResolveEffectiveWorkspacePermission).toHaveBeenCalledWith( - 'user-1', - 'workspace-1', - null, - undefined, - { forUpdate: undefined } - ) - }) -}) diff --git a/apps/sim/app/api/tools/file/manage/route.ts b/apps/sim/app/api/tools/file/manage/route.ts deleted file mode 100644 index 85df403dfcf..00000000000 --- a/apps/sim/app/api/tools/file/manage/route.ts +++ /dev/null @@ -1,1276 +0,0 @@ -import { Buffer, isUtf8 } from 'buffer' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateShortId } from '@sim/utils/id' -import { isRecordLike } from '@sim/utils/object' -import JSZip from 'jszip' -import { type NextRequest, NextResponse } from 'next/server' -import { fileManageContract } from '@/lib/api/contracts/tools/file' -import { parseRequest } from '@/lib/api/server' -import { AuthType, type AuthTypeValue, checkInternalAuth } from '@/lib/auth/hybrid' -import { splitWorkspaceFilePath } from '@/lib/copilot/tools/server/files/workspace-file' -import { acquireLock, releaseLock } from '@/lib/core/config/redis' -import { OrchestrationError } from '@/lib/core/orchestration/types' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { ensureAbsoluteUrl } from '@/lib/core/utils/urls' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { durableSecretProvenanceFromPrivateBundle } from '@/lib/execution/durable-secret-provenance' -import { - inspectPrivateSecretProvenanceRequest, - isPrivateSecretProvenanceBundleV1, -} from '@/lib/execution/model-input-provenance' -import { - PRIVATE_TOOL_METADATA_RESPONSE_HEADER, - RESOLVED_SECRET_PROVENANCE_FIELD, - RESOLVED_SECRET_PROVENANCE_METADATA_V1, - requestsPrivateToolMetadata, -} from '@/lib/execution/private-tool-metadata' -import { isSupportedFileType, parseBuffer } from '@/lib/file-parsers' -import { buildFolderPath } from '@/lib/folders/paths' -import { getSharesForResources, ShareValidationError } from '@/lib/public-shares/share-manager' -import { - ArchiveError, - type DecompressResult, - decompressArchiveBufferToWorkspaceFiles, - MAX_ARCHIVE_BYTES, - statusForArchiveError, -} from '@/lib/uploads/archive' -import type { getWorkspaceFile } from '@/lib/uploads/contexts/workspace/workspace-file-manager' -import { - getBoundWorkspaceFileSecretProvenance, - mergeWorkspaceFileSecretProvenance, - type WorkspaceFileSecretProvenance, - type WorkspaceFileSecretProvenanceIdentity, -} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' -import { getFileMetadataByKey } from '@/lib/uploads/server/metadata' -import { getFileExtension, getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils' -import { - downloadFileFromStorage, - downloadServableFileFromStorage, -} from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { buildZipEntryPaths } from '@/lib/uploads/zip-entry-path' -import { - admitCreateWorkspaceFile, - createWorkspaceFile, - createWorkspaceFileFromBuffer, -} from '@/lib/workspace-files/application/create-workspace-file' -import { createWorkspaceFileDelegatedPrincipal } from '@/lib/workspace-files/application/delegated-principal' -import { moveWorkspaceFileItemsOperation } from '@/lib/workspace-files/application/move-workspace-file-items' -import { fileOperations } from '@/lib/workspace-files/application/operations' -import { readWorkspaceFileContent } from '@/lib/workspace-files/application/read-workspace-file-content' -import { readWorkspaceFileMetadata } from '@/lib/workspace-files/application/read-workspace-file-metadata' -import { downloadWorkspaceFileRecord } from '@/lib/workspace-files/application/read-workspace-file-record' -import { resolveWorkspaceFileReference } from '@/lib/workspace-files/application/resolve-workspace-file-reference' -import { updateWorkspaceFileShare } from '@/lib/workspace-files/application/share-workspace-file' -import { updateWorkspaceFileContent } from '@/lib/workspace-files/application/update-workspace-file-content' -import { ensureWorkspaceFileFolderPathOperation } from '@/lib/workspace-files/application/workspace-file-folders' -import { MAX_WORKSPACE_FILE_CONTENT_BYTES } from '@/lib/workspace-files/orchestration' -import { isWorkspaceAccessDeniedError } from '@/lib/workspaces/permissions/utils' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import type { UserFile } from '@/executor/types' -import { - ResolvedSecretTraceProvenanceAccumulator, - type ResolvedSecretTraceProvenanceV1, - type ResolvedSecretTraceScopeV1, -} from '@/executor/utils/resolved-secret-trace-registry' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('FileManageAPI') - -function requireInternalPrincipal(auth: { userId?: string }, workspaceId: string) { - if (!auth.userId) throw new Error('Authenticated internal file operation is missing its user ID') - return createWorkspaceFileDelegatedPrincipal({ - serviceId: 'executor', - subjectUserId: auth.userId, - workspaceId, - delegationId: `internal-file-tool:${auth.userId}`, - }) -} - -const workspaceFileToUserFile = (file: Awaited>) => { - if (!file) return null - - return { - id: file.id, - name: file.name, - url: ensureAbsoluteUrl(file.path), - size: file.size, - type: file.type, - key: file.key, - context: 'workspace', - } -} - -const fileInputToUserFile = (fileInput: unknown) => { - if (!fileInput || typeof fileInput !== 'object' || Array.isArray(fileInput)) return null - - const record = fileInput as Record - const id = - typeof record.id === 'string' - ? record.id.trim() - : typeof record.fileId === 'string' - ? record.fileId.trim() - : '' - - // Objects with ids are resolved through workspace metadata. This fallback is for - // picker/upload values that only carry storage fields. - if (id) return null - - const key = typeof record.key === 'string' ? record.key.trim() : '' - const path = typeof record.path === 'string' ? record.path.trim() : '' - const url = typeof record.url === 'string' ? record.url.trim() : '' - const fileUrl = - url || path || (key ? `/api/files/serve/${encodeURIComponent(key)}?context=workspace` : '') - - if (!fileUrl && !key) return null - - return { - id: key || fileUrl, - name: - typeof record.name === 'string' && record.name.trim() ? record.name.trim() : 'workspace-file', - url: fileUrl ? ensureAbsoluteUrl(fileUrl) : '', - size: typeof record.size === 'number' ? record.size : 0, - type: - typeof record.type === 'string' && record.type.trim() - ? record.type.trim() - : 'application/octet-stream', - key, - context: 'workspace', - } -} - -const normalizeFileIdList = (value: unknown): string[] => { - if (typeof value === 'string') { - const trimmed = value.trim() - if (!trimmed) return [] - - try { - return normalizeFileIdList(JSON.parse(trimmed)) - } catch { - return [trimmed] - } - } - - if (!Array.isArray(value)) return [] - - return value - .map((item) => (typeof item === 'string' ? item.trim() : '')) - .filter((id) => id.length > 0) -} - -const extractUserFilesFromInput = (fileInput: unknown) => { - const inputs = Array.isArray(fileInput) ? fileInput : fileInput ? [fileInput] : [] - return inputs - .map((input) => fileInputToUserFile(input)) - .filter((file): file is NonNullable> => Boolean(file)) -} - -const extractFileIdsFromInput = (fileInput: unknown): string[] => { - const inputs = Array.isArray(fileInput) ? fileInput : fileInput ? [fileInput] : [] - - return inputs - .flatMap((input) => { - if (typeof input === 'string') return normalizeFileIdList(input) - if (input && typeof input === 'object') { - const record = input as Record - if (typeof record.id === 'string') return normalizeFileIdList(record.id) - if (typeof record.fileId === 'string') return normalizeFileIdList(record.fileId) - } - return [] - }) - .filter((id) => id.length > 0) -} - -/** Per-file download cap for the content operation. Aligned with the durable large-value ceiling. */ -const MAX_GET_CONTENT_FILE_BYTES = 64 * 1024 * 1024 -/** Combined extracted-text cap so the content array stays within the large-value-ref ceiling. */ -const MAX_GET_CONTENT_TOTAL_BYTES = 64 * 1024 * 1024 - -/** Per-file download cap for the compress operation. */ -const MAX_COMPRESS_FILE_BYTES = 100 * 1024 * 1024 -/** Combined input cap for the compress operation to bound in-memory archiving. */ -const MAX_COMPRESS_TOTAL_BYTES = 100 * 1024 * 1024 - -/** Ensure an archive name ends with a single `.zip` extension. */ -const ensureZipExtension = (name: string): string => - name.toLowerCase().endsWith('.zip') ? name : `${name}.zip` - -/** Strip the trailing extension from a file name (e.g., "report.pdf" -> "report"). */ -const stripExtension = (name: string): string => { - const dot = name.lastIndexOf('.') - return dot > 0 ? name.slice(0, dot) : name -} - -/** - * Reduce an arbitrary name to a safe, flat file name: takes the final path - * segment, drops directory and traversal components, and falls back when the - * result would be empty or a dot segment. Used for the compress archive name so - * untrusted input cannot introduce nested or zip-slip-style paths. - */ -const toFlatFileName = (name: string, fallback: string): string => { - const leaf = name.replace(/\\/g, '/').split('/').pop()?.trim() - if (!leaf || leaf === '.' || leaf === '..') return fallback - return leaf -} - -/** A file bound for a compress archive, paired with the workspace folder it lives in. */ -interface ArchiveEntry { - file: UserFile - folderPath: string | null -} - -const isLikelyTextBuffer = (buffer: Buffer): boolean => isUtf8(buffer) && !buffer.includes(0) - -/** - * Download a stored file and extract its text content. Parseable types (PDF, DOCX, - * CSV, etc.) go through the shared file-parsers; other UTF-8 files are returned as - * raw text; binary files yield a short placeholder rather than corrupt bytes. - */ -const extractUserFileTextContent = async ( - userFile: UserFile, - requestId: string -): Promise => { - const { buffer } = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_GET_CONTENT_FILE_BYTES, - }) - - const extension = getFileExtension(userFile.name) - if (extension && isSupportedFileType(extension)) { - try { - const result = await parseBuffer(buffer, extension) - return result.content ?? '' - } catch (error) { - logger.warn('Falling back to raw text after parser failure', { - name: userFile.name, - error: getErrorMessage(error, 'Unknown error'), - }) - } - } - - if (isLikelyTextBuffer(buffer)) { - return buffer.toString('utf-8') - } - - return `[Binary file: ${userFile.name} (${userFile.type || 'application/octet-stream'}, ${buffer.length} bytes). Cannot extract text content.]` -} - -interface FileContentSource { - file: UserFile - identity?: WorkspaceFileSecretProvenanceIdentity - ownerUserId?: string -} - -async function bindSelectedContentFile( - workspaceId: string, - file: UserFile -): Promise { - if (!file.key) return { file } - - const metadata = await getFileMetadataByKey(file.key, 'workspace') - if (!metadata || metadata.workspaceId !== workspaceId || metadata.context !== 'workspace') { - return { file } - } - - return { - file, - identity: { fileId: metadata.id, key: metadata.key, context: 'workspace' }, - ownerUserId: metadata.userId, - } -} - -async function getFileContentProvenance( - workspaceId: string, - sources: readonly FileContentSource[] -): Promise { - const ownerIds = new Set( - sources - .map((source) => source.ownerUserId) - .filter((ownerUserId): ownerUserId is string => Boolean(ownerUserId)) - ) - const ownerUserId = ownerIds.size === 1 ? ownerIds.values().next().value : undefined - const scope: ResolvedSecretTraceScopeV1 | undefined = ownerUserId - ? { userId: ownerUserId, workspaceId } - : undefined - const accumulator = new ResolvedSecretTraceProvenanceAccumulator(scope) - - for (const source of sources) { - if (!source.identity || !source.ownerUserId) { - accumulator.markIncomplete('file-source-unidentified') - continue - } - const provenance = await getBoundWorkspaceFileSecretProvenance(workspaceId, source.identity) - /** - * `unrecorded` is a more specific `unknown`, and this accumulator has not opted into the - * workspace file surface's policy, so it latches exactly as it did before. - */ - if (provenance.status !== 'exact') { - accumulator.markIncomplete('workspace-file-provenance-unknown') - continue - } - accumulator.record({ - version: 1, - complete: true, - entries: [...provenance.entries], - ...(scope ? { scope } : {}), - }) - } - - return accumulator.exportProvenance() -} - -type FileMutationProvenanceResolution = - | { - success: true - provenanceBySelection?: ReadonlyMap - } - | { success: false; error: string } - -/** Authenticates exact, causally selected file-mutation provenance from an internal caller. */ -function resolveFileMutationSecretProvenance(options: { - headers: Headers - payload: unknown - authType: AuthTypeValue | undefined - userId: string - workspaceId: string - selectionKeys: readonly string[] -}): FileMutationProvenanceResolution { - const inspection = inspectPrivateSecretProvenanceRequest(options.headers, options.payload) - if (inspection.status === 'unsupported') return { success: true } - if ( - inspection.status !== 'verified' || - options.authType !== AuthType.INTERNAL_JWT || - !isPrivateSecretProvenanceBundleV1(inspection.value) - ) { - return { success: false, error: 'Invalid file secret provenance' } - } - - const provenanceBySelection = new Map() - if (!inspection.value.complete) { - for (const selectionKey of options.selectionKeys) { - provenanceBySelection.set(selectionKey, { status: 'unknown' }) - } - return { success: true, provenanceBySelection } - } - if (inspection.value.selections.length !== options.selectionKeys.length) { - return { success: false, error: 'Invalid file secret provenance' } - } - - const destinationScope = { userId: options.userId, workspaceId: options.workspaceId } - for (const selectionKey of options.selectionKeys) { - const provenance = durableSecretProvenanceFromPrivateBundle( - inspection.value, - selectionKey, - destinationScope - ) - if (!provenance) { - return { success: false, error: 'Invalid file secret provenance' } - } - if (provenance.status === 'unknown') { - provenanceBySelection.set(selectionKey, provenance) - continue - } - if (provenance.entries.some((entry) => !entry.name || !entry.sourceUserId)) { - return { success: false, error: 'Invalid file secret provenance' } - } - provenanceBySelection.set(selectionKey, { - status: 'exact', - entries: provenance.entries.map((entry) => ({ - name: entry.name as string, - encryptedValue: entry.encryptedValue, - sourceUserId: entry.sourceUserId as string, - ...(entry.sourceWorkspaceId ? { sourceWorkspaceId: entry.sourceWorkspaceId } : {}), - })), - }) - } - return { success: true, provenanceBySelection } -} - -type FileWriteProvenanceResolution = - | { success: true; contentProvenance?: WorkspaceFileSecretProvenance } - | { success: false; error: string } - -/** Resolves file-content provenance before any folder or file mutation. */ -function resolveFileWriteSecretProvenance(options: { - headers: Headers - payload: unknown - authType: AuthTypeValue | undefined - userId: string - workspaceId: string -}): FileWriteProvenanceResolution { - const resolution = resolveFileMutationSecretProvenance({ - ...options, - selectionKeys: ['content'], - }) - if (!resolution.success || !resolution.provenanceBySelection) return resolution - const content = resolution.provenanceBySelection.get('content') - if (!content) { - return { success: false, error: 'Invalid file secret provenance' } - } - return { success: true, contentProvenance: content } -} - -async function deriveWorkspaceFileSecretProvenance(options: { - workspaceId: string - targetOwnerUserId: string - sources: readonly FileContentSource[] -}): Promise { - const provenances: WorkspaceFileSecretProvenance[] = [] - for (const source of options.sources) { - if (!source.identity || !source.ownerUserId) return { status: 'unknown' } - const provenance = await getBoundWorkspaceFileSecretProvenance( - options.workspaceId, - source.identity - ) - if ( - provenance.status === 'exact' && - provenance.entries.length > 0 && - source.ownerUserId !== options.targetOwnerUserId - ) { - return { status: 'unknown' } - } - provenances.push(provenance) - } - return mergeWorkspaceFileSecretProvenance(...provenances) -} - -function fileContentJsonResponse( - body: Record, - includePrivateProvenance: boolean, - init?: ResponseInit, - provenance: ResolvedSecretTraceProvenanceV1 = { version: 1, complete: true, entries: [] } -): NextResponse { - if (!includePrivateProvenance) return NextResponse.json(body, init) - - const headers = new Headers(init?.headers) - headers.delete('content-length') - headers.set(PRIVATE_TOOL_METADATA_RESPONSE_HEADER, RESOLVED_SECRET_PROVENANCE_METADATA_V1) - return NextResponse.json( - { ...body, [RESOLVED_SECRET_PROVENANCE_FIELD]: provenance }, - { ...init, headers } - ) -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!auth.success) { - return NextResponse.json({ success: false, error: auth.error }, { status: 401 }) - } - - const parsed = await parseRequest(fileManageContract, request, {}) - if (!parsed.success) return parsed.response - - const { query, body } = parsed.data - if (!auth.userId) throw new Error('Authenticated internal file operation is missing its user ID') - const userId = auth.userId - - const workspaceId = body.workspaceId || query.workspaceId - if (!workspaceId) { - return NextResponse.json({ success: false, error: 'workspaceId is required' }, { status: 400 }) - } - const principal = requireInternalPrincipal(auth, workspaceId) - const includePrivateContentProvenance = - body.operation === 'content' && - requestsPrivateToolMetadata(request.headers, RESOLVED_SECRET_PROVENANCE_METADATA_V1) - const contentResponse = ( - responseBody: Record, - init?: ResponseInit, - provenance?: ResolvedSecretTraceProvenanceV1 - ) => fileContentJsonResponse(responseBody, includePrivateContentProvenance, init, provenance) - - try { - switch (body.operation) { - case 'get': { - const { fileId, fileInput } = body - const selectedFileId = - fileId || - (isRecordLike(fileInput) - ? (() => { - const obj = fileInput as Record - return typeof obj.id === 'string' - ? obj.id - : typeof obj.fileId === 'string' - ? obj.fileId - : '' - })() - : '') - - if (!selectedFileId) { - return NextResponse.json({ success: false, error: 'File is required' }, { status: 400 }) - } - - let file: Awaited> - try { - file = ( - await readWorkspaceFileMetadata.execute({ - principal, - input: { fileId: selectedFileId, assertedWorkspaceId: workspaceId }, - request, - }) - ).file - } catch (error) { - if (error instanceof OrchestrationError && error.code === 'not_found') { - return NextResponse.json( - { success: false, error: `File not found: "${selectedFileId}"` }, - { status: 404 } - ) - } - throw error - } - - logger.info('File retrieved', { - fileId: file.id, - name: file.name, - }) - - return NextResponse.json({ - success: true, - data: { - file: workspaceFileToUserFile(file), - }, - }) - } - - case 'read': { - const { fileId, fileInput } = body - const selectedFileIds = Array.isArray(fileId) - ? fileId.map((id) => id.trim()).filter(Boolean) - : fileId - ? normalizeFileIdList(fileId) - : extractFileIdsFromInput(fileInput) - const selectedInputFiles = fileId ? [] : extractUserFilesFromInput(fileInput) - - if (selectedFileIds.length === 0 && selectedInputFiles.length === 0) { - return NextResponse.json({ success: false, error: 'File is required' }, { status: 400 }) - } - - const files = [] as Array>>> - for (const id of selectedFileIds) { - try { - files.push( - ( - await readWorkspaceFileMetadata.execute({ - principal, - input: { fileId: id, assertedWorkspaceId: workspaceId }, - request, - }) - ).file - ) - } catch (error) { - if (error instanceof OrchestrationError && error.code === 'not_found') { - return NextResponse.json( - { success: false, error: `File not found: "${id}"` }, - { status: 404 } - ) - } - throw error - } - } - - const shares = await getSharesForResources('file', selectedFileIds) - const privateReadShare = () => ({ - visibility: 'private' as const, - url: null, - allowedEmails: [] as string[], - }) - const toReadShare = (fileId: string) => { - const share = shares.get(fileId) - if (!share || !share.isActive) return privateReadShare() - return { - visibility: share.authType, - url: share.url, - allowedEmails: share.allowedEmails, - } - } - const userFiles = files - .map((file) => workspaceFileToUserFile(file)) - .filter((file): file is NonNullable> => - Boolean(file) - ) - .map((file) => ({ ...file, share: toReadShare(file.id) })) - // Picker/upload entries have only a synthetic id (storage key/URL), so they - // never carry a canonical share — mark them private without a lookup. - .concat(selectedInputFiles.map((file) => ({ ...file, share: privateReadShare() }))) - - logger.info('Files retrieved', { - count: userFiles.length, - fileIds: userFiles.map((file) => file.id), - }) - - return NextResponse.json({ - success: true, - data: { - file: userFiles[0], - files: userFiles, - }, - }) - } - - case 'content': { - const { fileId, fileInput } = body - const requestId = generateRequestId() - - const selectedFileIds = Array.isArray(fileId) - ? fileId.map((id) => id.trim()).filter(Boolean) - : fileId - ? normalizeFileIdList(fileId) - : extractFileIdsFromInput(fileInput) - const selectedInputFiles = fileId ? [] : extractUserFilesFromInput(fileInput) - - if (selectedFileIds.length === 0 && selectedInputFiles.length === 0) { - return contentResponse({ success: false, error: 'File is required' }, { status: 400 }) - } - - const workspaceFiles = [] as Array< - NonNullable>> - > - for (const id of selectedFileIds) { - try { - workspaceFiles.push( - ( - await readWorkspaceFileMetadata.execute({ - principal, - input: { fileId: id, assertedWorkspaceId: workspaceId }, - request, - }) - ).file - ) - } catch (error) { - if (error instanceof OrchestrationError && error.code === 'not_found') { - return contentResponse( - { success: false, error: `File not found: "${id}"` }, - { status: 404 } - ) - } - throw error - } - } - - const canonicalSources: FileContentSource[] = workspaceFiles.flatMap((file) => { - const userFile = workspaceFileToUserFile(file) - if (!file || !userFile) return [] - return [ - { - file: userFile, - identity: { fileId: file.id, key: file.key, context: 'workspace' }, - ownerUserId: file.uploadedBy, - }, - ] - }) - const selectedSources = await Promise.all( - selectedInputFiles.map((file) => bindSelectedContentFile(workspaceId, file)) - ) - const sources = canonicalSources.concat(selectedSources) - - const contents: string[] = [] - let totalBytes = 0 - for (const source of sources) { - const denied = await assertToolFileAccess(source.file.key, userId, requestId, logger) - if (denied) { - const deniedBody = (await denied.clone().json()) as Record - return contentResponse(deniedBody, { - status: denied.status, - statusText: denied.statusText, - headers: denied.headers, - }) - } - - const content = await extractUserFileTextContent(source.file, requestId) - totalBytes += Buffer.byteLength(content, 'utf8') - if (totalBytes > MAX_GET_CONTENT_TOTAL_BYTES) { - return contentResponse( - { - success: false, - error: `Combined file content is too large to return safely. Maximum is ${ - MAX_GET_CONTENT_TOTAL_BYTES / (1024 * 1024) - } MB.`, - }, - { status: 413 } - ) - } - contents.push(content) - } - - logger.info('File content extracted', { count: contents.length }) - const provenance = includePrivateContentProvenance - ? await getFileContentProvenance(workspaceId, sources) - : undefined - - return contentResponse({ success: true, data: { contents } }, undefined, provenance) - } - - case 'write': { - const { fileName, content, contentType } = body - const provenanceResolution = resolveFileWriteSecretProvenance({ - headers: request.headers, - payload: body, - authType: auth.authType, - userId, - workspaceId, - }) - if (!provenanceResolution.success) { - return NextResponse.json( - { success: false, error: provenanceResolution.error }, - { status: 400 } - ) - } - const { folderSegments, leafName } = splitWorkspaceFilePath(fileName) - await admitCreateWorkspaceFile(principal, workspaceId) - const { folderId } = await ensureWorkspaceFileFolderPathOperation.execute({ - principal, - input: { workspaceId, pathSegments: folderSegments }, - request, - }) - const mimeType = contentType || getMimeTypeFromExtension(getFileExtension(leafName)) - const result = await createWorkspaceFile.execute({ - principal, - input: { - workspaceId, - name: leafName, - contentType: mimeType, - content: content ?? '', - encoding: 'utf-8', - folderId, - exactName: false, - ...(provenanceResolution.contentProvenance - ? { secretProvenance: provenanceResolution.contentProvenance } - : {}), - }, - request, - }) - const fileBuffer = Buffer.from(content ?? '', 'utf-8') - - logger.info('File created', { - fileId: result.file.id, - name: fileName, - size: fileBuffer.length, - }) - - return NextResponse.json({ - success: true, - data: { - id: result.file.id, - name: result.file.name, - size: fileBuffer.length, - url: ensureAbsoluteUrl(result.file.url ?? result.file.path), - }, - }) - } - - case 'move': { - const { fileId, targetFolder } = body - const pathSegments = targetFolder.trim() - ? targetFolder - .trim() - .split('/') - .map((s) => s.trim()) - .filter(Boolean) - : [] - let targetFolderPath: string - try { - targetFolderPath = buildFolderPath(pathSegments) - } catch (error) { - throw new OrchestrationError('validation', getErrorMessage(error)) - } - await moveWorkspaceFileItemsOperation.execute({ - principal, - input: { - workspaceId, - fileIds: [fileId], - targetFolderPath, - }, - request, - }) - logger.info('File moved', { fileId, targetFolder: targetFolder || '(root)' }) - return NextResponse.json({ - success: true, - data: { fileId, targetFolder: targetFolder || '(root)' }, - }) - } - - case 'manage_sharing': { - const { fileId, fileInput, isActive, authType, password, allowedEmails } = body - - // Resolve the canonical file id. The basic file picker provides an object - // with a storage `key` but no id, so map the key to the workspace file row. - let resolvedFileId = typeof fileId === 'string' ? fileId : undefined - if (!resolvedFileId && fileInput) { - const single = Array.isArray(fileInput) ? fileInput[0] : fileInput - if (single && typeof single === 'object') { - const record = single as Record - if (typeof record.id === 'string' && record.id) resolvedFileId = record.id - else if (typeof record.fileId === 'string' && record.fileId) - resolvedFileId = record.fileId - else if (typeof record.key === 'string' && record.key) { - const meta = await getFileMetadataByKey(record.key, 'workspace') - resolvedFileId = meta?.id - } - } - } - if (!resolvedFileId) { - return NextResponse.json( - { success: false, error: 'A valid file is required to manage sharing' }, - { status: 400 } - ) - } - - const share = ( - await updateWorkspaceFileShare.execute({ - principal, - input: { - fileId: resolvedFileId, - assertedWorkspaceId: workspaceId, - isActive, - authType, - password, - allowedEmails, - }, - request, - }) - ).share - - logger.info('File sharing updated', { - fileId: resolvedFileId, - isActive, - authType: share.authType, - }) - - // A disabled link doesn't resolve, so don't hand back a dead URL. - const responseShare = share.isActive ? share : { ...share, url: '' } - return NextResponse.json({ success: true, data: { share: responseShare } }) - } - - case 'append': { - const { fileName, content } = body - - const existing = await resolveWorkspaceFileReference({ - principal, - operation: fileOperations.updateContent, - workspaceId, - reference: fileName, - }) - - const lockKey = `file-append:${workspaceId}:${existing.id}` - const lockValue = `${Date.now()}-${generateShortId()}` - const acquired = await acquireLock(lockKey, lockValue, 30) - if (!acquired) { - return NextResponse.json( - { success: false, error: 'File is busy, please retry' }, - { status: 409 } - ) - } - - try { - if (!existing.contentUpdatedAt) { - throw new Error('File content version is unavailable') - } - const existingProvenance = await getBoundWorkspaceFileSecretProvenance(workspaceId, { - fileId: existing.id, - key: existing.key, - context: 'workspace', - }) - const appendedResolution = resolveFileMutationSecretProvenance({ - headers: request.headers, - payload: body, - authType: auth.authType, - userId, - workspaceId, - selectionKeys: ['content'], - }) - if (!appendedResolution.success) { - return NextResponse.json( - { success: false, error: appendedResolution.error }, - { status: 400 } - ) - } - const appendedProvenance = appendedResolution.provenanceBySelection?.get('content') - const secretProvenance = - appendedProvenance?.status === 'exact' && - appendedProvenance.entries.length > 0 && - existing.uploadedBy !== userId - ? { status: 'unknown' as const } - : appendedProvenance - ? mergeWorkspaceFileSecretProvenance(existingProvenance, appendedProvenance) - : undefined - const { content: existingBuffer } = await readWorkspaceFileContent.execute({ - principal, - input: { - fileId: existing.id, - assertedWorkspaceId: workspaceId, - maxBytes: MAX_WORKSPACE_FILE_CONTENT_BYTES, - }, - }) - const finalContent = existingBuffer.toString('utf-8') + content - const fileBuffer = Buffer.from(finalContent, 'utf-8') - await updateWorkspaceFileContent.execute({ - principal, - input: { - fileId: existing.id, - assertedWorkspaceId: workspaceId, - content: finalContent, - encoding: 'utf-8', - expectedUpdatedAt: existing.contentUpdatedAt ?? undefined, - provenanceMode: secretProvenance ? undefined : 'preserve', - ...(secretProvenance ? { secretProvenance } : {}), - }, - request, - }) - - logger.info('File appended', { - fileId: existing.id, - name: existing.name, - size: fileBuffer.length, - }) - - return NextResponse.json({ - success: true, - data: { - id: existing.id, - name: existing.name, - size: fileBuffer.length, - url: ensureAbsoluteUrl(existing.path), - }, - }) - } finally { - await releaseLock(lockKey, lockValue) - } - } - - case 'compress': { - const { fileId, fileInput, archiveName } = body - const requestId = generateRequestId() - - const selectedFileIds = Array.isArray(fileId) - ? fileId.map((id) => id.trim()).filter(Boolean) - : fileId - ? normalizeFileIdList(fileId) - : extractFileIdsFromInput(fileInput) - const selectedInputFiles = fileId ? [] : extractUserFilesFromInput(fileInput) - - if (selectedFileIds.length === 0 && selectedInputFiles.length === 0) { - return NextResponse.json({ success: false, error: 'File is required' }, { status: 400 }) - } - await admitCreateWorkspaceFile(principal, workspaceId) - - const workspaceFiles = [] as Array< - NonNullable>> - > - for (const id of selectedFileIds) { - try { - workspaceFiles.push( - ( - await downloadWorkspaceFileRecord.execute({ - principal, - input: { fileId: id, assertedWorkspaceId: workspaceId }, - request, - }) - ).file - ) - } catch (error) { - if (error instanceof OrchestrationError && error.code === 'not_found') { - return NextResponse.json( - { success: false, error: `File not found: "${id}"` }, - { status: 404 } - ) - } - throw error - } - } - - const workspaceEntries: ArchiveEntry[] = workspaceFiles.flatMap((file) => { - const userFile = workspaceFileToUserFile(file) - return userFile ? [{ file: userFile, folderPath: file?.folderPath ?? null }] : [] - }) - - // Picker/upload values carry no workspace folder, so they archive at the root. - const archiveEntries = workspaceEntries.concat( - selectedInputFiles.map((file) => ({ file, folderPath: null })) - ) - const userFiles: UserFile[] = archiveEntries.map((entry) => entry.file) - const canonicalArchiveSources: FileContentSource[] = workspaceFiles.flatMap((file) => { - const userFile = workspaceFileToUserFile(file) - if (!file || !userFile) return [] - return [ - { - file: userFile, - identity: { fileId: file.id, key: file.key, context: 'workspace' }, - ownerUserId: file.uploadedBy, - }, - ] - }) - const selectedArchiveSources = await Promise.all( - selectedInputFiles.map((file) => bindSelectedContentFile(workspaceId, file)) - ) - const archiveProvenance = await deriveWorkspaceFileSecretProvenance({ - workspaceId, - targetOwnerUserId: userId, - sources: canonicalArchiveSources.concat(selectedArchiveSources), - }) - - // Mirror the workspace folder layout, dropping the ancestor chain the whole - // selection shares so archiving one folder does not nest it under its parents. - const entryPaths = buildZipEntryPaths( - archiveEntries.map((entry) => ({ name: entry.file.name, folderPath: entry.folderPath })), - { rebaseOnCommonFolder: true } - ) - - const zip = new JSZip() - let totalBytes = 0 - for (const [index, userFile] of userFiles.entries()) { - const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) - if (denied) return denied - - // Generated docs store their generation source, not the rendered binary, so - // the archive must carry the servable bytes instead of the raw source text. - // A still-compiling artifact throws, and the handler's catch turns that into - // the shared 409 via `docNotReadyResponse`. - const { buffer } = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_COMPRESS_FILE_BYTES, - }) - totalBytes += buffer.length - if (totalBytes > MAX_COMPRESS_TOTAL_BYTES) { - return NextResponse.json( - { - success: false, - error: `Combined input is too large to compress. Maximum is ${ - MAX_COMPRESS_TOTAL_BYTES / (1024 * 1024) - } MB.`, - }, - { status: 413 } - ) - } - zip.file(entryPaths[index], buffer) - } - - const zipBuffer = await zip.generateAsync({ - type: 'nodebuffer', - compression: 'DEFLATE', - compressionOptions: { level: 6 }, - }) - - const requestedName = typeof archiveName === 'string' ? archiveName.trim() : '' - const baseName = requestedName - ? toFlatFileName(requestedName, 'archive') - : userFiles.length === 1 - ? stripExtension(toFlatFileName(userFiles[0].name, 'archive')) - : 'archive' - const leafName = ensureZipExtension(baseName) - const result = await createWorkspaceFileFromBuffer.execute({ - principal, - input: { - workspaceId, - name: leafName, - contentType: 'application/zip', - content: zipBuffer, - folderId: null, - exactName: false, - secretProvenance: archiveProvenance, - }, - request, - }) - - const compressedFile: UserFile = { - ...result.file, - url: ensureAbsoluteUrl(result.file.url ?? result.file.path), - size: zipBuffer.length, - } - - logger.info('Files compressed', { - fileId: result.file.id, - name: result.file.name, - fileCount: userFiles.length, - size: zipBuffer.length, - }) - - return NextResponse.json({ - success: true, - data: { - id: compressedFile.id, - name: compressedFile.name, - size: compressedFile.size, - url: compressedFile.url, - files: [compressedFile], - }, - }) - } - - case 'decompress': { - const { fileId, fileInput } = body - const requestId = generateRequestId() - - const selectedFileIds = fileId ? [fileId] : extractFileIdsFromInput(fileInput) - const selectedInputFiles = fileId ? [] : extractUserFilesFromInput(fileInput) - - if (selectedFileIds.length === 0 && selectedInputFiles.length === 0) { - return NextResponse.json({ success: false, error: 'File is required' }, { status: 400 }) - } - if (selectedFileIds.length + selectedInputFiles.length > 1) { - return NextResponse.json( - { success: false, error: 'Decompress accepts a single .zip archive at a time' }, - { status: 400 } - ) - } - await admitCreateWorkspaceFile(principal, workspaceId) - - const workspaceFiles = [] as Array< - NonNullable>> - > - for (const id of selectedFileIds) { - try { - workspaceFiles.push( - ( - await downloadWorkspaceFileRecord.execute({ - principal, - input: { fileId: id, assertedWorkspaceId: workspaceId }, - request, - }) - ).file - ) - } catch (error) { - if (error instanceof OrchestrationError && error.code === 'not_found') { - return NextResponse.json( - { success: false, error: `File not found: "${id}"` }, - { status: 404 } - ) - } - throw error - } - } - - const archive = workspaceFiles - .map((file) => workspaceFileToUserFile(file)) - .filter((file): file is NonNullable> => - Boolean(file) - ) - .concat(selectedInputFiles)[0] - - if (!archive) { - return NextResponse.json({ success: false, error: 'File is required' }, { status: 400 }) - } - - const denied = await assertToolFileAccess(archive.key, userId, requestId, logger) - if (denied) return denied - - const canonicalArchiveSource: FileContentSource[] = workspaceFiles.flatMap((file) => { - const userFile = workspaceFileToUserFile(file) - if (!file || !userFile) return [] - return [ - { - file: userFile, - identity: { fileId: file.id, key: file.key, context: 'workspace' }, - ownerUserId: file.uploadedBy, - }, - ] - }) - const selectedArchiveSource = await Promise.all( - selectedInputFiles.map((file) => bindSelectedContentFile(workspaceId, file)) - ) - const archiveProvenance = await deriveWorkspaceFileSecretProvenance({ - workspaceId, - targetOwnerUserId: userId, - sources: canonicalArchiveSource.concat(selectedArchiveSource), - }) - - const archiveBuffer = await downloadFileFromStorage(archive, requestId, logger, { - maxBytes: MAX_ARCHIVE_BYTES, - }) - - let result: DecompressResult - try { - result = await decompressArchiveBufferToWorkspaceFiles(archiveBuffer, { - workspaceId, - principal, - secretProvenance: archiveProvenance, - }) - } catch (archiveError) { - if (archiveError instanceof ArchiveError) { - // The error message is single-sourced in ArchiveError (caps included); - // only the HTTP status is mapped here. - const status = statusForArchiveError(archiveError) - return NextResponse.json( - { success: false, error: `"${archive.name}": ${archiveError.message}` }, - { status } - ) - } - throw archiveError - } - - if (result.extracted.length === 0) { - return NextResponse.json( - { success: false, error: `No files could be extracted from "${archive.name}".` }, - { status: 422 } - ) - } - - const extractedFiles = result.extracted.map((file) => ({ - ...file, - url: ensureAbsoluteUrl(file.url), - })) - - if (result.skippedUnsafePaths.length > 0) { - logger.warn('Skipped unsafe archive entries', { - fileId: archive.id, - name: archive.name, - entryNames: result.skippedUnsafePaths, - }) - } - - logger.info('Archive decompressed', { - fileId: archive.id, - name: archive.name, - extractedCount: extractedFiles.length, - skippedCount: result.skipped, - }) - - return NextResponse.json({ - success: true, - data: { - files: extractedFiles, - }, - }) - } - } - } catch (error) { - if (isWorkspaceAccessDeniedError(error)) { - return contentResponse({ success: false, error: 'Workspace access denied' }, { status: 403 }) - } - if (error instanceof OrchestrationError) { - const status = - error.code === 'forbidden' - ? 403 - : error.code === 'not_found' - ? 404 - : error.code === 'conflict' - ? 409 - : error.code === 'payload_too_large' - ? 413 - : error.code === 'validation' - ? 400 - : 500 - return contentResponse({ success: false, error: error.message }, { status }) - } - const notReady = docNotReadyResponse(error) - if (notReady) { - if (!includePrivateContentProvenance) return notReady - const notReadyBody = (await notReady.clone().json()) as Record - return contentResponse(notReadyBody, { - status: notReady.status, - statusText: notReady.statusText, - headers: notReady.headers, - }) - } - // A file over its per-file cap is a size rejection, not a fault. Rendered - // documents can cross it even when the stored source was well under. - if (isPayloadSizeLimitError(error)) { - return contentResponse({ success: false, error: error.message }, { status: 413 }) - } - if (error instanceof ShareValidationError) { - return contentResponse({ success: false, error: error.message }, { status: 400 }) - } - const message = getErrorMessage(error, 'Unknown error') - logger.error('File operation failed', { operation: body.operation, error: message }) - return contentResponse({ success: false, error: message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/firecrawl/parse/route.test.ts b/apps/sim/app/api/tools/firecrawl/parse/route.test.ts deleted file mode 100644 index 529516f8c7c..00000000000 --- a/apps/sim/app/api/tools/firecrawl/parse/route.test.ts +++ /dev/null @@ -1,142 +0,0 @@ -/** - * @vitest-environment node - */ -import { createMockRequest, hybridAuthMockFns } from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' -import { PRIVATE_MODEL_INPUT_PROVENANCE_HEADER } from '@/lib/execution/model-input-provenance' -import { - RESOLVED_SECRET_PROVENANCE_FIELD, - RESOLVED_SECRET_PROVENANCE_METADATA_V1, -} from '@/lib/execution/private-tool-metadata' - -const { mockDownloadServableFile, mockIsModelSafeWorkspaceFileKey } = vi.hoisted(() => ({ - mockDownloadServableFile: vi.fn(), - mockIsModelSafeWorkspaceFileKey: vi.fn(), -})) - -vi.mock('@/app/api/files/authorization', () => ({ - assertToolFileAccess: vi.fn().mockResolvedValue(null), -})) -vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ - downloadServableFileFromStorage: mockDownloadServableFile, -})) -vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({ - isModelSafeWorkspaceFileKey: mockIsModelSafeWorkspaceFileKey, - MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE: - 'File cannot be sent to a model because its secret provenance is unavailable', -})) - -import { POST } from '@/app/api/tools/firecrawl/parse/route' - -const PDF_FILE = { - key: 'workspace/workspace-1/report.pdf', - name: 'report.pdf', - size: 4, - type: 'application/pdf', -} - -describe('POST /api/tools/firecrawl/parse', () => { - beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - mockIsModelSafeWorkspaceFileKey.mockResolvedValue(true) - mockDownloadServableFile.mockResolvedValue({ - buffer: Buffer.from('pdf'), - contentType: 'application/pdf', - }) - vi.stubGlobal( - 'fetch', - vi - .fn() - .mockResolvedValue( - Response.json({ success: true, data: { markdown: '# Parsed' } }, { status: 200 }) - ) - ) - }) - - it('does not apply model provenance guards to explicit text-only PDF parsing', async () => { - const response = await POST( - createMockRequest( - 'POST', - { - apiKey: 'firecrawl-key', - file: PDF_FILE, - options: { - formats: ['markdown'], - parsers: [{ type: 'pdf', mode: 'fast' }], - }, - [RESOLVED_SECRET_PROVENANCE_FIELD]: { - version: 1, - complete: false, - entries: [], - }, - }, - { [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1 } - ) - ) - - expect(response.status).toBe(200) - expect(mockIsModelSafeWorkspaceFileKey).not.toHaveBeenCalled() - expect(fetch).toHaveBeenCalledOnce() - }) - - it('rejects incomplete provenance when summary generation is requested', async () => { - const response = await POST( - createMockRequest( - 'POST', - { - apiKey: 'firecrawl-key', - file: { ...PDF_FILE, name: 'report.docx', type: 'application/vnd.ms-word' }, - options: { formats: ['summary'] }, - [RESOLVED_SECRET_PROVENANCE_FIELD]: { - version: 1, - complete: false, - entries: [], - }, - }, - { [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1 } - ) - ) - - expect(response.status).toBe(400) - expect(await response.json()).toEqual({ - success: false, - error: 'Model input provenance is unavailable', - }) - expect(mockDownloadServableFile).not.toHaveBeenCalled() - expect(fetch).not.toHaveBeenCalled() - }) - - it('applies durable file provenance to the default PDF auto parser', async () => { - mockIsModelSafeWorkspaceFileKey.mockResolvedValueOnce(false) - const response = await POST( - createMockRequest( - 'POST', - { - apiKey: 'firecrawl-key', - file: PDF_FILE, - options: { formats: ['markdown'] }, - [RESOLVED_SECRET_PROVENANCE_FIELD]: { - version: 1, - complete: true, - entries: [], - }, - }, - { [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1 } - ) - ) - - expect(response.status).toBe(400) - expect(await response.json()).toEqual({ - success: false, - error: 'File cannot be sent to a model because its secret provenance is unavailable', - }) - expect(mockIsModelSafeWorkspaceFileKey).toHaveBeenCalledWith(PDF_FILE.key) - expect(mockDownloadServableFile).not.toHaveBeenCalled() - expect(fetch).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/app/api/tools/firecrawl/parse/route.ts b/apps/sim/app/api/tools/firecrawl/parse/route.ts deleted file mode 100644 index a1b1fe8888f..00000000000 --- a/apps/sim/app/api/tools/firecrawl/parse/route.ts +++ /dev/null @@ -1,145 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { firecrawlParseContract } from '@/lib/api/contracts/tools' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' -import { - isModelSafeWorkspaceFileKey, - MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, -} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { hasFirecrawlParseModelInput } from '@/tools/firecrawl/model-input' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('FirecrawlParseAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Firecrawl parse attempt`, { - error: authResult.error || 'Missing userId', - }) - return NextResponse.json( - { success: false, error: authResult.error || 'Unauthorized' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(firecrawlParseContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - const hasModelInput = hasFirecrawlParseModelInput({ - file: validatedData.file, - formats: Array.isArray(validatedData.options?.formats) - ? validatedData.options.formats - : undefined, - parsers: Array.isArray(validatedData.options?.parsers) - ? validatedData.options.parsers - : undefined, - }) - if (hasModelInput) { - const modelInputProvenance = validateOpaqueModelInputProvenance({ - headers: request.headers, - payload: validatedData, - isInternalRequest: true, - }) - if (!modelInputProvenance.success) { - return NextResponse.json( - { success: false, error: modelInputProvenance.error }, - { status: modelInputProvenance.status } - ) - } - } - - const [userFile] = processFilesToUserFiles([validatedData.file], requestId, logger) - if (!userFile) { - return NextResponse.json({ success: false, error: 'File input is required' }, { status: 400 }) - } - - logger.info(`[${requestId}] Firecrawl parse request`, { - fileName: userFile.name, - size: userFile.size, - }) - - const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) - if (denied) return denied - if (hasModelInput && !(await isModelSafeWorkspaceFileKey(userFile.key))) { - return NextResponse.json( - { success: false, error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE }, - { status: 400 } - ) - } - - const { buffer, contentType } = await downloadServableFileFromStorage( - userFile, - requestId, - logger, - { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - } - ) - - const formData = new FormData() - const blob = new Blob([new Uint8Array(buffer)], { - type: contentType || userFile.type || 'application/octet-stream', - }) - formData.append('file', blob, userFile.name) - - if (validatedData.options && Object.keys(validatedData.options).length > 0) { - formData.append('options', JSON.stringify(validatedData.options)) - } - - const firecrawlResponse = await fetch('https://api.firecrawl.dev/v2/parse', { - method: 'POST', - headers: { - Authorization: `Bearer ${validatedData.apiKey}`, - }, - body: formData, - }) - - if (!firecrawlResponse.ok) { - const errorText = await firecrawlResponse.text() - logger.error(`[${requestId}] Firecrawl API error:`, errorText) - return NextResponse.json( - { - success: false, - error: `Firecrawl API error: ${errorText || firecrawlResponse.statusText}`, - }, - { status: firecrawlResponse.status } - ) - } - - const firecrawlData = await firecrawlResponse.json() - - logger.info(`[${requestId}] Firecrawl parse successful`) - - const document = firecrawlData.data ?? firecrawlData - return NextResponse.json({ - success: true, - output: - // Credits reported on the envelope would otherwise be dropped with it, - // leaving a paid parse with nothing to meter. - firecrawlData.creditsUsed != null - ? { ...document, creditsUsed: firecrawlData.creditsUsed } - : document, - }) - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - logger.error(`[${requestId}] Error in Firecrawl parse:`, error) - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/fireflies/upload-audio/route.test.ts b/apps/sim/app/api/tools/fireflies/upload-audio/route.test.ts deleted file mode 100644 index 697ca8e4590..00000000000 --- a/apps/sim/app/api/tools/fireflies/upload-audio/route.test.ts +++ /dev/null @@ -1,192 +0,0 @@ -/** - * @vitest-environment node - */ -import { createMockRequest, hybridAuthMockFns } from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' -import { PRIVATE_MODEL_INPUT_PROVENANCE_HEADER } from '@/lib/execution/model-input-provenance' -import { - RESOLVED_SECRET_PROVENANCE_FIELD, - RESOLVED_SECRET_PROVENANCE_METADATA_V1, -} from '@/lib/execution/private-tool-metadata' - -const { mockResolveFileInputToUrl } = vi.hoisted(() => ({ - mockResolveFileInputToUrl: vi.fn(), -})) - -vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ - resolveFileInputToUrl: mockResolveFileInputToUrl, -})) - -import { POST } from '@/app/api/tools/fireflies/upload-audio/route' - -const upstreamSuccess = { - data: { - uploadAudio: { - success: true, - title: 'Uploaded meeting', - message: 'Queued', - }, - }, -} - -describe('POST /api/tools/fireflies/upload-audio', () => { - beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - mockResolveFileInputToUrl.mockResolvedValue({ - fileUrl: 'https://media.example.com/audio.mp3', - }) - vi.stubGlobal( - 'fetch', - vi.fn().mockResolvedValue(Response.json(upstreamSuccess, { status: 200 })) - ) - }) - - it('keeps headerless external URL calls compatible and preserves the GraphQL response', async () => { - const response = await POST( - createMockRequest('POST', { - apiKey: 'fireflies-key', - audioUrl: 'https://media.example.com/audio.mp3', - title: 'Uploaded meeting', - attendees: [{ displayName: 'Ada', email: 'ada@example.com' }], - }) - ) - - expect(response.status).toBe(200) - expect(await response.json()).toEqual(upstreamSuccess) - expect(mockResolveFileInputToUrl).toHaveBeenCalledWith( - expect.objectContaining({ - filePath: 'https://media.example.com/audio.mp3', - userId: 'user-1', - modelEgress: true, - presignExpirySeconds: 3600, - }) - ) - expect(fetch).toHaveBeenCalledWith( - 'https://api.fireflies.ai/graphql', - expect.objectContaining({ - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer fireflies-key', - }, - }) - ) - const init = vi.mocked(fetch).mock.calls[0][1] - const upstreamBody = JSON.parse(String(init?.body)) - expect(upstreamBody.variables.input).toEqual({ - url: 'https://media.example.com/audio.mp3', - title: 'Uploaded meeting', - attendees: [{ displayName: 'Ada', email: 'ada@example.com' }], - }) - }) - - it('rejects an incomplete private provenance envelope before resolving the source', async () => { - const response = await POST( - createMockRequest( - 'POST', - { - apiKey: 'fireflies-key', - audioUrl: 'https://media.example.com/audio.mp3', - [RESOLVED_SECRET_PROVENANCE_FIELD]: { - version: 1, - complete: false, - entries: [], - }, - }, - { [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1 } - ) - ) - - expect(response.status).toBe(400) - expect(await response.json()).toEqual({ - errors: [{ message: 'Model input provenance is unavailable' }], - }) - expect(mockResolveFileInputToUrl).not.toHaveBeenCalled() - expect(fetch).not.toHaveBeenCalled() - }) - - it('rejects a model-unsafe stored file before contacting Fireflies', async () => { - mockResolveFileInputToUrl.mockResolvedValueOnce({ - error: { - status: 400, - message: 'File cannot be sent to a model because its secret provenance is unavailable', - }, - }) - - const response = await POST( - createMockRequest('POST', { - apiKey: 'fireflies-key', - audioFile: { - key: 'workspace/workspace-1/audio.mp3', - name: 'audio.mp3', - size: 42, - type: 'audio/mpeg', - }, - }) - ) - - expect(response.status).toBe(400) - expect(await response.json()).toEqual({ - errors: [ - { - message: 'File cannot be sent to a model because its secret provenance is unavailable', - }, - ], - }) - expect(mockResolveFileInputToUrl).toHaveBeenCalledWith( - expect.objectContaining({ - file: expect.objectContaining({ - key: 'workspace/workspace-1/audio.mp3', - name: 'audio.mp3', - size: 42, - }), - modelEgress: true, - }) - ) - expect(fetch).not.toHaveBeenCalled() - }) - - it('normalizes legacy key-only files and submits a fresh signed URL', async () => { - mockResolveFileInputToUrl.mockResolvedValueOnce({ - fileUrl: 'https://storage.example.com/signed-audio.mp3', - }) - - const response = await POST( - createMockRequest( - 'POST', - { - apiKey: 'fireflies-key', - audioFile: { key: 'workspace/workspace-1/audio.mp3' }, - [RESOLVED_SECRET_PROVENANCE_FIELD]: { - version: 1, - complete: true, - entries: [], - }, - }, - { [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1 } - ) - ) - - expect(response.status).toBe(200) - expect(mockResolveFileInputToUrl).toHaveBeenCalledWith( - expect.objectContaining({ - file: expect.objectContaining({ - key: 'workspace/workspace-1/audio.mp3', - name: 'audio', - size: 0, - }), - presignExpirySeconds: 3600, - modelEgress: true, - }) - ) - const init = vi.mocked(fetch).mock.calls[0][1] - expect(JSON.parse(String(init?.body)).variables.input.url).toBe( - 'https://storage.example.com/signed-audio.mp3' - ) - }) -}) diff --git a/apps/sim/app/api/tools/fireflies/upload-audio/route.ts b/apps/sim/app/api/tools/fireflies/upload-audio/route.ts deleted file mode 100644 index aedba25a335..00000000000 --- a/apps/sim/app/api/tools/fireflies/upload-audio/route.ts +++ /dev/null @@ -1,146 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import type { NextRequest } from 'next/server' -import { NextResponse } from 'next/server' -import { - type FirefliesUploadAudioBody, - firefliesUploadAudioContract, -} from '@/lib/api/contracts/tools/fireflies' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' -import type { RawFileInput } from '@/lib/uploads/utils/file-utils' -import { resolveFileInputToUrl } from '@/lib/uploads/utils/file-utils.server' - -const logger = createLogger('FirefliesUploadAudioAPI') -const FIREFLIES_API_URL = 'https://api.fireflies.ai/graphql' -const FIREFLIES_AUDIO_PRESIGN_EXPIRY_SECONDS = 60 * 60 -const UPLOAD_AUDIO_MUTATION = ` - mutation UploadAudio($input: AudioUploadInput) { - uploadAudio(input: $input) { - success - title - message - } - } -` - -function errorResponse(message: string, status: number): NextResponse { - return NextResponse.json({ errors: [{ message }] }, { status }) -} - -function normalizeStoredAudioFile(file: NonNullable) { - return { - ...file, - name: file.name || 'audio', - size: file.size ?? 0, - } satisfies RawFileInput -} - -async function resolveAudioUrl(options: { - body: FirefliesUploadAudioBody - userId: string - requestId: string -}): Promise<{ fileUrl?: string; error?: { status: number; message: string } }> { - const { body, userId, requestId } = options - const file = body.audioFile - - if (file?.key) { - return resolveFileInputToUrl({ - file: normalizeStoredAudioFile(file), - userId, - requestId, - logger, - presignExpirySeconds: FIREFLIES_AUDIO_PRESIGN_EXPIRY_SECONDS, - modelEgress: true, - }) - } - - return resolveFileInputToUrl({ - filePath: file?.url || file?.path || body.audioUrl, - userId, - requestId, - logger, - presignExpirySeconds: FIREFLIES_AUDIO_PRESIGN_EXPIRY_SECONDS, - modelEgress: true, - }) -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return errorResponse('Unauthorized', 401) - } - - const parsed = await parseRequest( - firefliesUploadAudioContract, - request, - {}, - { - validationErrorResponse: (error) => - errorResponse(getValidationErrorMessage(error, 'Invalid request data'), 400), - } - ) - if (!parsed.success) return parsed.response - - const body = parsed.data.body - const modelInputProvenance = validateOpaqueModelInputProvenance({ - headers: request.headers, - payload: body, - isInternalRequest: true, - }) - if (!modelInputProvenance.success) { - return errorResponse(modelInputProvenance.error, modelInputProvenance.status) - } - - const resolution = await resolveAudioUrl({ body, userId: authResult.userId, requestId }) - if (resolution.error) { - return errorResponse(resolution.error.message, resolution.error.status) - } - if (!resolution.fileUrl?.startsWith('https://')) { - return errorResponse('Audio URL must be a valid HTTPS URL', 400) - } - - const input: Record = { url: resolution.fileUrl } - if (body.title) input.title = body.title - if (body.webhook) input.webhook = body.webhook - if (body.language) input.custom_language = body.language - if (body.clientReferenceId) input.client_reference_id = body.clientReferenceId - if (body.attendees !== undefined) input.attendees = body.attendees - - const response = await fetch(FIREFLIES_API_URL, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${body.apiKey}`, - }, - body: JSON.stringify({ - query: UPLOAD_AUDIO_MUTATION, - variables: { input }, - }), - signal: request.signal, - }) - - logger.info(`[${requestId}] Fireflies upload request completed`, { - status: response.status, - }) - - return new Response(response.body, { - status: response.status, - statusText: response.statusText, - headers: { - 'Content-Type': response.headers.get('content-type') || 'application/json', - }, - }) - } catch (error) { - logger.error(`[${requestId}] Fireflies upload proxy failed`, { - error: getErrorMessage(error, 'Unknown error'), - }) - return errorResponse('Failed to upload audio', 500) - } -}) diff --git a/apps/sim/app/api/tools/github/latest-commit/route.ts b/apps/sim/app/api/tools/github/latest-commit/route.ts deleted file mode 100644 index ed9575c9976..00000000000 --- a/apps/sim/app/api/tools/github/latest-commit/route.ts +++ /dev/null @@ -1,191 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { githubLatestCommitContract } from '@/lib/api/contracts/tools/github' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('GitHubLatestCommitAPI') - -interface GitHubErrorResponse { - message?: string -} - -interface GitHubCommitResponse { - sha: string - html_url: string - commit: { - message: string - author: { name: string; email: string; date: string } - committer: { name: string; email: string; date: string } - } - author?: { login: string; avatar_url: string; html_url: string } - committer?: { login: string; avatar_url: string; html_url: string } - stats?: { additions: number; deletions: number; total: number } - files?: Array<{ - filename: string - status: string - additions: number - deletions: number - changes: number - patch?: string - raw_url?: string - blob_url?: string - }> -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized GitHub latest commit attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - const parsed = await parseRequest(githubLatestCommitContract, request, {}) - if (!parsed.success) return parsed.response - - const { owner, repo, branch, apiKey } = parsed.data.body - - const baseUrl = `https://api.github.com/repos/${owner}/${repo}` - const commitUrl = branch ? `${baseUrl}/commits/${branch}` : `${baseUrl}/commits/HEAD` - - logger.info(`[${requestId}] Fetching latest commit from GitHub`, { owner, repo, branch }) - - const urlValidation = await validateUrlWithDNS(commitUrl, 'commitUrl') - if (!urlValidation.isValid) { - return NextResponse.json({ success: false, error: urlValidation.error }, { status: 400 }) - } - - const response = await secureFetchWithPinnedIP(commitUrl, urlValidation.resolvedIP!, { - method: 'GET', - headers: { - Accept: 'application/vnd.github.v3+json', - Authorization: `Bearer ${apiKey}`, - 'X-GitHub-Api-Version': '2022-11-28', - }, - }) - - if (!response.ok) { - const errorData = (await response.json().catch(() => ({}))) as GitHubErrorResponse - logger.error(`[${requestId}] GitHub API error`, { - status: response.status, - error: errorData, - }) - return NextResponse.json( - { success: false, error: errorData.message || `GitHub API error: ${response.status}` }, - { status: 400 } - ) - } - - const data = (await response.json()) as GitHubCommitResponse - - const content = `Latest commit: "${data.commit.message}" by ${data.commit.author.name} on ${data.commit.author.date}. SHA: ${data.sha}` - - const files = data.files || [] - const fileDetailsWithContent = [] - - for (const file of files) { - const fileDetail: Record = { - filename: file.filename, - additions: file.additions, - deletions: file.deletions, - changes: file.changes, - status: file.status, - raw_url: file.raw_url, - blob_url: file.blob_url, - patch: file.patch, - content: undefined, - } - - if (file.status !== 'removed' && file.raw_url) { - try { - const rawUrlValidation = await validateUrlWithDNS(file.raw_url, 'rawUrl') - if (rawUrlValidation.isValid) { - const contentResponse = await secureFetchWithPinnedIP( - file.raw_url, - rawUrlValidation.resolvedIP!, - { - headers: { - Authorization: `Bearer ${apiKey}`, - 'X-GitHub-Api-Version': '2022-11-28', - }, - } - ) - - if (contentResponse.ok) { - fileDetail.content = await contentResponse.text() - } - } - } catch (error) { - logger.warn(`[${requestId}] Failed to fetch content for ${file.filename}:`, error) - } - } - - fileDetailsWithContent.push(fileDetail) - } - - logger.info(`[${requestId}] Latest commit fetched successfully`, { - sha: data.sha, - fileCount: files.length, - }) - - return NextResponse.json({ - success: true, - output: { - content, - metadata: { - sha: data.sha, - html_url: data.html_url, - commit_message: data.commit.message, - author: { - name: data.commit.author.name, - login: data.author?.login || 'Unknown', - avatar_url: data.author?.avatar_url || '', - html_url: data.author?.html_url || '', - }, - committer: { - name: data.commit.committer.name, - login: data.committer?.login || 'Unknown', - avatar_url: data.committer?.avatar_url || '', - html_url: data.committer?.html_url || '', - }, - stats: data.stats - ? { - additions: data.stats.additions, - deletions: data.stats.deletions, - total: data.stats.total, - } - : undefined, - files: fileDetailsWithContent.length > 0 ? fileDetailsWithContent : undefined, - }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error fetching GitHub latest commit:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/gmail/add-label/route.ts b/apps/sim/app/api/tools/gmail/add-label/route.ts deleted file mode 100644 index 41da810a2f0..00000000000 --- a/apps/sim/app/api/tools/gmail/add-label/route.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { gmailAddLabelContract } from '@/lib/api/contracts/tools/google' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId } from '@/lib/core/security/input-validation' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('GmailAddLabelAPI') - -const GMAIL_API_BASE = 'https://gmail.googleapis.com/gmail/v1/users/me' - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Gmail add label attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated Gmail add label request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const parsed = await parseRequest(gmailAddLabelContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Adding label(s) to Gmail email`, { - messageId: validatedData.messageId, - labelIds: validatedData.labelIds, - }) - - const labelIds = validatedData.labelIds - .split(',') - .map((id) => id.trim()) - .filter((id) => id.length > 0) - - for (const labelId of labelIds) { - const labelIdValidation = validateAlphanumericId(labelId, 'labelId', 255) - if (!labelIdValidation.isValid) { - logger.warn(`[${requestId}] Invalid label ID: ${labelIdValidation.error}`) - return NextResponse.json( - { - success: false, - error: labelIdValidation.error, - }, - { status: 400 } - ) - } - } - - const messageIdValidation = validateAlphanumericId(validatedData.messageId, 'messageId', 255) - if (!messageIdValidation.isValid) { - logger.warn(`[${requestId}] Invalid message ID: ${messageIdValidation.error}`) - return NextResponse.json( - { success: false, error: messageIdValidation.error }, - { status: 400 } - ) - } - - const gmailResponse = await fetch( - `${GMAIL_API_BASE}/messages/${validatedData.messageId}/modify`, - { - method: 'POST', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - addLabelIds: labelIds, - }), - } - ) - - if (!gmailResponse.ok) { - const errorText = await gmailResponse.text() - logger.error(`[${requestId}] Gmail API error:`, errorText) - return NextResponse.json( - { - success: false, - error: `Gmail API error: ${gmailResponse.statusText}`, - }, - { status: gmailResponse.status } - ) - } - - const data = await gmailResponse.json() - - logger.info(`[${requestId}] Label(s) added successfully`, { messageId: data.id }) - - return NextResponse.json({ - success: true, - output: { - content: `Successfully added ${labelIds.length} label(s) to email`, - metadata: { - id: data.id, - threadId: data.threadId, - labelIds: data.labelIds, - }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error adding label to Gmail email:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/gmail/archive/route.ts b/apps/sim/app/api/tools/gmail/archive/route.ts deleted file mode 100644 index 46161021ffc..00000000000 --- a/apps/sim/app/api/tools/gmail/archive/route.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { gmailArchiveContract } from '@/lib/api/contracts/tools/google' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('GmailArchiveAPI') - -const GMAIL_API_BASE = 'https://gmail.googleapis.com/gmail/v1/users/me' - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Gmail archive attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated Gmail archive request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const parsed = await parseRequest(gmailArchiveContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Archiving Gmail email`, { - messageId: validatedData.messageId, - }) - - const gmailResponse = await fetch( - `${GMAIL_API_BASE}/messages/${validatedData.messageId}/modify`, - { - method: 'POST', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - removeLabelIds: ['INBOX'], - }), - } - ) - - if (!gmailResponse.ok) { - const errorText = await gmailResponse.text() - logger.error(`[${requestId}] Gmail API error:`, errorText) - return NextResponse.json( - { - success: false, - error: `Gmail API error: ${gmailResponse.statusText}`, - }, - { status: gmailResponse.status } - ) - } - - const data = await gmailResponse.json() - - logger.info(`[${requestId}] Email archived successfully`, { messageId: data.id }) - - return NextResponse.json({ - success: true, - output: { - content: 'Email archived successfully', - metadata: { - id: data.id, - threadId: data.threadId, - labelIds: data.labelIds, - }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error archiving Gmail email:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/gmail/delete/route.ts b/apps/sim/app/api/tools/gmail/delete/route.ts deleted file mode 100644 index ba83d8bf740..00000000000 --- a/apps/sim/app/api/tools/gmail/delete/route.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { gmailDeleteContract } from '@/lib/api/contracts/tools/google' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('GmailDeleteAPI') - -const GMAIL_API_BASE = 'https://gmail.googleapis.com/gmail/v1/users/me' - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Gmail delete attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated Gmail delete request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const parsed = await parseRequest(gmailDeleteContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Deleting Gmail email`, { - messageId: validatedData.messageId, - }) - - const gmailResponse = await fetch( - `${GMAIL_API_BASE}/messages/${validatedData.messageId}/trash`, - { - method: 'POST', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': 'application/json', - }, - } - ) - - if (!gmailResponse.ok) { - const errorText = await gmailResponse.text() - logger.error(`[${requestId}] Gmail API error:`, errorText) - return NextResponse.json( - { - success: false, - error: `Gmail API error: ${gmailResponse.statusText}`, - }, - { status: gmailResponse.status } - ) - } - - const data = await gmailResponse.json() - - logger.info(`[${requestId}] Email deleted successfully`, { messageId: data.id }) - - return NextResponse.json({ - success: true, - output: { - content: 'Email moved to trash successfully', - metadata: { - id: data.id, - threadId: data.threadId, - labelIds: data.labelIds, - }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error deleting Gmail email:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/gmail/draft/route.ts b/apps/sim/app/api/tools/gmail/draft/route.ts deleted file mode 100644 index 255c5242967..00000000000 --- a/apps/sim/app/api/tools/gmail/draft/route.ts +++ /dev/null @@ -1,222 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { gmailDraftContract } from '@/lib/api/contracts/tools/google' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { - base64UrlEncode, - buildMimeMessage, - buildSimpleEmailMessage, - fetchThreadingHeaders, -} from '@/tools/gmail/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('GmailDraftAPI') - -const GMAIL_API_BASE = 'https://gmail.googleapis.com/gmail/v1/users/me' - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Gmail draft attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - const userId = authResult.userId - logger.info(`[${requestId}] Authenticated Gmail draft request via ${authResult.authType}`, { - userId, - }) - - const parsed = await parseRequest(gmailDraftContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Creating Gmail draft`, { - to: validatedData.to, - subject: validatedData.subject || '', - hasAttachments: !!(validatedData.attachments && validatedData.attachments.length > 0), - attachmentCount: validatedData.attachments?.length || 0, - }) - - const threadingHeaders = validatedData.replyToMessageId - ? await fetchThreadingHeaders(validatedData.replyToMessageId, validatedData.accessToken) - : {} - - const originalMessageId = threadingHeaders.messageId - const originalReferences = threadingHeaders.references - const originalSubject = threadingHeaders.subject - - let rawMessage: string | undefined - - if (validatedData.attachments && validatedData.attachments.length > 0) { - const rawAttachments = validatedData.attachments - logger.info(`[${requestId}] Processing ${rawAttachments.length} attachment(s)`) - - const attachments = processFilesToUserFiles(rawAttachments, requestId, logger) - - if (attachments.length === 0) { - logger.warn(`[${requestId}] No valid attachments found after processing`) - } else { - const totalSize = attachments.reduce((sum, file) => sum + file.size, 0) - const maxSize = 25 * 1024 * 1024 // 25MB - - if (totalSize > maxSize) { - const sizeMB = (totalSize / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`, - }, - { status: 400 } - ) - } - - const accessResults = await Promise.all( - attachments.map((file) => assertToolFileAccess(file.key, userId, requestId, logger)) - ) - const denied = accessResults.find((r) => r !== null) - if (denied) return denied - - let resolved: Array<{ buffer: Buffer; contentType: string }> - try { - resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, { - totalMaxBytes: maxSize, - label: 'Total attachment size', - }) - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - if (isPayloadSizeLimitError(error)) { - const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`, - }, - { status: 400 } - ) - } - logger.error(`[${requestId}] Failed to download an attachment:`, error) - return NextResponse.json( - { - success: false, - error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`, - }, - { status: 500 } - ) - } - - const attachmentBuffers = attachments.map((file, i) => ({ - filename: file.name, - mimeType: resolved[i].contentType || file.type || 'application/octet-stream', - content: resolved[i].buffer, - })) - - const mimeMessage = buildMimeMessage({ - to: validatedData.to, - cc: validatedData.cc ?? undefined, - bcc: validatedData.bcc ?? undefined, - subject: validatedData.subject || originalSubject || '', - body: validatedData.body, - contentType: validatedData.contentType || 'text', - inReplyTo: originalMessageId, - references: originalReferences, - attachments: attachmentBuffers, - }) - - logger.info(`[${requestId}] Built MIME message for draft (${mimeMessage.length} bytes)`) - rawMessage = base64UrlEncode(mimeMessage) - } - } - - if (!rawMessage) { - rawMessage = buildSimpleEmailMessage({ - to: validatedData.to, - cc: validatedData.cc, - bcc: validatedData.bcc, - subject: validatedData.subject || originalSubject, - body: validatedData.body, - contentType: validatedData.contentType || 'text', - inReplyTo: originalMessageId, - references: originalReferences, - }) - } - - const draftMessage: { raw: string; threadId?: string } = { raw: rawMessage } - - if (validatedData.threadId) { - draftMessage.threadId = validatedData.threadId - } - - const gmailResponse = await fetch(`${GMAIL_API_BASE}/drafts`, { - method: 'POST', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - message: draftMessage, - }), - }) - - if (!gmailResponse.ok) { - const errorText = await gmailResponse.text() - logger.error(`[${requestId}] Gmail API error:`, errorText) - return NextResponse.json( - { - success: false, - error: `Gmail API error: ${gmailResponse.statusText}`, - }, - { status: gmailResponse.status } - ) - } - - const data = await gmailResponse.json() - - logger.info(`[${requestId}] Draft created successfully`, { draftId: data.id }) - - return NextResponse.json({ - success: true, - output: { - content: 'Email drafted successfully', - metadata: { - id: data.id, - message: { - id: data.message?.id, - threadId: data.message?.threadId, - labelIds: data.message?.labelIds, - }, - }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error creating Gmail draft:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/gmail/edit-draft/route.ts b/apps/sim/app/api/tools/gmail/edit-draft/route.ts deleted file mode 100644 index 9e88ce6cdbd..00000000000 --- a/apps/sim/app/api/tools/gmail/edit-draft/route.ts +++ /dev/null @@ -1,215 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { gmailEditDraftContract } from '@/lib/api/contracts/tools/google' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { - base64UrlEncode, - buildMimeMessage, - buildSimpleEmailMessage, - fetchThreadingHeaders, - GMAIL_API_BASE, -} from '@/tools/gmail/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('GmailEditDraftAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Gmail edit draft attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - const userId = authResult.userId - logger.info( - `[${requestId}] Authenticated Gmail edit draft request via ${authResult.authType}`, - { userId } - ) - - const parsed = await parseRequest(gmailEditDraftContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Updating Gmail draft`, { - draftId: validatedData.draftId, - to: validatedData.to, - hasAttachments: !!(validatedData.attachments && validatedData.attachments.length > 0), - attachmentCount: validatedData.attachments?.length || 0, - }) - - const threadingHeaders = validatedData.replyToMessageId - ? await fetchThreadingHeaders(validatedData.replyToMessageId, validatedData.accessToken) - : {} - - const originalMessageId = threadingHeaders.messageId - const originalReferences = threadingHeaders.references - const originalSubject = threadingHeaders.subject - - let rawMessage: string | undefined - - if (validatedData.attachments && validatedData.attachments.length > 0) { - const rawAttachments = validatedData.attachments - const attachments = processFilesToUserFiles(rawAttachments, requestId, logger) - - if (attachments.length > 0) { - const totalSize = attachments.reduce((sum, file) => sum + file.size, 0) - const maxSize = 25 * 1024 * 1024 - - if (totalSize > maxSize) { - const sizeMB = (totalSize / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`, - }, - { status: 400 } - ) - } - - const accessResults = await Promise.all( - attachments.map((file) => assertToolFileAccess(file.key, userId, requestId, logger)) - ) - const denied = accessResults.find((r) => r !== null) - if (denied) return denied - - let resolved: Array<{ buffer: Buffer; contentType: string }> - try { - resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, { - totalMaxBytes: maxSize, - label: 'Total attachment size', - }) - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - if (isPayloadSizeLimitError(error)) { - const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`, - }, - { status: 400 } - ) - } - logger.error(`[${requestId}] Failed to download an attachment:`, error) - return NextResponse.json( - { - success: false, - error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`, - }, - { status: 500 } - ) - } - - const attachmentBuffers = attachments.map((file, i) => ({ - filename: file.name, - mimeType: resolved[i].contentType || file.type || 'application/octet-stream', - content: resolved[i].buffer, - })) - - const mimeMessage = buildMimeMessage({ - to: validatedData.to, - cc: validatedData.cc ?? undefined, - bcc: validatedData.bcc ?? undefined, - subject: validatedData.subject || originalSubject || '', - body: validatedData.body, - contentType: validatedData.contentType || 'text', - inReplyTo: originalMessageId, - references: originalReferences, - attachments: attachmentBuffers, - }) - - rawMessage = base64UrlEncode(mimeMessage) - } - } - - if (!rawMessage) { - rawMessage = buildSimpleEmailMessage({ - to: validatedData.to, - cc: validatedData.cc, - bcc: validatedData.bcc, - subject: validatedData.subject || originalSubject, - body: validatedData.body, - contentType: validatedData.contentType || 'text', - inReplyTo: originalMessageId, - references: originalReferences, - }) - } - - const draftMessage: { raw: string; threadId?: string } = { raw: rawMessage } - if (validatedData.threadId) { - draftMessage.threadId = validatedData.threadId - } - - const gmailResponse = await fetch( - `${GMAIL_API_BASE}/drafts/${encodeURIComponent(validatedData.draftId)}`, - { - method: 'PUT', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - id: validatedData.draftId, - message: draftMessage, - }), - } - ) - - if (!gmailResponse.ok) { - const errorText = await gmailResponse.text() - logger.error(`[${requestId}] Gmail API error:`, errorText) - return NextResponse.json( - { - success: false, - error: `Gmail API error: ${gmailResponse.statusText}`, - }, - { status: gmailResponse.status } - ) - } - - const data = await gmailResponse.json() - - logger.info(`[${requestId}] Draft updated successfully`, { draftId: data.id }) - - return NextResponse.json({ - success: true, - output: { - draftId: data.id ?? null, - messageId: data.message?.id ?? null, - threadId: data.message?.threadId ?? null, - labelIds: data.message?.labelIds ?? null, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error updating Gmail draft:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/gmail/label/route.ts b/apps/sim/app/api/tools/gmail/label/route.ts deleted file mode 100644 index f1abd52383c..00000000000 --- a/apps/sim/app/api/tools/gmail/label/route.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { gmailLabelSelectorContract } from '@/lib/api/contracts/selectors/google' -import { parseRequest } from '@/lib/api/server' -import { authorizeCredentialUse } from '@/lib/auth/credential-access' -import { validateAlphanumericId } from '@/lib/core/security/input-validation' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - refreshAccessTokenIfNeeded, - ServiceAccountTokenError, -} from '@/lib/oauth/credential-service' -import { getScopesForService } from '@/lib/oauth/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('GmailLabelAPI') - -export const GET = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const parsed = await parseRequest(gmailLabelSelectorContract, request, {}) - if (!parsed.success) return parsed.response - const { credentialId, labelId } = parsed.data.query - const impersonateEmail = parsed.data.query.impersonateEmail || undefined - - const labelIdValidation = validateAlphanumericId(labelId, 'labelId', 255) - if (!labelIdValidation.isValid) { - logger.warn(`[${requestId}] Invalid label ID: ${labelIdValidation.error}`) - return NextResponse.json({ error: labelIdValidation.error }, { status: 400 }) - } - - const credAccess = await authorizeCredentialUse(request, { - credentialId, - requireWorkflowIdForInternal: false, - }) - if (!credAccess.ok || !credAccess.credentialOwnerUserId) { - logger.warn(`[${requestId}] Credential access denied`, { error: credAccess.error }) - return NextResponse.json({ error: credAccess.error || 'Unauthorized' }, { status: 401 }) - } - - const accessToken = await refreshAccessTokenIfNeeded( - credentialId, - credAccess.credentialOwnerUserId, - requestId, - getScopesForService('gmail'), - impersonateEmail - ) - - if (!accessToken) { - return NextResponse.json({ error: 'Failed to obtain valid access token' }, { status: 401 }) - } - - logger.info(`[${requestId}] Fetching label ${labelId} from Gmail API`) - const response = await fetch( - `https://gmail.googleapis.com/gmail/v1/users/me/labels/${labelId}`, - { - headers: { - Authorization: `Bearer ${accessToken}`, - }, - } - ) - - logger.info(`[${requestId}] Gmail API response status: ${response.status}`) - - if (!response.ok) { - const errorText = await response.text() - logger.error(`[${requestId}] Gmail API error response: ${errorText}`) - - try { - const error = JSON.parse(errorText) - return NextResponse.json({ error }, { status: response.status }) - } catch (_e) { - return NextResponse.json({ error: errorText }, { status: response.status }) - } - } - - const label = await response.json() - - let formattedName = label.name - - if (label.type === 'system') { - formattedName = label.name.charAt(0).toUpperCase() + label.name.slice(1).toLowerCase() - } - - const formattedLabel = { - id: label.id, - name: formattedName, - type: label.type, - messagesTotal: label.messagesTotal || 0, - messagesUnread: label.messagesUnread || 0, - } - - return NextResponse.json({ label: formattedLabel }, { status: 200 }) - } catch (error) { - if (error instanceof ServiceAccountTokenError) { - return NextResponse.json({ error: error.message }, { status: 400 }) - } - logger.error(`[${requestId}] Error fetching Gmail label:`, error) - return NextResponse.json({ error: 'Failed to fetch Gmail label' }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/gmail/mark-read/route.ts b/apps/sim/app/api/tools/gmail/mark-read/route.ts deleted file mode 100644 index a9843788a65..00000000000 --- a/apps/sim/app/api/tools/gmail/mark-read/route.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { gmailMarkReadContract } from '@/lib/api/contracts/tools/google' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('GmailMarkReadAPI') - -const GMAIL_API_BASE = 'https://gmail.googleapis.com/gmail/v1/users/me' - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Gmail mark read attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated Gmail mark read request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const parsed = await parseRequest(gmailMarkReadContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Marking Gmail email as read`, { - messageId: validatedData.messageId, - }) - - const gmailResponse = await fetch( - `${GMAIL_API_BASE}/messages/${validatedData.messageId}/modify`, - { - method: 'POST', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - removeLabelIds: ['UNREAD'], - }), - } - ) - - if (!gmailResponse.ok) { - const errorText = await gmailResponse.text() - logger.error(`[${requestId}] Gmail API error:`, errorText) - return NextResponse.json( - { - success: false, - error: `Gmail API error: ${gmailResponse.statusText}`, - }, - { status: gmailResponse.status } - ) - } - - const data = await gmailResponse.json() - - logger.info(`[${requestId}] Email marked as read successfully`, { messageId: data.id }) - - return NextResponse.json({ - success: true, - output: { - content: 'Email marked as read successfully', - metadata: { - id: data.id, - threadId: data.threadId, - labelIds: data.labelIds, - }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error marking Gmail email as read:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/gmail/mark-unread/route.ts b/apps/sim/app/api/tools/gmail/mark-unread/route.ts deleted file mode 100644 index d56f8328c1f..00000000000 --- a/apps/sim/app/api/tools/gmail/mark-unread/route.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { gmailMarkUnreadContract } from '@/lib/api/contracts/tools/google' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('GmailMarkUnreadAPI') - -const GMAIL_API_BASE = 'https://gmail.googleapis.com/gmail/v1/users/me' - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Gmail mark unread attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info( - `[${requestId}] Authenticated Gmail mark unread request via ${authResult.authType}`, - { - userId: authResult.userId, - } - ) - - const parsed = await parseRequest(gmailMarkUnreadContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Marking Gmail email as unread`, { - messageId: validatedData.messageId, - }) - - const gmailResponse = await fetch( - `${GMAIL_API_BASE}/messages/${validatedData.messageId}/modify`, - { - method: 'POST', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - addLabelIds: ['UNREAD'], - }), - } - ) - - if (!gmailResponse.ok) { - const errorText = await gmailResponse.text() - logger.error(`[${requestId}] Gmail API error:`, errorText) - return NextResponse.json( - { - success: false, - error: `Gmail API error: ${gmailResponse.statusText}`, - }, - { status: gmailResponse.status } - ) - } - - const data = await gmailResponse.json() - - logger.info(`[${requestId}] Email marked as unread successfully`, { messageId: data.id }) - - return NextResponse.json({ - success: true, - output: { - content: 'Email marked as unread successfully', - metadata: { - id: data.id, - threadId: data.threadId, - labelIds: data.labelIds, - }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error marking Gmail email as unread:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/gmail/move/route.ts b/apps/sim/app/api/tools/gmail/move/route.ts deleted file mode 100644 index fb16b0c363e..00000000000 --- a/apps/sim/app/api/tools/gmail/move/route.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { gmailMoveContract } from '@/lib/api/contracts/tools/google' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('GmailMoveAPI') - -const GMAIL_API_BASE = 'https://gmail.googleapis.com/gmail/v1/users/me' - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Gmail move attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated Gmail move request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const parsed = await parseRequest(gmailMoveContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Moving Gmail email`, { - messageId: validatedData.messageId, - addLabelIds: validatedData.addLabelIds, - removeLabelIds: validatedData.removeLabelIds, - }) - - const addLabelIds = validatedData.addLabelIds - .split(',') - .map((id) => id.trim()) - .filter((id) => id.length > 0) - - const removeLabelIds = validatedData.removeLabelIds - ? validatedData.removeLabelIds - .split(',') - .map((id) => id.trim()) - .filter((id) => id.length > 0) - : [] - - const modifyBody: { addLabelIds?: string[]; removeLabelIds?: string[] } = {} - - if (addLabelIds.length > 0) { - modifyBody.addLabelIds = addLabelIds - } - - if (removeLabelIds.length > 0) { - modifyBody.removeLabelIds = removeLabelIds - } - - const gmailResponse = await fetch( - `${GMAIL_API_BASE}/messages/${validatedData.messageId}/modify`, - { - method: 'POST', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(modifyBody), - } - ) - - if (!gmailResponse.ok) { - const errorText = await gmailResponse.text() - logger.error(`[${requestId}] Gmail API error:`, errorText) - return NextResponse.json( - { - success: false, - error: `Gmail API error: ${gmailResponse.statusText}`, - }, - { status: gmailResponse.status } - ) - } - - const data = await gmailResponse.json() - - logger.info(`[${requestId}] Email moved successfully`, { messageId: data.id }) - - return NextResponse.json({ - success: true, - output: { - content: 'Email moved successfully', - metadata: { - id: data.id, - threadId: data.threadId, - labelIds: data.labelIds, - }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error moving Gmail email:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/gmail/remove-label/route.ts b/apps/sim/app/api/tools/gmail/remove-label/route.ts deleted file mode 100644 index 537f245217a..00000000000 --- a/apps/sim/app/api/tools/gmail/remove-label/route.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { gmailRemoveLabelContract } from '@/lib/api/contracts/tools/google' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId } from '@/lib/core/security/input-validation' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('GmailRemoveLabelAPI') - -const GMAIL_API_BASE = 'https://gmail.googleapis.com/gmail/v1/users/me' - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Gmail remove label attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info( - `[${requestId}] Authenticated Gmail remove label request via ${authResult.authType}`, - { - userId: authResult.userId, - } - ) - - const parsed = await parseRequest(gmailRemoveLabelContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Removing label(s) from Gmail email`, { - messageId: validatedData.messageId, - labelIds: validatedData.labelIds, - }) - - const labelIds = validatedData.labelIds - .split(',') - .map((id) => id.trim()) - .filter((id) => id.length > 0) - - for (const labelId of labelIds) { - const labelIdValidation = validateAlphanumericId(labelId, 'labelId', 255) - if (!labelIdValidation.isValid) { - logger.warn(`[${requestId}] Invalid label ID: ${labelIdValidation.error}`) - return NextResponse.json( - { - success: false, - error: labelIdValidation.error, - }, - { status: 400 } - ) - } - } - - const messageIdValidation = validateAlphanumericId(validatedData.messageId, 'messageId', 255) - if (!messageIdValidation.isValid) { - logger.warn(`[${requestId}] Invalid message ID: ${messageIdValidation.error}`) - return NextResponse.json( - { success: false, error: messageIdValidation.error }, - { status: 400 } - ) - } - - const gmailResponse = await fetch( - `${GMAIL_API_BASE}/messages/${validatedData.messageId}/modify`, - { - method: 'POST', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - removeLabelIds: labelIds, - }), - } - ) - - if (!gmailResponse.ok) { - const errorText = await gmailResponse.text() - logger.error(`[${requestId}] Gmail API error:`, errorText) - return NextResponse.json( - { - success: false, - error: `Gmail API error: ${gmailResponse.statusText}`, - }, - { status: gmailResponse.status } - ) - } - - const data = await gmailResponse.json() - - logger.info(`[${requestId}] Label(s) removed successfully`, { messageId: data.id }) - - return NextResponse.json({ - success: true, - output: { - content: `Successfully removed ${labelIds.length} label(s) from email`, - metadata: { - id: data.id, - threadId: data.threadId, - labelIds: data.labelIds, - }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error removing label from Gmail email:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/gmail/send/route.ts b/apps/sim/app/api/tools/gmail/send/route.ts deleted file mode 100644 index 62b10377e88..00000000000 --- a/apps/sim/app/api/tools/gmail/send/route.ts +++ /dev/null @@ -1,217 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { gmailSendContract } from '@/lib/api/contracts/tools/google' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { - base64UrlEncode, - buildMimeMessage, - buildSimpleEmailMessage, - fetchThreadingHeaders, -} from '@/tools/gmail/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('GmailSendAPI') - -const GMAIL_API_BASE = 'https://gmail.googleapis.com/gmail/v1/users/me' - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Gmail send attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - const userId = authResult.userId - logger.info(`[${requestId}] Authenticated Gmail send request via ${authResult.authType}`, { - userId, - }) - - const parsed = await parseRequest(gmailSendContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Sending Gmail email`, { - to: validatedData.to, - subject: validatedData.subject || '', - hasAttachments: !!(validatedData.attachments && validatedData.attachments.length > 0), - attachmentCount: validatedData.attachments?.length || 0, - }) - - const threadingHeaders = validatedData.replyToMessageId - ? await fetchThreadingHeaders(validatedData.replyToMessageId, validatedData.accessToken) - : {} - - const originalMessageId = threadingHeaders.messageId - const originalReferences = threadingHeaders.references - const originalSubject = threadingHeaders.subject - - let rawMessage: string | undefined - - if (validatedData.attachments && validatedData.attachments.length > 0) { - const rawAttachments = validatedData.attachments - logger.info(`[${requestId}] Processing ${rawAttachments.length} attachment(s)`) - - const attachments = processFilesToUserFiles(rawAttachments, requestId, logger) - - if (attachments.length === 0) { - logger.warn(`[${requestId}] No valid attachments found after processing`) - } else { - const totalSize = attachments.reduce((sum, file) => sum + file.size, 0) - const maxSize = 25 * 1024 * 1024 // 25MB - - if (totalSize > maxSize) { - const sizeMB = (totalSize / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`, - }, - { status: 400 } - ) - } - - const accessResults = await Promise.all( - attachments.map((file) => assertToolFileAccess(file.key, userId, requestId, logger)) - ) - const denied = accessResults.find((r) => r !== null) - if (denied) return denied - - let resolved: Array<{ buffer: Buffer; contentType: string }> - try { - resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, { - totalMaxBytes: maxSize, - label: 'Total attachment size', - }) - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - if (isPayloadSizeLimitError(error)) { - const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`, - }, - { status: 400 } - ) - } - logger.error(`[${requestId}] Failed to download an attachment:`, error) - return NextResponse.json( - { - success: false, - error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`, - }, - { status: 500 } - ) - } - - const attachmentBuffers = attachments.map((file, i) => ({ - filename: file.name, - mimeType: resolved[i].contentType || file.type || 'application/octet-stream', - content: resolved[i].buffer, - })) - - const mimeMessage = buildMimeMessage({ - to: validatedData.to, - cc: validatedData.cc ?? undefined, - bcc: validatedData.bcc ?? undefined, - subject: validatedData.subject || originalSubject || '', - body: validatedData.body, - contentType: validatedData.contentType || 'text', - inReplyTo: originalMessageId, - references: originalReferences, - attachments: attachmentBuffers, - }) - - logger.info(`[${requestId}] Built MIME message (${mimeMessage.length} bytes)`) - rawMessage = base64UrlEncode(mimeMessage) - } - } - - if (!rawMessage) { - rawMessage = buildSimpleEmailMessage({ - to: validatedData.to, - cc: validatedData.cc, - bcc: validatedData.bcc, - subject: validatedData.subject || originalSubject, - body: validatedData.body, - contentType: validatedData.contentType || 'text', - inReplyTo: originalMessageId, - references: originalReferences, - }) - } - - const requestBody: { raw: string; threadId?: string } = { raw: rawMessage } - - if (validatedData.threadId) { - requestBody.threadId = validatedData.threadId - } - - const gmailResponse = await fetch(`${GMAIL_API_BASE}/messages/send`, { - method: 'POST', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(requestBody), - }) - - if (!gmailResponse.ok) { - const errorText = await gmailResponse.text() - logger.error(`[${requestId}] Gmail API error:`, errorText) - return NextResponse.json( - { - success: false, - error: `Gmail API error: ${gmailResponse.statusText}`, - }, - { status: gmailResponse.status } - ) - } - - const data = await gmailResponse.json() - - logger.info(`[${requestId}] Email sent successfully`, { messageId: data.id }) - - return NextResponse.json({ - success: true, - output: { - content: 'Email sent successfully', - metadata: { - id: data.id, - threadId: data.threadId, - labelIds: data.labelIds, - }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error sending Gmail email:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/gmail/unarchive/route.ts b/apps/sim/app/api/tools/gmail/unarchive/route.ts deleted file mode 100644 index 3f81f633891..00000000000 --- a/apps/sim/app/api/tools/gmail/unarchive/route.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { gmailUnarchiveContract } from '@/lib/api/contracts/tools/google' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('GmailUnarchiveAPI') - -const GMAIL_API_BASE = 'https://gmail.googleapis.com/gmail/v1/users/me' - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Gmail unarchive attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated Gmail unarchive request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const parsed = await parseRequest(gmailUnarchiveContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Unarchiving Gmail email`, { - messageId: validatedData.messageId, - }) - - const gmailResponse = await fetch( - `${GMAIL_API_BASE}/messages/${validatedData.messageId}/modify`, - { - method: 'POST', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - addLabelIds: ['INBOX'], - }), - } - ) - - if (!gmailResponse.ok) { - const errorText = await gmailResponse.text() - logger.error(`[${requestId}] Gmail API error:`, errorText) - return NextResponse.json( - { - success: false, - error: `Gmail API error: ${gmailResponse.statusText}`, - }, - { status: gmailResponse.status } - ) - } - - const data = await gmailResponse.json() - - logger.info(`[${requestId}] Email unarchived successfully`, { messageId: data.id }) - - return NextResponse.json({ - success: true, - output: { - content: 'Email moved back to inbox successfully', - metadata: { - id: data.id, - threadId: data.threadId, - labelIds: data.labelIds, - }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error unarchiving Gmail email:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/google_drive/download/route.test.ts b/apps/sim/app/api/tools/google_drive/download/route.test.ts deleted file mode 100644 index a9ec28346de..00000000000 --- a/apps/sim/app/api/tools/google_drive/download/route.test.ts +++ /dev/null @@ -1,178 +0,0 @@ -/** - * @vitest-environment node - */ -import { - createMockRequest, - hybridAuthMockFns, - inputValidationMock, - inputValidationMockFns, -} from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' -import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' - -vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) - -import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { POST } from '@/app/api/tools/google_drive/download/route' - -const { mockValidateUrlWithDNS, mockSecureFetchWithPinnedIP } = inputValidationMockFns - -const PINNED_IP = '93.184.216.34' - -const baseBody = { - accessToken: 'token-123', - fileId: 'file-abc', -} - -function jsonResponse(body: unknown, ok = true) { - return { - ok, - status: ok ? 200 : 400, - statusText: '', - headers: new Headers(), - body: null, - text: async () => JSON.stringify(body), - json: async () => body, - arrayBuffer: async () => new ArrayBuffer(0), - } -} - -function fileResponse(bytes: number, ok = true) { - return { - ok, - status: ok ? 200 : 400, - statusText: '', - headers: new Headers(), - body: null, - text: async () => '', - json: async () => ({}), - arrayBuffer: async () => new ArrayBuffer(bytes), - } -} - -beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - mockValidateUrlWithDNS.mockResolvedValue({ - isValid: true, - resolvedIP: PINNED_IP, - originalHostname: 'www.googleapis.com', - }) -}) - -describe('POST /api/tools/google_drive/download', () => { - it('downloads a normal file under the size cap', async () => { - mockSecureFetchWithPinnedIP - .mockResolvedValueOnce( - jsonResponse({ - id: 'file-abc', - name: 'report.pdf', - mimeType: 'application/pdf', - size: '1024', - capabilities: { canReadRevisions: false }, - }) - ) - .mockResolvedValueOnce(fileResponse(1024)) - - const response = await POST(createMockRequest('POST', baseBody)) - expect(response.status).toBe(200) - const data = (await response.json()) as { success: boolean; output: { file: { size: number } } } - expect(data.success).toBe(true) - expect(data.output.file.size).toBe(1024) - - const downloadCall = mockSecureFetchWithPinnedIP.mock.calls[1] - expect(downloadCall[2]).toMatchObject({ maxResponseBytes: MAX_FILE_SIZE }) - }) - - it('rejects the download before fetching content when metadata size exceeds the cap', async () => { - mockSecureFetchWithPinnedIP.mockResolvedValueOnce( - jsonResponse({ - id: 'file-abc', - name: 'huge.bin', - mimeType: 'application/octet-stream', - size: String(MAX_FILE_SIZE + 1), - capabilities: { canReadRevisions: false }, - }) - ) - - const response = await POST(createMockRequest('POST', baseBody)) - expect(response.status).toBe(413) - const data = (await response.json()) as { success: boolean; error: string } - expect(data.success).toBe(false) - expect(data.error).toContain('exceeds maximum size') - - // Content download must never be initiated once metadata size trips the check. - expect(mockSecureFetchWithPinnedIP).toHaveBeenCalledTimes(1) - }) - - it('surfaces a clean 413 when the streamed content exceeds the cap', async () => { - mockSecureFetchWithPinnedIP - .mockResolvedValueOnce( - jsonResponse({ - id: 'file-abc', - name: 'report.pdf', - mimeType: 'application/pdf', - capabilities: { canReadRevisions: false }, - }) - ) - .mockRejectedValueOnce( - new PayloadSizeLimitError({ - label: 'response body', - maxBytes: MAX_FILE_SIZE, - observedBytes: MAX_FILE_SIZE + 1, - }) - ) - - const response = await POST(createMockRequest('POST', baseBody)) - expect(response.status).toBe(413) - const data = (await response.json()) as { success: boolean } - expect(data.success).toBe(false) - }) - - it('proceeds to the streamed download when metadata size is malformed', async () => { - mockSecureFetchWithPinnedIP - .mockResolvedValueOnce( - jsonResponse({ - id: 'file-abc', - name: 'report.pdf', - mimeType: 'application/pdf', - size: 'not-a-number', - capabilities: { canReadRevisions: false }, - }) - ) - .mockResolvedValueOnce(fileResponse(1024)) - - const response = await POST(createMockRequest('POST', baseBody)) - expect(response.status).toBe(200) - const data = (await response.json()) as { success: boolean; output: { file: { size: number } } } - expect(data.success).toBe(true) - expect(data.output.file.size).toBe(1024) - - // The early size check should be skipped, but the streaming cap must still apply. - const downloadCall = mockSecureFetchWithPinnedIP.mock.calls[1] - expect(downloadCall[2]).toMatchObject({ maxResponseBytes: MAX_FILE_SIZE }) - }) - - it('does not require a metadata size for Google Workspace exports', async () => { - mockSecureFetchWithPinnedIP - .mockResolvedValueOnce( - jsonResponse({ - id: 'doc-1', - name: 'My Doc', - mimeType: 'application/vnd.google-apps.document', - capabilities: { canReadRevisions: false }, - }) - ) - .mockResolvedValueOnce(fileResponse(2048)) - - const response = await POST(createMockRequest('POST', baseBody)) - expect(response.status).toBe(200) - - const exportCall = mockSecureFetchWithPinnedIP.mock.calls[1] - expect(exportCall[2]).toMatchObject({ maxResponseBytes: MAX_FILE_SIZE }) - }) -}) diff --git a/apps/sim/app/api/tools/google_drive/download/route.ts b/apps/sim/app/api/tools/google_drive/download/route.ts deleted file mode 100644 index 788a7de418b..00000000000 --- a/apps/sim/app/api/tools/google_drive/download/route.ts +++ /dev/null @@ -1,289 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { googleDriveDownloadContract } from '@/lib/api/contracts/tools/google' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { assertKnownSizeWithinLimit, isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' -import type { GoogleDriveFile, GoogleDriveRevision } from '@/tools/google_drive/types' -import { - ALL_FILE_FIELDS, - ALL_REVISION_FIELDS, - DEFAULT_EXPORT_FORMATS, - GOOGLE_WORKSPACE_MIME_TYPES, - VALID_EXPORT_FORMATS, -} from '@/tools/google_drive/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('GoogleDriveDownloadAPI') - -/** Google API error response structure */ -interface GoogleApiErrorResponse { - error?: { - message?: string - code?: number - status?: string - } -} - -/** Google Drive revisions list response */ -interface GoogleDriveRevisionsResponse { - revisions?: GoogleDriveRevision[] - nextPageToken?: string -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Google Drive download attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - googleDriveDownloadContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Invalid request') }, - { status: 400 } - ), - } - ) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const { - accessToken, - fileId, - mimeType: rawExportMimeType, - fileName, - includeRevisions, - } = validatedData - const exportMimeType = - rawExportMimeType && rawExportMimeType !== 'auto' ? rawExportMimeType : null - const authHeader = `Bearer ${accessToken}` - - logger.info(`[${requestId}] Getting file metadata from Google Drive`, { fileId }) - - const metadataUrl = `https://www.googleapis.com/drive/v3/files/${fileId}?fields=${ALL_FILE_FIELDS}&supportsAllDrives=true` - const metadataUrlValidation = await validateUrlWithDNS(metadataUrl, 'metadataUrl') - if (!metadataUrlValidation.isValid) { - return NextResponse.json( - { success: false, error: metadataUrlValidation.error }, - { status: 400 } - ) - } - - const metadataResponse = await secureFetchWithPinnedIP( - metadataUrl, - metadataUrlValidation.resolvedIP!, - { - headers: { Authorization: authHeader }, - } - ) - - if (!metadataResponse.ok) { - const errorDetails = (await metadataResponse - .json() - .catch(() => ({}))) as GoogleApiErrorResponse - logger.error(`[${requestId}] Failed to get file metadata`, { - status: metadataResponse.status, - error: errorDetails, - }) - return NextResponse.json( - { success: false, error: errorDetails.error?.message || 'Failed to get file metadata' }, - { status: 400 } - ) - } - - const metadata = (await metadataResponse.json()) as GoogleDriveFile - const fileMimeType = metadata.mimeType - - let fileBuffer: Buffer - let finalMimeType = fileMimeType - - if (GOOGLE_WORKSPACE_MIME_TYPES.includes(fileMimeType)) { - const exportFormat = exportMimeType || DEFAULT_EXPORT_FORMATS[fileMimeType] || 'text/plain' - - const validFormats = VALID_EXPORT_FORMATS[fileMimeType] - if (validFormats && !validFormats.includes(exportFormat)) { - logger.warn(`[${requestId}] Unsupported export format requested`, { - fileId, - fileMimeType, - requestedFormat: exportFormat, - validFormats, - }) - return NextResponse.json( - { - success: false, - error: `Export format "${exportFormat}" is not supported for this file type. Supported formats: ${validFormats.join(', ')}`, - }, - { status: 400 } - ) - } - - finalMimeType = exportFormat - - logger.info(`[${requestId}] Exporting Google Workspace file`, { - fileId, - mimeType: fileMimeType, - exportFormat, - }) - - const exportUrl = `https://www.googleapis.com/drive/v3/files/${fileId}/export?mimeType=${encodeURIComponent(exportFormat)}&supportsAllDrives=true` - const exportUrlValidation = await validateUrlWithDNS(exportUrl, 'exportUrl') - if (!exportUrlValidation.isValid) { - return NextResponse.json( - { success: false, error: exportUrlValidation.error }, - { status: 400 } - ) - } - - const exportResponse = await secureFetchWithPinnedIP( - exportUrl, - exportUrlValidation.resolvedIP!, - { headers: { Authorization: authHeader }, maxResponseBytes: MAX_FILE_SIZE } - ) - - if (!exportResponse.ok) { - const exportError = (await exportResponse - .json() - .catch(() => ({}))) as GoogleApiErrorResponse - logger.error(`[${requestId}] Failed to export file`, { - status: exportResponse.status, - error: exportError, - }) - return NextResponse.json( - { - success: false, - error: exportError.error?.message || 'Failed to export Google Workspace file', - }, - { status: 400 } - ) - } - - const arrayBuffer = await exportResponse.arrayBuffer() - fileBuffer = Buffer.from(arrayBuffer) - } else { - logger.info(`[${requestId}] Downloading regular file`, { fileId, mimeType: fileMimeType }) - - if (metadata.size) { - const parsedSize = Number.parseInt(metadata.size, 10) - if (Number.isFinite(parsedSize)) { - assertKnownSizeWithinLimit(parsedSize, MAX_FILE_SIZE, `Google Drive file ${fileId}`) - } - } - - const downloadUrl = `https://www.googleapis.com/drive/v3/files/${fileId}?alt=media&supportsAllDrives=true` - const downloadUrlValidation = await validateUrlWithDNS(downloadUrl, 'downloadUrl') - if (!downloadUrlValidation.isValid) { - return NextResponse.json( - { success: false, error: downloadUrlValidation.error }, - { status: 400 } - ) - } - - const downloadResponse = await secureFetchWithPinnedIP( - downloadUrl, - downloadUrlValidation.resolvedIP!, - { headers: { Authorization: authHeader }, maxResponseBytes: MAX_FILE_SIZE } - ) - - if (!downloadResponse.ok) { - const downloadError = (await downloadResponse - .json() - .catch(() => ({}))) as GoogleApiErrorResponse - logger.error(`[${requestId}] Failed to download file`, { - status: downloadResponse.status, - error: downloadError, - }) - return NextResponse.json( - { success: false, error: downloadError.error?.message || 'Failed to download file' }, - { status: 400 } - ) - } - - const arrayBuffer = await downloadResponse.arrayBuffer() - fileBuffer = Buffer.from(arrayBuffer) - } - - const canReadRevisions = metadata.capabilities?.canReadRevisions === true - if (includeRevisions && canReadRevisions) { - try { - const revisionsUrl = `https://www.googleapis.com/drive/v3/files/${fileId}/revisions?fields=revisions(${ALL_REVISION_FIELDS})&pageSize=100` - const revisionsUrlValidation = await validateUrlWithDNS(revisionsUrl, 'revisionsUrl') - if (revisionsUrlValidation.isValid) { - const revisionsResponse = await secureFetchWithPinnedIP( - revisionsUrl, - revisionsUrlValidation.resolvedIP!, - { headers: { Authorization: authHeader } } - ) - - if (revisionsResponse.ok) { - const revisionsData = (await revisionsResponse.json()) as GoogleDriveRevisionsResponse - metadata.revisions = revisionsData.revisions - logger.info(`[${requestId}] Fetched file revisions`, { - fileId, - revisionCount: metadata.revisions?.length || 0, - }) - } - } - } catch (error) { - logger.warn(`[${requestId}] Error fetching revisions, continuing without them`, { error }) - } - } - - const resolvedName = fileName || metadata.name || 'download' - - logger.info(`[${requestId}] File downloaded successfully`, { - fileId, - name: resolvedName, - size: fileBuffer.length, - mimeType: finalMimeType, - }) - - const base64Data = fileBuffer.toString('base64') - - return NextResponse.json({ - success: true, - output: { - file: { - name: resolvedName, - mimeType: finalMimeType, - data: base64Data, - size: fileBuffer.length, - }, - metadata, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error downloading Google Drive file:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/google_drive/export/route.ts b/apps/sim/app/api/tools/google_drive/export/route.ts deleted file mode 100644 index 40b48057def..00000000000 --- a/apps/sim/app/api/tools/google_drive/export/route.ts +++ /dev/null @@ -1,209 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { googleDriveExportContract } from '@/lib/api/contracts/tools/google' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { GoogleDriveFile } from '@/tools/google_drive/types' -import { - ALL_FILE_FIELDS, - GOOGLE_WORKSPACE_MIME_TYPES, - MAX_EXPORT_BYTES, - VALID_EXPORT_FORMATS, -} from '@/tools/google_drive/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('GoogleDriveExportAPI') - -/** Google API error response structure */ -interface GoogleApiErrorResponse { - error?: { - message?: string - code?: number - status?: string - } -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Google Drive export attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - googleDriveExportContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Invalid request') }, - { status: 400 } - ), - } - ) - if (!parsed.success) return parsed.response - - const { accessToken, fileId, mimeType: exportMimeType, fileName } = parsed.data.body - const authHeader = `Bearer ${accessToken}` - - logger.info(`[${requestId}] Getting file metadata from Google Drive`, { fileId }) - - const metadataUrl = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}?fields=${ALL_FILE_FIELDS}&supportsAllDrives=true` - const metadataUrlValidation = await validateUrlWithDNS(metadataUrl, 'metadataUrl') - if (!metadataUrlValidation.isValid) { - return NextResponse.json( - { success: false, error: metadataUrlValidation.error }, - { status: 400 } - ) - } - - const metadataResponse = await secureFetchWithPinnedIP( - metadataUrl, - metadataUrlValidation.resolvedIP!, - { headers: { Authorization: authHeader } } - ) - - if (!metadataResponse.ok) { - const errorDetails = (await metadataResponse - .json() - .catch(() => ({}))) as GoogleApiErrorResponse - logger.error(`[${requestId}] Failed to get file metadata`, { - status: metadataResponse.status, - error: errorDetails, - }) - return NextResponse.json( - { success: false, error: errorDetails.error?.message || 'Failed to get file metadata' }, - { status: 400 } - ) - } - - const metadata = (await metadataResponse.json()) as GoogleDriveFile - const fileMimeType = metadata.mimeType - - if (!GOOGLE_WORKSPACE_MIME_TYPES.includes(fileMimeType)) { - return NextResponse.json( - { - success: false, - error: `Export only supports Google Workspace files (Docs, Sheets, Slides, Drawings). This file is "${fileMimeType}" — use the Download operation instead.`, - }, - { status: 400 } - ) - } - - const validFormats = VALID_EXPORT_FORMATS[fileMimeType] - if (validFormats && !validFormats.includes(exportMimeType)) { - return NextResponse.json( - { - success: false, - error: `Export format "${exportMimeType}" is not supported for this file type. Supported formats: ${validFormats.join(', ')}`, - }, - { status: 400 } - ) - } - - logger.info(`[${requestId}] Exporting Google Workspace file`, { - fileId, - mimeType: fileMimeType, - exportFormat: exportMimeType, - }) - - const exportUrl = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}/export?mimeType=${encodeURIComponent(exportMimeType)}` - const exportUrlValidation = await validateUrlWithDNS(exportUrl, 'exportUrl') - if (!exportUrlValidation.isValid) { - return NextResponse.json( - { success: false, error: exportUrlValidation.error }, - { status: 400 } - ) - } - - const exportResponse = await secureFetchWithPinnedIP( - exportUrl, - exportUrlValidation.resolvedIP!, - { - headers: { Authorization: authHeader }, - } - ) - - if (!exportResponse.ok) { - const exportError = (await exportResponse.json().catch(() => ({}))) as GoogleApiErrorResponse - logger.error(`[${requestId}] Failed to export file`, { - status: exportResponse.status, - error: exportError, - }) - return NextResponse.json( - { - success: false, - error: exportError.error?.message || 'Failed to export Google Workspace file', - }, - { status: 400 } - ) - } - - const declaredSize = Number(exportResponse.headers.get('content-length')) - if (Number.isFinite(declaredSize) && declaredSize > MAX_EXPORT_BYTES) { - return NextResponse.json( - { - success: false, - error: `Exported content (${declaredSize} bytes) exceeds the ${MAX_EXPORT_BYTES}-byte export limit.`, - }, - { status: 400 } - ) - } - - const arrayBuffer = await exportResponse.arrayBuffer() - if (arrayBuffer.byteLength > MAX_EXPORT_BYTES) { - return NextResponse.json( - { - success: false, - error: `Exported content (${arrayBuffer.byteLength} bytes) exceeds the ${MAX_EXPORT_BYTES}-byte export limit.`, - }, - { status: 400 } - ) - } - const fileBuffer = Buffer.from(arrayBuffer) - - const resolvedName = fileName || metadata.name || 'export' - - logger.info(`[${requestId}] File exported successfully`, { - fileId, - name: resolvedName, - size: fileBuffer.length, - mimeType: exportMimeType, - }) - - return NextResponse.json({ - success: true, - output: { - file: { - name: resolvedName, - mimeType: exportMimeType, - data: fileBuffer.toString('base64'), - size: fileBuffer.length, - }, - exportedMimeType: exportMimeType, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error exporting Google Drive file:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error occurred') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/google_drive/upload/route.ts b/apps/sim/app/api/tools/google_drive/upload/route.ts deleted file mode 100644 index ca134da06cc..00000000000 --- a/apps/sim/app/api/tools/google_drive/upload/route.ts +++ /dev/null @@ -1,300 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateShortId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { googleDriveUploadContract } from '@/lib/api/contracts/tools/google' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { - GOOGLE_WORKSPACE_MIME_TYPES, - handleSheetsFormat, - SOURCE_MIME_TYPES, -} from '@/tools/google_drive/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('GoogleDriveUploadAPI') - -const GOOGLE_DRIVE_API_BASE = 'https://www.googleapis.com/upload/drive/v3/files' - -/** - * Build multipart upload body for Google Drive API - */ -function buildMultipartBody( - metadata: Record, - fileBuffer: Buffer, - mimeType: string, - boundary: string -): string { - const parts: string[] = [] - - parts.push(`--${boundary}`) - parts.push('Content-Type: application/json; charset=UTF-8') - parts.push('') - parts.push(JSON.stringify(metadata)) - - parts.push(`--${boundary}`) - parts.push(`Content-Type: ${mimeType}`) - parts.push('Content-Transfer-Encoding: base64') - parts.push('') - parts.push(fileBuffer.toString('base64')) - - parts.push(`--${boundary}--`) - - return parts.join('\r\n') -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Google Drive upload attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info( - `[${requestId}] Authenticated Google Drive upload request via ${authResult.authType}`, - { - userId: authResult.userId, - } - ) - - const parsed = await parseRequest(googleDriveUploadContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Uploading file to Google Drive`, { - fileName: validatedData.fileName, - mimeType: validatedData.mimeType, - folderId: validatedData.folderId, - hasFile: !!validatedData.file, - }) - - if (!validatedData.file) { - return NextResponse.json( - { - success: false, - error: 'No file provided. Use the text content field for text-only uploads.', - }, - { status: 400 } - ) - } - - // Process file - convert to UserFile format if needed - const fileData = validatedData.file - - let userFile - try { - userFile = processSingleFileToUserFile(fileData, requestId, logger) - } catch (error) { - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Failed to process file'), - }, - { status: 400 } - ) - } - - logger.info(`[${requestId}] Downloading file from storage`, { - fileName: userFile.name, - key: userFile.key, - size: userFile.size, - }) - - const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) - if (denied) return denied - - let fileBuffer: Buffer - let downloadedContentType = '' - - try { - const result = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - }) - fileBuffer = result.buffer - downloadedContentType = result.contentType - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - logger.error(`[${requestId}] Failed to download file:`, error) - return NextResponse.json( - { - success: false, - error: `Failed to download file: ${getErrorMessage(error, 'Unknown error')}`, - }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } - - let uploadMimeType = - validatedData.mimeType || downloadedContentType || userFile.type || 'application/octet-stream' - const requestedMimeType = - validatedData.mimeType || downloadedContentType || userFile.type || 'application/octet-stream' - - if (GOOGLE_WORKSPACE_MIME_TYPES.includes(requestedMimeType)) { - uploadMimeType = SOURCE_MIME_TYPES[requestedMimeType] || 'text/plain' - logger.info(`[${requestId}] Converting to Google Workspace type`, { - requestedMimeType, - uploadMimeType, - }) - } - - if (requestedMimeType === 'application/vnd.google-apps.spreadsheet') { - try { - const textContent = fileBuffer.toString('utf-8') - const { csv } = handleSheetsFormat(textContent) - if (csv !== undefined) { - fileBuffer = Buffer.from(csv, 'utf-8') - uploadMimeType = 'text/csv' - logger.info(`[${requestId}] Converted to CSV for Google Sheets upload`) - } - } catch (error) { - logger.warn(`[${requestId}] Could not convert to CSV, uploading as-is:`, error) - } - } - - const metadata: { - name: string - mimeType: string - parents?: string[] - } = { - name: validatedData.fileName, - mimeType: requestedMimeType, - } - - if (validatedData.folderId && validatedData.folderId.trim() !== '') { - metadata.parents = [validatedData.folderId.trim()] - } - - const boundary = `boundary_${Date.now()}_${generateShortId(7)}` - - const multipartBody = buildMultipartBody(metadata, fileBuffer, uploadMimeType, boundary) - - logger.info(`[${requestId}] Uploading to Google Drive via multipart upload`, { - fileName: validatedData.fileName, - size: fileBuffer.length, - uploadMimeType, - requestedMimeType, - }) - - const uploadResponse = await fetch( - `${GOOGLE_DRIVE_API_BASE}?uploadType=multipart&supportsAllDrives=true`, - { - method: 'POST', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': `multipart/related; boundary=${boundary}`, - 'Content-Length': Buffer.byteLength(multipartBody, 'utf-8').toString(), - }, - body: multipartBody, - } - ) - - if (!uploadResponse.ok) { - const errorText = await uploadResponse.text() - logger.error(`[${requestId}] Google Drive API error:`, { - status: uploadResponse.status, - statusText: uploadResponse.statusText, - error: errorText, - }) - return NextResponse.json( - { - success: false, - error: `Google Drive API error: ${uploadResponse.statusText}`, - }, - { status: uploadResponse.status } - ) - } - - const uploadData = await uploadResponse.json() - const fileId = uploadData.id - - logger.info(`[${requestId}] File uploaded successfully`, { fileId }) - - if (GOOGLE_WORKSPACE_MIME_TYPES.includes(requestedMimeType)) { - logger.info(`[${requestId}] Updating file name to ensure it persists after conversion`) - - const updateNameResponse = await fetch( - `https://www.googleapis.com/drive/v3/files/${fileId}?supportsAllDrives=true`, - { - method: 'PATCH', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - name: validatedData.fileName, - }), - } - ) - - if (!updateNameResponse.ok) { - logger.warn( - `[${requestId}] Failed to update filename after conversion, but content was uploaded` - ) - } - } - - const finalFileResponse = await fetch( - `https://www.googleapis.com/drive/v3/files/${fileId}?supportsAllDrives=true&fields=id,name,mimeType,webViewLink,webContentLink,size,createdTime,modifiedTime,parents`, - { - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - }, - } - ) - - const finalFile = await finalFileResponse.json() - - logger.info(`[${requestId}] Upload complete`, { - fileId: finalFile.id, - fileName: finalFile.name, - webViewLink: finalFile.webViewLink, - }) - - return NextResponse.json({ - success: true, - output: { - file: { - id: finalFile.id, - name: finalFile.name, - mimeType: finalFile.mimeType, - webViewLink: finalFile.webViewLink, - webContentLink: finalFile.webContentLink, - size: finalFile.size, - createdTime: finalFile.createdTime, - modifiedTime: finalFile.modifiedTime, - parents: finalFile.parents, - }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error uploading file to Google Drive:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/google_slides/export-presentation/route.ts b/apps/sim/app/api/tools/google_slides/export-presentation/route.ts deleted file mode 100644 index 5c36785f8eb..00000000000 --- a/apps/sim/app/api/tools/google_slides/export-presentation/route.ts +++ /dev/null @@ -1,176 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { googleSlidesExportPresentationContract } from '@/lib/api/contracts/tools/google' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { - DEFAULT_MAX_ERROR_BODY_BYTES, - isPayloadSizeLimitError, - readResponseTextWithLimit, - readResponseToBufferWithLimit, -} from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { uploadCopilotFile } from '@/lib/uploads/contexts/copilot' -import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' -import { presentationUrl } from '@/tools/google_slides/utils' - -const logger = createLogger('GoogleSlidesExportAPI') -const MAX_GOOGLE_SLIDES_EXPORT_BYTES = 10 * 1024 * 1024 -const MAX_LEGACY_INLINE_EXPORT_BYTES = 7 * 1024 * 1024 - -const FORMAT_TO_MIME = { - PDF: 'application/pdf', - PPTX: 'application/vnd.openxmlformats-officedocument.presentationml.presentation', - ODP: 'application/vnd.oasis.opendocument.presentation', - TXT: 'text/plain', - PNG: 'image/png', - JPEG: 'image/jpeg', - SVG: 'image/svg+xml', -} as const - -export const dynamic = 'force-dynamic' - -function buildExportUrl(presentationId: string, exportFormat: keyof typeof FORMAT_TO_MIME): string { - const mimeType = FORMAT_TO_MIME[exportFormat] - return `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(presentationId)}/export?mimeType=${encodeURIComponent(mimeType)}` -} - -function buildExportFilename( - presentationId: string, - exportFormat: keyof typeof FORMAT_TO_MIME -): string { - return `${presentationId}.${exportFormat.toLowerCase()}` -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest( - googleSlidesExportPresentationContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Invalid request data') }, - { status: 400 } - ), - } - ) - if (!parsed.success) return parsed.response - - try { - const body = parsed.data.body - const exportFormat = body.exportFormat ?? 'PDF' - const mimeType = FORMAT_TO_MIME[exportFormat] - const exportUrl = buildExportUrl(body.presentationId, exportFormat) - const urlValidation = await validateUrlWithDNS(exportUrl, 'googleSlidesExportUrl') - if (!urlValidation.isValid) { - return NextResponse.json( - { success: false, error: urlValidation.error || 'Invalid Google Slides export URL' }, - { status: 400 } - ) - } - - const response = await secureFetchWithPinnedIP(exportUrl, urlValidation.resolvedIP!, { - headers: { Authorization: `Bearer ${body.accessToken}` }, - maxResponseBytes: MAX_GOOGLE_SLIDES_EXPORT_BYTES, - }) - - if (!response.ok) { - const errorText = await readResponseTextWithLimit(response, { - maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, - label: 'Google Slides export error response', - }).catch(() => '') - return NextResponse.json( - { - success: false, - error: `Failed to export presentation: ${response.status} ${errorText}`, - }, - { status: response.status } - ) - } - - const buffer = await readResponseToBufferWithLimit(response, { - maxBytes: MAX_GOOGLE_SLIDES_EXPORT_BYTES, - label: 'Google Slides export response', - }) - const filename = buildExportFilename(body.presentationId, exportFormat) - const legacyInlineContent = - buffer.length <= MAX_LEGACY_INLINE_EXPORT_BYTES - ? { contentBase64: buffer.toString('base64') } - : {} - const executionContext = - body.workspaceId && body.workflowId && body.executionId - ? { - workspaceId: body.workspaceId, - workflowId: body.workflowId, - executionId: body.executionId, - } - : undefined - - if (executionContext) { - const file = await uploadExecutionFile( - executionContext, - buffer, - filename, - mimeType, - authResult.userId - ) - return NextResponse.json({ - success: true, - output: { - file: { ...file, mimeType }, - exportFormat, - mimeType, - sizeBytes: buffer.length, - exportUrl: file.url, - ...legacyInlineContent, - metadata: { - presentationId: body.presentationId, - url: presentationUrl(body.presentationId), - exportFormat, - }, - }, - }) - } - - const file = await uploadCopilotFile({ - buffer, - fileName: filename, - contentType: mimeType, - userId: authResult.userId, - }) - - return NextResponse.json({ - success: true, - output: { - file, - exportUrl: file.url, - exportFormat, - mimeType, - sizeBytes: buffer.length, - ...legacyInlineContent, - metadata: { - presentationId: body.presentationId, - url: presentationUrl(body.presentationId), - exportFormat, - }, - }, - }) - } catch (error) { - logger.error('Google Slides export failed', { error }) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Failed to export presentation') }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/google_vault/download-export-file/route.ts b/apps/sim/app/api/tools/google_vault/download-export-file/route.ts deleted file mode 100644 index 9a492698afa..00000000000 --- a/apps/sim/app/api/tools/google_vault/download-export-file/route.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { googleVaultDownloadExportFileContract } from '@/lib/api/contracts/tools/google' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { enhanceGoogleVaultError } from '@/tools/google_vault/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('GoogleVaultDownloadExportFileAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Google Vault download attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - const parsed = await parseRequest(googleVaultDownloadExportFileContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const { accessToken, bucketName, objectName, fileName } = validatedData - - const bucket = encodeURIComponent(bucketName) - const object = encodeURIComponent(objectName) - const downloadUrl = `https://storage.googleapis.com/storage/v1/b/${bucket}/o/${object}?alt=media` - - logger.info(`[${requestId}] Downloading file from Google Vault`, { bucketName, objectName }) - - const urlValidation = await validateUrlWithDNS(downloadUrl, 'downloadUrl') - if (!urlValidation.isValid) { - return NextResponse.json( - { success: false, error: enhanceGoogleVaultError(urlValidation.error || 'Invalid URL') }, - { status: 400 } - ) - } - - const downloadResponse = await secureFetchWithPinnedIP(downloadUrl, urlValidation.resolvedIP!, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!downloadResponse.ok) { - const errorText = await downloadResponse.text().catch(() => '') - const errorMessage = `Failed to download file: ${errorText || downloadResponse.statusText}` - logger.error(`[${requestId}] Failed to download Vault export file`, { - status: downloadResponse.status, - error: errorText, - }) - return NextResponse.json( - { success: false, error: enhanceGoogleVaultError(errorMessage) }, - { status: 400 } - ) - } - - const contentType = downloadResponse.headers.get('content-type') || 'application/octet-stream' - const disposition = downloadResponse.headers.get('content-disposition') || '' - const match = disposition.match(/filename\*=UTF-8''([^;]+)|filename="([^"]+)"/) - - let resolvedName = fileName - if (!resolvedName) { - if (match?.[1]) { - try { - resolvedName = decodeURIComponent(match[1]) - } catch { - resolvedName = match[1] - } - } else if (match?.[2]) { - resolvedName = match[2] - } else if (objectName) { - const parts = objectName.split('/') - resolvedName = parts[parts.length - 1] || 'vault-export.bin' - } else { - resolvedName = 'vault-export.bin' - } - } - - const arrayBuffer = await downloadResponse.arrayBuffer() - const buffer = Buffer.from(arrayBuffer) - - logger.info(`[${requestId}] Vault export file downloaded successfully`, { - name: resolvedName, - size: buffer.length, - mimeType: contentType, - }) - - return NextResponse.json({ - success: true, - output: { - file: { - name: resolvedName, - mimeType: contentType, - data: buffer.toString('base64'), - size: buffer.length, - }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error downloading Google Vault export file:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/grafana/check_data_source_health/route.test.ts b/apps/sim/app/api/tools/grafana/check_data_source_health/route.test.ts deleted file mode 100644 index f92fbe8c0de..00000000000 --- a/apps/sim/app/api/tools/grafana/check_data_source_health/route.test.ts +++ /dev/null @@ -1,131 +0,0 @@ -/** - * @vitest-environment node - */ -import { createMockRequest, hybridAuthMockFns } from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { mockSecureFetch, mockValidateUrl, MOCK_MAX_JSON_BYTES } = vi.hoisted(() => ({ - mockSecureFetch: vi.fn(), - mockValidateUrl: vi.fn(), - MOCK_MAX_JSON_BYTES: 10 * 1024 * 1024, -})) - -vi.mock('@/lib/core/security/input-validation.server', () => ({ - secureFetchWithPinnedIP: mockSecureFetch, - validateUrlWithDNS: mockValidateUrl, - MAX_JSON_API_RESPONSE_BYTES: MOCK_MAX_JSON_BYTES, -})) - -import { POST } from '@/app/api/tools/grafana/check_data_source_health/route' - -const baseBody = { - apiKey: 'glsa_token', - baseUrl: 'https://grafana.example.com', - dataSourceUid: 'P1234AB5678', -} - -function grafanaResponse(body: unknown, status: number) { - return { - ok: status >= 200 && status < 300, - status, - statusText: '', - headers: new Headers(), - text: async () => (typeof body === 'string' ? body : JSON.stringify(body)), - } -} - -function post(body: Record = baseBody) { - return POST(createMockRequest('POST', body) as never, undefined as never) -} - -describe('POST /api/tools/grafana/check_data_source_health', () => { - beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ success: true, userId: 'user-1' }) - mockValidateUrl.mockResolvedValue({ isValid: true, resolvedIP: '203.0.113.10' }) - }) - - it('reports a healthy data source', async () => { - mockSecureFetch.mockResolvedValue( - grafanaResponse({ status: 'OK', message: 'Data source is working' }, 200) - ) - - const response = await post() - const data = await response.json() - - expect(data.success).toBe(true) - expect(data.output).toEqual({ status: 'OK', message: 'Data source is working' }) - }) - - it('reports an UNHEALTHY data source, which Grafana answers with HTTP 400', async () => { - mockSecureFetch.mockResolvedValue( - grafanaResponse({ status: 'ERROR', message: 'dial tcp: connection refused' }, 400) - ) - - const response = await post() - const data = await response.json() - - expect(data.success).toBe(true) - expect(data.output.status).toBe('ERROR') - expect(data.output.message).toBe('dial tcp: connection refused') - }) - - it('surfaces the plugin details when Grafana supplies them', async () => { - mockSecureFetch.mockResolvedValue( - grafanaResponse( - { status: 'ERROR', message: 'bad query', details: { verboseMessage: 'x' } }, - 400 - ) - ) - - const response = await post() - const data = await response.json() - - expect(data.output.details).toEqual({ verboseMessage: 'x' }) - }) - - it('treats a failure with no health verdict as a real request failure', async () => { - mockSecureFetch.mockResolvedValue(grafanaResponse({ message: 'Data source not found' }, 404)) - - const response = await post() - const data = await response.json() - - expect(data.success).toBe(false) - expect(data.error).toContain('404') - }) - - it('bounds and protects the outbound call', async () => { - mockSecureFetch.mockResolvedValue(grafanaResponse({ status: 'OK', message: 'ok' }, 200)) - - await post() - - const [url, resolvedIP, options] = mockSecureFetch.mock.calls[0] - expect(resolvedIP).toBe('203.0.113.10') - expect(url).toBe('https://grafana.example.com/api/datasources/uid/P1234AB5678/health') - expect(options.maxResponseBytes).toBe(MOCK_MAX_JSON_BYTES) - expect(options.timeout).toBeGreaterThan(0) - expect(options.stripAuthOnRedirect).toBe(true) - expect(options.headers.Authorization).toBe('Bearer glsa_token') - }) - - it('encodes the UID so it cannot re-target the request path', async () => { - mockSecureFetch.mockResolvedValue(grafanaResponse({ status: 'OK', message: 'ok' }, 200)) - - await post({ ...baseBody, dataSourceUid: 'a/../../admin' }) - - const [url] = mockSecureFetch.mock.calls[0] - expect(url).toBe('https://grafana.example.com/api/datasources/uid/a%2F..%2F..%2Fadmin/health') - }) - - it('rejects an unauthenticated request before reaching Grafana', async () => { - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: false, - error: 'Authentication required', - }) - - const response = await post() - - expect(response.status).toBe(401) - expect(mockSecureFetch).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/app/api/tools/grafana/check_data_source_health/route.ts b/apps/sim/app/api/tools/grafana/check_data_source_health/route.ts deleted file mode 100644 index 970fce774f5..00000000000 --- a/apps/sim/app/api/tools/grafana/check_data_source_health/route.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { truncate } from '@sim/utils/string' -import { type NextRequest, NextResponse } from 'next/server' -import { grafanaCheckDataSourceHealthContract } from '@/lib/api/contracts/tools/grafana' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - MAX_JSON_API_RESPONSE_BYTES, - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('GrafanaCheckDataSourceHealthAPI') - -const OUTBOUND_FETCH_TIMEOUT_MS = 30_000 -const MAX_ERROR_MESSAGE_LENGTH = 2000 - -/** - * Runs a data source health check. - * - * Grafana answers an *unhealthy* data source with HTTP 400 carrying the same - * `{status, message}` payload it uses for a healthy one, so the diagnostic the - * caller actually wants only exists on the failure status. A plain tool would - * have that converted into an opaque tool error, making the check able to report - * health and never ill-health — hence this route, which reads the payload off - * either status and reports it as a successful check. - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Grafana health check: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - grafanaCheckDataSourceHealthContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const baseUrl = params.baseUrl.replace(/\/$/, '') - const healthUrl = `${baseUrl}/api/datasources/uid/${encodeURIComponent( - params.dataSourceUid.trim() - )}/health` - - const urlValidation = await validateUrlWithDNS(healthUrl, 'baseUrl') - if (!urlValidation.isValid || !urlValidation.resolvedIP) { - return NextResponse.json({ - success: false, - error: `Invalid Grafana baseUrl: ${urlValidation.error}`, - }) - } - - const headers: Record = { - Accept: 'application/json', - Authorization: `Bearer ${params.apiKey}`, - } - if (params.organizationId) { - headers['X-Grafana-Org-Id'] = params.organizationId - } - - const response = await secureFetchWithPinnedIP(healthUrl, urlValidation.resolvedIP, { - method: 'GET', - headers, - maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, - timeout: OUTBOUND_FETCH_TIMEOUT_MS, - stripAuthOnRedirect: true, - }) - - const raw = await response.text() - let body: unknown = null - if (raw.length > 0) { - try { - body = JSON.parse(raw) - } catch { - body = null - } - } - - const payload = - body && typeof body === 'object' - ? (body as { status?: unknown; message?: unknown; details?: unknown }) - : null - - /** - * A `status` in the body means Grafana ran the check and reported a verdict, - * whatever the HTTP status. Anything else — an auth failure, a missing data - * source, a plugin with no health endpoint — is a genuine request failure. - */ - if (payload && typeof payload.status === 'string') { - return NextResponse.json({ - success: true, - output: { - status: payload.status, - message: typeof payload.message === 'string' ? payload.message : null, - ...(payload.details === undefined ? {} : { details: payload.details }), - }, - }) - } - - logger.warn(`[${requestId}] Grafana health check did not report a status (${response.status})`) - return NextResponse.json({ - success: false, - error: `Failed to check data source health: HTTP ${response.status} ${truncate( - raw, - MAX_ERROR_MESSAGE_LENGTH - )}`, - }) - } catch (error) { - logger.error(`[${requestId}] Error checking Grafana data source health:`, error) - return NextResponse.json({ success: false, error: getErrorMessage(error) }) - } -}) diff --git a/apps/sim/app/api/tools/grafana/update_alert_rule/route.ts b/apps/sim/app/api/tools/grafana/update_alert_rule/route.ts deleted file mode 100644 index e9a79380e0e..00000000000 --- a/apps/sim/app/api/tools/grafana/update_alert_rule/route.ts +++ /dev/null @@ -1,242 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { truncate } from '@sim/utils/string' -import { type NextRequest, NextResponse } from 'next/server' -import { grafanaUpdateAlertRuleContract } from '@/lib/api/contracts/tools/grafana' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - MAX_JSON_API_RESPONSE_BYTES, - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { mapAlertRule } from '@/tools/grafana/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('GrafanaUpdateAlertRuleAPI') - -/** Grafana is reached over two sequential hops, so each one needs its own bound. */ -const OUTBOUND_FETCH_TIMEOUT_MS = 30_000 -/** Upstream error bodies can be a full HTML page; only a prefix is useful. */ -const MAX_ERROR_MESSAGE_LENGTH = 2000 - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn( - `[${requestId}] Unauthorized Grafana update alert rule attempt: ${authResult.error}` - ) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - grafanaUpdateAlertRuleContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const baseUrl = params.baseUrl.replace(/\/$/, '') - - const getHeaders: Record = { - 'Content-Type': 'application/json', - Authorization: `Bearer ${params.apiKey}`, - } - if (params.organizationId) { - getHeaders['X-Grafana-Org-Id'] = params.organizationId - } - - const getUrl = `${baseUrl}/api/v1/provisioning/alert-rules/${encodeURIComponent(params.alertRuleUid.trim())}` - const getValidation = await validateUrlWithDNS(getUrl, 'baseUrl') - if (!getValidation.isValid || !getValidation.resolvedIP) { - return NextResponse.json({ - success: false, - output: {}, - error: `Invalid Grafana baseUrl: ${getValidation.error}`, - }) - } - - const getResponse = await secureFetchWithPinnedIP(getUrl, getValidation.resolvedIP, { - method: 'GET', - headers: getHeaders, - maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, - timeout: OUTBOUND_FETCH_TIMEOUT_MS, - stripAuthOnRedirect: true, - }) - - if (!getResponse.ok) { - const errorText = truncate(await getResponse.text(), MAX_ERROR_MESSAGE_LENGTH) - return NextResponse.json({ - success: false, - output: {}, - error: `Failed to fetch existing alert rule: ${errorText}`, - }) - } - - const existingRule = (await getResponse.json()) as Record - - if (!existingRule || !existingRule.uid) { - return NextResponse.json({ - success: false, - output: {}, - error: 'Failed to fetch existing alert rule', - }) - } - - const updatedRule: Record = { - ...existingRule, - } - - if (params.title) updatedRule.title = params.title - if (params.folderUid) updatedRule.folderUID = params.folderUid - if (params.ruleGroup) updatedRule.ruleGroup = params.ruleGroup - if (params.condition) updatedRule.condition = params.condition - if (params.forDuration) updatedRule.for = params.forDuration - if (params.noDataState) updatedRule.noDataState = params.noDataState - if (params.execErrState) updatedRule.execErrState = params.execErrState - if (params.isPaused !== undefined) updatedRule.isPaused = params.isPaused - if (params.keepFiringFor) updatedRule.keep_firing_for = params.keepFiringFor - if (params.missingSeriesEvalsToResolve !== undefined) { - updatedRule.missingSeriesEvalsToResolve = params.missingSeriesEvalsToResolve - } - - if (params.notificationSettings) { - try { - updatedRule.notification_settings = JSON.parse(params.notificationSettings) - } catch { - return NextResponse.json({ - success: false, - output: {}, - error: 'Invalid JSON for notificationSettings parameter', - }) - } - } - - if (params.record) { - try { - updatedRule.record = JSON.parse(params.record) - } catch { - return NextResponse.json({ - success: false, - output: {}, - error: 'Invalid JSON for record parameter', - }) - } - } - - if (params.data) { - try { - updatedRule.data = JSON.parse(params.data) - } catch { - return NextResponse.json({ - success: false, - output: {}, - error: 'Invalid JSON for data parameter', - }) - } - } - - if (params.annotations) { - try { - updatedRule.annotations = { - ...(existingRule.annotations || {}), - ...JSON.parse(params.annotations), - } - } catch { - return NextResponse.json({ - success: false, - output: {}, - error: 'Invalid JSON for annotations parameter', - }) - } - } - - if (params.labels) { - try { - updatedRule.labels = { - ...(existingRule.labels || {}), - ...JSON.parse(params.labels), - } - } catch { - return NextResponse.json({ - success: false, - output: {}, - error: 'Invalid JSON for labels parameter', - }) - } - } - - const headers: Record = { - 'Content-Type': 'application/json', - Authorization: `Bearer ${params.apiKey}`, - } - if (params.organizationId) { - headers['X-Grafana-Org-Id'] = params.organizationId - } - if (params.disableProvenance) { - headers['X-Disable-Provenance'] = 'true' - } - - const updateUrl = `${baseUrl}/api/v1/provisioning/alert-rules/${encodeURIComponent(params.alertRuleUid.trim())}` - const urlValidation = await validateUrlWithDNS(updateUrl, 'baseUrl') - if (!urlValidation.isValid || !urlValidation.resolvedIP) { - return NextResponse.json({ - success: false, - output: {}, - error: `Invalid Grafana baseUrl: ${urlValidation.error}`, - }) - } - - const updateResponse = await secureFetchWithPinnedIP(updateUrl, urlValidation.resolvedIP, { - method: 'PUT', - headers, - body: JSON.stringify(updatedRule), - maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, - timeout: OUTBOUND_FETCH_TIMEOUT_MS, - stripAuthOnRedirect: true, - }) - - if (!updateResponse.ok) { - const errorText = truncate(await updateResponse.text(), MAX_ERROR_MESSAGE_LENGTH) - return NextResponse.json({ - success: false, - output: {}, - error: `Failed to update alert rule: ${errorText}`, - }) - } - - const data = (await updateResponse.json()) as Record - return NextResponse.json({ success: true, output: mapAlertRule(data) }) - } catch (error) { - logger.error(`[${requestId}] Error updating Grafana alert rule:`, error) - return NextResponse.json({ - success: false, - output: {}, - error: getErrorMessage(error), - }) - } -}) diff --git a/apps/sim/app/api/tools/grafana/update_dashboard/route.ts b/apps/sim/app/api/tools/grafana/update_dashboard/route.ts deleted file mode 100644 index e7ceebf7f4a..00000000000 --- a/apps/sim/app/api/tools/grafana/update_dashboard/route.ts +++ /dev/null @@ -1,229 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { truncate } from '@sim/utils/string' -import { type NextRequest, NextResponse } from 'next/server' -import { grafanaUpdateDashboardContract } from '@/lib/api/contracts/tools/grafana' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - MAX_JSON_API_RESPONSE_BYTES, - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('GrafanaUpdateDashboardAPI') - -/** Grafana is reached over two sequential hops, so each one needs its own bound. */ -const OUTBOUND_FETCH_TIMEOUT_MS = 30_000 -/** Upstream error bodies can be a full HTML page; only a prefix is useful. */ -const MAX_ERROR_MESSAGE_LENGTH = 2000 - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn( - `[${requestId}] Unauthorized Grafana update dashboard attempt: ${authResult.error}` - ) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - grafanaUpdateDashboardContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const baseUrl = params.baseUrl.replace(/\/$/, '') - - const getHeaders: Record = { - 'Content-Type': 'application/json', - Authorization: `Bearer ${params.apiKey}`, - } - if (params.organizationId) { - getHeaders['X-Grafana-Org-Id'] = params.organizationId - } - - const getUrl = `${baseUrl}/api/dashboards/uid/${encodeURIComponent(params.dashboardUid.trim())}` - const getValidation = await validateUrlWithDNS(getUrl, 'baseUrl') - if (!getValidation.isValid || !getValidation.resolvedIP) { - return NextResponse.json({ - success: false, - output: {}, - error: `Invalid Grafana baseUrl: ${getValidation.error}`, - }) - } - - const getResponse = await secureFetchWithPinnedIP(getUrl, getValidation.resolvedIP, { - method: 'GET', - headers: getHeaders, - maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, - timeout: OUTBOUND_FETCH_TIMEOUT_MS, - stripAuthOnRedirect: true, - }) - - if (!getResponse.ok) { - const errorText = truncate(await getResponse.text(), MAX_ERROR_MESSAGE_LENGTH) - return NextResponse.json({ - success: false, - output: {}, - error: `Failed to fetch existing dashboard: ${errorText}`, - }) - } - - /** - * `GET /api/dashboards/uid/:uid` answers `{dashboard, meta}`. Only the few - * fields this route reads are narrowed — the rest of the dashboard is - * arbitrary user JSON that is spread through untouched. - */ - const existing = (await getResponse.json()) as { - dashboard?: Record - meta?: { folderUid?: string } - } - const existingDashboard = existing.dashboard - const existingMeta = existing.meta - - if (!existingDashboard || !existingDashboard.uid) { - return NextResponse.json({ - success: false, - output: {}, - error: 'Failed to fetch existing dashboard', - }) - } - - const updatedDashboard: Record = { - ...existingDashboard, - } - - if (params.title) updatedDashboard.title = params.title - if (params.timezone) updatedDashboard.timezone = params.timezone - if (params.refresh) updatedDashboard.refresh = params.refresh - - if (params.tags) { - updatedDashboard.tags = params.tags - .split(',') - .map((t) => t.trim()) - .filter((t) => t) - } - - if (params.panels) { - try { - updatedDashboard.panels = JSON.parse(params.panels) - } catch { - return NextResponse.json({ - success: false, - output: {}, - error: 'Invalid JSON for panels parameter', - }) - } - } - - if (existingDashboard.version) { - updatedDashboard.version = existingDashboard.version - } - - const body: Record = { - dashboard: updatedDashboard, - overwrite: params.overwrite === true, - } - - if (params.folderUid) { - body.folderUid = params.folderUid - } else if (existingMeta?.folderUid) { - body.folderUid = existingMeta.folderUid - } - - if (params.message) { - body.message = params.message - } - - const headers: Record = { - 'Content-Type': 'application/json', - Authorization: `Bearer ${params.apiKey}`, - } - if (params.organizationId) { - headers['X-Grafana-Org-Id'] = params.organizationId - } - - const updateUrl = `${baseUrl}/api/dashboards/db` - const urlValidation = await validateUrlWithDNS(updateUrl, 'baseUrl') - if (!urlValidation.isValid || !urlValidation.resolvedIP) { - return NextResponse.json({ - success: false, - output: {}, - error: `Invalid Grafana baseUrl: ${urlValidation.error}`, - }) - } - - const updateResponse = await secureFetchWithPinnedIP(updateUrl, urlValidation.resolvedIP, { - method: 'POST', - headers, - body: JSON.stringify(body), - maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, - timeout: OUTBOUND_FETCH_TIMEOUT_MS, - stripAuthOnRedirect: true, - }) - - if (!updateResponse.ok) { - const errorText = truncate(await updateResponse.text(), MAX_ERROR_MESSAGE_LENGTH) - return NextResponse.json({ - success: false, - output: {}, - error: `Failed to update dashboard: ${errorText}`, - }) - } - - const data = (await updateResponse.json()) as { - id?: number - uid?: string - url?: string - status?: string - version?: number - slug?: string - } - - return NextResponse.json({ - success: true, - output: { - id: data.id, - uid: data.uid, - url: data.url, - status: data.status, - version: data.version, - slug: data.slug, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error updating Grafana dashboard:`, error) - return NextResponse.json({ - success: false, - output: {}, - error: getErrorMessage(error), - }) - } -}) diff --git a/apps/sim/app/api/tools/grafana/update_folder/route.ts b/apps/sim/app/api/tools/grafana/update_folder/route.ts deleted file mode 100644 index 9623cef77fe..00000000000 --- a/apps/sim/app/api/tools/grafana/update_folder/route.ts +++ /dev/null @@ -1,167 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { truncate } from '@sim/utils/string' -import { type NextRequest, NextResponse } from 'next/server' -import { grafanaUpdateFolderContract } from '@/lib/api/contracts/tools/grafana' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - MAX_JSON_API_RESPONSE_BYTES, - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('GrafanaUpdateFolderAPI') - -/** Grafana is reached over two sequential hops, so each one needs its own bound. */ -const OUTBOUND_FETCH_TIMEOUT_MS = 30_000 -/** Upstream error bodies can be a full HTML page; only a prefix is useful. */ -const MAX_ERROR_MESSAGE_LENGTH = 2000 - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Grafana update folder attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - grafanaUpdateFolderContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const baseUrl = params.baseUrl.replace(/\/$/, '') - - const headers: Record = { - 'Content-Type': 'application/json', - Authorization: `Bearer ${params.apiKey}`, - } - if (params.organizationId) { - headers['X-Grafana-Org-Id'] = params.organizationId - } - - const folderUrl = `${baseUrl}/api/folders/${encodeURIComponent(params.folderUid.trim())}` - const urlValidation = await validateUrlWithDNS(folderUrl, 'baseUrl') - if (!urlValidation.isValid || !urlValidation.resolvedIP) { - return NextResponse.json({ - success: false, - output: {}, - error: `Invalid Grafana baseUrl: ${urlValidation.error}`, - }) - } - - const getResponse = await secureFetchWithPinnedIP(folderUrl, urlValidation.resolvedIP, { - method: 'GET', - headers, - maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, - timeout: OUTBOUND_FETCH_TIMEOUT_MS, - stripAuthOnRedirect: true, - }) - - if (!getResponse.ok) { - const errorText = truncate(await getResponse.text(), MAX_ERROR_MESSAGE_LENGTH) - return NextResponse.json({ - success: false, - output: {}, - error: `Failed to fetch existing folder: ${errorText}`, - }) - } - - const existingFolder = (await getResponse.json()) as Record - - if (!existingFolder || !existingFolder.uid) { - return NextResponse.json({ - success: false, - output: {}, - error: 'Failed to fetch existing folder', - }) - } - - /** - * Grafana treats `version` and `overwrite` as alternatives: `version` is - * "not needed if overwrite=true". Sending both made the version we just - * fetched decorative and silently clobbered a concurrent rename, so only - * the version is sent and a conflicting edit surfaces as Grafana's 412 - * instead of being lost. - */ - const body: Record = { - title: params.title, - version: existingFolder.version, - } - - const updateResponse = await secureFetchWithPinnedIP(folderUrl, urlValidation.resolvedIP, { - method: 'PUT', - headers, - body: JSON.stringify(body), - maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, - timeout: OUTBOUND_FETCH_TIMEOUT_MS, - stripAuthOnRedirect: true, - }) - - if (!updateResponse.ok) { - const errorText = truncate(await updateResponse.text(), MAX_ERROR_MESSAGE_LENGTH) - return NextResponse.json({ - success: false, - output: {}, - error: `Failed to update folder: ${errorText}`, - }) - } - - const data = (await updateResponse.json()) as Record - - return NextResponse.json({ - success: true, - output: { - id: (data.id as number) ?? null, - uid: (data.uid as string) ?? null, - title: (data.title as string) ?? null, - url: (data.url as string) ?? null, - parentUid: (data.parentUid as string) ?? null, - parents: (data.parents as { uid: string; title: string; url: string }[]) ?? [], - hasAcl: (data.hasAcl as boolean) ?? null, - canSave: (data.canSave as boolean) ?? null, - canEdit: (data.canEdit as boolean) ?? null, - canAdmin: (data.canAdmin as boolean) ?? null, - createdBy: (data.createdBy as string) ?? null, - created: (data.created as string) ?? null, - updatedBy: (data.updatedBy as string) ?? null, - updated: (data.updated as string) ?? null, - version: (data.version as number) ?? null, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error updating Grafana folder:`, error) - return NextResponse.json({ - success: false, - output: {}, - error: getErrorMessage(error), - }) - } -}) diff --git a/apps/sim/app/api/tools/iam/add-user-to-group/route.ts b/apps/sim/app/api/tools/iam/add-user-to-group/route.ts deleted file mode 100644 index e0b0825368d..00000000000 --- a/apps/sim/app/api/tools/iam/add-user-to-group/route.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIamAddUserToGroupContract } from '@/lib/api/contracts/tools/aws/iam-add-user-to-group' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { addUserToGroup, createIAMClient } from '../utils' - -const logger = createLogger('IAMAddUserToGroupAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIamAddUserToGroupContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Adding user "${params.userName}" to group "${params.groupName}"`) - - const client = createIAMClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - await addUserToGroup(client, params.userName, params.groupName) - logger.info(`Successfully added user "${params.userName}" to group "${params.groupName}"`) - return NextResponse.json({ - message: `User "${params.userName}" added to group "${params.groupName}"`, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error(`Failed to add user to group:`, error) - return NextResponse.json( - { error: `Failed to add user to group: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/iam/attach-role-policy/route.ts b/apps/sim/app/api/tools/iam/attach-role-policy/route.ts deleted file mode 100644 index 9ebed7c6438..00000000000 --- a/apps/sim/app/api/tools/iam/attach-role-policy/route.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIamAttachRolePolicyContract } from '@/lib/api/contracts/tools/aws/iam-attach-role-policy' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { attachRolePolicy, createIAMClient } from '../utils' - -const logger = createLogger('IAMAttachRolePolicyAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIamAttachRolePolicyContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Attaching policy to IAM role "${params.roleName}"`) - - const client = createIAMClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - await attachRolePolicy(client, params.roleName, params.policyArn) - logger.info(`Successfully attached policy to IAM role "${params.roleName}"`) - return NextResponse.json({ - message: `Policy "${params.policyArn}" attached to role "${params.roleName}"`, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error(`Failed to attach role policy:`, error) - return NextResponse.json( - { error: `Failed to attach role policy: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/iam/attach-user-policy/route.ts b/apps/sim/app/api/tools/iam/attach-user-policy/route.ts deleted file mode 100644 index f5c238808e0..00000000000 --- a/apps/sim/app/api/tools/iam/attach-user-policy/route.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIamAttachUserPolicyContract } from '@/lib/api/contracts/tools/aws/iam-attach-user-policy' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { attachUserPolicy, createIAMClient } from '../utils' - -const logger = createLogger('IAMAttachUserPolicyAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIamAttachUserPolicyContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Attaching policy to IAM user "${params.userName}"`) - - const client = createIAMClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - await attachUserPolicy(client, params.userName, params.policyArn) - logger.info(`Successfully attached policy to IAM user "${params.userName}"`) - return NextResponse.json({ - message: `Policy "${params.policyArn}" attached to user "${params.userName}"`, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error(`Failed to attach user policy:`, error) - return NextResponse.json( - { error: `Failed to attach user policy: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/iam/create-access-key/route.ts b/apps/sim/app/api/tools/iam/create-access-key/route.ts deleted file mode 100644 index 1c0dded954e..00000000000 --- a/apps/sim/app/api/tools/iam/create-access-key/route.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIamCreateAccessKeyContract } from '@/lib/api/contracts/tools/aws/iam-create-access-key' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createAccessKey, createIAMClient } from '../utils' - -const logger = createLogger('IAMCreateAccessKeyAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIamCreateAccessKeyContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Creating IAM access key`) - - const client = createIAMClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await createAccessKey(client, params.userName) - logger.info(`Successfully created access key for user "${result.userName}"`) - return NextResponse.json({ - message: `Access key created for user "${result.userName}"`, - ...result, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error(`Failed to create access key:`, error) - return NextResponse.json( - { error: `Failed to create access key: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/iam/create-role/route.ts b/apps/sim/app/api/tools/iam/create-role/route.ts deleted file mode 100644 index bb8f857a764..00000000000 --- a/apps/sim/app/api/tools/iam/create-role/route.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIamCreateRoleContract } from '@/lib/api/contracts/tools/aws/iam-create-role' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createIAMClient, createRole } from '../utils' - -const logger = createLogger('IAMCreateRoleAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIamCreateRoleContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Creating IAM role "${params.roleName}"`) - - const client = createIAMClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await createRole( - client, - params.roleName, - params.assumeRolePolicyDocument, - params.description, - params.path, - params.maxSessionDuration - ) - logger.info(`Successfully created IAM role "${result.roleName}"`) - return NextResponse.json({ - message: `Role "${result.roleName}" created successfully`, - ...result, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error(`Failed to create IAM role:`, error) - return NextResponse.json( - { error: `Failed to create IAM role: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/iam/create-user/route.ts b/apps/sim/app/api/tools/iam/create-user/route.ts deleted file mode 100644 index 73c82caad72..00000000000 --- a/apps/sim/app/api/tools/iam/create-user/route.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIamCreateUserContract } from '@/lib/api/contracts/tools/aws/iam-create-user' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createIAMClient, createUser } from '../utils' - -const logger = createLogger('IAMCreateUserAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIamCreateUserContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Creating IAM user "${params.userName}"`) - - const client = createIAMClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await createUser(client, params.userName, params.path) - logger.info(`Successfully created IAM user "${result.userName}"`) - return NextResponse.json({ - message: `User "${result.userName}" created successfully`, - ...result, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error(`Failed to create IAM user:`, error) - return NextResponse.json( - { error: `Failed to create IAM user: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/iam/delete-access-key/route.ts b/apps/sim/app/api/tools/iam/delete-access-key/route.ts deleted file mode 100644 index edf6acc38a7..00000000000 --- a/apps/sim/app/api/tools/iam/delete-access-key/route.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIamDeleteAccessKeyContract } from '@/lib/api/contracts/tools/aws/iam-delete-access-key' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createIAMClient, deleteAccessKey } from '../utils' - -const logger = createLogger('IAMDeleteAccessKeyAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIamDeleteAccessKeyContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Deleting IAM access key "${params.accessKeyIdToDelete}"`) - - const client = createIAMClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - await deleteAccessKey(client, params.accessKeyIdToDelete, params.userName) - logger.info(`Successfully deleted access key "${params.accessKeyIdToDelete}"`) - return NextResponse.json({ message: `Access key "${params.accessKeyIdToDelete}" deleted` }) - } finally { - client.destroy() - } - } catch (error) { - logger.error(`Failed to delete access key:`, error) - return NextResponse.json( - { error: `Failed to delete access key: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/iam/delete-role/route.ts b/apps/sim/app/api/tools/iam/delete-role/route.ts deleted file mode 100644 index 77a7d2c8184..00000000000 --- a/apps/sim/app/api/tools/iam/delete-role/route.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIamDeleteRoleContract } from '@/lib/api/contracts/tools/aws/iam-delete-role' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createIAMClient, deleteRole } from '../utils' - -const logger = createLogger('IAMDeleteRoleAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIamDeleteRoleContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Deleting IAM role "${params.roleName}"`) - - const client = createIAMClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - await deleteRole(client, params.roleName) - logger.info(`Successfully deleted IAM role "${params.roleName}"`) - return NextResponse.json({ message: `Role "${params.roleName}" deleted successfully` }) - } finally { - client.destroy() - } - } catch (error) { - logger.error(`Failed to delete IAM role:`, error) - return NextResponse.json( - { error: `Failed to delete IAM role: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/iam/delete-user/route.ts b/apps/sim/app/api/tools/iam/delete-user/route.ts deleted file mode 100644 index 6ed84011d1d..00000000000 --- a/apps/sim/app/api/tools/iam/delete-user/route.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIamDeleteUserContract } from '@/lib/api/contracts/tools/aws/iam-delete-user' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createIAMClient, deleteUser } from '../utils' - -const logger = createLogger('IAMDeleteUserAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIamDeleteUserContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Deleting IAM user "${params.userName}"`) - - const client = createIAMClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - await deleteUser(client, params.userName) - logger.info(`Successfully deleted IAM user "${params.userName}"`) - return NextResponse.json({ message: `User "${params.userName}" deleted successfully` }) - } finally { - client.destroy() - } - } catch (error) { - logger.error(`Failed to delete IAM user:`, error) - return NextResponse.json( - { error: `Failed to delete IAM user: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/iam/detach-role-policy/route.ts b/apps/sim/app/api/tools/iam/detach-role-policy/route.ts deleted file mode 100644 index e16e69843b0..00000000000 --- a/apps/sim/app/api/tools/iam/detach-role-policy/route.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIamDetachRolePolicyContract } from '@/lib/api/contracts/tools/aws/iam-detach-role-policy' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createIAMClient, detachRolePolicy } from '../utils' - -const logger = createLogger('IAMDetachRolePolicyAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIamDetachRolePolicyContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Detaching policy from IAM role "${params.roleName}"`) - - const client = createIAMClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - await detachRolePolicy(client, params.roleName, params.policyArn) - logger.info(`Successfully detached policy from IAM role "${params.roleName}"`) - return NextResponse.json({ - message: `Policy "${params.policyArn}" detached from role "${params.roleName}"`, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error(`Failed to detach role policy:`, error) - return NextResponse.json( - { error: `Failed to detach role policy: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/iam/detach-user-policy/route.ts b/apps/sim/app/api/tools/iam/detach-user-policy/route.ts deleted file mode 100644 index 9968b2f9b4c..00000000000 --- a/apps/sim/app/api/tools/iam/detach-user-policy/route.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIamDetachUserPolicyContract } from '@/lib/api/contracts/tools/aws/iam-detach-user-policy' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createIAMClient, detachUserPolicy } from '../utils' - -const logger = createLogger('IAMDetachUserPolicyAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIamDetachUserPolicyContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Detaching policy from IAM user "${params.userName}"`) - - const client = createIAMClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - await detachUserPolicy(client, params.userName, params.policyArn) - logger.info(`Successfully detached policy from IAM user "${params.userName}"`) - return NextResponse.json({ - message: `Policy "${params.policyArn}" detached from user "${params.userName}"`, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error(`Failed to detach user policy:`, error) - return NextResponse.json( - { error: `Failed to detach user policy: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/iam/get-role/route.ts b/apps/sim/app/api/tools/iam/get-role/route.ts deleted file mode 100644 index 4be677db5e7..00000000000 --- a/apps/sim/app/api/tools/iam/get-role/route.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIamGetRoleContract } from '@/lib/api/contracts/tools/aws/iam-get-role' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createIAMClient, getRole } from '../utils' - -const logger = createLogger('IAMGetRoleAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIamGetRoleContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Getting IAM role "${params.roleName}"`) - - const client = createIAMClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await getRole(client, params.roleName) - logger.info(`Successfully retrieved IAM role "${params.roleName}"`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error(`Failed to get IAM role:`, error) - return NextResponse.json( - { error: `Failed to get IAM role: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/iam/get-user/route.ts b/apps/sim/app/api/tools/iam/get-user/route.ts deleted file mode 100644 index b734fc8aa53..00000000000 --- a/apps/sim/app/api/tools/iam/get-user/route.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIamGetUserContract } from '@/lib/api/contracts/tools/aws/iam-get-user' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createIAMClient, getUser } from '../utils' - -const logger = createLogger('IAMGetUserAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIamGetUserContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Getting IAM user "${params.userName}"`) - - const client = createIAMClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await getUser(client, params.userName) - logger.info(`Successfully retrieved IAM user "${params.userName ?? 'caller'}"`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error(`Failed to get IAM user:`, error) - return NextResponse.json( - { error: `Failed to get IAM user: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/iam/list-attached-role-policies/route.ts b/apps/sim/app/api/tools/iam/list-attached-role-policies/route.ts deleted file mode 100644 index d49445e9bc4..00000000000 --- a/apps/sim/app/api/tools/iam/list-attached-role-policies/route.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIamListAttachedRolePoliciesContract } from '@/lib/api/contracts/tools/aws/iam-list-attached-role-policies' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createIAMClient, listAttachedRolePolicies } from '../utils' - -const logger = createLogger('IAMListAttachedRolePoliciesAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIamListAttachedRolePoliciesContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Listing policies attached to IAM role "${params.roleName}"`) - - const client = createIAMClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await listAttachedRolePolicies( - client, - params.roleName, - params.pathPrefix, - params.maxItems, - params.marker - ) - logger.info(`Found ${result.count} policies attached to role "${params.roleName}"`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error(`Failed to list attached role policies:`, error) - return NextResponse.json( - { error: `Failed to list attached role policies: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/iam/list-attached-user-policies/route.ts b/apps/sim/app/api/tools/iam/list-attached-user-policies/route.ts deleted file mode 100644 index 9e3da6d05e5..00000000000 --- a/apps/sim/app/api/tools/iam/list-attached-user-policies/route.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIamListAttachedUserPoliciesContract } from '@/lib/api/contracts/tools/aws/iam-list-attached-user-policies' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createIAMClient, listAttachedUserPolicies } from '../utils' - -const logger = createLogger('IAMListAttachedUserPoliciesAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIamListAttachedUserPoliciesContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Listing policies attached to IAM user "${params.userName}"`) - - const client = createIAMClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await listAttachedUserPolicies( - client, - params.userName, - params.pathPrefix, - params.maxItems, - params.marker - ) - logger.info(`Found ${result.count} policies attached to user "${params.userName}"`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error(`Failed to list attached user policies:`, error) - return NextResponse.json( - { error: `Failed to list attached user policies: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/iam/list-groups/route.ts b/apps/sim/app/api/tools/iam/list-groups/route.ts deleted file mode 100644 index ebf2214d0d9..00000000000 --- a/apps/sim/app/api/tools/iam/list-groups/route.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIamListGroupsContract } from '@/lib/api/contracts/tools/aws/iam-list-groups' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createIAMClient, listGroups } from '../utils' - -const logger = createLogger('IAMListGroupsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIamListGroupsContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Listing IAM groups`) - - const client = createIAMClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await listGroups(client, params.pathPrefix, params.maxItems, params.marker) - logger.info(`Successfully listed ${result.count} IAM groups`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error(`Failed to list IAM groups:`, error) - return NextResponse.json( - { error: `Failed to list IAM groups: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/iam/list-policies/route.ts b/apps/sim/app/api/tools/iam/list-policies/route.ts deleted file mode 100644 index 4e160687c20..00000000000 --- a/apps/sim/app/api/tools/iam/list-policies/route.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIamListPoliciesContract } from '@/lib/api/contracts/tools/aws/iam-list-policies' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createIAMClient, listPolicies } from '../utils' - -const logger = createLogger('IAMListPoliciesAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIamListPoliciesContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Listing IAM policies`) - - const client = createIAMClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await listPolicies( - client, - params.scope, - params.onlyAttached, - params.pathPrefix, - params.maxItems, - params.marker - ) - logger.info(`Successfully listed ${result.count} IAM policies`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error(`Failed to list IAM policies:`, error) - return NextResponse.json( - { error: `Failed to list IAM policies: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/iam/list-roles/route.ts b/apps/sim/app/api/tools/iam/list-roles/route.ts deleted file mode 100644 index 8d38bf0baa5..00000000000 --- a/apps/sim/app/api/tools/iam/list-roles/route.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIamListRolesContract } from '@/lib/api/contracts/tools/aws/iam-list-roles' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createIAMClient, listRoles } from '../utils' - -const logger = createLogger('IAMListRolesAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIamListRolesContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Listing IAM roles`) - - const client = createIAMClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await listRoles(client, params.pathPrefix, params.maxItems, params.marker) - logger.info(`Successfully listed ${result.count} IAM roles`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error(`Failed to list IAM roles:`, error) - return NextResponse.json( - { error: `Failed to list IAM roles: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/iam/list-users/route.ts b/apps/sim/app/api/tools/iam/list-users/route.ts deleted file mode 100644 index 831d2cd9a62..00000000000 --- a/apps/sim/app/api/tools/iam/list-users/route.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIamListUsersContract } from '@/lib/api/contracts/tools/aws/iam-list-users' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createIAMClient, listUsers } from '../utils' - -const logger = createLogger('IAMListUsersAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIamListUsersContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Listing IAM users`) - - const client = createIAMClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await listUsers(client, params.pathPrefix, params.maxItems, params.marker) - logger.info(`Successfully listed ${result.count} IAM users`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error(`Failed to list IAM users:`, error) - return NextResponse.json( - { error: `Failed to list IAM users: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/iam/remove-user-from-group/route.ts b/apps/sim/app/api/tools/iam/remove-user-from-group/route.ts deleted file mode 100644 index da2728bf9d5..00000000000 --- a/apps/sim/app/api/tools/iam/remove-user-from-group/route.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIamRemoveUserFromGroupContract } from '@/lib/api/contracts/tools/aws/iam-remove-user-from-group' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createIAMClient, removeUserFromGroup } from '../utils' - -const logger = createLogger('IAMRemoveUserFromGroupAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIamRemoveUserFromGroupContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Removing user "${params.userName}" from group "${params.groupName}"`) - - const client = createIAMClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - await removeUserFromGroup(client, params.userName, params.groupName) - logger.info(`Successfully removed user "${params.userName}" from group "${params.groupName}"`) - return NextResponse.json({ - message: `User "${params.userName}" removed from group "${params.groupName}"`, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error(`Failed to remove user from group:`, error) - return NextResponse.json( - { error: `Failed to remove user from group: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/iam/simulate-principal-policy/route.ts b/apps/sim/app/api/tools/iam/simulate-principal-policy/route.ts deleted file mode 100644 index 42dffe952a5..00000000000 --- a/apps/sim/app/api/tools/iam/simulate-principal-policy/route.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIamSimulatePrincipalPolicyContract } from '@/lib/api/contracts/tools/aws/iam-simulate-principal-policy' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createIAMClient, simulatePrincipalPolicy } from '../utils' - -const logger = createLogger('IAMSimulatePrincipalPolicyAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIamSimulatePrincipalPolicyContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `Simulating principal policy for "${params.policySourceArn}" on actions: ${params.actionNames}` - ) - - const client = createIAMClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await simulatePrincipalPolicy( - client, - params.policySourceArn, - params.actionNames, - params.resourceArns, - params.maxResults, - params.marker - ) - logger.info(`Simulation complete: ${result.count} results`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error(`Failed to simulate principal policy:`, error) - return NextResponse.json( - { error: `Failed to simulate principal policy: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/iam/utils.ts b/apps/sim/app/api/tools/iam/utils.ts deleted file mode 100644 index 9c937dd5a91..00000000000 --- a/apps/sim/app/api/tools/iam/utils.ts +++ /dev/null @@ -1,453 +0,0 @@ -import type { - AttachedPolicy, - Group, - Policy, - PolicyScopeType, - Role, - User, -} from '@aws-sdk/client-iam' -import { - AddUserToGroupCommand, - AttachRolePolicyCommand, - AttachUserPolicyCommand, - CreateAccessKeyCommand, - CreateRoleCommand, - CreateUserCommand, - DeleteAccessKeyCommand, - DeleteRoleCommand, - DeleteUserCommand, - DetachRolePolicyCommand, - DetachUserPolicyCommand, - GetRoleCommand, - GetUserCommand, - IAMClient, - ListAttachedRolePoliciesCommand, - ListAttachedUserPoliciesCommand, - ListGroupsCommand, - ListPoliciesCommand, - ListRolesCommand, - ListUsersCommand, - RemoveUserFromGroupCommand, - SimulatePrincipalPolicyCommand, -} from '@aws-sdk/client-iam' -import type { IAMConnectionConfig } from '@/tools/iam/types' - -export function createIAMClient(config: IAMConnectionConfig): IAMClient { - return new IAMClient({ - region: config.region, - credentials: { - accessKeyId: config.accessKeyId, - secretAccessKey: config.secretAccessKey, - }, - }) -} - -export async function listUsers( - client: IAMClient, - pathPrefix?: string | null, - maxItems?: number | null, - marker?: string | null -) { - const command = new ListUsersCommand({ - ...(pathPrefix ? { PathPrefix: pathPrefix } : {}), - ...(maxItems ? { MaxItems: maxItems } : {}), - ...(marker ? { Marker: marker } : {}), - }) - - const response = await client.send(command) - const users = (response.Users ?? []).map((user: User) => ({ - userName: user.UserName ?? '', - userId: user.UserId ?? '', - arn: user.Arn ?? '', - path: user.Path ?? '', - createDate: user.CreateDate?.toISOString() ?? null, - passwordLastUsed: user.PasswordLastUsed?.toISOString() ?? null, - })) - - return { - users, - isTruncated: response.IsTruncated ?? false, - marker: response.Marker ?? null, - count: users.length, - } -} - -export async function getUser(client: IAMClient, userName?: string | null) { - const command = new GetUserCommand(userName ? { UserName: userName } : {}) - const response = await client.send(command) - const user = response.User - - return { - userName: user?.UserName ?? '', - userId: user?.UserId ?? '', - arn: user?.Arn ?? '', - path: user?.Path ?? '', - createDate: user?.CreateDate?.toISOString() ?? null, - passwordLastUsed: user?.PasswordLastUsed?.toISOString() ?? null, - permissionsBoundaryArn: user?.PermissionsBoundary?.PermissionsBoundaryArn ?? null, - tags: user?.Tags?.map((t) => ({ key: t.Key ?? '', value: t.Value ?? '' })) ?? [], - } -} - -export async function createUser(client: IAMClient, userName: string, path?: string | null) { - const command = new CreateUserCommand({ - UserName: userName, - ...(path ? { Path: path } : {}), - }) - - const response = await client.send(command) - const user = response.User - - return { - userName: user?.UserName ?? '', - userId: user?.UserId ?? '', - arn: user?.Arn ?? '', - path: user?.Path ?? '', - createDate: user?.CreateDate?.toISOString() ?? null, - } -} - -export async function deleteUser(client: IAMClient, userName: string) { - const command = new DeleteUserCommand({ UserName: userName }) - await client.send(command) -} - -export async function listRoles( - client: IAMClient, - pathPrefix?: string | null, - maxItems?: number | null, - marker?: string | null -) { - const command = new ListRolesCommand({ - ...(pathPrefix ? { PathPrefix: pathPrefix } : {}), - ...(maxItems ? { MaxItems: maxItems } : {}), - ...(marker ? { Marker: marker } : {}), - }) - - const response = await client.send(command) - const roles = (response.Roles ?? []).map((role: Role) => ({ - roleName: role.RoleName ?? '', - roleId: role.RoleId ?? '', - arn: role.Arn ?? '', - path: role.Path ?? '', - createDate: role.CreateDate?.toISOString() ?? null, - description: role.Description ?? null, - maxSessionDuration: role.MaxSessionDuration ?? null, - })) - - return { - roles, - isTruncated: response.IsTruncated ?? false, - marker: response.Marker ?? null, - count: roles.length, - } -} - -export async function getRole(client: IAMClient, roleName: string) { - const command = new GetRoleCommand({ RoleName: roleName }) - const response = await client.send(command) - const role = response.Role - - let policyDocument: string | null = null - if (role?.AssumeRolePolicyDocument) { - try { - policyDocument = decodeURIComponent(role.AssumeRolePolicyDocument) - } catch { - policyDocument = role.AssumeRolePolicyDocument - } - } - - return { - roleName: role?.RoleName ?? '', - roleId: role?.RoleId ?? '', - arn: role?.Arn ?? '', - path: role?.Path ?? '', - createDate: role?.CreateDate?.toISOString() ?? null, - description: role?.Description ?? null, - maxSessionDuration: role?.MaxSessionDuration ?? null, - assumeRolePolicyDocument: policyDocument, - roleLastUsedDate: role?.RoleLastUsed?.LastUsedDate?.toISOString() ?? null, - roleLastUsedRegion: role?.RoleLastUsed?.Region ?? null, - } -} - -export async function createRole( - client: IAMClient, - roleName: string, - assumeRolePolicyDocument: string, - description?: string | null, - path?: string | null, - maxSessionDuration?: number | null -) { - const command = new CreateRoleCommand({ - RoleName: roleName, - AssumeRolePolicyDocument: assumeRolePolicyDocument, - ...(description ? { Description: description } : {}), - ...(path ? { Path: path } : {}), - ...(maxSessionDuration ? { MaxSessionDuration: maxSessionDuration } : {}), - }) - - const response = await client.send(command) - const role = response.Role - - return { - roleName: role?.RoleName ?? '', - roleId: role?.RoleId ?? '', - arn: role?.Arn ?? '', - path: role?.Path ?? '', - createDate: role?.CreateDate?.toISOString() ?? null, - } -} - -export async function deleteRole(client: IAMClient, roleName: string) { - const command = new DeleteRoleCommand({ RoleName: roleName }) - await client.send(command) -} - -export async function attachUserPolicy(client: IAMClient, userName: string, policyArn: string) { - const command = new AttachUserPolicyCommand({ - UserName: userName, - PolicyArn: policyArn, - }) - await client.send(command) -} - -export async function detachUserPolicy(client: IAMClient, userName: string, policyArn: string) { - const command = new DetachUserPolicyCommand({ - UserName: userName, - PolicyArn: policyArn, - }) - await client.send(command) -} - -export async function attachRolePolicy(client: IAMClient, roleName: string, policyArn: string) { - const command = new AttachRolePolicyCommand({ - RoleName: roleName, - PolicyArn: policyArn, - }) - await client.send(command) -} - -export async function detachRolePolicy(client: IAMClient, roleName: string, policyArn: string) { - const command = new DetachRolePolicyCommand({ - RoleName: roleName, - PolicyArn: policyArn, - }) - await client.send(command) -} - -export async function listPolicies( - client: IAMClient, - scope?: string | null, - onlyAttached?: boolean | null, - pathPrefix?: string | null, - maxItems?: number | null, - marker?: string | null -) { - const command = new ListPoliciesCommand({ - ...(scope ? { Scope: scope as PolicyScopeType } : {}), - ...(onlyAttached != null ? { OnlyAttached: onlyAttached } : {}), - ...(pathPrefix ? { PathPrefix: pathPrefix } : {}), - ...(maxItems ? { MaxItems: maxItems } : {}), - ...(marker ? { Marker: marker } : {}), - }) - - const response = await client.send(command) - const policies = (response.Policies ?? []).map((policy: Policy) => ({ - policyName: policy.PolicyName ?? '', - policyId: policy.PolicyId ?? '', - arn: policy.Arn ?? '', - path: policy.Path ?? '', - attachmentCount: policy.AttachmentCount ?? 0, - isAttachable: policy.IsAttachable ?? false, - createDate: policy.CreateDate?.toISOString() ?? null, - updateDate: policy.UpdateDate?.toISOString() ?? null, - description: policy.Description ?? null, - defaultVersionId: policy.DefaultVersionId ?? null, - permissionsBoundaryUsageCount: policy.PermissionsBoundaryUsageCount ?? 0, - })) - - return { - policies, - isTruncated: response.IsTruncated ?? false, - marker: response.Marker ?? null, - count: policies.length, - } -} - -export async function createAccessKey(client: IAMClient, userName?: string | null) { - const command = new CreateAccessKeyCommand({ - ...(userName ? { UserName: userName } : {}), - }) - - const response = await client.send(command) - const key = response.AccessKey - - return { - accessKeyId: key?.AccessKeyId ?? '', - secretAccessKey: key?.SecretAccessKey ?? '', - userName: key?.UserName ?? '', - status: key?.Status ?? '', - createDate: key?.CreateDate?.toISOString() ?? null, - } -} - -export async function deleteAccessKey( - client: IAMClient, - accessKeyIdToDelete: string, - userName?: string | null -) { - const command = new DeleteAccessKeyCommand({ - AccessKeyId: accessKeyIdToDelete, - ...(userName ? { UserName: userName } : {}), - }) - await client.send(command) -} - -export async function listGroups( - client: IAMClient, - pathPrefix?: string | null, - maxItems?: number | null, - marker?: string | null -) { - const command = new ListGroupsCommand({ - ...(pathPrefix ? { PathPrefix: pathPrefix } : {}), - ...(maxItems ? { MaxItems: maxItems } : {}), - ...(marker ? { Marker: marker } : {}), - }) - - const response = await client.send(command) - const groups = (response.Groups ?? []).map((group: Group) => ({ - groupName: group.GroupName ?? '', - groupId: group.GroupId ?? '', - arn: group.Arn ?? '', - path: group.Path ?? '', - createDate: group.CreateDate?.toISOString() ?? null, - })) - - return { - groups, - isTruncated: response.IsTruncated ?? false, - marker: response.Marker ?? null, - count: groups.length, - } -} - -export async function addUserToGroup(client: IAMClient, userName: string, groupName: string) { - const command = new AddUserToGroupCommand({ - UserName: userName, - GroupName: groupName, - }) - await client.send(command) -} - -export async function removeUserFromGroup(client: IAMClient, userName: string, groupName: string) { - const command = new RemoveUserFromGroupCommand({ - UserName: userName, - GroupName: groupName, - }) - await client.send(command) -} - -export async function listAttachedRolePolicies( - client: IAMClient, - roleName: string, - pathPrefix?: string | null, - maxItems?: number | null, - marker?: string | null -) { - const command = new ListAttachedRolePoliciesCommand({ - RoleName: roleName, - ...(pathPrefix ? { PathPrefix: pathPrefix } : {}), - ...(maxItems ? { MaxItems: maxItems } : {}), - ...(marker ? { Marker: marker } : {}), - }) - - const response = await client.send(command) - const attachedPolicies = (response.AttachedPolicies ?? []).map((p: AttachedPolicy) => ({ - policyName: p.PolicyName ?? '', - policyArn: p.PolicyArn ?? '', - })) - - return { - attachedPolicies, - isTruncated: response.IsTruncated ?? false, - marker: response.Marker ?? null, - count: attachedPolicies.length, - } -} - -export async function listAttachedUserPolicies( - client: IAMClient, - userName: string, - pathPrefix?: string | null, - maxItems?: number | null, - marker?: string | null -) { - const command = new ListAttachedUserPoliciesCommand({ - UserName: userName, - ...(pathPrefix ? { PathPrefix: pathPrefix } : {}), - ...(maxItems ? { MaxItems: maxItems } : {}), - ...(marker ? { Marker: marker } : {}), - }) - - const response = await client.send(command) - const attachedPolicies = (response.AttachedPolicies ?? []).map((p: AttachedPolicy) => ({ - policyName: p.PolicyName ?? '', - policyArn: p.PolicyArn ?? '', - })) - - return { - attachedPolicies, - isTruncated: response.IsTruncated ?? false, - marker: response.Marker ?? null, - count: attachedPolicies.length, - } -} - -export async function simulatePrincipalPolicy( - client: IAMClient, - policySourceArn: string, - actionNames: string, - resourceArns?: string | null, - maxResults?: number | null, - marker?: string | null -) { - const actions = actionNames - .split(',') - .map((a) => a.trim()) - .filter(Boolean) - const resources = resourceArns - ? resourceArns - .split(',') - .map((r) => r.trim()) - .filter(Boolean) - : ['*'] - - const command = new SimulatePrincipalPolicyCommand({ - PolicySourceArn: policySourceArn, - ActionNames: actions, - ResourceArns: resources, - ...(maxResults ? { MaxItems: maxResults } : {}), - ...(marker ? { Marker: marker } : {}), - }) - - const response = await client.send(command) - const evaluationResults = (response.EvaluationResults ?? []).map((r) => ({ - evalActionName: r.EvalActionName ?? '', - evalResourceName: r.EvalResourceName ?? '', - evalDecision: r.EvalDecision ?? '', - matchedStatements: (r.MatchedStatements ?? []).map((s) => ({ - sourcePolicyId: s.SourcePolicyId ?? '', - sourcePolicyType: s.SourcePolicyType ?? '', - })), - missingContextValues: (r.MissingContextValues ?? []).map((v) => String(v)), - })) - - return { - evaluationResults, - isTruncated: response.IsTruncated ?? false, - marker: response.Marker ?? null, - count: evaluationResults.length, - } -} diff --git a/apps/sim/app/api/tools/identity-center/check-assignment-deletion-status/route.ts b/apps/sim/app/api/tools/identity-center/check-assignment-deletion-status/route.ts deleted file mode 100644 index 3d7c6dc0a05..00000000000 --- a/apps/sim/app/api/tools/identity-center/check-assignment-deletion-status/route.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIdentityCenterCheckAssignmentDeletionStatusContract } from '@/lib/api/contracts/tools/aws/identity-center-check-assignment-deletion-status' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { checkAssignmentDeletionStatus, createSSOAdminClient } from '../utils' - -const logger = createLogger('IdentityCenterCheckAssignmentDeletionStatusAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest( - awsIdentityCenterCheckAssignmentDeletionStatusContract, - request, - { - errorFormat: 'details', - logger, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Checking assignment deletion status for request ${params.requestId}`) - - const client = createSSOAdminClient(params) - try { - const result = await checkAssignmentDeletionStatus( - client, - params.instanceArn, - params.requestId - ) - logger.info(`Assignment deletion status: ${result.status}`) - return NextResponse.json({ - message: `Assignment deletion status: ${result.status}`, - ...result, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to check assignment deletion status:', error) - return NextResponse.json( - { error: `Failed to check assignment deletion status: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/identity-center/check-assignment-status/route.ts b/apps/sim/app/api/tools/identity-center/check-assignment-status/route.ts deleted file mode 100644 index dc39e6fb7a3..00000000000 --- a/apps/sim/app/api/tools/identity-center/check-assignment-status/route.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIdentityCenterCheckAssignmentStatusContract } from '@/lib/api/contracts/tools/aws/identity-center-check-assignment-status' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { checkAssignmentCreationStatus, createSSOAdminClient } from '../utils' - -const logger = createLogger('IdentityCenterCheckAssignmentStatusAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIdentityCenterCheckAssignmentStatusContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Checking assignment status for request ${params.requestId}`) - - const client = createSSOAdminClient(params) - try { - const result = await checkAssignmentCreationStatus( - client, - params.instanceArn, - params.requestId - ) - logger.info(`Assignment status: ${result.status}`) - return NextResponse.json({ - message: `Assignment status: ${result.status}`, - ...result, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to check assignment status:', error) - return NextResponse.json( - { error: `Failed to check assignment status: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/identity-center/create-account-assignment/route.ts b/apps/sim/app/api/tools/identity-center/create-account-assignment/route.ts deleted file mode 100644 index 82524418fea..00000000000 --- a/apps/sim/app/api/tools/identity-center/create-account-assignment/route.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { - CreateAccountAssignmentCommand, - type PrincipalType, - type TargetType, -} from '@aws-sdk/client-sso-admin' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIdentityCenterCreateAccountAssignmentContract } from '@/lib/api/contracts/tools/aws/identity-center-create-account-assignment' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSSOAdminClient, mapAssignmentStatus } from '../utils' - -const logger = createLogger('IdentityCenterCreateAccountAssignmentAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest( - awsIdentityCenterCreateAccountAssignmentContract, - request, - { - errorFormat: 'details', - logger, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `Creating account assignment for ${params.principalType} ${params.principalId} on account ${params.accountId}` - ) - - const client = createSSOAdminClient(params) - try { - const command = new CreateAccountAssignmentCommand({ - InstanceArn: params.instanceArn, - TargetId: params.accountId, - TargetType: 'AWS_ACCOUNT' as TargetType, - PermissionSetArn: params.permissionSetArn, - PrincipalType: params.principalType as PrincipalType, - PrincipalId: params.principalId, - }) - const response = await client.send(command) - const status = response.AccountAssignmentCreationStatus ?? {} - const result = mapAssignmentStatus(status) - - logger.info( - `Account assignment creation initiated with status ${result.status}, requestId ${result.requestId}` - ) - - return NextResponse.json({ - message: `Account assignment creation ${result.status === 'SUCCEEDED' ? 'succeeded' : 'initiated'}`, - ...result, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to create account assignment:', error) - return NextResponse.json( - { error: `Failed to create account assignment: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/identity-center/delete-account-assignment/route.ts b/apps/sim/app/api/tools/identity-center/delete-account-assignment/route.ts deleted file mode 100644 index ddf80ba8319..00000000000 --- a/apps/sim/app/api/tools/identity-center/delete-account-assignment/route.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { - DeleteAccountAssignmentCommand, - type PrincipalType, - type TargetType, -} from '@aws-sdk/client-sso-admin' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIdentityCenterDeleteAccountAssignmentContract } from '@/lib/api/contracts/tools/aws/identity-center-delete-account-assignment' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSSOAdminClient, mapAssignmentStatus } from '../utils' - -const logger = createLogger('IdentityCenterDeleteAccountAssignmentAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest( - awsIdentityCenterDeleteAccountAssignmentContract, - request, - { - errorFormat: 'details', - logger, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `Deleting account assignment for ${params.principalType} ${params.principalId} on account ${params.accountId}` - ) - - const client = createSSOAdminClient(params) - try { - const command = new DeleteAccountAssignmentCommand({ - InstanceArn: params.instanceArn, - TargetId: params.accountId, - TargetType: 'AWS_ACCOUNT' as TargetType, - PermissionSetArn: params.permissionSetArn, - PrincipalType: params.principalType as PrincipalType, - PrincipalId: params.principalId, - }) - const response = await client.send(command) - const status = response.AccountAssignmentDeletionStatus ?? {} - const result = mapAssignmentStatus(status) - - logger.info( - `Account assignment deletion initiated with status ${result.status}, requestId ${result.requestId}` - ) - - return NextResponse.json({ - message: `Account assignment deletion ${result.status === 'SUCCEEDED' ? 'succeeded' : 'initiated'}`, - ...result, - }) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to delete account assignment:', error) - return NextResponse.json( - { error: `Failed to delete account assignment: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/identity-center/describe-account/route.ts b/apps/sim/app/api/tools/identity-center/describe-account/route.ts deleted file mode 100644 index ca0c28eb931..00000000000 --- a/apps/sim/app/api/tools/identity-center/describe-account/route.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIdentityCenterDescribeAccountContract } from '@/lib/api/contracts/tools/aws/identity-center-describe-account' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createOrganizationsClient, describeAccount } from '../utils' - -const logger = createLogger('IdentityCenterDescribeAccountAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIdentityCenterDescribeAccountContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Describing AWS account ${params.accountId}`) - - const client = createOrganizationsClient(params) - try { - const result = await describeAccount(client, params.accountId) - logger.info(`Successfully described account ${result.name}`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to describe account:', error) - return NextResponse.json( - { error: `Failed to describe account: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/identity-center/get-group/route.ts b/apps/sim/app/api/tools/identity-center/get-group/route.ts deleted file mode 100644 index e9c2fb0b44c..00000000000 --- a/apps/sim/app/api/tools/identity-center/get-group/route.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIdentityCenterGetGroupContract } from '@/lib/api/contracts/tools/aws/identity-center-get-group' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createIdentityStoreClient, getGroupByDisplayName } from '../utils' - -const logger = createLogger('IdentityCenterGetGroupAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIdentityCenterGetGroupContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `Looking up group "${params.displayName}" in identity store ${params.identityStoreId}` - ) - - const client = createIdentityStoreClient(params) - try { - const result = await getGroupByDisplayName(client, params.identityStoreId, params.displayName) - logger.info(`Successfully found group ${result.groupId}`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to get group:', error) - return NextResponse.json( - { error: `Failed to get group: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/identity-center/get-user/route.ts b/apps/sim/app/api/tools/identity-center/get-user/route.ts deleted file mode 100644 index 965222bf5f6..00000000000 --- a/apps/sim/app/api/tools/identity-center/get-user/route.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIdentityCenterGetUserContract } from '@/lib/api/contracts/tools/aws/identity-center-get-user' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createIdentityStoreClient, getUserByEmail } from '../utils' - -const logger = createLogger('IdentityCenterGetUserAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIdentityCenterGetUserContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Looking up user by email in identity store ${params.identityStoreId}`) - - const client = createIdentityStoreClient(params) - try { - const result = await getUserByEmail(client, params.identityStoreId, params.email) - logger.info(`Successfully found user ${result.userId}`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to get user:', error) - return NextResponse.json( - { error: `Failed to get user: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/identity-center/list-account-assignments/route.ts b/apps/sim/app/api/tools/identity-center/list-account-assignments/route.ts deleted file mode 100644 index 8528d277108..00000000000 --- a/apps/sim/app/api/tools/identity-center/list-account-assignments/route.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIdentityCenterListAccountAssignmentsContract } from '@/lib/api/contracts/tools/aws/identity-center-list-account-assignments' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSSOAdminClient, listAccountAssignmentsForPrincipal } from '../utils' - -const logger = createLogger('IdentityCenterListAccountAssignmentsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest( - awsIdentityCenterListAccountAssignmentsContract, - request, - { - errorFormat: 'details', - logger, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Listing account assignments for ${params.principalType} ${params.principalId}`) - - const client = createSSOAdminClient(params) - try { - const result = await listAccountAssignmentsForPrincipal( - client, - params.instanceArn, - params.principalId, - params.principalType, - params.maxResults, - params.nextToken - ) - logger.info(`Successfully listed ${result.count} account assignments`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to list account assignments:', error) - return NextResponse.json( - { error: `Failed to list account assignments: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/identity-center/list-accounts/route.ts b/apps/sim/app/api/tools/identity-center/list-accounts/route.ts deleted file mode 100644 index 18be304cbbc..00000000000 --- a/apps/sim/app/api/tools/identity-center/list-accounts/route.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIdentityCenterListAccountsContract } from '@/lib/api/contracts/tools/aws/identity-center-list-accounts' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createOrganizationsClient, listAccounts } from '../utils' - -const logger = createLogger('IdentityCenterListAccountsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIdentityCenterListAccountsContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info('Listing AWS accounts') - - const client = createOrganizationsClient(params) - try { - const result = await listAccounts(client, params.maxResults, params.nextToken) - logger.info(`Successfully listed ${result.count} accounts`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to list AWS accounts:', error) - return NextResponse.json( - { error: `Failed to list AWS accounts: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/identity-center/list-groups/route.ts b/apps/sim/app/api/tools/identity-center/list-groups/route.ts deleted file mode 100644 index 512a6444f5d..00000000000 --- a/apps/sim/app/api/tools/identity-center/list-groups/route.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIdentityCenterListGroupsContract } from '@/lib/api/contracts/tools/aws/identity-center-list-groups' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createIdentityStoreClient, listGroups } from '../utils' - -const logger = createLogger('IdentityCenterListGroupsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIdentityCenterListGroupsContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Listing groups in identity store ${params.identityStoreId}`) - - const client = createIdentityStoreClient(params) - try { - const result = await listGroups( - client, - params.identityStoreId, - params.maxResults, - params.nextToken - ) - logger.info(`Successfully listed ${result.count} groups`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to list groups:', error) - return NextResponse.json( - { error: `Failed to list groups: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/identity-center/list-instances/route.ts b/apps/sim/app/api/tools/identity-center/list-instances/route.ts deleted file mode 100644 index 0c059a841f4..00000000000 --- a/apps/sim/app/api/tools/identity-center/list-instances/route.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIdentityCenterListInstancesContract } from '@/lib/api/contracts/tools/aws/identity-center-list-instances' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSSOAdminClient, listInstances } from '../utils' - -const logger = createLogger('IdentityCenterListInstancesAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIdentityCenterListInstancesContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info('Listing Identity Center instances') - - const client = createSSOAdminClient(params) - try { - const result = await listInstances(client, params.maxResults, params.nextToken) - logger.info(`Successfully listed ${result.count} instances`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to list Identity Center instances:', error) - return NextResponse.json( - { error: `Failed to list Identity Center instances: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/identity-center/list-permission-sets/route.ts b/apps/sim/app/api/tools/identity-center/list-permission-sets/route.ts deleted file mode 100644 index f72e80b3991..00000000000 --- a/apps/sim/app/api/tools/identity-center/list-permission-sets/route.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsIdentityCenterListPermissionSetsContract } from '@/lib/api/contracts/tools/aws/identity-center-list-permission-sets' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSSOAdminClient, listPermissionSets } from '../utils' - -const logger = createLogger('IdentityCenterListPermissionSetsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsIdentityCenterListPermissionSetsContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Listing permission sets for instance ${params.instanceArn}`) - - const client = createSSOAdminClient(params) - try { - const result = await listPermissionSets( - client, - params.instanceArn, - params.maxResults, - params.nextToken - ) - logger.info(`Successfully listed ${result.count} permission sets`) - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to list permission sets:', error) - return NextResponse.json( - { error: `Failed to list permission sets: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/identity-center/utils.ts b/apps/sim/app/api/tools/identity-center/utils.ts deleted file mode 100644 index eb7301f96d3..00000000000 --- a/apps/sim/app/api/tools/identity-center/utils.ts +++ /dev/null @@ -1,304 +0,0 @@ -import { - DescribeGroupCommand, - DescribeUserCommand, - GetGroupIdCommand, - GetUserIdCommand, - IdentitystoreClient, - ListGroupsCommand, -} from '@aws-sdk/client-identitystore' -import { - DescribeAccountCommand, - ListAccountsCommand, - OrganizationsClient, -} from '@aws-sdk/client-organizations' -import { - type AccountAssignmentOperationStatus, - DescribeAccountAssignmentCreationStatusCommand, - DescribeAccountAssignmentDeletionStatusCommand, - DescribePermissionSetCommand, - ListAccountAssignmentsForPrincipalCommand, - ListInstancesCommand, - ListPermissionSetsCommand, - type PrincipalType, - SSOAdminClient, -} from '@aws-sdk/client-sso-admin' -import type { IdentityCenterConnectionConfig } from '@/tools/identity_center/types' - -export function createSSOAdminClient(config: IdentityCenterConnectionConfig): SSOAdminClient { - return new SSOAdminClient({ - region: config.region, - credentials: { - accessKeyId: config.accessKeyId, - secretAccessKey: config.secretAccessKey, - }, - }) -} - -export function createIdentityStoreClient( - config: IdentityCenterConnectionConfig -): IdentitystoreClient { - return new IdentitystoreClient({ - region: config.region, - credentials: { - accessKeyId: config.accessKeyId, - secretAccessKey: config.secretAccessKey, - }, - }) -} - -export function createOrganizationsClient(config: IdentityCenterConnectionConfig) { - return new OrganizationsClient({ - region: 'us-east-1', // Organizations API only available in us-east-1 - credentials: { - accessKeyId: config.accessKeyId, - secretAccessKey: config.secretAccessKey, - }, - }) -} - -export async function listInstances( - client: SSOAdminClient, - maxResults?: number | null, - nextToken?: string | null -) { - const command = new ListInstancesCommand({ - ...(maxResults ? { MaxResults: maxResults } : {}), - ...(nextToken ? { NextToken: nextToken } : {}), - }) - const response = await client.send(command) - const instances = (response.Instances ?? []).map((instance) => ({ - instanceArn: instance.InstanceArn ?? '', - identityStoreId: instance.IdentityStoreId ?? '', - name: instance.Name ?? null, - status: instance.Status ?? '', - statusReason: instance.StatusReason ?? null, - ownerAccountId: instance.OwnerAccountId ?? null, - createdDate: instance.CreatedDate?.toISOString() ?? null, - })) - return { instances, nextToken: response.NextToken ?? null, count: instances.length } -} - -export async function listAccounts( - client: OrganizationsClient, - maxResults?: number | null, - nextToken?: string | null -) { - const command = new ListAccountsCommand({ - ...(maxResults ? { MaxResults: maxResults } : {}), - ...(nextToken ? { NextToken: nextToken } : {}), - }) - const response = await client.send(command) - const accounts = (response.Accounts ?? []).map((account) => ({ - id: account.Id ?? '', - arn: account.Arn ?? '', - name: account.Name ?? '', - email: account.Email ?? '', - status: account.State ?? '', - joinedTimestamp: account.JoinedTimestamp?.toISOString() ?? null, - })) - return { accounts, nextToken: response.NextToken ?? null, count: accounts.length } -} - -export async function listPermissionSets( - client: SSOAdminClient, - instanceArn: string, - maxResults?: number | null, - nextToken?: string | null -) { - const listCommand = new ListPermissionSetsCommand({ - InstanceArn: instanceArn, - ...(maxResults ? { MaxResults: maxResults } : {}), - ...(nextToken ? { NextToken: nextToken } : {}), - }) - const listResponse = await client.send(listCommand) - const permissionSetArns = listResponse.PermissionSets ?? [] - - const permissionSets = await Promise.all( - permissionSetArns.map(async (arn) => { - const describeCommand = new DescribePermissionSetCommand({ - InstanceArn: instanceArn, - PermissionSetArn: arn, - }) - const describeResponse = await client.send(describeCommand) - const ps = describeResponse.PermissionSet - return { - permissionSetArn: ps?.PermissionSetArn ?? arn, - name: ps?.Name ?? '', - description: ps?.Description ?? null, - sessionDuration: ps?.SessionDuration ?? null, - createdDate: ps?.CreatedDate?.toISOString() ?? null, - } - }) - ) - - return { - permissionSets, - nextToken: listResponse.NextToken ?? null, - count: permissionSets.length, - } -} - -export async function getUserByEmail( - ssoClient: IdentitystoreClient, - identityStoreId: string, - email: string -) { - const getUserIdCommand = new GetUserIdCommand({ - IdentityStoreId: identityStoreId, - AlternateIdentifier: { - UniqueAttribute: { - AttributePath: 'emails.value', - AttributeValue: email, - }, - }, - }) - const getUserIdResponse = await ssoClient.send(getUserIdCommand) - const userId = getUserIdResponse.UserId ?? '' - - const describeCommand = new DescribeUserCommand({ - IdentityStoreId: identityStoreId, - UserId: userId, - }) - const describeResponse = await ssoClient.send(describeCommand) - - const primaryEmail = - describeResponse.Emails?.find((e) => e.Primary)?.Value ?? - describeResponse.Emails?.[0]?.Value ?? - null - - return { - userId, - userName: describeResponse.UserName ?? '', - displayName: describeResponse.DisplayName ?? null, - email: primaryEmail, - } -} - -export function mapAssignmentStatus(status: AccountAssignmentOperationStatus) { - return { - status: status.Status ?? '', - requestId: status.RequestId ?? '', - accountId: status.TargetId ?? null, - permissionSetArn: status.PermissionSetArn ?? null, - principalType: status.PrincipalType ?? null, - principalId: status.PrincipalId ?? null, - failureReason: status.FailureReason ?? null, - createdDate: status.CreatedDate?.toISOString() ?? null, - } -} - -export async function checkAssignmentCreationStatus( - client: SSOAdminClient, - instanceArn: string, - requestId: string -) { - const command = new DescribeAccountAssignmentCreationStatusCommand({ - InstanceArn: instanceArn, - AccountAssignmentCreationRequestId: requestId, - }) - const response = await client.send(command) - return mapAssignmentStatus(response.AccountAssignmentCreationStatus ?? {}) -} - -export async function checkAssignmentDeletionStatus( - client: SSOAdminClient, - instanceArn: string, - requestId: string -) { - const command = new DescribeAccountAssignmentDeletionStatusCommand({ - InstanceArn: instanceArn, - AccountAssignmentDeletionRequestId: requestId, - }) - const response = await client.send(command) - return mapAssignmentStatus(response.AccountAssignmentDeletionStatus ?? {}) -} - -export async function listGroups( - client: IdentitystoreClient, - identityStoreId: string, - maxResults?: number | null, - nextToken?: string | null -) { - const command = new ListGroupsCommand({ - IdentityStoreId: identityStoreId, - ...(maxResults ? { MaxResults: maxResults } : {}), - ...(nextToken ? { NextToken: nextToken } : {}), - }) - const response = await client.send(command) - const groups = (response.Groups ?? []).map((group) => ({ - groupId: group.GroupId ?? '', - displayName: group.DisplayName ?? null, - description: group.Description ?? null, - externalIds: group.ExternalIds?.map((e) => ({ issuer: e.Issuer ?? '', id: e.Id ?? '' })) ?? [], - })) - return { groups, nextToken: response.NextToken ?? null, count: groups.length } -} - -export async function getGroupByDisplayName( - client: IdentitystoreClient, - identityStoreId: string, - displayName: string -) { - const getGroupIdCommand = new GetGroupIdCommand({ - IdentityStoreId: identityStoreId, - AlternateIdentifier: { - UniqueAttribute: { - AttributePath: 'displayName', - AttributeValue: displayName, - }, - }, - }) - const getGroupIdResponse = await client.send(getGroupIdCommand) - const groupId = getGroupIdResponse.GroupId ?? '' - - const describeCommand = new DescribeGroupCommand({ - IdentityStoreId: identityStoreId, - GroupId: groupId, - }) - const describeResponse = await client.send(describeCommand) - - return { - groupId, - displayName: describeResponse.DisplayName ?? null, - description: describeResponse.Description ?? null, - } -} - -export async function describeAccount(client: OrganizationsClient, accountId: string) { - const command = new DescribeAccountCommand({ AccountId: accountId }) - const response = await client.send(command) - const account = response.Account - return { - id: account?.Id ?? '', - arn: account?.Arn ?? '', - name: account?.Name ?? '', - email: account?.Email ?? '', - status: account?.State ?? '', - joinedTimestamp: account?.JoinedTimestamp?.toISOString() ?? null, - } -} - -export async function listAccountAssignmentsForPrincipal( - client: SSOAdminClient, - instanceArn: string, - principalId: string, - principalType: string, - maxResults?: number | null, - nextToken?: string | null -) { - const command = new ListAccountAssignmentsForPrincipalCommand({ - InstanceArn: instanceArn, - PrincipalId: principalId, - PrincipalType: principalType as PrincipalType, - ...(maxResults ? { MaxResults: maxResults } : {}), - ...(nextToken ? { NextToken: nextToken } : {}), - }) - const response = await client.send(command) - const assignments = (response.AccountAssignments ?? []).map((a) => ({ - accountId: a.AccountId ?? '', - permissionSetArn: a.PermissionSetArn ?? '', - principalType: a.PrincipalType ?? '', - principalId: a.PrincipalId ?? '', - })) - return { assignments, nextToken: response.NextToken ?? null, count: assignments.length } -} diff --git a/apps/sim/app/api/tools/image/route.ts b/apps/sim/app/api/tools/image/route.ts deleted file mode 100644 index f01e0c16159..00000000000 --- a/apps/sim/app/api/tools/image/route.ts +++ /dev/null @@ -1,1045 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { sleep } from '@sim/utils/helpers' -import { isRecordLike } from '@sim/utils/object' -import { type NextRequest, NextResponse } from 'next/server' -import { - type ImageToolBody, - type imageProviders, - imageProxyQuerySchema, - imageToolContract, -} from '@/lib/api/contracts/tools/media/image' -import { - getValidationErrorMessage, - parseRequest, - searchParamsToObject, - validationErrorResponse, -} from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { - assertKnownSizeWithinLimit, - consumeOrCancelBody, - DEFAULT_MAX_ERROR_BODY_BYTES, - isPayloadSizeLimitError, - readResponseJsonWithLimit, - readResponseTextWithLimit, - readResponseToBufferWithLimit, -} from '@/lib/core/utils/stream-limits' -import { getBaseUrl } from '@/lib/core/utils/urls' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { type FalAICostMetadata, getFalAICostMetadata } from '@/lib/tools/falai-pricing' - -const logger = createLogger('ImageProxyAPI') -const MAX_IMAGE_BYTES = 25 * 1024 * 1024 -const MAX_IMAGE_JSON_BYTES = Math.ceil((MAX_IMAGE_BYTES * 4) / 3) + 256 * 1024 - -export const dynamic = 'force-dynamic' -/** - * Mirrors the hosted workflow execution ceiling (7 days) used by - * `getMaxExecutionTimeout()` for the provider polling loop below. Next.js requires a - * static literal for `maxDuration`, so this value must be kept in sync with that source. - */ -export const maxDuration = 604800 - -type ImageProvider = (typeof imageProviders)[number] - -interface GeneratedImageResult { - buffer: Buffer - contentType: string - fileName: string - provider: ImageProvider - model: string - sourceUrl?: string - description?: string - revisedPrompt?: string - seed?: number - jobId?: string - falaiCost?: FalAICostMetadata -} - -interface StoredImageResponse { - content: string - imageUrl: string - imageFile?: unknown - fileName: string - contentType: string - provider: ImageProvider - model: string - metadata: { - provider: ImageProvider - model: string - description?: string - revisedPrompt?: string - seed?: number - jobId?: string - contentType: string - } - __falaiCostDollars?: number - __falaiBilling?: FalAICostMetadata -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - logger.info(`[${requestId}] Image generation request started`) - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest( - imageToolContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid image generation request:`, error.issues) - return validationErrorResponse( - error, - getValidationErrorMessage(error, 'Invalid request data') - ) - }, - } - ) - if (!parsed.success) return parsed.response - - const body = parsed.data.body - const provider = body.provider as ImageProvider - const { apiKey, model, prompt } = body - - if (prompt.length < 3 || prompt.length > 4000) { - return NextResponse.json( - { error: 'Prompt must be between 3 and 4000 characters' }, - { status: 400 } - ) - } - - logger.info(`[${requestId}] Generating image with ${provider}, model: ${model || 'default'}`) - - let imageResult: GeneratedImageResult - try { - if (provider === 'openai') { - imageResult = await generateWithOpenAI(apiKey, body, requestId, logger) - } else if (provider === 'gemini') { - imageResult = await generateWithGemini(apiKey, body, requestId, logger) - } else if (provider === 'falai') { - imageResult = await generateWithFalAI(apiKey, body, requestId, logger) - } else { - return NextResponse.json({ error: `Unknown provider: ${provider}` }, { status: 400 }) - } - } catch (error) { - logger.error(`[${requestId}] Image generation failed:`, error) - const errorMessage = getErrorMessage(error, 'Image generation failed') - return NextResponse.json( - { error: errorMessage }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } - - const storedImage = await storeGeneratedImage(imageResult, body, authResult.userId, requestId) - - logger.info(`[${requestId}] Image generation completed successfully`, { - provider, - model: storedImage.model, - contentType: storedImage.contentType, - }) - - return NextResponse.json(storedImage) - } catch (error) { - logger.error(`[${requestId}] Image generation route error:`, error) - const errorMessage = getErrorMessage(error, 'Unknown error') - return NextResponse.json( - { error: errorMessage }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } -}) - -/** - * Proxy for fetching images - * This allows client-side requests to fetch images from various sources while avoiding CORS issues - */ -export const GET = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - logger.error(`[${requestId}] Authentication failed for image proxy:`, authResult.error) - return new NextResponse('Unauthorized', { status: 401 }) - } - - const queryResult = imageProxyQuerySchema.safeParse( - searchParamsToObject(request.nextUrl.searchParams) - ) - if (!queryResult.success) { - const error = getValidationErrorMessage(queryResult.error, 'Missing URL parameter') - logger.error(`[${requestId}] ${error}`) - return new NextResponse(error, { status: 400 }) - } - const { url: imageUrl } = queryResult.data - - const urlValidation = await validateUrlWithDNS(imageUrl, 'imageUrl') - if (!urlValidation.isValid) { - logger.warn(`[${requestId}] Blocked image proxy request`, { - url: imageUrl.substring(0, 100), - error: urlValidation.error, - }) - return new NextResponse(urlValidation.error || 'Invalid image URL', { status: 403 }) - } - - logger.info(`[${requestId}] Proxying image request for: ${imageUrl}`) - - try { - const imageResponse = await secureFetchWithPinnedIP(imageUrl, urlValidation.resolvedIP!, { - method: 'GET', - maxResponseBytes: MAX_IMAGE_BYTES, - headers: { - 'User-Agent': - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', - Accept: 'image/webp,image/avif,image/apng,image/svg+xml,image/*,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.9', - 'Accept-Encoding': 'gzip, deflate, br', - Referer: 'https://sim.ai/', - 'Sec-Fetch-Dest': 'image', - 'Sec-Fetch-Mode': 'no-cors', - 'Sec-Fetch-Site': 'cross-site', - }, - }) - - if (!imageResponse.ok) { - await consumeOrCancelBody(imageResponse) - logger.error(`[${requestId}] Image fetch failed:`, { - status: imageResponse.status, - statusText: imageResponse.statusText, - }) - return new NextResponse(`Failed to fetch image: ${imageResponse.statusText}`, { - status: imageResponse.status, - }) - } - - const contentType = imageResponse.headers.get('content-type') || 'image/jpeg' - - const imageBuffer = await readResponseToBufferWithLimit(imageResponse, { - maxBytes: MAX_IMAGE_BYTES, - label: 'image proxy response', - }) - - if (imageBuffer.length === 0) { - logger.error(`[${requestId}] Empty image received`) - return new NextResponse('Empty image received', { status: 404 }) - } - - return new NextResponse(new Uint8Array(imageBuffer), { - headers: { - 'Content-Type': contentType, - 'Access-Control-Allow-Origin': '*', - 'Cache-Control': 'public, max-age=86400', // Cache for 24 hours - }, - }) - } catch (error) { - const errorMessage = toError(error).message - logger.error(`[${requestId}] Image proxy error:`, { error: errorMessage }) - - return new NextResponse(`Failed to proxy image: ${errorMessage}`, { - status: isPayloadSizeLimitError(error) ? 413 : 500, - }) - } -}) - -const OPENAI_IMAGE_MODELS = [ - 'gpt-image-2', - 'gpt-image-1.5', - 'gpt-image-1', - 'gpt-image-1-mini', -] as const -const OPENAI_IMAGE_SIZES = ['auto', '1024x1024', '1536x1024', '1024x1536'] as const -const OPENAI_IMAGE_2_SIZES = [...OPENAI_IMAGE_SIZES, '2560x1440', '3840x2160'] as const -const OPENAI_IMAGE_QUALITIES = ['auto', 'low', 'medium', 'high'] as const -const OPENAI_IMAGE_BACKGROUNDS = ['auto', 'transparent', 'opaque'] as const -const IMAGE_OUTPUT_FORMATS = ['png', 'jpeg', 'webp'] as const -const OPENAI_MODERATION_LEVELS = ['auto', 'low'] as const - -const GEMINI_IMAGE_MODELS = [ - 'gemini-3.1-flash-image-preview', - 'gemini-3-pro-image-preview', - 'gemini-2.5-flash-image', -] as const -const GEMINI_BASE_ASPECT_RATIOS = [ - '1:1', - '2:3', - '3:2', - '3:4', - '4:3', - '4:5', - '5:4', - '9:16', - '16:9', - '21:9', -] as const -const GEMINI_EXTREME_ASPECT_RATIOS = ['1:4', '1:8', '4:1', '8:1'] as const -const GEMINI_IMAGE_SIZES = ['512', '1K', '2K', '4K'] as const -const GEMINI_PRO_IMAGE_SIZES = ['1K', '2K', '4K'] as const - -interface FalAIImageModelConfig { - endpoint: string - defaultSize?: string - sizeOptions?: readonly string[] - defaultAspectRatio?: string - aspectRatios?: readonly string[] - defaultResolution?: string - resolutionOptions?: readonly string[] - defaultOutputFormat?: string - outputFormats?: readonly string[] - defaultQuality?: string - qualityOptions?: readonly string[] - defaultBackground?: string - backgroundOptions?: readonly string[] - defaultSafetyTolerance?: string - safetyToleranceOptions?: readonly string[] - maxNumImages?: number - supportsSeed?: boolean - supportsEnableSafetyChecker?: boolean - supportsEnableWebSearch?: boolean - supportsThinkingLevel?: boolean -} - -const FALAI_NANO_BANANA_ASPECT_RATIOS = [ - 'auto', - '21:9', - '16:9', - '3:2', - '4:3', - '5:4', - '1:1', - '4:5', - '3:4', - '2:3', - '9:16', -] as const -const FALAI_EXTREME_ASPECT_RATIOS = ['4:1', '1:4', '8:1', '1:8'] as const -const FALAI_STANDARD_IMAGE_SIZES = [ - 'square_hd', - 'square', - 'portrait_4_3', - 'portrait_16_9', - 'landscape_4_3', - 'landscape_16_9', -] as const -const FALAI_SEEDREAM_IMAGE_SIZES = [...FALAI_STANDARD_IMAGE_SIZES, 'auto_2K', 'auto_4K'] as const - -const FALAI_IMAGE_MODEL_CONFIGS: Record = { - 'nano-banana-2': { - endpoint: 'fal-ai/nano-banana-2', - defaultAspectRatio: 'auto', - aspectRatios: [...FALAI_NANO_BANANA_ASPECT_RATIOS, ...FALAI_EXTREME_ASPECT_RATIOS], - defaultResolution: '1K', - resolutionOptions: ['0.5K', '1K', '2K', '4K'], - defaultOutputFormat: 'png', - outputFormats: IMAGE_OUTPUT_FORMATS, - defaultSafetyTolerance: '4', - safetyToleranceOptions: ['1', '2', '3', '4', '5', '6'], - maxNumImages: 4, - supportsSeed: true, - supportsEnableWebSearch: true, - supportsThinkingLevel: true, - }, - 'nano-banana-pro': { - endpoint: 'fal-ai/nano-banana-pro', - defaultAspectRatio: '1:1', - aspectRatios: FALAI_NANO_BANANA_ASPECT_RATIOS, - defaultResolution: '1K', - resolutionOptions: ['1K', '2K', '4K'], - defaultOutputFormat: 'png', - outputFormats: IMAGE_OUTPUT_FORMATS, - defaultSafetyTolerance: '4', - safetyToleranceOptions: ['1', '2', '3', '4', '5', '6'], - maxNumImages: 4, - supportsSeed: true, - supportsEnableWebSearch: true, - }, - 'nano-banana': { - endpoint: 'fal-ai/nano-banana', - defaultAspectRatio: '1:1', - aspectRatios: FALAI_NANO_BANANA_ASPECT_RATIOS.filter((ratio) => ratio !== 'auto'), - defaultOutputFormat: 'png', - outputFormats: IMAGE_OUTPUT_FORMATS, - defaultSafetyTolerance: '4', - safetyToleranceOptions: ['1', '2', '3', '4', '5', '6'], - maxNumImages: 4, - supportsSeed: true, - }, - 'gpt-image-1.5': { - endpoint: 'fal-ai/gpt-image-1.5', - defaultSize: '1024x1024', - sizeOptions: ['1024x1024', '1536x1024', '1024x1536'], - defaultQuality: 'high', - qualityOptions: ['low', 'medium', 'high'], - defaultBackground: 'auto', - backgroundOptions: OPENAI_IMAGE_BACKGROUNDS, - defaultOutputFormat: 'png', - outputFormats: IMAGE_OUTPUT_FORMATS, - maxNumImages: 4, - }, - 'seedream-v4.5': { - endpoint: 'fal-ai/bytedance/seedream/v4.5/text-to-image', - defaultSize: 'auto_2K', - sizeOptions: FALAI_SEEDREAM_IMAGE_SIZES, - maxNumImages: 6, - supportsSeed: true, - supportsEnableSafetyChecker: true, - }, - 'flux-2-pro': { - endpoint: 'fal-ai/flux-2-pro', - defaultSize: 'landscape_4_3', - sizeOptions: FALAI_STANDARD_IMAGE_SIZES, - defaultOutputFormat: 'jpeg', - outputFormats: ['jpeg', 'png'], - defaultSafetyTolerance: '2', - safetyToleranceOptions: ['1', '2', '3', '4', '5'], - supportsSeed: true, - supportsEnableSafetyChecker: true, - }, - 'grok-imagine-image': { - endpoint: 'xai/grok-imagine-image', - defaultAspectRatio: '1:1', - aspectRatios: [ - '2:1', - '20:9', - '19.5:9', - '16:9', - '4:3', - '3:2', - '1:1', - '2:3', - '3:4', - '9:16', - '9:19.5', - '9:20', - '1:2', - ], - defaultResolution: '1k', - resolutionOptions: ['1k', '2k'], - defaultOutputFormat: 'jpeg', - outputFormats: IMAGE_OUTPUT_FORMATS, - maxNumImages: 4, - }, -} - -function getStringProperty( - record: Record | undefined, - key: string -): string | undefined { - const value = record?.[key] - return typeof value === 'string' ? value : undefined -} - -function getNumberProperty( - record: Record | undefined, - key: string -): number | undefined { - const value = record?.[key] - return typeof value === 'number' ? value : undefined -} - -function firstRecord(value: unknown): Record | undefined { - return Array.isArray(value) ? value.find(isRecordLike) : undefined -} - -function pickAllowed( - value: string | undefined, - allowed: readonly string[], - fallback: string -): string { - return value && allowed.includes(value) ? value : fallback -} - -function clampInteger( - value: number | undefined, - min: number, - max: number, - fallback: number -): number { - if (typeof value !== 'number' || !Number.isInteger(value)) return fallback - return Math.min(Math.max(value, min), max) -} - -function getContentTypeForFormat(format: string | undefined): string { - if (format === 'jpeg') return 'image/jpeg' - if (format === 'webp') return 'image/webp' - return 'image/png' -} - -function extensionFromContentType(contentType: string): string { - if (contentType.includes('jpeg') || contentType.includes('jpg')) return 'jpg' - if (contentType.includes('webp')) return 'webp' - return 'png' -} - -async function bufferFromImageUrl(url: string): Promise<{ buffer: Buffer; contentType: string }> { - if (url.startsWith('data:')) { - const match = /^data:([^;]+);base64,(.+)$/u.exec(url) - if (!match) throw new Error('Invalid data URI image response') - const buffer = Buffer.from(match[2], 'base64') - assertKnownSizeWithinLimit(buffer.length, MAX_IMAGE_BYTES, 'inline image response') - return { - contentType: match[1], - buffer, - } - } - - const urlValidation = await validateUrlWithDNS(url, 'imageUrl') - if (!urlValidation.isValid || !urlValidation.resolvedIP) { - throw new Error(urlValidation.error || 'Generated image URL failed validation') - } - - const imageResponse = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP, { - method: 'GET', - maxResponseBytes: MAX_IMAGE_BYTES, - }) - if (!imageResponse.ok) { - await readResponseTextWithLimit(imageResponse, { - maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, - label: 'generated image error response', - }).catch(() => '') - throw new Error(`Failed to download generated image: ${imageResponse.status}`) - } - - const contentType = imageResponse.headers.get('content-type') || 'image/png' - const buffer = await readResponseToBufferWithLimit(imageResponse, { - maxBytes: MAX_IMAGE_BYTES, - label: 'generated image download', - }) - return { buffer, contentType } -} - -async function generateWithOpenAI( - apiKey: string, - body: ImageToolBody, - requestId: string, - logger: ReturnType -): Promise { - const model = pickAllowed(body.model, OPENAI_IMAGE_MODELS, 'gpt-image-1.5') - const size = - model === 'gpt-image-2' - ? pickAllowed(body.size, OPENAI_IMAGE_2_SIZES, 'auto') - : pickAllowed(body.size, OPENAI_IMAGE_SIZES, 'auto') - const outputFormat = pickAllowed(body.outputFormat, IMAGE_OUTPUT_FORMATS, 'png') - const requestBody: Record = { - model, - prompt: body.prompt, - size, - n: 1, - } - - if (body.quality) { - requestBody.quality = pickAllowed(body.quality, OPENAI_IMAGE_QUALITIES, 'auto') - } - if (body.background) { - requestBody.background = pickAllowed(body.background, OPENAI_IMAGE_BACKGROUNDS, 'auto') - } - if (body.outputFormat) { - requestBody.output_format = outputFormat - } - if (body.moderation) { - requestBody.moderation = pickAllowed(body.moderation, OPENAI_MODERATION_LEVELS, 'auto') - } - - const openaiResponse = await fetch('https://api.openai.com/v1/images/generations', { - method: 'POST', - headers: { - Authorization: `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(requestBody), - }) - - if (!openaiResponse.ok) { - const error = await readResponseTextWithLimit(openaiResponse, { - maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, - label: 'OpenAI image error response', - }) - throw new Error(`OpenAI API error: ${openaiResponse.status} - ${error}`) - } - - const data = await readResponseJsonWithLimit(openaiResponse, { - maxBytes: MAX_IMAGE_JSON_BYTES, - label: 'OpenAI image response', - }) - if (!isRecordLike(data)) { - throw new Error('Invalid OpenAI image response') - } - - const firstImage = firstRecord(data.data) - const base64Image = getStringProperty(firstImage, 'b64_json') - const imageUrl = getStringProperty(firstImage, 'url') - const revisedPrompt = getStringProperty(firstImage, 'revised_prompt') - let buffer: Buffer - let contentType = getContentTypeForFormat(outputFormat) - - if (base64Image) { - buffer = Buffer.from(base64Image, 'base64') - assertKnownSizeWithinLimit(buffer.length, MAX_IMAGE_BYTES, 'OpenAI image response') - } else if (imageUrl) { - const downloaded = await bufferFromImageUrl(imageUrl) - buffer = downloaded.buffer - contentType = downloaded.contentType - } else { - logger.error(`[${requestId}] OpenAI response missing image payload`) - throw new Error('No image data found in OpenAI response') - } - - return { - buffer, - contentType, - fileName: `openai-${model}.${extensionFromContentType(contentType)}`, - provider: 'openai', - model, - sourceUrl: imageUrl, - revisedPrompt, - } -} - -async function generateWithGemini( - apiKey: string, - body: ImageToolBody, - requestId: string, - logger: ReturnType -): Promise { - const model = pickAllowed(body.model, GEMINI_IMAGE_MODELS, 'gemini-3.1-flash-image-preview') - const aspectRatios = - model === 'gemini-3.1-flash-image-preview' - ? [...GEMINI_BASE_ASPECT_RATIOS, ...GEMINI_EXTREME_ASPECT_RATIOS] - : GEMINI_BASE_ASPECT_RATIOS - const imageConfig: Record = {} - - if (body.aspectRatio) { - imageConfig.aspectRatio = pickAllowed(body.aspectRatio, aspectRatios, '1:1') - } - - if (model === 'gemini-3.1-flash-image-preview' && body.resolution) { - imageConfig.imageSize = pickAllowed(body.resolution, GEMINI_IMAGE_SIZES, '1K') - } else if (model === 'gemini-3-pro-image-preview' && body.resolution) { - imageConfig.imageSize = pickAllowed(body.resolution, GEMINI_PRO_IMAGE_SIZES, '1K') - } - - const requestBody: Record = { - contents: [ - { - parts: [{ text: body.prompt }], - }, - ], - } - - requestBody.generationConfig = { - responseModalities: ['TEXT', 'IMAGE'], - ...(Object.keys(imageConfig).length > 0 && { imageConfig }), - } - - const geminiResponse = await fetch( - `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent`, - { - method: 'POST', - headers: { - 'x-goog-api-key': apiKey, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(requestBody), - } - ) - - if (!geminiResponse.ok) { - const error = await readResponseTextWithLimit(geminiResponse, { - maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, - label: 'Gemini image error response', - }) - throw new Error(`Gemini API error: ${geminiResponse.status} - ${error}`) - } - - const data = await readResponseJsonWithLimit(geminiResponse, { - maxBytes: MAX_IMAGE_JSON_BYTES, - label: 'Gemini image response', - }) - if (!isRecordLike(data)) { - throw new Error('Invalid Gemini image response') - } - - const candidate = firstRecord(data.candidates) - const content = isRecordLike(candidate?.content) ? candidate.content : undefined - const parts = Array.isArray(content?.parts) ? content.parts : [] - const textPart = parts.find((part) => isRecordLike(part) && typeof part.text === 'string') - const imagePart = parts.find((part) => { - if (!isRecordLike(part)) return false - return isRecordLike(part.inlineData) || isRecordLike(part.inline_data) - }) - - if (!isRecordLike(imagePart)) { - logger.error(`[${requestId}] Gemini response missing image part`) - throw new Error('No image data found in Gemini response') - } - - const inlineData = isRecordLike(imagePart.inlineData) - ? imagePart.inlineData - : isRecordLike(imagePart.inline_data) - ? imagePart.inline_data - : undefined - const base64Image = getStringProperty(inlineData, 'data') - const contentType = - getStringProperty(inlineData, 'mimeType') || - getStringProperty(inlineData, 'mime_type') || - 'image/png' - - if (!base64Image) { - throw new Error('Gemini image response missing inline image data') - } - - return { - buffer: (() => { - const buffer = Buffer.from(base64Image, 'base64') - assertKnownSizeWithinLimit(buffer.length, MAX_IMAGE_BYTES, 'Gemini image response') - return buffer - })(), - contentType, - fileName: `gemini-${model}.${extensionFromContentType(contentType)}`, - provider: 'gemini', - model, - description: isRecordLike(textPart) ? getStringProperty(textPart, 'text') : undefined, - } -} - -function buildFalAIQueueUrl(endpoint: string, requestId: string, path: 'status' | 'response') { - return `https://queue.fal.run/${endpoint}/requests/${requestId}/${path}` -} - -function getFalAIErrorMessage(error: unknown): string { - if (typeof error === 'string') return error - if (isRecordLike(error)) { - return ( - getStringProperty(error, 'message') || - getStringProperty(error, 'detail') || - JSON.stringify(error) - ) - } - return 'Unknown Fal.ai error' -} - -async function generateWithFalAI( - apiKey: string, - body: ImageToolBody, - requestId: string, - logger: ReturnType -): Promise { - const model = body.model || 'nano-banana-2' - const modelConfig = FALAI_IMAGE_MODEL_CONFIGS[model] - if (!modelConfig) { - throw new Error(`Unknown Fal.ai image model: ${model}`) - } - - const requestBody: Record = { - prompt: body.prompt, - sync_mode: false, - } - - if (modelConfig.maxNumImages) { - requestBody.num_images = clampInteger(body.numImages, 1, modelConfig.maxNumImages, 1) - } - if (modelConfig.supportsSeed && body.seed !== undefined) { - requestBody.seed = body.seed - } - if (modelConfig.sizeOptions && modelConfig.defaultSize) { - requestBody.image_size = pickAllowed( - body.size, - modelConfig.sizeOptions, - modelConfig.defaultSize - ) - } - if (modelConfig.aspectRatios && modelConfig.defaultAspectRatio) { - requestBody.aspect_ratio = pickAllowed( - body.aspectRatio, - modelConfig.aspectRatios, - modelConfig.defaultAspectRatio - ) - } - if (modelConfig.resolutionOptions && modelConfig.defaultResolution) { - requestBody.resolution = pickAllowed( - body.resolution, - modelConfig.resolutionOptions, - modelConfig.defaultResolution - ) - } - if (modelConfig.outputFormats && modelConfig.defaultOutputFormat) { - requestBody.output_format = pickAllowed( - body.outputFormat, - modelConfig.outputFormats, - modelConfig.defaultOutputFormat - ) - } - if (modelConfig.qualityOptions && modelConfig.defaultQuality) { - requestBody.quality = pickAllowed( - body.quality, - modelConfig.qualityOptions, - modelConfig.defaultQuality - ) - } - if (modelConfig.backgroundOptions && modelConfig.defaultBackground) { - requestBody.background = pickAllowed( - body.background, - modelConfig.backgroundOptions, - modelConfig.defaultBackground - ) - } - if (modelConfig.safetyToleranceOptions && modelConfig.defaultSafetyTolerance) { - requestBody.safety_tolerance = pickAllowed( - body.safetyTolerance, - modelConfig.safetyToleranceOptions, - modelConfig.defaultSafetyTolerance - ) - } - if (modelConfig.supportsEnableSafetyChecker && body.enableSafetyChecker !== undefined) { - requestBody.enable_safety_checker = body.enableSafetyChecker - } - if (modelConfig.supportsEnableWebSearch && body.enableWebSearch !== undefined) { - requestBody.enable_web_search = body.enableWebSearch - } - if (modelConfig.supportsThinkingLevel && body.thinkingLevel) { - requestBody.thinking_level = pickAllowed(body.thinkingLevel, ['minimal', 'high'], 'minimal') - } - - const createResponse = await fetch(`https://queue.fal.run/${modelConfig.endpoint}`, { - method: 'POST', - headers: { - Authorization: `Key ${apiKey}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(requestBody), - }) - - if (!createResponse.ok) { - const error = await readResponseTextWithLimit(createResponse, { - maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, - label: 'Fal.ai create error response', - }) - throw new Error(`Fal.ai API error: ${createResponse.status} - ${error}`) - } - - const createData = await readResponseJsonWithLimit(createResponse, { - maxBytes: MAX_IMAGE_JSON_BYTES, - label: 'Fal.ai create response', - }) - if (!isRecordLike(createData)) { - throw new Error('Invalid Fal.ai queue response') - } - - const falRequestId = getStringProperty(createData, 'request_id') - if (!falRequestId) { - throw new Error('Fal.ai queue response missing request_id') - } - - const statusUrl = - getStringProperty(createData, 'status_url') || - buildFalAIQueueUrl(modelConfig.endpoint, falRequestId, 'status') - const responseUrl = - getStringProperty(createData, 'response_url') || - buildFalAIQueueUrl(modelConfig.endpoint, falRequestId, 'response') - - logger.info(`[${requestId}] Fal.ai image request created: ${falRequestId}`) - - const pollIntervalMs = 3000 - const maxAttempts = Math.ceil(getMaxExecutionTimeout() / pollIntervalMs) - let attempts = 0 - - while (attempts < maxAttempts) { - await sleep(pollIntervalMs) - - const statusResponse = await fetch(statusUrl, { - headers: { - Authorization: `Key ${apiKey}`, - }, - }) - - if (!statusResponse.ok) { - await readResponseTextWithLimit(statusResponse, { - maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, - label: 'Fal.ai status error response', - }).catch(() => '') - throw new Error(`Fal.ai status check failed: ${statusResponse.status}`) - } - - const statusData = await readResponseJsonWithLimit(statusResponse, { - maxBytes: MAX_IMAGE_JSON_BYTES, - label: 'Fal.ai status response', - }) - if (!isRecordLike(statusData)) { - throw new Error('Invalid Fal.ai status response') - } - - const status = getStringProperty(statusData, 'status') - if (status === 'COMPLETED') { - const statusError = statusData.error - if (statusError) { - throw new Error(`Fal.ai generation failed: ${getFalAIErrorMessage(statusError)}`) - } - - const resultResponse = await fetch( - getStringProperty(statusData, 'response_url') || responseUrl, - { - headers: { - Authorization: `Key ${apiKey}`, - }, - } - ) - - if (!resultResponse.ok) { - await readResponseTextWithLimit(resultResponse, { - maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, - label: 'Fal.ai result error response', - }).catch(() => '') - throw new Error(`Failed to fetch Fal.ai result: ${resultResponse.status}`) - } - - const resultData = await readResponseJsonWithLimit(resultResponse, { - maxBytes: MAX_IMAGE_JSON_BYTES, - label: 'Fal.ai result response', - }) - if (!isRecordLike(resultData)) { - throw new Error('Invalid Fal.ai result response') - } - - const firstImage = firstRecord(resultData.images) - const imageUrl = - getStringProperty(firstImage, 'url') || - getStringProperty(firstImage, 'data') || - getStringProperty(firstImage, 'content') - if (!imageUrl) { - throw new Error('No image URL in Fal.ai response') - } - - const downloaded = await bufferFromImageUrl(imageUrl) - const contentType = - getStringProperty(firstImage, 'content_type') || - getStringProperty(firstImage, 'contentType') || - downloaded.contentType - const fileName = - getStringProperty(firstImage, 'file_name') || - getStringProperty(firstImage, 'fileName') || - `falai-${model}.${extensionFromContentType(contentType)}` - - return { - buffer: downloaded.buffer, - contentType, - fileName, - provider: 'falai', - model, - sourceUrl: imageUrl.startsWith('data:') ? undefined : imageUrl, - description: getStringProperty(resultData, 'description'), - revisedPrompt: getStringProperty(resultData, 'revised_prompt'), - seed: getNumberProperty(resultData, 'seed'), - jobId: falRequestId, - falaiCost: body.useHostedCostTracking - ? await getFalAICostMetadata({ - apiKey, - endpointId: modelConfig.endpoint, - requestId: falRequestId, - }) - : undefined, - } - } - - if (['ERROR', 'FAILED', 'CANCELLED'].includes(status || '')) { - throw new Error(`Fal.ai generation failed: ${getFalAIErrorMessage(statusData.error)}`) - } - - attempts += 1 - } - - throw new Error('Fal.ai image generation timed out') -} - -async function storeGeneratedImage( - imageResult: GeneratedImageResult, - body: ImageToolBody, - userId: string, - requestId: string -): Promise { - const timestamp = Date.now() - const safeFileName = imageResult.fileName || `image-${imageResult.provider}-${timestamp}.png` - const executionContext = - body.workspaceId && body.workflowId && body.executionId - ? { - workspaceId: body.workspaceId, - workflowId: body.workflowId, - executionId: body.executionId, - } - : null - - if (executionContext) { - const { uploadExecutionFile } = await import('@/lib/uploads/contexts/execution') - const imageFile = await uploadExecutionFile( - executionContext, - imageResult.buffer, - safeFileName, - imageResult.contentType, - userId - ) - - return { - content: imageFile.url, - imageUrl: imageFile.url, - imageFile, - fileName: safeFileName, - contentType: imageResult.contentType, - provider: imageResult.provider, - model: imageResult.model, - metadata: { - provider: imageResult.provider, - model: imageResult.model, - description: imageResult.description, - revisedPrompt: imageResult.revisedPrompt, - seed: imageResult.seed, - jobId: imageResult.jobId, - contentType: imageResult.contentType, - }, - __falaiCostDollars: imageResult.falaiCost?.costDollars, - __falaiBilling: imageResult.falaiCost, - } - } - - const { StorageService } = await import('@/lib/uploads') - const fileInfo = await StorageService.uploadFile({ - file: imageResult.buffer, - fileName: safeFileName, - contentType: imageResult.contentType, - context: 'copilot', - }) - const imageUrl = `${getBaseUrl()}${fileInfo.path}` - logger.info(`[${requestId}] Stored generated image fallback`, { - fileName: safeFileName, - size: imageResult.buffer.length, - }) - - return { - content: imageUrl, - imageUrl, - fileName: safeFileName, - contentType: imageResult.contentType, - provider: imageResult.provider, - model: imageResult.model, - metadata: { - provider: imageResult.provider, - model: imageResult.model, - description: imageResult.description, - revisedPrompt: imageResult.revisedPrompt, - seed: imageResult.seed, - jobId: imageResult.jobId, - contentType: imageResult.contentType, - }, - __falaiCostDollars: imageResult.falaiCost?.costDollars, - __falaiBilling: imageResult.falaiCost, - } -} diff --git a/apps/sim/app/api/tools/instagram/download-media/route.test.ts b/apps/sim/app/api/tools/instagram/download-media/route.test.ts deleted file mode 100644 index 31885fdc83c..00000000000 --- a/apps/sim/app/api/tools/instagram/download-media/route.test.ts +++ /dev/null @@ -1,353 +0,0 @@ -/** - * @vitest-environment node - */ -import { createMockRequest, hybridAuthMockFns } from '@sim/testing' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -const { - mockDeleteFileMetadata, - mockDeleteFiles, - mockDownloadFileFromUrl, - mockUploadExecutionFile, - mockUploadCopilotFile, -} = vi.hoisted(() => ({ - mockDeleteFileMetadata: vi.fn(), - mockDeleteFiles: vi.fn(), - mockDownloadFileFromUrl: vi.fn(), - mockUploadExecutionFile: vi.fn(), - mockUploadCopilotFile: vi.fn(), -})) - -vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ - downloadFileFromUrl: mockDownloadFileFromUrl, -})) -vi.mock('@/lib/uploads/contexts/execution', () => ({ - uploadExecutionFile: mockUploadExecutionFile, -})) -vi.mock('@/lib/uploads/contexts/copilot', () => ({ - uploadCopilotFile: mockUploadCopilotFile, -})) -vi.mock('@/lib/uploads/core/storage-service', () => ({ - deleteFiles: mockDeleteFiles, -})) -vi.mock('@/lib/uploads/server/metadata', () => ({ - deleteFileMetadata: mockDeleteFileMetadata, -})) - -import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' -import { POST } from '@/app/api/tools/instagram/download-media/route' -import { instagramDownloadMediaTool } from '@/tools/instagram/download_media' - -const mockFetch = vi.fn() -const JPEG_BYTES = Buffer.from([0xff, 0xd8, 0xff, 0x01]) - -function executionFile(name: string, type: string, size: number) { - return { - id: `file-${name}`, - name, - url: `/api/files/serve/execution/${name}`, - size, - type, - key: `execution/workflow-1/execution-1/${name}`, - context: 'execution', - } -} - -beforeEach(() => { - vi.clearAllMocks() - vi.stubGlobal('fetch', mockFetch) - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - mockDownloadFileFromUrl.mockResolvedValue(Buffer.from('instagram-media')) - mockDeleteFiles.mockResolvedValue({ deleted: 0, failed: [] }) - mockDeleteFileMetadata.mockResolvedValue(true) - mockUploadExecutionFile.mockImplementation( - async ( - _context: { workspaceId: string; workflowId: string; executionId: string }, - buffer: Buffer, - name: string, - type: string - ) => executionFile(name, type, buffer.length) - ) -}) - -afterEach(() => { - vi.unstubAllGlobals() -}) - -describe('POST /api/tools/instagram/download-media', () => { - it('stores a single download as an execution-scoped UserFile', async () => { - mockFetch.mockResolvedValueOnce( - Response.json({ - id: 'media-1', - media_type: 'IMAGE', - media_url: 'https://scontent.example.com/media-1.jpg', - }) - ) - mockDownloadFileFromUrl.mockResolvedValueOnce(JPEG_BYTES) - - const response = await POST( - createMockRequest('POST', { - accessToken: 'instagram-token', - mediaId: 'media-1', - filename: 'campaign-cover.png', - workspaceId: 'workspace-1', - workflowId: 'workflow-1', - executionId: 'execution-1', - }) - ) - - expect(response.status).toBe(200) - const data = await response.json() - expect(data).toEqual({ - success: true, - output: { - files: [executionFile('campaign-cover.jpg', 'image/jpeg', JPEG_BYTES.length)], - mediaId: 'media-1', - mediaType: 'IMAGE', - downloadedCount: 1, - }, - }) - expect(mockDownloadFileFromUrl).toHaveBeenCalledWith( - 'https://scontent.example.com/media-1.jpg', - expect.objectContaining({ - maxBytes: MAX_FILE_SIZE, - signal: expect.any(AbortSignal), - userId: 'user-1', - }) - ) - expect(mockUploadExecutionFile).toHaveBeenCalledWith( - { - workspaceId: 'workspace-1', - workflowId: 'workflow-1', - executionId: 'execution-1', - }, - JPEG_BYTES, - 'campaign-cover.jpg', - 'image/jpeg', - 'user-1' - ) - expect(mockUploadCopilotFile).not.toHaveBeenCalled() - }) - - it('downloads carousel children sequentially and preserves their order', async () => { - mockFetch - .mockResolvedValueOnce( - Response.json({ - id: 'carousel-1', - media_type: 'CAROUSEL_ALBUM', - children: { data: [{ id: 'child-image' }, { id: 'child-video' }] }, - }) - ) - .mockResolvedValueOnce( - Response.json({ - id: 'child-image', - media_type: 'IMAGE', - media_url: 'https://scontent.example.com/child-image.jpg', - }) - ) - .mockResolvedValueOnce( - Response.json({ - id: 'child-video', - media_type: 'VIDEO', - media_url: 'https://scontent.example.com/child-video.mp4', - }) - ) - mockDownloadFileFromUrl - .mockResolvedValueOnce(JPEG_BYTES) - .mockResolvedValueOnce(Buffer.from('video')) - - const response = await POST( - createMockRequest('POST', { - accessToken: 'instagram-token', - mediaId: 'carousel-1', - filename: 'launch', - workspaceId: 'workspace-1', - workflowId: 'workflow-1', - executionId: 'execution-1', - }) - ) - - expect(response.status).toBe(200) - const data = await response.json() - expect(data.output).toEqual({ - files: [ - executionFile('launch-1.jpg', 'image/jpeg', JPEG_BYTES.length), - executionFile('launch-2.mp4', 'video/mp4', 5), - ], - mediaId: 'carousel-1', - mediaType: 'CAROUSEL_ALBUM', - downloadedCount: 2, - }) - expect(mockDownloadFileFromUrl.mock.calls.map(([url]) => url)).toEqual([ - 'https://scontent.example.com/child-image.jpg', - 'https://scontent.example.com/child-video.mp4', - ]) - expect(mockUploadExecutionFile.mock.calls.map(([, , name]) => name)).toEqual([ - 'launch-1.jpg', - 'launch-2.mp4', - ]) - expect(mockUploadExecutionFile.mock.invocationCallOrder[0]).toBeLessThan( - mockFetch.mock.invocationCallOrder[2] - ) - }) - - it('rolls back earlier carousel files when a later child cannot be downloaded', async () => { - mockFetch - .mockResolvedValueOnce( - Response.json({ - id: 'carousel-1', - media_type: 'CAROUSEL_ALBUM', - children: { data: [{ id: 'child-image' }, { id: 'child-missing' }] }, - }) - ) - .mockResolvedValueOnce( - Response.json({ - id: 'child-image', - media_type: 'IMAGE', - media_url: 'https://scontent.example.com/child-image.jpg', - }) - ) - .mockResolvedValueOnce( - Response.json( - { error: { message: 'The second carousel item is unavailable' } }, - { status: 404 } - ) - ) - mockDownloadFileFromUrl.mockResolvedValueOnce(JPEG_BYTES) - mockDeleteFiles.mockResolvedValueOnce({ deleted: 1, failed: [] }) - - const response = await POST( - createMockRequest('POST', { - accessToken: 'instagram-token', - mediaId: 'carousel-1', - filename: 'launch', - workspaceId: 'workspace-1', - workflowId: 'workflow-1', - executionId: 'execution-1', - }) - ) - - const storedFile = executionFile('launch-1.jpg', 'image/jpeg', JPEG_BYTES.length) - expect(response.status).toBe(404) - expect(mockDeleteFiles).toHaveBeenCalledWith([storedFile.key], 'execution') - expect(mockDeleteFileMetadata).toHaveBeenCalledWith(storedFile.key) - }) - - it('does not preserve an image MIME type when the downloaded bytes are not a raster image', async () => { - mockFetch.mockResolvedValueOnce( - Response.json({ - id: 'media-invalid-image', - media_type: 'IMAGE', - media_url: 'https://scontent.example.com/media-invalid-image.jpg', - }) - ) - const invalidImage = Buffer.from('not an image') - mockDownloadFileFromUrl.mockResolvedValueOnce(invalidImage) - - const response = await POST( - createMockRequest('POST', { - accessToken: 'instagram-token', - mediaId: 'media-invalid-image', - workspaceId: 'workspace-1', - workflowId: 'workflow-1', - executionId: 'execution-1', - }) - ) - - expect(response.status).toBe(200) - expect(mockUploadExecutionFile).toHaveBeenCalledWith( - expect.any(Object), - invalidImage, - 'instagram-media-invalid-image.bin', - 'application/octet-stream', - 'user-1' - ) - }) - - it('returns 413 when a media download exceeds the size cap', async () => { - mockFetch.mockResolvedValueOnce( - Response.json({ - id: 'media-large', - media_type: 'VIDEO', - media_url: 'https://scontent.example.com/media-large.mp4', - }) - ) - mockDownloadFileFromUrl.mockRejectedValueOnce( - new PayloadSizeLimitError({ - label: 'Instagram media download', - maxBytes: MAX_FILE_SIZE, - observedBytes: MAX_FILE_SIZE + 1, - }) - ) - - const response = await POST( - createMockRequest('POST', { - accessToken: 'instagram-token', - mediaId: 'media-large', - }) - ) - - expect(response.status).toBe(413) - expect(await response.json()).toEqual({ - success: false, - error: 'Instagram media exceeds the 100 MB canonical User File limit', - }) - expect(mockUploadExecutionFile).not.toHaveBeenCalled() - expect(mockUploadCopilotFile).not.toHaveBeenCalled() - }) -}) - -describe('instagramDownloadMediaTool', () => { - it('forwards execution context and returns canonical file-array output', async () => { - const body = instagramDownloadMediaTool.request.body?.({ - accessToken: 'instagram-token', - mediaId: 'media-1', - filename: 'campaign-cover', - _context: { - workspaceId: 'workspace-1', - workflowId: 'workflow-1', - executionId: 'execution-1', - }, - }) - expect(body).toEqual({ - accessToken: 'instagram-token', - mediaId: 'media-1', - filename: 'campaign-cover', - workspaceId: 'workspace-1', - workflowId: 'workflow-1', - executionId: 'execution-1', - }) - - const file = executionFile('campaign-cover.jpg', 'image/jpeg', 11) - const result = await instagramDownloadMediaTool.transformResponse?.( - Response.json({ - success: true, - output: { - files: [file], - mediaId: 'media-1', - mediaType: 'IMAGE', - downloadedCount: 1, - }, - }), - { - accessToken: 'instagram-token', - mediaId: 'media-1', - } - ) - - expect(result).toEqual({ - success: true, - output: { - files: [file], - mediaId: 'media-1', - mediaType: 'IMAGE', - downloadedCount: 1, - }, - }) - }) -}) diff --git a/apps/sim/app/api/tools/instagram/download-media/route.ts b/apps/sim/app/api/tools/instagram/download-media/route.ts deleted file mode 100644 index c4611a4d488..00000000000 --- a/apps/sim/app/api/tools/instagram/download-media/route.ts +++ /dev/null @@ -1,380 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { isRecordLike } from '@sim/utils/object' -import { type NextRequest, NextResponse } from 'next/server' -import { - type InstagramDownloadMediaRouteResponse, - instagramDownloadMediaContract, - instagramDownloadMediaOutputSchema, -} from '@/lib/api/contracts/tools/instagram' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { uploadCopilotFile } from '@/lib/uploads/contexts/copilot' -import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' -import { deleteFiles } from '@/lib/uploads/core/storage-service' -import { deleteFileMetadata } from '@/lib/uploads/server/metadata' -import type { StorageContext } from '@/lib/uploads/shared/types' -import { - getExtensionFromMimeType, - getFileExtension, - getMimeTypeFromExtension, -} from '@/lib/uploads/utils/file-utils' -import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server' -import { MAX_FILE_SIZE, sniffImageContentType } from '@/lib/uploads/utils/validation' -import { sanitizeFileName } from '@/executor/constants' -import type { UserFile } from '@/executor/types' -import { bearerHeaders, graphUrl, idString, readGraphError } from '@/tools/instagram/utils' - -const logger = createLogger('InstagramDownloadMediaAPI') -const MAX_GRAPH_METADATA_BYTES = 256 * 1024 -const MAX_CAROUSEL_ITEMS = 10 -const ROOT_MEDIA_FIELDS = 'id,media_type,media_url,children{id}' -const CHILD_MEDIA_FIELDS = 'id,media_type,media_url' - -export const dynamic = 'force-dynamic' -export const maxDuration = 900 - -interface InstagramMediaMetadata { - id: string - mediaType: string | null - mediaUrl: string | null - childIds: string[] -} - -type InstagramMediaMetadataResult = - | { success: true; data: InstagramMediaMetadata } - | { success: false; error: string; status: number } - -function failureResponse(error: string, status: number) { - const body = { success: false, error } satisfies InstagramDownloadMediaRouteResponse - return NextResponse.json(body, { status }) -} - -function normalizedId(value: unknown): string | null { - return typeof value === 'string' || typeof value === 'number' ? idString(value) : null -} - -function parseMediaMetadata(data: unknown): InstagramMediaMetadataResult { - if (!isRecordLike(data)) { - return { success: false, error: 'Instagram returned invalid media metadata', status: 502 } - } - - const id = normalizedId(data.id) - if (!id) { - return { success: false, error: 'Instagram media metadata did not include an ID', status: 502 } - } - - const mediaType = typeof data.media_type === 'string' ? data.media_type : null - const mediaUrl = - typeof data.media_url === 'string' && data.media_url.length > 0 ? data.media_url : null - const children = data.children - - if (children === undefined) { - return { success: true, data: { id, mediaType, mediaUrl, childIds: [] } } - } - - if (!isRecordLike(children) || !Array.isArray(children.data)) { - return { success: false, error: 'Instagram returned invalid carousel metadata', status: 502 } - } - - if (children.data.length > MAX_CAROUSEL_ITEMS) { - return { - success: false, - error: `Instagram carousel exceeds the ${MAX_CAROUSEL_ITEMS}-item download limit`, - status: 502, - } - } - - const childIds: string[] = [] - for (const child of children.data) { - if (!isRecordLike(child)) { - return { success: false, error: 'Instagram returned an invalid carousel item', status: 502 } - } - const childId = normalizedId(child.id) - if (!childId) { - return { - success: false, - error: 'Instagram carousel item did not include an ID', - status: 502, - } - } - childIds.push(childId) - } - - return { success: true, data: { id, mediaType, mediaUrl, childIds } } -} - -async function fetchMediaMetadata({ - accessToken, - mediaId, - fields, - signal, -}: { - accessToken: string - mediaId: string - fields: string - signal: AbortSignal -}): Promise { - const response = await fetch(graphUrl(`/${encodeURIComponent(mediaId)}`, { fields }), { - headers: bearerHeaders(accessToken), - signal, - }) - - if (!response.ok) { - return { - success: false, - error: await readGraphError(response), - status: response.status >= 400 && response.status < 500 ? response.status : 502, - } - } - - const data = await readResponseJsonWithLimit(response, { - maxBytes: MAX_GRAPH_METADATA_BYTES, - label: `Instagram media ${mediaId} metadata`, - signal, - }) - return parseMediaMetadata(data) -} - -function inferContentType(mediaUrl: string, mediaType: string | null): string { - if (mediaType === 'VIDEO') return 'video/mp4' - if (mediaType === 'IMAGE') return 'image/jpeg' - - let extension = '' - try { - extension = getFileExtension(new URL(mediaUrl).pathname) - } catch { - extension = '' - } - - const mimeType = getMimeTypeFromExtension(extension) - if (mimeType !== 'application/octet-stream') return mimeType - return 'application/octet-stream' -} - -function resolveDownloadedContentType( - buffer: Buffer, - mediaUrl: string, - mediaType: string | null -): string { - const inferred = inferContentType(mediaUrl, mediaType) - if (mediaType === 'IMAGE' || inferred.startsWith('image/')) { - return sniffImageContentType(buffer) ?? 'application/octet-stream' - } - return inferred -} - -function buildFilename({ - filename, - mediaId, - contentType, - itemIndex, - itemCount, -}: { - filename?: string - mediaId: string - contentType: string - itemIndex: number - itemCount: number -}): string { - const extension = getExtensionFromMimeType(contentType) ?? 'bin' - if (!filename) return sanitizeFileName(`instagram-${mediaId}.${extension}`) - - const sanitized = sanitizeFileName(filename).replace(/^\.+/, '') - const lastDot = sanitized.lastIndexOf('.') - const base = (lastDot > 0 ? sanitized.slice(0, lastDot) : sanitized) || `instagram-${mediaId}` - const suffix = itemCount > 1 ? `-${itemIndex + 1}` : '' - return `${base}${suffix}.${extension}` -} - -async function downloadAndStoreMedia({ - metadata, - filename, - itemIndex, - itemCount, - userId, - executionContext, - signal, -}: { - metadata: InstagramMediaMetadata - filename?: string - itemIndex: number - itemCount: number - userId: string - executionContext?: { workspaceId: string; workflowId: string; executionId: string } - signal: AbortSignal -}): Promise { - if (!metadata.mediaUrl) { - throw new Error(`Instagram media ${metadata.id} did not include a downloadable URL`) - } - - const buffer = await downloadFileFromUrl(metadata.mediaUrl, { - maxBytes: MAX_FILE_SIZE, - signal, - userId, - }) - const contentType = resolveDownloadedContentType(buffer, metadata.mediaUrl, metadata.mediaType) - const storedFilename = buildFilename({ - filename, - mediaId: metadata.id, - contentType, - itemIndex, - itemCount, - }) - if (executionContext) { - return uploadExecutionFile(executionContext, buffer, storedFilename, contentType, userId) - } - - return uploadCopilotFile({ - buffer, - fileName: storedFilename, - contentType, - userId, - }) -} - -/** Removes successfully stored files when a multi-item download cannot return a complete result. */ -async function rollbackStoredFiles(files: UserFile[], context: StorageContext): Promise { - if (files.length === 0) return - - const keys = files.map((file) => file.key) - let failedKeys: Set - try { - const deletion = await deleteFiles(keys, context) - failedKeys = new Set(deletion.failed.map((failure) => failure.key)) - if (deletion.failed.length > 0) { - logger.warn('Instagram media rollback could not delete every stored object', { - context, - failedKeys: [...failedKeys], - }) - } - } catch (error) { - logger.warn('Instagram media rollback failed before metadata cleanup', { - context, - error: getErrorMessage(error), - keys, - }) - return - } - - for (const key of keys) { - if (failedKeys.has(key)) continue - try { - await deleteFileMetadata(key) - } catch (error) { - logger.warn('Instagram media rollback could not delete file metadata', { - error: getErrorMessage(error), - key, - }) - } - } -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return failureResponse(authResult.error || 'Unauthorized', 401) - } - - const parsed = await parseRequest( - instagramDownloadMediaContract, - request, - {}, - { - validationErrorResponse: (error) => - failureResponse(getValidationErrorMessage(error, 'Invalid request data'), 400), - } - ) - if (!parsed.success) return parsed.response - - const files: UserFile[] = [] - let storageContext: StorageContext = 'copilot' - try { - const body = parsed.data.body - const rootResult = await fetchMediaMetadata({ - accessToken: body.accessToken, - mediaId: body.mediaId, - fields: ROOT_MEDIA_FIELDS, - signal: request.signal, - }) - if (!rootResult.success) return failureResponse(rootResult.error, rootResult.status) - - const rootMedia = rootResult.data - const itemCount = rootMedia.childIds.length || 1 - const executionContext = - body.workspaceId && body.workflowId && body.executionId - ? { - workspaceId: body.workspaceId, - workflowId: body.workflowId, - executionId: body.executionId, - } - : undefined - storageContext = executionContext ? 'execution' : 'copilot' - - if (rootMedia.childIds.length === 0) { - files.push( - await downloadAndStoreMedia({ - metadata: rootMedia, - filename: body.filename, - itemIndex: 0, - itemCount, - userId: authResult.userId, - executionContext, - signal: request.signal, - }) - ) - } else { - for (const [itemIndex, childId] of rootMedia.childIds.entries()) { - const childResult = await fetchMediaMetadata({ - accessToken: body.accessToken, - mediaId: childId, - fields: CHILD_MEDIA_FIELDS, - signal: request.signal, - }) - if (!childResult.success) { - await rollbackStoredFiles(files, storageContext) - return failureResponse(childResult.error, childResult.status) - } - - files.push( - await downloadAndStoreMedia({ - metadata: childResult.data, - filename: body.filename, - itemIndex, - itemCount, - userId: authResult.userId, - executionContext, - signal: request.signal, - }) - ) - } - } - - const output = instagramDownloadMediaOutputSchema.parse({ - files, - mediaId: rootMedia.id, - mediaType: rootMedia.mediaType, - downloadedCount: files.length, - }) - const responseBody = { - success: true, - output, - } satisfies InstagramDownloadMediaRouteResponse - - return NextResponse.json(responseBody) - } catch (error) { - await rollbackStoredFiles(files, storageContext) - logger.error('Instagram media download failed', { error }) - - if (isPayloadSizeLimitError(error) && error.maxBytes === MAX_FILE_SIZE) { - return failureResponse('Instagram media exceeds the 100 MB canonical User File limit', 413) - } - - return failureResponse( - getErrorMessage(error, 'Failed to download Instagram media'), - isPayloadSizeLimitError(error) ? 413 : 500 - ) - } -}) diff --git a/apps/sim/app/api/tools/instagram/publish-carousel/route.ts b/apps/sim/app/api/tools/instagram/publish-carousel/route.ts deleted file mode 100644 index 7c79758e5b5..00000000000 --- a/apps/sim/app/api/tools/instagram/publish-carousel/route.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { createLogger, getRequestContext } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { instagramPublishCarouselContract } from '@/lib/api/contracts/tools/instagram' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createMediaContainer, - publishMediaContainer, - resolveIgUserId, - resolveInstagramCarouselMedia, - waitForContainerReady, -} from '@/app/api/tools/instagram/server-utils' - -export const dynamic = 'force-dynamic' -/** - * Children are polled in parallel, so the worst case is one five-minute poll - * window for the children plus another for the parent container. - */ -export const maxDuration = 900 - -const logger = createLogger('InstagramPublishCarouselAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = getRequestContext()?.requestId ?? 'unknown' - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn('Unauthorized Instagram publish carousel', { error: authResult.error }) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(instagramPublishCarouselContract, request, {}) - if (!parsed.success) return parsed.response - const body = parsed.data.body - - const resolved = await resolveInstagramCarouselMedia( - body.media, - authResult.userId, - requestId, - logger - ) - if (resolved.error || !resolved.items) { - return NextResponse.json( - { - success: false, - error: resolved.error?.message || 'Failed to resolve carousel media', - output: { containerId: null, mediaId: null, statusCode: null }, - }, - { status: resolved.error?.status || 400 } - ) - } - - const igUserId = await resolveIgUserId( - body.accessToken, - body.igUserId ?? undefined, - request.signal - ) - - const childIds: string[] = [] - for (const item of resolved.items) { - const childBody: Record = { - is_carousel_item: true, - } - if (item.kind === 'video') { - childBody.media_type = 'VIDEO' - childBody.video_url = item.url - } else { - childBody.image_url = item.url - } - childIds.push( - await createMediaContainer(body.accessToken, igUserId, childBody, request.signal) - ) - } - - const childResults = await Promise.allSettled( - childIds.map((childId) => waitForContainerReady(body.accessToken, childId, request.signal)) - ) - const failedChild = childResults.find( - (result): result is PromiseRejectedResult => result.status === 'rejected' - ) - if (failedChild) { - throw failedChild.reason - } - - const parentBody: Record = { - media_type: 'CAROUSEL', - children: childIds.join(','), - } - if (body.caption) parentBody.caption = body.caption - - const containerId = await createMediaContainer( - body.accessToken, - igUserId, - parentBody, - request.signal - ) - const { statusCode } = await waitForContainerReady( - body.accessToken, - containerId, - request.signal - ) - const mediaId = await publishMediaContainer( - body.accessToken, - igUserId, - containerId, - request.signal - ) - - return NextResponse.json({ - success: true, - output: { containerId, mediaId, statusCode }, - }) - } catch (error) { - logger.error('Instagram publish carousel failed', { error }) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Failed to publish carousel'), - output: { containerId: null, mediaId: null, statusCode: null }, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/instagram/publish-image/route.ts b/apps/sim/app/api/tools/instagram/publish-image/route.ts deleted file mode 100644 index fe02954c420..00000000000 --- a/apps/sim/app/api/tools/instagram/publish-image/route.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { createLogger, getRequestContext } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { instagramPublishImageContract } from '@/lib/api/contracts/tools/instagram' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createMediaContainer, - publishMediaContainer, - resolveIgUserId, - resolveInstagramMedia, - waitForContainerReady, -} from '@/app/api/tools/instagram/server-utils' - -export const dynamic = 'force-dynamic' -/** Meta may poll container status once per minute for up to five minutes. */ -export const maxDuration = 600 - -const logger = createLogger('InstagramPublishImageAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = getRequestContext()?.requestId ?? 'unknown' - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn('Unauthorized Instagram publish image', { error: authResult.error }) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(instagramPublishImageContract, request, {}) - if (!parsed.success) return parsed.response - const body = parsed.data.body - - const resolved = await resolveInstagramMedia({ - input: body.image, - userId: authResult.userId, - requestId, - logger, - role: 'image', - label: 'Image', - }) - if (resolved.error || !resolved.media) { - return NextResponse.json( - { - success: false, - error: resolved.error?.message || 'Failed to resolve image', - output: { containerId: null, mediaId: null, statusCode: null }, - }, - { status: resolved.error?.status || 400 } - ) - } - - const igUserId = await resolveIgUserId( - body.accessToken, - body.igUserId ?? undefined, - request.signal - ) - const containerBody: Record = { - image_url: resolved.media.url, - } - if (body.caption) containerBody.caption = body.caption - if (body.altText) containerBody.alt_text = body.altText - if (body.isAiGenerated === true) containerBody.is_ai_generated = true - - const containerId = await createMediaContainer( - body.accessToken, - igUserId, - containerBody, - request.signal - ) - const { statusCode } = await waitForContainerReady( - body.accessToken, - containerId, - request.signal - ) - const mediaId = await publishMediaContainer( - body.accessToken, - igUserId, - containerId, - request.signal - ) - - return NextResponse.json({ - success: true, - output: { containerId, mediaId, statusCode }, - }) - } catch (error) { - logger.error('Instagram publish image failed', { error }) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Failed to publish image'), - output: { containerId: null, mediaId: null, statusCode: null }, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/instagram/publish-reel/route.ts b/apps/sim/app/api/tools/instagram/publish-reel/route.ts deleted file mode 100644 index 048befcd8cf..00000000000 --- a/apps/sim/app/api/tools/instagram/publish-reel/route.ts +++ /dev/null @@ -1,131 +0,0 @@ -import { createLogger, getRequestContext } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { instagramPublishReelContract } from '@/lib/api/contracts/tools/instagram' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createMediaContainer, - publishMediaContainer, - resolveIgUserId, - resolveInstagramMedia, - waitForContainerReady, -} from '@/app/api/tools/instagram/server-utils' - -export const dynamic = 'force-dynamic' -/** Meta may poll container status once per minute for up to five minutes. */ -export const maxDuration = 600 - -const logger = createLogger('InstagramPublishReelAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = getRequestContext()?.requestId ?? 'unknown' - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn('Unauthorized Instagram publish reel', { error: authResult.error }) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(instagramPublishReelContract, request, {}) - if (!parsed.success) return parsed.response - const body = parsed.data.body - - const resolvedVideo = await resolveInstagramMedia({ - input: body.video, - userId: authResult.userId, - requestId, - logger, - role: 'video', - label: 'Video', - }) - if (resolvedVideo.error || !resolvedVideo.media) { - return NextResponse.json( - { - success: false, - error: resolvedVideo.error?.message || 'Failed to resolve video', - output: { containerId: null, mediaId: null, statusCode: null }, - }, - { status: resolvedVideo.error?.status || 400 } - ) - } - - let coverUrl: string | undefined - if (body.cover != null) { - const resolvedCover = await resolveInstagramMedia({ - input: body.cover, - userId: authResult.userId, - requestId, - logger, - role: 'cover', - required: false, - label: 'Cover image', - }) - if (resolvedCover.error) { - return NextResponse.json( - { - success: false, - error: resolvedCover.error.message, - output: { containerId: null, mediaId: null, statusCode: null }, - }, - { status: resolvedCover.error.status } - ) - } - coverUrl = resolvedCover.media?.url - } - - const igUserId = await resolveIgUserId( - body.accessToken, - body.igUserId ?? undefined, - request.signal - ) - const containerBody: Record = { - media_type: 'REELS', - video_url: resolvedVideo.media.url, - } - if (body.caption) containerBody.caption = body.caption - if (coverUrl) containerBody.cover_url = coverUrl - if (body.shareToFeed !== undefined && body.shareToFeed !== null) { - containerBody.share_to_feed = body.shareToFeed - } - if (body.thumbOffset != null) containerBody.thumb_offset = body.thumbOffset - - const containerId = await createMediaContainer( - body.accessToken, - igUserId, - containerBody, - request.signal - ) - const { statusCode } = await waitForContainerReady( - body.accessToken, - containerId, - request.signal - ) - const mediaId = await publishMediaContainer( - body.accessToken, - igUserId, - containerId, - request.signal - ) - - return NextResponse.json({ - success: true, - output: { containerId, mediaId, statusCode }, - }) - } catch (error) { - logger.error('Instagram publish reel failed', { error }) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Failed to publish reel'), - output: { containerId: null, mediaId: null, statusCode: null }, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/instagram/publish-story/route.ts b/apps/sim/app/api/tools/instagram/publish-story/route.ts deleted file mode 100644 index 3732caea7c7..00000000000 --- a/apps/sim/app/api/tools/instagram/publish-story/route.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { createLogger, getRequestContext } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { instagramPublishStoryContract } from '@/lib/api/contracts/tools/instagram' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createMediaContainer, - publishMediaContainer, - resolveIgUserId, - resolveInstagramMedia, - waitForContainerReady, -} from '@/app/api/tools/instagram/server-utils' - -export const dynamic = 'force-dynamic' -/** Meta may poll container status once per minute for up to five minutes. */ -export const maxDuration = 600 - -const logger = createLogger('InstagramPublishStoryAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = getRequestContext()?.requestId ?? 'unknown' - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn('Unauthorized Instagram publish story', { error: authResult.error }) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(instagramPublishStoryContract, request, {}) - if (!parsed.success) return parsed.response - const body = parsed.data.body - - const resolved = await resolveInstagramMedia({ - input: body.media, - userId: authResult.userId, - requestId, - logger, - role: 'story', - label: 'Story media', - }) - if (resolved.error || !resolved.media) { - return NextResponse.json( - { - success: false, - error: resolved.error?.message || 'Failed to resolve story media', - output: { containerId: null, mediaId: null, statusCode: null }, - }, - { status: resolved.error?.status || 400 } - ) - } - - const igUserId = await resolveIgUserId( - body.accessToken, - body.igUserId ?? undefined, - request.signal - ) - const containerBody: Record = { - media_type: 'STORIES', - } - if (resolved.media.kind === 'video') { - containerBody.video_url = resolved.media.url - } else { - containerBody.image_url = resolved.media.url - } - - const containerId = await createMediaContainer( - body.accessToken, - igUserId, - containerBody, - request.signal - ) - const { statusCode } = await waitForContainerReady( - body.accessToken, - containerId, - request.signal - ) - const mediaId = await publishMediaContainer( - body.accessToken, - igUserId, - containerId, - request.signal - ) - - return NextResponse.json({ - success: true, - output: { containerId, mediaId, statusCode }, - }) - } catch (error) { - logger.error('Instagram publish story failed', { error }) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Failed to publish story'), - output: { containerId: null, mediaId: null, statusCode: null }, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/instagram/publish-video/route.ts b/apps/sim/app/api/tools/instagram/publish-video/route.ts deleted file mode 100644 index 98b4d5eca85..00000000000 --- a/apps/sim/app/api/tools/instagram/publish-video/route.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { createLogger, getRequestContext } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { instagramPublishVideoContract } from '@/lib/api/contracts/tools/instagram' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createMediaContainer, - publishMediaContainer, - resolveIgUserId, - resolveInstagramMedia, - waitForContainerReady, -} from '@/app/api/tools/instagram/server-utils' - -export const dynamic = 'force-dynamic' -/** Meta may poll container status once per minute for up to five minutes. */ -export const maxDuration = 600 - -const logger = createLogger('InstagramPublishVideoAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = getRequestContext()?.requestId ?? 'unknown' - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn('Unauthorized Instagram publish video', { error: authResult.error }) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(instagramPublishVideoContract, request, {}) - if (!parsed.success) return parsed.response - const body = parsed.data.body - - const resolvedVideo = await resolveInstagramMedia({ - input: body.video, - userId: authResult.userId, - requestId, - logger, - role: 'video', - label: 'Video', - }) - if (resolvedVideo.error || !resolvedVideo.media) { - return NextResponse.json( - { - success: false, - error: resolvedVideo.error?.message || 'Failed to resolve video', - output: { containerId: null, mediaId: null, statusCode: null }, - }, - { status: resolvedVideo.error?.status || 400 } - ) - } - - let coverUrl: string | undefined - if (body.cover != null) { - const resolvedCover = await resolveInstagramMedia({ - input: body.cover, - userId: authResult.userId, - requestId, - logger, - role: 'cover', - required: false, - label: 'Cover image', - }) - if (resolvedCover.error) { - return NextResponse.json( - { - success: false, - error: resolvedCover.error.message, - output: { containerId: null, mediaId: null, statusCode: null }, - }, - { status: resolvedCover.error.status } - ) - } - coverUrl = resolvedCover.media?.url - } - - const igUserId = await resolveIgUserId( - body.accessToken, - body.igUserId ?? undefined, - request.signal - ) - const containerBody: Record = { - media_type: 'REELS', - video_url: resolvedVideo.media.url, - share_to_feed: true, - } - if (body.caption) containerBody.caption = body.caption - if (coverUrl) containerBody.cover_url = coverUrl - - const containerId = await createMediaContainer( - body.accessToken, - igUserId, - containerBody, - request.signal - ) - const { statusCode } = await waitForContainerReady( - body.accessToken, - containerId, - request.signal - ) - const mediaId = await publishMediaContainer( - body.accessToken, - igUserId, - containerId, - request.signal - ) - - return NextResponse.json({ - success: true, - output: { containerId, mediaId, statusCode }, - }) - } catch (error) { - logger.error('Instagram publish video failed', { error }) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Failed to publish video'), - output: { containerId: null, mediaId: null, statusCode: null }, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/instagram/server-utils.test.ts b/apps/sim/app/api/tools/instagram/server-utils.test.ts deleted file mode 100644 index 7fd1a53a80a..00000000000 --- a/apps/sim/app/api/tools/instagram/server-utils.test.ts +++ /dev/null @@ -1,245 +0,0 @@ -/** - * @vitest-environment node - */ -import type { Logger } from '@sim/logger' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -const { mockHasCloudStorage, mockResolveFileInputToUrl } = vi.hoisted(() => ({ - mockHasCloudStorage: vi.fn(), - mockResolveFileInputToUrl: vi.fn(), -})) - -vi.mock('@/lib/uploads/core/storage-service', () => ({ - hasCloudStorage: mockHasCloudStorage, -})) - -vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ - resolveFileInputToUrl: mockResolveFileInputToUrl, -})) - -import { - createMediaContainer, - INSTAGRAM_MEDIA_URL_TTL_SECONDS, - publishMediaContainer, - resolveIgUserId, - resolveInstagramCarouselMedia, - resolveInstagramMedia, -} from '@/app/api/tools/instagram/server-utils' - -const logger = {} as Logger -const context = { - userId: 'user-1', - requestId: 'request-1', - logger, -} - -function uploadedFile(overrides: Record = {}) { - return { - id: 'file-1', - key: 'execution/workflow-1/execution-1/photo.jpg', - name: 'photo.jpg', - size: 1024, - type: 'image/jpeg', - ...overrides, - } -} - -beforeEach(() => { - vi.clearAllMocks() - mockHasCloudStorage.mockReturnValue(true) - mockResolveFileInputToUrl.mockImplementation(async ({ file }: { file?: { name?: string } }) => ({ - fileUrl: `https://signed.example.com/${file?.name || 'media'}`, - })) -}) - -afterEach(() => { - vi.unstubAllGlobals() -}) - -describe('resolveInstagramMedia', () => { - it('rejects non-file inputs before resolving them', async () => { - const result = await resolveInstagramMedia({ - ...context, - input: 'https://cdn.example.com/photo.jpg', - role: 'image', - }) - - expect(result.error).toEqual({ - status: 400, - message: 'Media must be a Sim file', - }) - expect(mockResolveFileInputToUrl).not.toHaveBeenCalled() - }) - - it('resolves an uploaded file with the Instagram publishing URL lifetime', async () => { - const file = uploadedFile() - const result = await resolveInstagramMedia({ ...context, input: file, role: 'image' }) - - expect(result.media).toEqual({ - url: 'https://signed.example.com/photo.jpg', - kind: 'image', - mimeType: 'image/jpeg', - size: 1024, - name: 'photo.jpg', - }) - expect(mockResolveFileInputToUrl).toHaveBeenCalledWith({ - file, - ...context, - presignExpirySeconds: INSTAGRAM_MEDIA_URL_TTL_SECONDS, - }) - }) - - it('requires cloud storage for publishing files', async () => { - mockHasCloudStorage.mockReturnValue(false) - - const result = await resolveInstagramMedia({ ...context, input: uploadedFile(), role: 'image' }) - expect(result.error).toEqual({ - status: 400, - message: expect.stringContaining('Cloud storage is required'), - }) - expect(mockResolveFileInputToUrl).not.toHaveBeenCalled() - }) - - it('validates JPEG MIME type and size without loading file bytes', async () => { - const invalidType = await resolveInstagramMedia({ - ...context, - input: uploadedFile({ name: 'photo.png', type: 'image/png' }), - role: 'image', - label: 'Image', - }) - const oversized = await resolveInstagramMedia({ - ...context, - input: uploadedFile({ size: 8 * 1024 * 1024 + 1 }), - role: 'image', - label: 'Image', - }) - - expect(invalidType.error?.message).toBe('Image must be a JPEG image (got image/png)') - expect(oversized.error?.message).toContain("Instagram's 8MB JPEG limit") - }) - - it.each([ - { role: 'video' as const, maxBytes: 300 * 1024 * 1024, label: 'Video' }, - { role: 'story' as const, maxBytes: 100 * 1024 * 1024, label: 'Story' }, - ])('enforces the $role video size limit', async ({ role, maxBytes, label }) => { - const result = await resolveInstagramMedia({ - ...context, - input: uploadedFile({ - key: 'execution/workflow-1/execution-1/video.mp4', - name: 'video.mp4', - size: maxBytes + 1, - type: 'video/mp4', - }), - role, - label, - }) - - expect(result.error?.message).toContain(`video limit for ${role}`) - }) - - it('rejects unsupported video formats', async () => { - const result = await resolveInstagramMedia({ - ...context, - input: uploadedFile({ name: 'video.webm', type: 'video/webm' }), - role: 'video', - label: 'Video', - }) - - expect(result.error?.message).toBe('Video must be an MP4 or MOV video (got video/webm)') - }) -}) - -describe('resolveInstagramCarouselMedia', () => { - it('resolves canonical files in order and infers image and video types sequentially', async () => { - let activeResolutions = 0 - let maxActiveResolutions = 0 - mockResolveFileInputToUrl.mockImplementation(async ({ file }: { file?: { name?: string } }) => { - activeResolutions += 1 - maxActiveResolutions = Math.max(maxActiveResolutions, activeResolutions) - await Promise.resolve() - activeResolutions -= 1 - return { fileUrl: `https://signed.example.com/${file?.name}` } - }) - - const result = await resolveInstagramCarouselMedia( - [ - uploadedFile({ name: 'carousel-1.jpg' }), - uploadedFile({ name: 'carousel-2.mp4', type: 'video/mp4' }), - ], - context.userId, - context.requestId, - logger - ) - - expect(result.items?.map(({ url, kind }) => ({ url, kind }))).toEqual([ - { url: 'https://signed.example.com/carousel-1.jpg', kind: 'image' }, - { url: 'https://signed.example.com/carousel-2.mp4', kind: 'video' }, - ]) - expect(maxActiveResolutions).toBe(1) - }) - - it.each([ - { count: 1, label: 'too few' }, - { count: 11, label: 'too many' }, - ])('rejects $label carousel items before resolving them', async ({ count }) => { - const input = Array.from({ length: count }, (_, index) => - uploadedFile({ id: `file-${index + 1}`, name: `carousel-${index + 1}.jpg` }) - ) - - const result = await resolveInstagramCarouselMedia( - input, - context.userId, - context.requestId, - logger - ) - - expect(result.error).toEqual({ - status: 400, - message: 'Carousels require between 2 and 10 items', - }) - expect(mockResolveFileInputToUrl).not.toHaveBeenCalled() - }) - - it('rejects non-file string inputs', async () => { - const result = await resolveInstagramCarouselMedia( - 'https://example.com/one.jpg,https://example.com/two.jpg', - context.userId, - context.requestId, - logger - ) - - expect(result.error).toEqual({ status: 400, message: 'Carousel media is required' }) - }) -}) - -describe('Instagram publishing requests', () => { - it('resolves the connected account when no override is supplied', async () => { - const fetchMock = vi.fn().mockResolvedValue(Response.json({ user_id: 123 }, { status: 200 })) - vi.stubGlobal('fetch', fetchMock) - - await expect(resolveIgUserId('token')).resolves.toBe('123') - expect(fetchMock).toHaveBeenCalledOnce() - }) - - it('creates and publishes form-encoded containers', async () => { - const fetchMock = vi - .fn() - .mockResolvedValueOnce(Response.json({ id: 'container-1' }, { status: 200 })) - .mockResolvedValueOnce(Response.json({ id: 'media-1' }, { status: 200 })) - vi.stubGlobal('fetch', fetchMock) - - await expect( - createMediaContainer('token', 'user-1', { image_url: 'https://signed.example/image.jpg' }) - ).resolves.toBe('container-1') - await expect(publishMediaContainer('token', 'user-1', 'container-1')).resolves.toBe('media-1') - - expect(fetchMock.mock.calls[0]?.[1]).toMatchObject({ - method: 'POST', - body: 'image_url=https%3A%2F%2Fsigned.example%2Fimage.jpg', - }) - expect(fetchMock.mock.calls[1]?.[1]).toMatchObject({ - method: 'POST', - body: 'creation_id=container-1', - }) - }) -}) diff --git a/apps/sim/app/api/tools/jira/add-attachment/route.ts b/apps/sim/app/api/tools/jira/add-attachment/route.ts deleted file mode 100644 index 70218256ffc..00000000000 --- a/apps/sim/app/api/tools/jira/add-attachment/route.ts +++ /dev/null @@ -1,131 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jiraAddAttachmentContract } from '@/lib/api/contracts/selectors/jira' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' - -const logger = createLogger('JiraAddAttachmentAPI') - -export const dynamic = 'force-dynamic' - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = `jira-attach-${Date.now()}` - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json( - { success: false, error: authResult.error || 'Unauthorized' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(jiraAddAttachmentContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const userFiles = processFilesToUserFiles(validatedData.files, requestId, logger) - if (userFiles.length === 0) { - return NextResponse.json( - { success: false, error: 'No valid files provided for upload' }, - { status: 400 } - ) - } - - const cloudId = - validatedData.cloudId || - (await getJiraCloudId(validatedData.domain, validatedData.accessToken)) - - const formData = new FormData() - // Every attachment lands in the same multipart body, so the ceiling covers the - // set rather than each file on its own. - let remainingBytes = MAX_BUFFERED_TRANSFER_BYTES - - for (const file of userFiles) { - const denied = await assertToolFileAccess(file.key, authResult.userId, requestId, logger) - if (denied) return denied - let buffer: Buffer - let downloadedContentType = '' - try { - const result = await downloadServableFileFromStorage(file, requestId, logger, { - maxBytes: remainingBytes, - }) - buffer = result.buffer - downloadedContentType = result.contentType - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - throw error - } - remainingBytes -= buffer.length - const blob = new Blob([new Uint8Array(buffer)], { - type: downloadedContentType || file.type || 'application/octet-stream', - }) - formData.append('file', blob, file.name) - } - - const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${validatedData.issueKey}/attachments` - - const response = await fetch(url, { - method: 'POST', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'X-Atlassian-Token': 'no-check', - }, - body: formData, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error(`[${requestId}] Jira attachment upload failed`, { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - return NextResponse.json( - { - success: false, - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - }, - { status: response.status } - ) - } - - const jiraAttachments = await response.json() - const attachmentsList = Array.isArray(jiraAttachments) ? jiraAttachments : [] - - const attachmentIds = attachmentsList.map((att: any) => att.id).filter(Boolean) - const attachments = attachmentsList.map((att: any) => ({ - id: att.id ?? '', - filename: att.filename ?? '', - mimeType: att.mimeType ?? '', - size: att.size ?? 0, - content: att.content ?? '', - })) - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - issueKey: validatedData.issueKey, - attachments, - attachmentIds, - files: userFiles, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Jira attachment upload error`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Internal server error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jira/update/route.ts b/apps/sim/app/api/tools/jira/update/route.ts deleted file mode 100644 index f8357b18bd4..00000000000 --- a/apps/sim/app/api/tools/jira/update/route.ts +++ /dev/null @@ -1,176 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jiraUpdateContract } from '@/lib/api/contracts/selectors/jira' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId, validateJiraIssueKey } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage, toAdf } from '@/tools/jira/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JiraUpdateAPI') - -export const PUT = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(jiraUpdateContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - issueKey, - summary, - title, - description, - priority, - assignee, - labels, - components, - duedate, - fixVersions, - environment, - customFieldId, - customFieldValue, - notifyUsers, - cloudId: providedCloudId, - } = parsed.data.body - - const cloudId = providedCloudId || (await getJiraCloudId(domain, accessToken)) - logger.info('Using cloud ID:', cloudId) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const issueKeyValidation = validateJiraIssueKey(issueKey, 'issueKey') - if (!issueKeyValidation.isValid) { - return NextResponse.json({ error: issueKeyValidation.error }, { status: 400 }) - } - - const notifyParam = - notifyUsers === false ? '?notifyUsers=false' : notifyUsers === true ? '?notifyUsers=true' : '' - const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${issueKey}${notifyParam}` - - logger.info('Updating Jira issue at:', url) - - const summaryValue = summary || title - const fields: Record = {} - - if (summaryValue !== undefined && summaryValue !== null && summaryValue !== '') { - fields.summary = summaryValue - } - - if (description !== undefined && description !== null && description !== '') { - fields.description = toAdf(description) - } - - if (priority !== undefined && priority !== null && priority !== '') { - const isNumericId = /^\d+$/.test(priority) - fields.priority = isNumericId ? { id: priority } : { name: priority } - } - - if (assignee !== undefined && assignee !== null && assignee !== '') { - fields.assignee = { - accountId: assignee, - } - } - - if (labels !== undefined && labels !== null && labels.length > 0) { - fields.labels = labels - } - - if (components !== undefined && components !== null && components.length > 0) { - fields.components = components.map((name) => ({ name })) - } - - if (duedate !== undefined && duedate !== null && duedate !== '') { - fields.duedate = duedate - } - - if (fixVersions !== undefined && fixVersions !== null && fixVersions.length > 0) { - fields.fixVersions = fixVersions.map((name) => ({ name })) - } - - if (environment !== undefined && environment !== null && environment !== '') { - fields.environment = toAdf(environment) - } - - if ( - customFieldId !== undefined && - customFieldId !== null && - customFieldId !== '' && - customFieldValue !== undefined && - customFieldValue !== null && - customFieldValue !== '' - ) { - const fieldId = customFieldId.startsWith('customfield_') - ? customFieldId - : `customfield_${customFieldId}` - fields[fieldId] = customFieldValue - } - - const requestBody = { fields } - - const response = await fetch(url, { - method: 'PUT', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify(requestBody), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Jira API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const responseData = - response.status === 204 ? {} : await response.json().catch(() => ({}) as Record) - logger.info('Successfully updated Jira issue:', issueKey) - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - issueKey: responseData.key || issueKey, - summary: responseData.fields?.summary || summaryValue || 'Issue updated', - success: true, - }, - }) - } catch (error: any) { - logger.error('Error updating Jira issue:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jira/write/route.ts b/apps/sim/app/api/tools/jira/write/route.ts deleted file mode 100644 index 916ee57c5ac..00000000000 --- a/apps/sim/app/api/tools/jira/write/route.ts +++ /dev/null @@ -1,251 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jiraWriteContract } from '@/lib/api/contracts/selectors/jira' -import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage, toAdf } from '@/tools/jira/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JiraWriteAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const auth = await checkSessionOrInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(jiraWriteContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - projectId, - summary, - description, - priority, - assignee, - cloudId: providedCloudId, - issueType, - parent, - labels, - duedate, - reporter, - environment, - customFieldId, - customFieldValue, - components, - fixVersions, - } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!projectId) { - logger.error('Missing project ID in request') - return NextResponse.json({ error: 'Project ID is required' }, { status: 400 }) - } - - if (!summary) { - logger.error('Missing summary in request') - return NextResponse.json({ error: 'Summary is required' }, { status: 400 }) - } - - const normalizedIssueType = issueType || 'Task' - - const cloudId = providedCloudId || (await getJiraCloudId(domain, accessToken)) - logger.info('Using cloud ID:', cloudId) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const projectIdValidation = validateAlphanumericId(projectId, 'projectId', 100) - if (!projectIdValidation.isValid) { - return NextResponse.json({ error: projectIdValidation.error }, { status: 400 }) - } - - const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue` - - logger.info('Creating Jira issue at:', url) - - const isNumericProjectId = /^\d+$/.test(projectId) - const fields: Record = { - project: isNumericProjectId ? { id: projectId } : { key: projectId }, - issuetype: { - name: normalizedIssueType, - }, - summary: summary, - } - - if (description !== undefined && description !== null && description !== '') { - fields.description = toAdf(description) - } - - if (parent !== undefined && parent !== null && parent !== '') { - if (typeof parent === 'string') { - fields.parent = /^\d+$/.test(parent) ? { id: parent } : { key: parent } - } else if (typeof parent === 'object') { - fields.parent = parent - } - } - - if (priority !== undefined && priority !== null && priority !== '') { - const isNumericId = /^\d+$/.test(priority) - fields.priority = isNumericId ? { id: priority } : { name: priority } - } - - if (labels !== undefined && labels !== null && Array.isArray(labels) && labels.length > 0) { - fields.labels = labels - } - - if ( - components !== undefined && - components !== null && - Array.isArray(components) && - components.length > 0 - ) { - fields.components = components.map((name: string) => ({ name })) - } - - if (duedate !== undefined && duedate !== null && duedate !== '') { - fields.duedate = duedate - } - - if ( - fixVersions !== undefined && - fixVersions !== null && - Array.isArray(fixVersions) && - fixVersions.length > 0 - ) { - fields.fixVersions = fixVersions.map((name: string) => ({ name })) - } - - if (reporter !== undefined && reporter !== null && reporter !== '') { - fields.reporter = { - accountId: reporter, - } - } - - if (environment !== undefined && environment !== null && environment !== '') { - fields.environment = toAdf(environment) - } - - if ( - customFieldId !== undefined && - customFieldId !== null && - customFieldId !== '' && - customFieldValue !== undefined && - customFieldValue !== null && - customFieldValue !== '' - ) { - const fieldId = customFieldId.startsWith('customfield_') - ? customFieldId - : `customfield_${customFieldId}` - - fields[fieldId] = customFieldValue - } - - const body = { fields } - - const response = await fetch(url, { - method: 'POST', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify(body), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Jira API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const responseData = await response.json() - const issueKey = responseData.key || 'unknown' - logger.info('Successfully created Jira issue:', issueKey) - - let assigneeId: string | undefined - if (assignee !== undefined && assignee !== null && assignee !== '') { - const assignUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${issueKey}/assignee` - logger.info('Assigning issue to:', assignee) - - const assignResponse = await fetch(assignUrl, { - method: 'PUT', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - accountId: assignee, - }), - }) - - if (!assignResponse.ok) { - const assignErrorText = await assignResponse.text() - logger.warn('Failed to assign issue (issue was created successfully):', { - status: assignResponse.status, - error: assignErrorText, - }) - } else { - assigneeId = assignee - logger.info('Successfully assigned issue to:', assignee) - } - } - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - id: responseData.id || '', - issueKey: issueKey, - self: responseData.self || '', - summary: responseData.fields?.summary || summary || 'Issue created', - success: true, - url: `https://${domain}/browse/${issueKey}`, - ...(assigneeId && { assigneeId }), - }, - }) - } catch (error: any) { - logger.error('Error creating Jira issue:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/approvals/route.ts b/apps/sim/app/api/tools/jsm/approvals/route.ts deleted file mode 100644 index ba8f2692045..00000000000 --- a/apps/sim/app/api/tools/jsm/approvals/route.ts +++ /dev/null @@ -1,220 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmApprovalsContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - validateAlphanumericId, - validateEnum, - validateJiraCloudId, - validateJiraIssueKey, -} from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmApprovalsAPI') - -const VALID_ACTIONS = ['get', 'answer'] as const -const VALID_DECISIONS = ['approve', 'decline'] as const - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmApprovalsContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - action, - issueIdOrKey, - approvalId, - decision, - start, - limit, - } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!issueIdOrKey) { - logger.error('Missing issueIdOrKey in request') - return NextResponse.json({ error: 'Issue ID or key is required' }, { status: 400 }) - } - - if (!action) { - logger.error('Missing action in request') - return NextResponse.json({ error: 'Action is required' }, { status: 400 }) - } - - const actionValidation = validateEnum(action, VALID_ACTIONS, 'action') - if (!actionValidation.isValid) { - return NextResponse.json({ error: actionValidation.error }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const issueIdOrKeyValidation = validateJiraIssueKey(issueIdOrKey, 'issueIdOrKey') - if (!issueIdOrKeyValidation.isValid) { - return NextResponse.json({ error: issueIdOrKeyValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmApiBaseUrl(cloudId) - - if (action === 'get') { - const params = new URLSearchParams() - if (start) params.append('start', start) - if (limit) params.append('limit', limit) - - const url = `${baseUrl}/request/${issueIdOrKey}/approval${params.toString() ? `?${params.toString()}` : ''}` - - logger.info('Fetching approvals from:', url) - - const response = await fetch(url, { - method: 'GET', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - issueIdOrKey, - approvals: data.values || [], - total: data.size || 0, - isLastPage: data.isLastPage ?? true, - }, - }) - } - if (action === 'answer') { - if (!approvalId) { - logger.error('Missing approvalId in request') - return NextResponse.json({ error: 'Approval ID is required' }, { status: 400 }) - } - - const approvalIdValidation = validateAlphanumericId(approvalId, 'approvalId') - if (!approvalIdValidation.isValid) { - return NextResponse.json({ error: approvalIdValidation.error }, { status: 400 }) - } - - const decisionValidation = validateEnum(decision, VALID_DECISIONS, 'decision') - if (!decisionValidation.isValid) { - return NextResponse.json({ error: decisionValidation.error }, { status: 400 }) - } - - const url = `${baseUrl}/request/${issueIdOrKey}/approval/${approvalId}` - - logger.info('Answering approval:', { issueIdOrKey, approvalId, decision }) - - const response = await fetch(url, { - method: 'POST', - headers: getJsmHeaders(accessToken), - body: JSON.stringify({ decision }), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - issueIdOrKey, - approvalId, - decision, - id: data.id ?? null, - name: data.name ?? null, - finalDecision: data.finalDecision ?? null, - canAnswerApproval: data.canAnswerApproval ?? null, - approvers: (data.approvers ?? []).map((a: Record) => { - const approver = a.approver as Record | undefined - return { - approver: { - accountId: approver?.accountId ?? null, - displayName: approver?.displayName ?? null, - emailAddress: approver?.emailAddress ?? null, - active: approver?.active ?? null, - }, - approverDecision: a.approverDecision ?? null, - } - }), - createdDate: data.createdDate ?? null, - completedDate: data.completedDate ?? null, - approval: data, - success: true, - }, - }) - } - - return NextResponse.json({ error: 'Invalid action' }, { status: 400 }) - } catch (error) { - logger.error('Error in approvals operation:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/assets/attributes/route.ts b/apps/sim/app/api/tools/jsm/assets/attributes/route.ts deleted file mode 100644 index 28804c427f2..00000000000 --- a/apps/sim/app/api/tools/jsm/assets/attributes/route.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmObjectTypeAttributesContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - validateAssetsWorkspaceId, - validateJiraCloudId, -} from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getAssetsApiBaseUrl, getJsmHeaders, resolveAssetsContext } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmAssetsAttributesAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmObjectTypeAttributesContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - workspaceId: workspaceIdParam, - objectTypeId, - onlyValueEditable, - query: searchQuery, - } = parsed.data.body - - const { cloudId, workspaceId } = await resolveAssetsContext( - domain, - accessToken, - cloudIdParam, - workspaceIdParam - ) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const workspaceIdValidation = validateAssetsWorkspaceId(workspaceId, 'workspaceId') - if (!workspaceIdValidation.isValid) { - return NextResponse.json({ error: workspaceIdValidation.error }, { status: 400 }) - } - - const query = new URLSearchParams() - if (onlyValueEditable !== undefined) { - query.append('onlyValueEditable', String(onlyValueEditable)) - } - if (searchQuery) query.append('query', searchQuery) - - const url = `${getAssetsApiBaseUrl(cloudId, workspaceId)}/objecttype/${encodeURIComponent( - objectTypeId - )}/attributes${query.toString() ? `?${query.toString()}` : ''}` - - const response = await fetch(url, { method: 'GET', headers: getJsmHeaders(accessToken) }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Assets API error getting attributes', { status: response.status, errorText }) - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - const attributes = Array.isArray(data) ? data : (data.values ?? []) - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - attributes, - total: attributes.length, - }, - }) - } catch (error) { - logger.error('Error getting Assets attributes', { error: toError(error).message }) - return NextResponse.json( - { error: getErrorMessage(error, 'Internal server error'), success: false }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/assets/object-types/route.ts b/apps/sim/app/api/tools/jsm/assets/object-types/route.ts deleted file mode 100644 index 328ccaa8c69..00000000000 --- a/apps/sim/app/api/tools/jsm/assets/object-types/route.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmListObjectTypesContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - validateAssetsWorkspaceId, - validateJiraCloudId, -} from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getAssetsApiBaseUrl, getJsmHeaders, resolveAssetsContext } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmAssetsObjectTypesAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmListObjectTypesContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - workspaceId: workspaceIdParam, - schemaId, - excludeAbstract, - } = parsed.data.body - - const { cloudId, workspaceId } = await resolveAssetsContext( - domain, - accessToken, - cloudIdParam, - workspaceIdParam - ) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const workspaceIdValidation = validateAssetsWorkspaceId(workspaceId, 'workspaceId') - if (!workspaceIdValidation.isValid) { - return NextResponse.json({ error: workspaceIdValidation.error }, { status: 400 }) - } - - const query = new URLSearchParams() - if (excludeAbstract !== undefined) query.append('excludeAbstract', String(excludeAbstract)) - - const url = `${getAssetsApiBaseUrl(cloudId, workspaceId)}/objectschema/${encodeURIComponent( - schemaId - )}/objecttypes${query.toString() ? `?${query.toString()}` : ''}` - - const response = await fetch(url, { method: 'GET', headers: getJsmHeaders(accessToken) }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Assets API error listing object types', { - status: response.status, - errorText, - }) - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - const objectTypes = Array.isArray(data) ? data : (data.values ?? []) - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - objectTypes, - total: objectTypes.length, - }, - }) - } catch (error) { - logger.error('Error listing Assets object types', { error: toError(error).message }) - return NextResponse.json( - { error: getErrorMessage(error, 'Internal server error'), success: false }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/assets/object/create/route.ts b/apps/sim/app/api/tools/jsm/assets/object/create/route.ts deleted file mode 100644 index 7c85f69a0fd..00000000000 --- a/apps/sim/app/api/tools/jsm/assets/object/create/route.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmCreateObjectContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - validateAssetsWorkspaceId, - validateJiraCloudId, -} from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { - getAssetsApiBaseUrl, - getJsmHeaders, - mapAssetObject, - resolveAssetsContext, -} from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmAssetsCreateObjectAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmCreateObjectContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - workspaceId: workspaceIdParam, - objectTypeId, - attributes, - } = parsed.data.body - - const { cloudId, workspaceId } = await resolveAssetsContext( - domain, - accessToken, - cloudIdParam, - workspaceIdParam - ) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const workspaceIdValidation = validateAssetsWorkspaceId(workspaceId, 'workspaceId') - if (!workspaceIdValidation.isValid) { - return NextResponse.json({ error: workspaceIdValidation.error }, { status: 400 }) - } - - const url = `${getAssetsApiBaseUrl(cloudId, workspaceId)}/object/create` - - const response = await fetch(url, { - method: 'POST', - headers: getJsmHeaders(accessToken), - body: JSON.stringify({ objectTypeId, attributes }), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Assets API error creating object', { status: response.status, errorText }) - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { ts: new Date().toISOString(), object: mapAssetObject(data) }, - }) - } catch (error) { - logger.error('Error creating Assets object', { error: toError(error).message }) - return NextResponse.json( - { error: getErrorMessage(error, 'Internal server error'), success: false }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/assets/object/delete/route.ts b/apps/sim/app/api/tools/jsm/assets/object/delete/route.ts deleted file mode 100644 index cf4dd4b1d4a..00000000000 --- a/apps/sim/app/api/tools/jsm/assets/object/delete/route.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmDeleteObjectContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - validateAssetsWorkspaceId, - validateJiraCloudId, -} from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getAssetsApiBaseUrl, getJsmHeaders, resolveAssetsContext } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmAssetsDeleteObjectAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmDeleteObjectContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - workspaceId: workspaceIdParam, - objectId, - } = parsed.data.body - - const { cloudId, workspaceId } = await resolveAssetsContext( - domain, - accessToken, - cloudIdParam, - workspaceIdParam - ) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const workspaceIdValidation = validateAssetsWorkspaceId(workspaceId, 'workspaceId') - if (!workspaceIdValidation.isValid) { - return NextResponse.json({ error: workspaceIdValidation.error }, { status: 400 }) - } - - const url = `${getAssetsApiBaseUrl(cloudId, workspaceId)}/object/${encodeURIComponent(objectId)}` - - const response = await fetch(url, { method: 'DELETE', headers: getJsmHeaders(accessToken) }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Assets API error deleting object', { status: response.status, errorText }) - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - return NextResponse.json({ - success: true, - output: { ts: new Date().toISOString(), objectId, deleted: true }, - }) - } catch (error) { - logger.error('Error deleting Assets object', { error: toError(error).message }) - return NextResponse.json( - { error: getErrorMessage(error, 'Internal server error'), success: false }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/assets/object/get/route.ts b/apps/sim/app/api/tools/jsm/assets/object/get/route.ts deleted file mode 100644 index fa7fa3759e9..00000000000 --- a/apps/sim/app/api/tools/jsm/assets/object/get/route.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmGetObjectContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - validateAssetsWorkspaceId, - validateJiraCloudId, -} from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { - getAssetsApiBaseUrl, - getJsmHeaders, - mapAssetObject, - resolveAssetsContext, -} from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmAssetsGetObjectAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmGetObjectContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - workspaceId: workspaceIdParam, - objectId, - } = parsed.data.body - - const { cloudId, workspaceId } = await resolveAssetsContext( - domain, - accessToken, - cloudIdParam, - workspaceIdParam - ) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const workspaceIdValidation = validateAssetsWorkspaceId(workspaceId, 'workspaceId') - if (!workspaceIdValidation.isValid) { - return NextResponse.json({ error: workspaceIdValidation.error }, { status: 400 }) - } - - const url = `${getAssetsApiBaseUrl(cloudId, workspaceId)}/object/${encodeURIComponent(objectId)}` - - const response = await fetch(url, { method: 'GET', headers: getJsmHeaders(accessToken) }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Assets API error getting object', { status: response.status, errorText }) - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { ts: new Date().toISOString(), object: mapAssetObject(data) }, - }) - } catch (error) { - logger.error('Error getting Assets object', { error: toError(error).message }) - return NextResponse.json( - { error: getErrorMessage(error, 'Internal server error'), success: false }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/assets/object/update/route.ts b/apps/sim/app/api/tools/jsm/assets/object/update/route.ts deleted file mode 100644 index bfc7a6286a8..00000000000 --- a/apps/sim/app/api/tools/jsm/assets/object/update/route.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmUpdateObjectContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - validateAssetsWorkspaceId, - validateJiraCloudId, -} from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { - getAssetsApiBaseUrl, - getJsmHeaders, - mapAssetObject, - resolveAssetsContext, -} from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmAssetsUpdateObjectAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmUpdateObjectContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - workspaceId: workspaceIdParam, - objectId, - objectTypeId, - attributes, - } = parsed.data.body - - const { cloudId, workspaceId } = await resolveAssetsContext( - domain, - accessToken, - cloudIdParam, - workspaceIdParam - ) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const workspaceIdValidation = validateAssetsWorkspaceId(workspaceId, 'workspaceId') - if (!workspaceIdValidation.isValid) { - return NextResponse.json({ error: workspaceIdValidation.error }, { status: 400 }) - } - - const url = `${getAssetsApiBaseUrl(cloudId, workspaceId)}/object/${encodeURIComponent(objectId)}` - - const body: Record = { attributes } - if (objectTypeId) body.objectTypeId = objectTypeId - - const response = await fetch(url, { - method: 'PUT', - headers: getJsmHeaders(accessToken), - body: JSON.stringify(body), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Assets API error updating object', { status: response.status, errorText }) - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { ts: new Date().toISOString(), object: mapAssetObject(data) }, - }) - } catch (error) { - logger.error('Error updating Assets object', { error: toError(error).message }) - return NextResponse.json( - { error: getErrorMessage(error, 'Internal server error'), success: false }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/assets/schema/route.ts b/apps/sim/app/api/tools/jsm/assets/schema/route.ts deleted file mode 100644 index d2eb97d01c7..00000000000 --- a/apps/sim/app/api/tools/jsm/assets/schema/route.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmGetObjectSchemaContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - validateAssetsWorkspaceId, - validateJiraCloudId, -} from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getAssetsApiBaseUrl, getJsmHeaders, resolveAssetsContext } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmAssetsSchemaAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmGetObjectSchemaContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - workspaceId: workspaceIdParam, - schemaId, - } = parsed.data.body - - const { cloudId, workspaceId } = await resolveAssetsContext( - domain, - accessToken, - cloudIdParam, - workspaceIdParam - ) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const workspaceIdValidation = validateAssetsWorkspaceId(workspaceId, 'workspaceId') - if (!workspaceIdValidation.isValid) { - return NextResponse.json({ error: workspaceIdValidation.error }, { status: 400 }) - } - - const url = `${getAssetsApiBaseUrl(cloudId, workspaceId)}/objectschema/${encodeURIComponent(schemaId)}` - - const response = await fetch(url, { method: 'GET', headers: getJsmHeaders(accessToken) }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Assets API error getting schema', { status: response.status, errorText }) - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { ts: new Date().toISOString(), schema: data ?? null }, - }) - } catch (error) { - logger.error('Error getting Assets schema', { error: toError(error).message }) - return NextResponse.json( - { error: getErrorMessage(error, 'Internal server error'), success: false }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/assets/schemas/route.ts b/apps/sim/app/api/tools/jsm/assets/schemas/route.ts deleted file mode 100644 index 5219e97853d..00000000000 --- a/apps/sim/app/api/tools/jsm/assets/schemas/route.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmListObjectSchemasContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - validateAssetsWorkspaceId, - validateJiraCloudId, -} from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getAssetsApiBaseUrl, getJsmHeaders, resolveAssetsContext } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmAssetsSchemasAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmListObjectSchemasContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - workspaceId: workspaceIdParam, - startAt, - maxResults, - includeCounts, - } = parsed.data.body - - const { cloudId, workspaceId } = await resolveAssetsContext( - domain, - accessToken, - cloudIdParam, - workspaceIdParam - ) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const workspaceIdValidation = validateAssetsWorkspaceId(workspaceId, 'workspaceId') - if (!workspaceIdValidation.isValid) { - return NextResponse.json({ error: workspaceIdValidation.error }, { status: 400 }) - } - - const query = new URLSearchParams() - if (startAt !== undefined) query.append('startAt', String(startAt)) - if (maxResults !== undefined) query.append('maxResults', String(maxResults)) - if (includeCounts !== undefined) query.append('includeCounts', String(includeCounts)) - - const url = `${getAssetsApiBaseUrl(cloudId, workspaceId)}/objectschema/list${ - query.toString() ? `?${query.toString()}` : '' - }` - - const response = await fetch(url, { method: 'GET', headers: getJsmHeaders(accessToken) }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Assets API error listing schemas', { status: response.status, errorText }) - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - schemas: data.values ?? [], - total: data.total ?? (data.values?.length || 0), - isLast: data.isLast ?? data.last ?? true, - }, - }) - } catch (error) { - logger.error('Error listing Assets schemas', { error: toError(error).message }) - return NextResponse.json( - { error: getErrorMessage(error, 'Internal server error'), success: false }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/assets/search/route.ts b/apps/sim/app/api/tools/jsm/assets/search/route.ts deleted file mode 100644 index 3ddad2dae78..00000000000 --- a/apps/sim/app/api/tools/jsm/assets/search/route.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmSearchObjectsAqlContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - validateAssetsWorkspaceId, - validateJiraCloudId, -} from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { - getAssetsApiBaseUrl, - getJsmHeaders, - mapAssetObject, - resolveAssetsContext, -} from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmAssetsSearchAPI') - -/** Coerce a string|number|boolean param into a number, falling back when unset */ -function toNumber(value: string | number | undefined, fallback: number): number { - if (value === undefined) return fallback - const parsed = typeof value === 'number' ? value : Number(value) - return Number.isFinite(parsed) ? parsed : fallback -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmSearchObjectsAqlContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - workspaceId: workspaceIdParam, - qlQuery, - page, - resultsPerPage, - includeAttributes, - objectTypeId, - objectSchemaId, - } = parsed.data.body - - const { cloudId, workspaceId } = await resolveAssetsContext( - domain, - accessToken, - cloudIdParam, - workspaceIdParam - ) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const workspaceIdValidation = validateAssetsWorkspaceId(workspaceId, 'workspaceId') - if (!workspaceIdValidation.isValid) { - return NextResponse.json({ error: workspaceIdValidation.error }, { status: 400 }) - } - - const includeAttrs = - includeAttributes === undefined ? true : String(includeAttributes) === 'true' - - const body: Record = { - qlQuery, - page: toNumber(page, 1), - resultsPerPage: toNumber(resultsPerPage, 25), - includeAttributes: includeAttrs, - } - if (objectTypeId) body.objectTypeId = objectTypeId - if (objectSchemaId) body.objectSchemaId = objectSchemaId - - const url = `${getAssetsApiBaseUrl(cloudId, workspaceId)}/object/aql` - - const response = await fetch(url, { - method: 'POST', - headers: getJsmHeaders(accessToken), - body: JSON.stringify(body), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('Assets API error running AQL search', { status: response.status, errorText }) - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - objects: Array.isArray(data.objectEntries) ? data.objectEntries.map(mapAssetObject) : [], - total: data.totalFilterCount ?? (data.objectEntries?.length || 0), - pageNumber: data.pageNumber ?? 1, - pageSize: data.pageSize ?? (data.objectEntries?.length || 0), - }, - }) - } catch (error) { - logger.error('Error running Assets AQL search', { error: toError(error).message }) - return NextResponse.json( - { error: getErrorMessage(error, 'Internal server error'), success: false }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/comment/route.ts b/apps/sim/app/api/tools/jsm/comment/route.ts deleted file mode 100644 index fb48eef55c6..00000000000 --- a/apps/sim/app/api/tools/jsm/comment/route.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmCommentContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId, validateJiraIssueKey } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmCommentAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmCommentContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: providedCloudId, - issueIdOrKey, - body: commentBody, - isPublic, - } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!issueIdOrKey) { - logger.error('Missing issueIdOrKey in request') - return NextResponse.json({ error: 'Issue ID or key is required' }, { status: 400 }) - } - - if (!commentBody) { - logger.error('Missing comment body in request') - return NextResponse.json({ error: 'Comment body is required' }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const issueIdOrKeyValidation = validateJiraIssueKey(issueIdOrKey, 'issueIdOrKey') - if (!issueIdOrKeyValidation.isValid) { - return NextResponse.json({ error: issueIdOrKeyValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmApiBaseUrl(cloudId) - - const url = `${baseUrl}/request/${issueIdOrKey}/comment` - - logger.info('Adding comment to:', url) - - const response = await fetch(url, { - method: 'POST', - headers: getJsmHeaders(accessToken), - body: JSON.stringify({ - body: commentBody, - public: isPublic ?? true, - }), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - issueIdOrKey, - commentId: data.id, - body: data.body, - isPublic: data.public, - author: data.author - ? { - accountId: data.author.accountId ?? null, - displayName: data.author.displayName ?? null, - emailAddress: data.author.emailAddress ?? null, - } - : null, - createdDate: data.created ?? null, - success: true, - }, - }) - } catch (error) { - logger.error('Error adding comment:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/comments/route.ts b/apps/sim/app/api/tools/jsm/comments/route.ts deleted file mode 100644 index 8741ff4366b..00000000000 --- a/apps/sim/app/api/tools/jsm/comments/route.ts +++ /dev/null @@ -1,126 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmCommentsContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId, validateJiraIssueKey } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmCommentsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmCommentsContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - issueIdOrKey, - isPublic, - internal, - expand, - start, - limit, - } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!issueIdOrKey) { - logger.error('Missing issueIdOrKey in request') - return NextResponse.json({ error: 'Issue ID or key is required' }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const issueIdOrKeyValidation = validateJiraIssueKey(issueIdOrKey, 'issueIdOrKey') - if (!issueIdOrKeyValidation.isValid) { - return NextResponse.json({ error: issueIdOrKeyValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmApiBaseUrl(cloudId) - - const params = new URLSearchParams() - if (isPublic !== undefined) params.append('public', String(isPublic)) - if (internal !== undefined) params.append('internal', String(internal)) - if (expand) params.append('expand', expand) - if (start) params.append('start', start) - if (limit) params.append('limit', limit) - - const url = `${baseUrl}/request/${issueIdOrKey}/comment${params.toString() ? `?${params.toString()}` : ''}` - - logger.info('Fetching comments from:', url) - - const response = await fetch(url, { - method: 'GET', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - issueIdOrKey, - comments: data.values || [], - total: data.size || 0, - isLastPage: data.isLastPage ?? true, - }, - }) - } catch (error) { - logger.error('Error fetching comments:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/customers/route.ts b/apps/sim/app/api/tools/jsm/customers/route.ts deleted file mode 100644 index 4b52715ea00..00000000000 --- a/apps/sim/app/api/tools/jsm/customers/route.ts +++ /dev/null @@ -1,186 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmCustomersContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmCustomersAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmCustomersContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - serviceDeskId, - query, - start, - limit, - accountIds, - emails, - } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!serviceDeskId) { - logger.error('Missing serviceDeskId in request') - return NextResponse.json({ error: 'Service Desk ID is required' }, { status: 400 }) - } - - if (emails !== undefined) { - return NextResponse.json( - { - error: - 'The `emails` parameter is no longer supported. Use `accountIds` (Atlassian account IDs) instead.', - }, - { status: 400 } - ) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const serviceDeskIdValidation = validateAlphanumericId(serviceDeskId, 'serviceDeskId') - if (!serviceDeskIdValidation.isValid) { - return NextResponse.json({ error: serviceDeskIdValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmApiBaseUrl(cloudId) - - const splitCsv = (value: unknown): string[] => - value - ? typeof value === 'string' - ? value - .split(',') - .map((v: string) => v.trim()) - .filter((v: string) => v) - : Array.isArray(value) - ? (value as string[]) - : [] - : [] - - const parsedAccountIds = splitCsv(accountIds) - - if (parsedAccountIds.length > 0) { - const url = `${baseUrl}/servicedesk/${serviceDeskId}/customer` - - logger.info('Adding customers to:', url, { - accountIds: parsedAccountIds, - }) - - const response = await fetch(url, { - method: 'POST', - headers: getJsmHeaders(accessToken), - body: JSON.stringify({ accountIds: parsedAccountIds }), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - serviceDeskId, - success: true, - }, - }) - } - const params = new URLSearchParams() - if (query) params.append('query', query) - if (start) params.append('start', start) - if (limit) params.append('limit', limit) - - const url = `${baseUrl}/servicedesk/${serviceDeskId}/customer${params.toString() ? `?${params.toString()}` : ''}` - - logger.info('Fetching customers from:', url) - - const response = await fetch(url, { - method: 'GET', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - customers: data.values || [], - total: data.size || 0, - isLastPage: data.isLastPage ?? true, - }, - }) - } catch (error) { - logger.error('Error with customers operation:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/forms/answers/route.ts b/apps/sim/app/api/tools/jsm/forms/answers/route.ts deleted file mode 100644 index 7e435308527..00000000000 --- a/apps/sim/app/api/tools/jsm/forms/answers/route.ts +++ /dev/null @@ -1,117 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmFormAnswersContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId, validateJiraIssueKey } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmFormsApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmGetFormAnswersAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmFormAnswersContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, cloudId: cloudIdParam, issueIdOrKey, formId } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!issueIdOrKey) { - logger.error('Missing issueIdOrKey in request') - return NextResponse.json({ error: 'Issue ID or key is required' }, { status: 400 }) - } - - if (!formId) { - logger.error('Missing formId in request') - return NextResponse.json({ error: 'Form ID is required' }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const issueIdOrKeyValidation = validateJiraIssueKey(issueIdOrKey, 'issueIdOrKey') - if (!issueIdOrKeyValidation.isValid) { - return NextResponse.json({ error: issueIdOrKeyValidation.error }, { status: 400 }) - } - - const formIdValidation = validateJiraCloudId(formId, 'formId') - if (!formIdValidation.isValid) { - return NextResponse.json({ error: formIdValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmFormsApiBaseUrl(cloudId) - const url = `${baseUrl}/issue/${encodeURIComponent(issueIdOrKey)}/form/${encodeURIComponent(formId)}/format/answers` - - logger.info('Getting form answers:', { url, issueIdOrKey, formId }) - - const response = await fetch(url, { - method: 'GET', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM Forms API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - issueIdOrKey, - formId, - answers: data ?? null, - }, - }) - } catch (error) { - logger.error('Error getting form answers:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/forms/attach/route.ts b/apps/sim/app/api/tools/jsm/forms/attach/route.ts deleted file mode 100644 index 0a25aee3746..00000000000 --- a/apps/sim/app/api/tools/jsm/forms/attach/route.ts +++ /dev/null @@ -1,131 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmAttachFormContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId, validateJiraIssueKey } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmFormsApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmAttachFormAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmAttachFormContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - issueIdOrKey, - formTemplateId, - } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!issueIdOrKey) { - logger.error('Missing issueIdOrKey in request') - return NextResponse.json({ error: 'Issue ID or key is required' }, { status: 400 }) - } - - if (!formTemplateId) { - logger.error('Missing formTemplateId in request') - return NextResponse.json({ error: 'Form template ID is required' }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const issueIdOrKeyValidation = validateJiraIssueKey(issueIdOrKey, 'issueIdOrKey') - if (!issueIdOrKeyValidation.isValid) { - return NextResponse.json({ error: issueIdOrKeyValidation.error }, { status: 400 }) - } - - const formTemplateIdValidation = validateJiraCloudId(formTemplateId, 'formTemplateId') - if (!formTemplateIdValidation.isValid) { - return NextResponse.json({ error: formTemplateIdValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmFormsApiBaseUrl(cloudId) - const url = `${baseUrl}/issue/${encodeURIComponent(issueIdOrKey)}/form` - - logger.info('Attaching form to issue:', { url, issueIdOrKey, formTemplateId }) - - const response = await fetch(url, { - method: 'POST', - headers: getJsmHeaders(accessToken), - body: JSON.stringify({ - formTemplate: { id: formTemplateId }, - }), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM Forms API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - issueIdOrKey, - id: data.id ?? null, - name: data.name ?? null, - updated: data.updated ?? null, - submitted: data.submitted ?? false, - lock: data.lock ?? false, - internal: data.internal ?? null, - formTemplateId: (data.formTemplate as Record)?.id ?? null, - }, - }) - } catch (error) { - logger.error('Error attaching form:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/forms/copy/route.ts b/apps/sim/app/api/tools/jsm/forms/copy/route.ts deleted file mode 100644 index dc32fec761a..00000000000 --- a/apps/sim/app/api/tools/jsm/forms/copy/route.ts +++ /dev/null @@ -1,132 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmCopyFormsContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId, validateJiraIssueKey } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmFormsApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmCopyFormsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmCopyFormsContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - sourceIssueIdOrKey, - targetIssueIdOrKey, - formIds, - } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!sourceIssueIdOrKey) { - logger.error('Missing sourceIssueIdOrKey in request') - return NextResponse.json({ error: 'Source issue ID or key is required' }, { status: 400 }) - } - - if (!targetIssueIdOrKey) { - logger.error('Missing targetIssueIdOrKey in request') - return NextResponse.json({ error: 'Target issue ID or key is required' }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const sourceValidation = validateJiraIssueKey(sourceIssueIdOrKey, 'sourceIssueIdOrKey') - if (!sourceValidation.isValid) { - return NextResponse.json({ error: sourceValidation.error }, { status: 400 }) - } - - const targetValidation = validateJiraIssueKey(targetIssueIdOrKey, 'targetIssueIdOrKey') - if (!targetValidation.isValid) { - return NextResponse.json({ error: targetValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmFormsApiBaseUrl(cloudId) - const url = `${baseUrl}/issue/${encodeURIComponent(sourceIssueIdOrKey)}/form/copy/${encodeURIComponent(targetIssueIdOrKey)}` - - if (formIds !== undefined && !Array.isArray(formIds)) { - return NextResponse.json({ error: 'formIds must be an array of form UUIDs' }, { status: 400 }) - } - - const requestBody = Array.isArray(formIds) && formIds.length > 0 ? { ids: formIds } : {} - - logger.info('Copying forms:', { url, sourceIssueIdOrKey, targetIssueIdOrKey, formIds }) - - const response = await fetch(url, { - method: 'POST', - headers: getJsmHeaders(accessToken), - body: JSON.stringify(requestBody), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM Forms API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - sourceIssueIdOrKey, - targetIssueIdOrKey, - copiedForms: data.copiedForms ?? [], - errors: data.errors ?? [], - }, - }) - } catch (error) { - logger.error('Error copying forms:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/forms/delete/route.ts b/apps/sim/app/api/tools/jsm/forms/delete/route.ts deleted file mode 100644 index 67c8f5fc66c..00000000000 --- a/apps/sim/app/api/tools/jsm/forms/delete/route.ts +++ /dev/null @@ -1,117 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmDeleteFormContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId, validateJiraIssueKey } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmFormsApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmDeleteFormAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmDeleteFormContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, cloudId: cloudIdParam, issueIdOrKey, formId } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!issueIdOrKey) { - logger.error('Missing issueIdOrKey in request') - return NextResponse.json({ error: 'Issue ID or key is required' }, { status: 400 }) - } - - if (!formId) { - logger.error('Missing formId in request') - return NextResponse.json({ error: 'Form ID is required' }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const issueIdOrKeyValidation = validateJiraIssueKey(issueIdOrKey, 'issueIdOrKey') - if (!issueIdOrKeyValidation.isValid) { - return NextResponse.json({ error: issueIdOrKeyValidation.error }, { status: 400 }) - } - - const formIdValidation = validateJiraCloudId(formId, 'formId') - if (!formIdValidation.isValid) { - return NextResponse.json({ error: formIdValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmFormsApiBaseUrl(cloudId) - const url = `${baseUrl}/issue/${encodeURIComponent(issueIdOrKey)}/form/${encodeURIComponent(formId)}` - - logger.info('Deleting form:', { url, issueIdOrKey, formId }) - - const response = await fetch(url, { - method: 'DELETE', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM Forms API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - await response.text() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - issueIdOrKey, - formId, - deleted: true, - }, - }) - } catch (error) { - logger.error('Error deleting form:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/forms/externalise/route.ts b/apps/sim/app/api/tools/jsm/forms/externalise/route.ts deleted file mode 100644 index a41823e211d..00000000000 --- a/apps/sim/app/api/tools/jsm/forms/externalise/route.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmExternaliseFormContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId, validateJiraIssueKey } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmFormsApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmExternaliseFormAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmExternaliseFormContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, cloudId: cloudIdParam, issueIdOrKey, formId } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!issueIdOrKey) { - logger.error('Missing issueIdOrKey in request') - return NextResponse.json({ error: 'Issue ID or key is required' }, { status: 400 }) - } - - if (!formId) { - logger.error('Missing formId in request') - return NextResponse.json({ error: 'Form ID is required' }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const issueIdOrKeyValidation = validateJiraIssueKey(issueIdOrKey, 'issueIdOrKey') - if (!issueIdOrKeyValidation.isValid) { - return NextResponse.json({ error: issueIdOrKeyValidation.error }, { status: 400 }) - } - - const formIdValidation = validateJiraCloudId(formId, 'formId') - if (!formIdValidation.isValid) { - return NextResponse.json({ error: formIdValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmFormsApiBaseUrl(cloudId) - const url = `${baseUrl}/issue/${encodeURIComponent(issueIdOrKey)}/form/${encodeURIComponent(formId)}/action/external` - - logger.info('Externalising form:', { url, issueIdOrKey, formId }) - - const response = await fetch(url, { - method: 'PUT', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM Forms API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const bodyText = await response.text() - const data = bodyText ? JSON.parse(bodyText) : {} - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - issueIdOrKey, - formId, - visibility: data.visibility ?? 'external', - }, - }) - } catch (error) { - logger.error('Error externalising form:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/forms/get/route.ts b/apps/sim/app/api/tools/jsm/forms/get/route.ts deleted file mode 100644 index 278cd6b1ca8..00000000000 --- a/apps/sim/app/api/tools/jsm/forms/get/route.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmGetFormContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId, validateJiraIssueKey } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmFormsApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmGetFormAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmGetFormContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, cloudId: cloudIdParam, issueIdOrKey, formId } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!issueIdOrKey) { - logger.error('Missing issueIdOrKey in request') - return NextResponse.json({ error: 'Issue ID or key is required' }, { status: 400 }) - } - - if (!formId) { - logger.error('Missing formId in request') - return NextResponse.json({ error: 'Form ID is required' }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const issueIdOrKeyValidation = validateJiraIssueKey(issueIdOrKey, 'issueIdOrKey') - if (!issueIdOrKeyValidation.isValid) { - return NextResponse.json({ error: issueIdOrKeyValidation.error }, { status: 400 }) - } - - const formIdValidation = validateJiraCloudId(formId, 'formId') - if (!formIdValidation.isValid) { - return NextResponse.json({ error: formIdValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmFormsApiBaseUrl(cloudId) - const url = `${baseUrl}/issue/${encodeURIComponent(issueIdOrKey)}/form/${encodeURIComponent(formId)}` - - logger.info('Getting form:', { url, issueIdOrKey, formId }) - - const response = await fetch(url, { - method: 'GET', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM Forms API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - issueIdOrKey, - formId, - design: data.design ?? null, - state: data.state ?? null, - updated: data.updated ?? null, - }, - }) - } catch (error) { - logger.error('Error getting form:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/forms/internalise/route.ts b/apps/sim/app/api/tools/jsm/forms/internalise/route.ts deleted file mode 100644 index d38975aa89e..00000000000 --- a/apps/sim/app/api/tools/jsm/forms/internalise/route.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmInternaliseFormContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId, validateJiraIssueKey } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmFormsApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmInternaliseFormAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmInternaliseFormContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, cloudId: cloudIdParam, issueIdOrKey, formId } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!issueIdOrKey) { - logger.error('Missing issueIdOrKey in request') - return NextResponse.json({ error: 'Issue ID or key is required' }, { status: 400 }) - } - - if (!formId) { - logger.error('Missing formId in request') - return NextResponse.json({ error: 'Form ID is required' }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const issueIdOrKeyValidation = validateJiraIssueKey(issueIdOrKey, 'issueIdOrKey') - if (!issueIdOrKeyValidation.isValid) { - return NextResponse.json({ error: issueIdOrKeyValidation.error }, { status: 400 }) - } - - const formIdValidation = validateJiraCloudId(formId, 'formId') - if (!formIdValidation.isValid) { - return NextResponse.json({ error: formIdValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmFormsApiBaseUrl(cloudId) - const url = `${baseUrl}/issue/${encodeURIComponent(issueIdOrKey)}/form/${encodeURIComponent(formId)}/action/internal` - - logger.info('Internalising form:', { url, issueIdOrKey, formId }) - - const response = await fetch(url, { - method: 'PUT', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM Forms API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const bodyText = await response.text() - const data = bodyText ? JSON.parse(bodyText) : {} - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - issueIdOrKey, - formId, - visibility: data.visibility ?? 'internal', - }, - }) - } catch (error) { - logger.error('Error internalising form:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/forms/issue/route.ts b/apps/sim/app/api/tools/jsm/forms/issue/route.ts deleted file mode 100644 index 2f944aabc3d..00000000000 --- a/apps/sim/app/api/tools/jsm/forms/issue/route.ts +++ /dev/null @@ -1,117 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmIssueFormsContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId, validateJiraIssueKey } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmFormsApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmIssueFormsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmIssueFormsContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, cloudId: cloudIdParam, issueIdOrKey } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!issueIdOrKey) { - logger.error('Missing issueIdOrKey in request') - return NextResponse.json({ error: 'Issue ID or key is required' }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const issueIdOrKeyValidation = validateJiraIssueKey(issueIdOrKey, 'issueIdOrKey') - if (!issueIdOrKeyValidation.isValid) { - return NextResponse.json({ error: issueIdOrKeyValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmFormsApiBaseUrl(cloudId) - const url = `${baseUrl}/issue/${encodeURIComponent(issueIdOrKey)}/form` - - logger.info('Fetching issue forms from:', { url, issueIdOrKey }) - - const response = await fetch(url, { - method: 'GET', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM Forms API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - const forms = Array.isArray(data) ? data : (data.values ?? data.forms ?? []) - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - issueIdOrKey, - forms: forms.map((form: Record) => ({ - id: form.id ?? null, - name: form.name ?? null, - updated: form.updated ?? null, - submitted: form.submitted ?? false, - lock: form.lock ?? false, - internal: form.internal ?? null, - formTemplateId: (form.formTemplate as Record)?.id ?? null, - })), - total: forms.length, - }, - }) - } catch (error) { - logger.error('Error fetching issue forms:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/forms/reopen/route.ts b/apps/sim/app/api/tools/jsm/forms/reopen/route.ts deleted file mode 100644 index 718a1ff7282..00000000000 --- a/apps/sim/app/api/tools/jsm/forms/reopen/route.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmReopenFormContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId, validateJiraIssueKey } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmFormsApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmReopenFormAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmReopenFormContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, cloudId: cloudIdParam, issueIdOrKey, formId } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!issueIdOrKey) { - logger.error('Missing issueIdOrKey in request') - return NextResponse.json({ error: 'Issue ID or key is required' }, { status: 400 }) - } - - if (!formId) { - logger.error('Missing formId in request') - return NextResponse.json({ error: 'Form ID is required' }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const issueIdOrKeyValidation = validateJiraIssueKey(issueIdOrKey, 'issueIdOrKey') - if (!issueIdOrKeyValidation.isValid) { - return NextResponse.json({ error: issueIdOrKeyValidation.error }, { status: 400 }) - } - - const formIdValidation = validateJiraCloudId(formId, 'formId') - if (!formIdValidation.isValid) { - return NextResponse.json({ error: formIdValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmFormsApiBaseUrl(cloudId) - const url = `${baseUrl}/issue/${encodeURIComponent(issueIdOrKey)}/form/${encodeURIComponent(formId)}/action/reopen` - - logger.info('Reopening form:', { url, issueIdOrKey, formId }) - - const response = await fetch(url, { - method: 'PUT', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM Forms API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const bodyText = await response.text() - const data = bodyText ? JSON.parse(bodyText) : {} - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - issueIdOrKey, - formId, - status: data.status ?? 'open', - }, - }) - } catch (error) { - logger.error('Error reopening form:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/forms/save/route.ts b/apps/sim/app/api/tools/jsm/forms/save/route.ts deleted file mode 100644 index 76d1000da31..00000000000 --- a/apps/sim/app/api/tools/jsm/forms/save/route.ts +++ /dev/null @@ -1,131 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmSaveFormAnswersContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId, validateJiraIssueKey } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmFormsApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmSaveFormAnswersAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmSaveFormAnswersContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - issueIdOrKey, - formId, - answers, - } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!issueIdOrKey) { - logger.error('Missing issueIdOrKey in request') - return NextResponse.json({ error: 'Issue ID or key is required' }, { status: 400 }) - } - - if (!formId) { - logger.error('Missing formId in request') - return NextResponse.json({ error: 'Form ID is required' }, { status: 400 }) - } - - if (!answers || typeof answers !== 'object' || Array.isArray(answers)) { - logger.error('Missing or invalid answers in request') - return NextResponse.json({ error: 'Answers object is required' }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const issueIdOrKeyValidation = validateJiraIssueKey(issueIdOrKey, 'issueIdOrKey') - if (!issueIdOrKeyValidation.isValid) { - return NextResponse.json({ error: issueIdOrKeyValidation.error }, { status: 400 }) - } - - const formIdValidation = validateJiraCloudId(formId, 'formId') - if (!formIdValidation.isValid) { - return NextResponse.json({ error: formIdValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmFormsApiBaseUrl(cloudId) - const url = `${baseUrl}/issue/${encodeURIComponent(issueIdOrKey)}/form/${encodeURIComponent(formId)}` - - logger.info('Saving form answers:', { url, issueIdOrKey, formId }) - - const response = await fetch(url, { - method: 'PUT', - headers: getJsmHeaders(accessToken), - body: JSON.stringify({ answers }), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM Forms API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - issueIdOrKey, - formId, - state: data.state ?? null, - updated: data.updated ?? null, - }, - }) - } catch (error) { - logger.error('Error saving form answers:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/forms/structure/route.ts b/apps/sim/app/api/tools/jsm/forms/structure/route.ts deleted file mode 100644 index b82b1f1ea49..00000000000 --- a/apps/sim/app/api/tools/jsm/forms/structure/route.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmProjectFormStructureContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId, validateJiraIssueKey } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmFormsApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmFormStructureAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmProjectFormStructureContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, cloudId: cloudIdParam, projectIdOrKey, formId } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!projectIdOrKey) { - logger.error('Missing projectIdOrKey in request') - return NextResponse.json({ error: 'Project ID or key is required' }, { status: 400 }) - } - - if (!formId) { - logger.error('Missing formId in request') - return NextResponse.json({ error: 'Form ID is required' }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const projectIdOrKeyValidation = validateJiraIssueKey(projectIdOrKey, 'projectIdOrKey') - if (!projectIdOrKeyValidation.isValid) { - return NextResponse.json({ error: projectIdOrKeyValidation.error }, { status: 400 }) - } - - const formIdValidation = validateJiraCloudId(formId, 'formId') - if (!formIdValidation.isValid) { - return NextResponse.json({ error: formIdValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmFormsApiBaseUrl(cloudId) - const url = `${baseUrl}/project/${encodeURIComponent(projectIdOrKey)}/form/${encodeURIComponent(formId)}` - - logger.info('Fetching form template from:', { url, projectIdOrKey, formId }) - - const response = await fetch(url, { - method: 'GET', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM Forms API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - projectIdOrKey, - formId, - design: data.design ?? null, - updated: data.updated ?? null, - publish: data.publish ?? null, - }, - }) - } catch (error) { - logger.error('Error fetching form structure:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/forms/submit/route.ts b/apps/sim/app/api/tools/jsm/forms/submit/route.ts deleted file mode 100644 index a23e98c7d3f..00000000000 --- a/apps/sim/app/api/tools/jsm/forms/submit/route.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmSubmitFormContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId, validateJiraIssueKey } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmFormsApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmSubmitFormAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmSubmitFormContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, cloudId: cloudIdParam, issueIdOrKey, formId } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!issueIdOrKey) { - logger.error('Missing issueIdOrKey in request') - return NextResponse.json({ error: 'Issue ID or key is required' }, { status: 400 }) - } - - if (!formId) { - logger.error('Missing formId in request') - return NextResponse.json({ error: 'Form ID is required' }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const issueIdOrKeyValidation = validateJiraIssueKey(issueIdOrKey, 'issueIdOrKey') - if (!issueIdOrKeyValidation.isValid) { - return NextResponse.json({ error: issueIdOrKeyValidation.error }, { status: 400 }) - } - - const formIdValidation = validateJiraCloudId(formId, 'formId') - if (!formIdValidation.isValid) { - return NextResponse.json({ error: formIdValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmFormsApiBaseUrl(cloudId) - const url = `${baseUrl}/issue/${encodeURIComponent(issueIdOrKey)}/form/${encodeURIComponent(formId)}/action/submit` - - logger.info('Submitting form:', { url, issueIdOrKey, formId }) - - const response = await fetch(url, { - method: 'PUT', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM Forms API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const bodyText = await response.text() - const data = bodyText ? JSON.parse(bodyText) : {} - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - issueIdOrKey, - formId, - status: data.status ?? 'submitted', - }, - }) - } catch (error) { - logger.error('Error submitting form:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/forms/templates/route.ts b/apps/sim/app/api/tools/jsm/forms/templates/route.ts deleted file mode 100644 index 681ff6903c9..00000000000 --- a/apps/sim/app/api/tools/jsm/forms/templates/route.ts +++ /dev/null @@ -1,117 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmProjectFormTemplatesContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId, validateJiraIssueKey } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmFormsApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmFormTemplatesAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmProjectFormTemplatesContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, cloudId: cloudIdParam, projectIdOrKey } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!projectIdOrKey) { - logger.error('Missing projectIdOrKey in request') - return NextResponse.json({ error: 'Project ID or key is required' }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const projectIdOrKeyValidation = validateJiraIssueKey(projectIdOrKey, 'projectIdOrKey') - if (!projectIdOrKeyValidation.isValid) { - return NextResponse.json({ error: projectIdOrKeyValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmFormsApiBaseUrl(cloudId) - const url = `${baseUrl}/project/${encodeURIComponent(projectIdOrKey)}/form` - - logger.info('Fetching project form templates from:', { url, projectIdOrKey }) - - const response = await fetch(url, { - method: 'GET', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM Forms API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - const templates = Array.isArray(data) ? data : (data.values ?? []) - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - projectIdOrKey, - templates: templates.map((template: Record) => ({ - id: template.id ?? null, - name: template.name ?? null, - updated: template.updated ?? null, - issueCreateIssueTypeIds: template.issueCreateIssueTypeIds ?? [], - issueCreateRequestTypeIds: template.issueCreateRequestTypeIds ?? [], - portalRequestTypeIds: template.portalRequestTypeIds ?? [], - recommendedIssueRequestTypeIds: template.recommendedIssueRequestTypeIds ?? [], - })), - total: templates.length, - }, - }) - } catch (error) { - logger.error('Error fetching form templates:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/organization/route.ts b/apps/sim/app/api/tools/jsm/organization/route.ts deleted file mode 100644 index 3e0fb3ccd5c..00000000000 --- a/apps/sim/app/api/tools/jsm/organization/route.ts +++ /dev/null @@ -1,198 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmOrganizationContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - validateAlphanumericId, - validateEnum, - validateJiraCloudId, -} from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmOrganizationAPI') - -const VALID_ACTIONS = ['create', 'add_to_service_desk'] as const - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmOrganizationContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - action, - name, - serviceDeskId, - organizationId, - } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!action) { - logger.error('Missing action in request') - return NextResponse.json({ error: 'Action is required' }, { status: 400 }) - } - - const actionValidation = validateEnum(action, VALID_ACTIONS, 'action') - if (!actionValidation.isValid) { - return NextResponse.json({ error: actionValidation.error }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmApiBaseUrl(cloudId) - - if (action === 'create') { - if (!name) { - logger.error('Missing organization name in request') - return NextResponse.json({ error: 'Organization name is required' }, { status: 400 }) - } - - const url = `${baseUrl}/organization` - - logger.info('Creating organization:', { name }) - - const response = await fetch(url, { - method: 'POST', - headers: getJsmHeaders(accessToken), - body: JSON.stringify({ name }), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - organizationId: data.id, - name: data.name, - success: true, - }, - }) - } - if (action === 'add_to_service_desk') { - if (!serviceDeskId) { - logger.error('Missing serviceDeskId in request') - return NextResponse.json({ error: 'Service Desk ID is required' }, { status: 400 }) - } - - if (!organizationId) { - logger.error('Missing organizationId in request') - return NextResponse.json({ error: 'Organization ID is required' }, { status: 400 }) - } - - const serviceDeskIdValidation = validateAlphanumericId(serviceDeskId, 'serviceDeskId') - if (!serviceDeskIdValidation.isValid) { - return NextResponse.json({ error: serviceDeskIdValidation.error }, { status: 400 }) - } - - const organizationIdValidation = validateAlphanumericId(organizationId, 'organizationId') - if (!organizationIdValidation.isValid) { - return NextResponse.json({ error: organizationIdValidation.error }, { status: 400 }) - } - - const orgIdNumeric = Number.parseInt(String(organizationId).trim(), 10) - if (!Number.isFinite(orgIdNumeric) || orgIdNumeric <= 0) { - return NextResponse.json( - { error: 'organizationId must be a positive integer' }, - { status: 400 } - ) - } - - const url = `${baseUrl}/servicedesk/${serviceDeskId}/organization` - - logger.info('Adding organization to service desk:', { serviceDeskId, organizationId }) - - const response = await fetch(url, { - method: 'POST', - headers: getJsmHeaders(accessToken), - body: JSON.stringify({ organizationId: orgIdNumeric }), - }) - - if (response.status === 204 || response.ok) { - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - serviceDeskId, - organizationId, - success: true, - }, - }) - } - - const errorText = await response.text() - logger.error('JSM API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - return NextResponse.json({ error: 'Invalid action' }, { status: 400 }) - } catch (error) { - logger.error('Error in organization operation:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/organizations/route.ts b/apps/sim/app/api/tools/jsm/organizations/route.ts deleted file mode 100644 index 769dfec7eb1..00000000000 --- a/apps/sim/app/api/tools/jsm/organizations/route.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmOrganizationsContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmOrganizationsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmOrganizationsContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - serviceDeskId, - start, - limit, - } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!serviceDeskId) { - logger.error('Missing serviceDeskId in request') - return NextResponse.json({ error: 'Service Desk ID is required' }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const serviceDeskIdValidation = validateAlphanumericId(serviceDeskId, 'serviceDeskId') - if (!serviceDeskIdValidation.isValid) { - return NextResponse.json({ error: serviceDeskIdValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmApiBaseUrl(cloudId) - - const params = new URLSearchParams() - if (start) params.append('start', start) - if (limit) params.append('limit', limit) - - const url = `${baseUrl}/servicedesk/${serviceDeskId}/organization${params.toString() ? `?${params.toString()}` : ''}` - - logger.info('Fetching organizations from:', url) - - const response = await fetch(url, { - method: 'GET', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - organizations: data.values || [], - total: data.size || 0, - isLastPage: data.isLastPage ?? true, - }, - }) - } catch (error) { - logger.error('Error fetching organizations:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/participants/route.ts b/apps/sim/app/api/tools/jsm/participants/route.ts deleted file mode 100644 index 496c587c81c..00000000000 --- a/apps/sim/app/api/tools/jsm/participants/route.ts +++ /dev/null @@ -1,195 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmParticipantsContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - validateEnum, - validateJiraCloudId, - validateJiraIssueKey, -} from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmParticipantsAPI') - -const VALID_ACTIONS = ['get', 'add'] as const - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmParticipantsContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - action, - issueIdOrKey, - accountIds, - start, - limit, - } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!issueIdOrKey) { - logger.error('Missing issueIdOrKey in request') - return NextResponse.json({ error: 'Issue ID or key is required' }, { status: 400 }) - } - - if (!action) { - logger.error('Missing action in request') - return NextResponse.json({ error: 'Action is required' }, { status: 400 }) - } - - const actionValidation = validateEnum(action, VALID_ACTIONS, 'action') - if (!actionValidation.isValid) { - return NextResponse.json({ error: actionValidation.error }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const issueIdOrKeyValidation = validateJiraIssueKey(issueIdOrKey, 'issueIdOrKey') - if (!issueIdOrKeyValidation.isValid) { - return NextResponse.json({ error: issueIdOrKeyValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmApiBaseUrl(cloudId) - - if (action === 'get') { - const params = new URLSearchParams() - if (start) params.append('start', start) - if (limit) params.append('limit', limit) - - const url = `${baseUrl}/request/${issueIdOrKey}/participant${params.toString() ? `?${params.toString()}` : ''}` - - logger.info('Fetching participants from:', url) - - const response = await fetch(url, { - method: 'GET', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - issueIdOrKey, - participants: data.values || [], - total: data.size || 0, - isLastPage: data.isLastPage ?? true, - }, - }) - } - if (action === 'add') { - if (!accountIds) { - logger.error('Missing accountIds in request') - return NextResponse.json({ error: 'Account IDs are required' }, { status: 400 }) - } - - const parsedAccountIds = - typeof accountIds === 'string' - ? accountIds - .split(',') - .map((id: string) => id.trim()) - .filter((id: string) => id) - : accountIds - - const url = `${baseUrl}/request/${issueIdOrKey}/participant` - - logger.info('Adding participants to:', url, { accountIds: parsedAccountIds }) - - const response = await fetch(url, { - method: 'POST', - headers: getJsmHeaders(accessToken), - body: JSON.stringify({ accountIds: parsedAccountIds }), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - issueIdOrKey, - participants: data.values || [], - success: true, - }, - }) - } - - return NextResponse.json({ error: 'Invalid action' }, { status: 400 }) - } catch (error) { - logger.error('Error in participants operation:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/queues/route.ts b/apps/sim/app/api/tools/jsm/queues/route.ts deleted file mode 100644 index d966c682f03..00000000000 --- a/apps/sim/app/api/tools/jsm/queues/route.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmQueuesContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmQueuesAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmQueuesContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - serviceDeskId, - includeCount, - start, - limit, - } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!serviceDeskId) { - logger.error('Missing serviceDeskId in request') - return NextResponse.json({ error: 'Service Desk ID is required' }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const serviceDeskIdValidation = validateAlphanumericId(serviceDeskId, 'serviceDeskId') - if (!serviceDeskIdValidation.isValid) { - return NextResponse.json({ error: serviceDeskIdValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmApiBaseUrl(cloudId) - - const params = new URLSearchParams() - if (includeCount) params.append('includeCount', includeCount) - if (start) params.append('start', start) - if (limit) params.append('limit', limit) - - const url = `${baseUrl}/servicedesk/${serviceDeskId}/queue${params.toString() ? `?${params.toString()}` : ''}` - - logger.info('Fetching queues from:', url) - - const response = await fetch(url, { - method: 'GET', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - queues: data.values || [], - total: data.size || 0, - isLastPage: data.isLastPage ?? true, - }, - }) - } catch (error) { - logger.error('Error fetching queues:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/request/route.ts b/apps/sim/app/api/tools/jsm/request/route.ts deleted file mode 100644 index 3b2e9922df0..00000000000 --- a/apps/sim/app/api/tools/jsm/request/route.ts +++ /dev/null @@ -1,271 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmRequestContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - validateAlphanumericId, - validateJiraCloudId, - validateJiraIssueKey, -} from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmRequestAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmRequestContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - issueIdOrKey, - serviceDeskId, - requestTypeId, - summary, - description, - raiseOnBehalfOf, - requestFieldValues, - formAnswers, - requestParticipants, - channel, - expand, - } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmApiBaseUrl(cloudId) - - const isCreateOperation = serviceDeskId && requestTypeId && (summary || formAnswers) - - if (isCreateOperation) { - const serviceDeskIdValidation = validateAlphanumericId(serviceDeskId, 'serviceDeskId') - if (!serviceDeskIdValidation.isValid) { - return NextResponse.json({ error: serviceDeskIdValidation.error }, { status: 400 }) - } - - const requestTypeIdValidation = validateAlphanumericId(requestTypeId, 'requestTypeId') - if (!requestTypeIdValidation.isValid) { - return NextResponse.json({ error: requestTypeIdValidation.error }, { status: 400 }) - } - const url = `${baseUrl}/request` - - logger.info('Creating request at:', { url, serviceDeskId, requestTypeId }) - - const requestBody: Record = { - serviceDeskId, - requestTypeId, - } - - if (formAnswers && typeof formAnswers === 'object') { - // When form answers are provided, use them as the primary data source. - // Per Atlassian docs, fields linked to form questions must NOT also appear - // in requestFieldValues — doing so causes a 400 error. - requestBody.form = { answers: formAnswers } - - // Only include explicit requestFieldValues if the caller provided them - // (they know which fields are safe to include alongside form answers). - if (requestFieldValues && typeof requestFieldValues === 'object') { - requestBody.requestFieldValues = requestFieldValues - } - } else if (summary || description || requestFieldValues) { - const fieldValues = - requestFieldValues && typeof requestFieldValues === 'object' - ? { - ...(!requestFieldValues.summary && summary ? { summary } : {}), - ...(!requestFieldValues.description && description ? { description } : {}), - ...requestFieldValues, - } - : { - ...(summary && { summary }), - ...(description && { description }), - } - requestBody.requestFieldValues = fieldValues - } - - if (raiseOnBehalfOf) { - requestBody.raiseOnBehalfOf = raiseOnBehalfOf - } - if (requestParticipants) { - requestBody.requestParticipants = Array.isArray(requestParticipants) - ? requestParticipants - : typeof requestParticipants === 'string' - ? requestParticipants - .split(',') - .map((id: string) => id.trim()) - .filter(Boolean) - : [] - } - if (channel) { - requestBody.channel = channel - } - - const response = await fetch(url, { - method: 'POST', - headers: getJsmHeaders(accessToken), - body: JSON.stringify(requestBody), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - issueId: data.issueId, - issueKey: data.issueKey, - requestTypeId: data.requestTypeId, - serviceDeskId: data.serviceDeskId, - createdDate: data.createdDate ?? null, - currentStatus: data.currentStatus - ? { - status: data.currentStatus.status ?? null, - statusCategory: data.currentStatus.statusCategory ?? null, - statusDate: data.currentStatus.statusDate ?? null, - } - : null, - reporter: data.reporter - ? { - accountId: data.reporter.accountId ?? null, - displayName: data.reporter.displayName ?? null, - emailAddress: data.reporter.emailAddress ?? null, - } - : null, - success: true, - url: `https://${domain}/browse/${data.issueKey}`, - }, - }) - } - if (!issueIdOrKey) { - logger.error('Missing issueIdOrKey in request') - return NextResponse.json({ error: 'Issue ID or key is required' }, { status: 400 }) - } - - const issueIdOrKeyValidation = validateJiraIssueKey(issueIdOrKey, 'issueIdOrKey') - if (!issueIdOrKeyValidation.isValid) { - return NextResponse.json({ error: issueIdOrKeyValidation.error }, { status: 400 }) - } - - const params = new URLSearchParams() - if (expand) params.append('expand', expand) - - const url = `${baseUrl}/request/${issueIdOrKey}${params.toString() ? `?${params.toString()}` : ''}` - - logger.info('Fetching request from:', url) - - const response = await fetch(url, { - method: 'GET', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - issueId: data.issueId ?? null, - issueKey: data.issueKey ?? null, - requestTypeId: data.requestTypeId ?? null, - serviceDeskId: data.serviceDeskId ?? null, - createdDate: data.createdDate ?? null, - currentStatus: data.currentStatus - ? { - status: data.currentStatus.status ?? null, - statusCategory: data.currentStatus.statusCategory ?? null, - statusDate: data.currentStatus.statusDate ?? null, - } - : null, - reporter: data.reporter - ? { - accountId: data.reporter.accountId ?? null, - displayName: data.reporter.displayName ?? null, - emailAddress: data.reporter.emailAddress ?? null, - active: data.reporter.active ?? true, - } - : null, - requestFieldValues: (data.requestFieldValues ?? []).map((fv: Record) => ({ - fieldId: fv.fieldId ?? null, - label: fv.label ?? null, - value: fv.value ?? null, - })), - url: `https://${domain}/browse/${data.issueKey}`, - request: data, - }, - }) - } catch (error) { - logger.error('Error with request operation:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/requests/route.ts b/apps/sim/app/api/tools/jsm/requests/route.ts deleted file mode 100644 index 449d0d52253..00000000000 --- a/apps/sim/app/api/tools/jsm/requests/route.ts +++ /dev/null @@ -1,161 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmRequestsContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - validateAlphanumericId, - validateEnum, - validateJiraCloudId, -} from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmRequestsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmRequestsContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - serviceDeskId, - requestOwnership, - requestStatus, - requestTypeId, - searchTerm, - expand, - start, - limit, - } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - if (serviceDeskId) { - const serviceDeskIdValidation = validateAlphanumericId(serviceDeskId, 'serviceDeskId') - if (!serviceDeskIdValidation.isValid) { - return NextResponse.json({ error: serviceDeskIdValidation.error }, { status: 400 }) - } - } - - const VALID_REQUEST_OWNERSHIP = [ - 'OWNED_REQUESTS', - 'PARTICIPATED_REQUESTS', - 'APPROVER', - 'ALL_REQUESTS', - ] as const - const VALID_REQUEST_STATUS = ['OPEN_REQUESTS', 'CLOSED_REQUESTS', 'ALL_REQUESTS'] as const - - if (requestOwnership) { - const ownershipValidation = validateEnum( - requestOwnership, - VALID_REQUEST_OWNERSHIP, - 'requestOwnership' - ) - if (!ownershipValidation.isValid) { - return NextResponse.json({ error: ownershipValidation.error }, { status: 400 }) - } - } - - if (requestStatus) { - const statusValidation = validateEnum(requestStatus, VALID_REQUEST_STATUS, 'requestStatus') - if (!statusValidation.isValid) { - return NextResponse.json({ error: statusValidation.error }, { status: 400 }) - } - } - - const baseUrl = getJsmApiBaseUrl(cloudId) - - const params = new URLSearchParams() - if (serviceDeskId) params.append('serviceDeskId', serviceDeskId) - if (requestOwnership) { - params.append('requestOwnership', requestOwnership) - } - if (requestStatus) { - params.append('requestStatus', requestStatus) - } - if (requestTypeId) params.append('requestTypeId', requestTypeId) - if (searchTerm) params.append('searchTerm', searchTerm) - if (expand) params.append('expand', expand) - if (start) params.append('start', start) - if (limit) params.append('limit', limit) - - const url = `${baseUrl}/request${params.toString() ? `?${params.toString()}` : ''}` - - logger.info('Fetching requests from:', url) - - const response = await fetch(url, { - method: 'GET', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - requests: data.values || [], - total: data.size || 0, - isLastPage: data.isLastPage ?? true, - }, - }) - } catch (error) { - logger.error('Error fetching requests:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/requesttypefields/route.ts b/apps/sim/app/api/tools/jsm/requesttypefields/route.ts deleted file mode 100644 index cf6d0439439..00000000000 --- a/apps/sim/app/api/tools/jsm/requesttypefields/route.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmRequestTypeFieldsContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmRequestTypeFieldsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmRequestTypeFieldsContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - serviceDeskId, - requestTypeId, - } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!serviceDeskId) { - logger.error('Missing serviceDeskId in request') - return NextResponse.json({ error: 'Service Desk ID is required' }, { status: 400 }) - } - - if (!requestTypeId) { - logger.error('Missing requestTypeId in request') - return NextResponse.json({ error: 'Request Type ID is required' }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const serviceDeskIdValidation = validateAlphanumericId(serviceDeskId, 'serviceDeskId') - if (!serviceDeskIdValidation.isValid) { - return NextResponse.json({ error: serviceDeskIdValidation.error }, { status: 400 }) - } - - const requestTypeIdValidation = validateAlphanumericId(requestTypeId, 'requestTypeId') - if (!requestTypeIdValidation.isValid) { - return NextResponse.json({ error: requestTypeIdValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmApiBaseUrl(cloudId) - const url = `${baseUrl}/servicedesk/${serviceDeskId}/requesttype/${requestTypeId}/field` - - logger.info('Fetching request type fields from:', url) - - const response = await fetch(url, { - method: 'GET', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - serviceDeskId, - requestTypeId, - canAddRequestParticipants: data.canAddRequestParticipants ?? false, - canRaiseOnBehalfOf: data.canRaiseOnBehalfOf ?? false, - requestTypeFields: (data.requestTypeFields ?? []).map((field: Record) => ({ - fieldId: field.fieldId ?? null, - name: field.name ?? null, - description: field.description ?? null, - required: field.required ?? false, - visible: field.visible ?? true, - validValues: field.validValues ?? [], - presetValues: field.presetValues ?? [], - defaultValues: field.defaultValues ?? [], - jiraSchema: field.jiraSchema ?? null, - })), - }, - }) - } catch (error) { - logger.error('Error fetching request type fields:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/requesttypes/route.ts b/apps/sim/app/api/tools/jsm/requesttypes/route.ts deleted file mode 100644 index e16a39022ee..00000000000 --- a/apps/sim/app/api/tools/jsm/requesttypes/route.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmRequestTypesContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmRequestTypesAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmRequestTypesContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - serviceDeskId, - searchQuery, - groupId, - expand, - start, - limit, - } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!serviceDeskId) { - logger.error('Missing serviceDeskId in request') - return NextResponse.json({ error: 'Service Desk ID is required' }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const serviceDeskIdValidation = validateAlphanumericId(serviceDeskId, 'serviceDeskId') - if (!serviceDeskIdValidation.isValid) { - return NextResponse.json({ error: serviceDeskIdValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmApiBaseUrl(cloudId) - - const params = new URLSearchParams() - if (searchQuery) params.append('searchQuery', searchQuery) - if (groupId) params.append('groupId', groupId) - if (expand) params.append('expand', expand) - if (start) params.append('start', start) - if (limit) params.append('limit', limit) - - const url = `${baseUrl}/servicedesk/${serviceDeskId}/requesttype${params.toString() ? `?${params.toString()}` : ''}` - - logger.info('Fetching request types from:', url) - - const response = await fetch(url, { - method: 'GET', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - requestTypes: data.values || [], - total: data.size || 0, - isLastPage: data.isLastPage ?? true, - }, - }) - } catch (error) { - logger.error('Error fetching request types:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/selector-requesttypes/route.ts b/apps/sim/app/api/tools/jsm/selector-requesttypes/route.ts index 705dc2be2e0..771a85f3f24 100644 --- a/apps/sim/app/api/tools/jsm/selector-requesttypes/route.ts +++ b/apps/sim/app/api/tools/jsm/selector-requesttypes/route.ts @@ -1,177 +1,51 @@ import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' import { type NextRequest, NextResponse } from 'next/server' import { jsmRequestTypesSelectorContract } from '@/lib/api/contracts/selectors/jsm' import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' -import { validateAlphanumericId, validateJiraCloudId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { JsmOperationError } from '@/lib/internal/jsm/errors' +import { listJsmRequestTypeOptions } from '@/lib/internal/jsm/service-desk' import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' const logger = createLogger('JsmSelectorRequestTypesAPI') export const dynamic = 'force-dynamic' -const JSM_REQUEST_TYPES_PAGE_SIZE = 100 -const MAX_JSM_REQUEST_TYPES_PAGES = 50 - -interface JsmPagedResponse { - values?: T[] - isLastPage?: boolean - _links?: { next?: string } -} - -interface JsmRequestTypeValue { - id: string - name: string -} - -/** - * Drains the offset-paginated JSM `/servicedesk/{id}/requesttype` endpoint, - * advancing `start` by the number of rows actually returned until - * `isLastPage === true` (or `_links.next` is absent, or a page comes back - * empty). Advancing by the real row count — not the requested `limit` — - * prevents skipping items if the server returns a short non-final page. Bounded - * by `MAX_JSM_REQUEST_TYPES_PAGES`; emits a `logger.warn` and returns the - * partial set rather than looping unbounded when the cap is hit. - */ -async function fetchAllJsmRequestTypes( - requestTypeUrl: string, - accessToken: string -): Promise<{ values: JsmRequestTypeValue[]; lastResponse: Response }> { - const values: JsmRequestTypeValue[] = [] - let start = 0 - let lastResponse: Response - - for (let page = 0; page < MAX_JSM_REQUEST_TYPES_PAGES; page++) { - const url = `${requestTypeUrl}?start=${start}&limit=${JSM_REQUEST_TYPES_PAGE_SIZE}` - - const response = await fetch(url, { - method: 'GET', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - return { values, lastResponse: response } - } - - const data = (await response.json()) as JsmPagedResponse - lastResponse = response - - const pageValues = data.values ?? [] - values.push(...pageValues) - - if (data.isLastPage === true || !data._links?.next || pageValues.length === 0) { - return { values, lastResponse } - } - - start += pageValues.length - - if (page === MAX_JSM_REQUEST_TYPES_PAGES - 1) { - logger.warn('JSM request type list hit pagination cap; list may be incomplete', { - pages: MAX_JSM_REQUEST_TYPES_PAGES, - collected: values.length, - }) - } - } - - return { values, lastResponse: lastResponse! } -} - export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() try { const parsed = await parseRequest(jsmRequestTypesSelectorContract, request, {}) if (!parsed.success) return parsed.response - const { credential, workflowId, domain, serviceDeskId } = parsed.data.body - - if (!credential) { - logger.error('Missing credential in request') - return NextResponse.json({ error: 'Credential is required' }, { status: 400 }) - } - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!serviceDeskId) { - return NextResponse.json({ error: 'Service Desk ID is required' }, { status: 400 }) - } - - const serviceDeskIdValidation = validateAlphanumericId(serviceDeskId, 'serviceDeskId') - if (!serviceDeskIdValidation.isValid) { - return NextResponse.json({ error: serviceDeskIdValidation.error }, { status: 400 }) - } - - const authz = await authorizeCredentialUse(request, { - credentialId: credential, - workflowId, - }) + const authz = await authorizeCredentialUse(request, { credentialId: credential, workflowId }) if (!authz.ok || !authz.credentialOwnerUserId) { return NextResponse.json({ error: authz.error || 'Unauthorized' }, { status: 403 }) } - const accessToken = await refreshAccessTokenIfNeeded( credential, authz.credentialOwnerUserId, requestId ) if (!accessToken) { - logger.error('Failed to get access token', { - credentialId: credential, - userId: authz.credentialOwnerUserId, - }) return NextResponse.json( { error: 'Could not retrieve access token', authRequired: true }, { status: 401 } ) } - - const cloudId = await getJiraCloudId(domain, accessToken) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmApiBaseUrl(cloudIdValidation.sanitized!) - const requestTypeUrl = `${baseUrl}/servicedesk/${serviceDeskIdValidation.sanitized}/requesttype` - - const { values, lastResponse } = await fetchAllJsmRequestTypes(requestTypeUrl, accessToken) - - if (!lastResponse.ok) { - const errorText = await lastResponse.text() - logger.error('JSM API error:', { - status: lastResponse.status, - statusText: lastResponse.statusText, - error: errorText, - }) - return NextResponse.json( - { - error: parseAtlassianErrorMessage( - lastResponse.status, - lastResponse.statusText, - errorText - ), - }, - { status: lastResponse.status } - ) - } - - const requestTypes = values.map((rt) => ({ - id: rt.id, - name: rt.name, - })) - + const requestTypes = await listJsmRequestTypeOptions( + { domain, accessToken, serviceDeskId }, + request.signal + ) return NextResponse.json({ requestTypes }) } catch (error) { + request.signal.throwIfAborted() logger.error('Error listing JSM request types:', error) return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } + { error: getErrorMessage(error, 'Internal server error') }, + { status: error instanceof JsmOperationError ? error.status : 500 } ) } }) diff --git a/apps/sim/app/api/tools/jsm/selector-servicedesks/route.ts b/apps/sim/app/api/tools/jsm/selector-servicedesks/route.ts index 9dc55ea0a83..c6dc4091023 100644 --- a/apps/sim/app/api/tools/jsm/selector-servicedesks/route.ts +++ b/apps/sim/app/api/tools/jsm/selector-servicedesks/route.ts @@ -1,167 +1,48 @@ import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' import { type NextRequest, NextResponse } from 'next/server' import { jsmServiceDesksSelectorContract } from '@/lib/api/contracts/selectors/jsm' import { parseRequest } from '@/lib/api/server' import { authorizeCredentialUse } from '@/lib/auth/credential-access' -import { validateJiraCloudId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { JsmOperationError } from '@/lib/internal/jsm/errors' +import { listJsmServiceDeskOptions } from '@/lib/internal/jsm/service-desk' import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' const logger = createLogger('JsmSelectorServiceDesksAPI') export const dynamic = 'force-dynamic' -const JSM_SERVICE_DESKS_PAGE_SIZE = 100 -const MAX_JSM_SERVICE_DESKS_PAGES = 50 - -interface JsmPagedResponse { - values?: T[] - isLastPage?: boolean - _links?: { next?: string } -} - -interface JsmServiceDeskValue { - id: string - projectName: string -} - -/** - * Drains the offset-paginated JSM `/servicedesk` endpoint, advancing `start` by - * the number of rows actually returned until `isLastPage === true` (or - * `_links.next` is absent, or a page comes back empty). Advancing by the real - * row count — not the requested `limit` — prevents skipping items if the server - * returns a short non-final page. Bounded by `MAX_JSM_SERVICE_DESKS_PAGES`; - * emits a `logger.warn` and returns the partial set rather than looping - * unbounded when the cap is hit. - */ -async function fetchAllJsmServiceDesks( - baseUrl: string, - accessToken: string -): Promise<{ values: JsmServiceDeskValue[]; lastResponse: Response }> { - const values: JsmServiceDeskValue[] = [] - let start = 0 - let lastResponse: Response - - for (let page = 0; page < MAX_JSM_SERVICE_DESKS_PAGES; page++) { - const url = `${baseUrl}/servicedesk?start=${start}&limit=${JSM_SERVICE_DESKS_PAGE_SIZE}` - - const response = await fetch(url, { - method: 'GET', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - return { values, lastResponse: response } - } - - const data = (await response.json()) as JsmPagedResponse - lastResponse = response - - const pageValues = data.values ?? [] - values.push(...pageValues) - - if (data.isLastPage === true || !data._links?.next || pageValues.length === 0) { - return { values, lastResponse } - } - - start += pageValues.length - - if (page === MAX_JSM_SERVICE_DESKS_PAGES - 1) { - logger.warn('JSM service desk list hit pagination cap; list may be incomplete', { - pages: MAX_JSM_SERVICE_DESKS_PAGES, - collected: values.length, - }) - } - } - - return { values, lastResponse: lastResponse! } -} - export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() try { const parsed = await parseRequest(jsmServiceDesksSelectorContract, request, {}) if (!parsed.success) return parsed.response - const { credential, workflowId, domain } = parsed.data.body - - if (!credential) { - logger.error('Missing credential in request') - return NextResponse.json({ error: 'Credential is required' }, { status: 400 }) - } - - if (!domain) { - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - const authz = await authorizeCredentialUse(request, { - credentialId: credential, - workflowId, - }) + const authz = await authorizeCredentialUse(request, { credentialId: credential, workflowId }) if (!authz.ok || !authz.credentialOwnerUserId) { return NextResponse.json({ error: authz.error || 'Unauthorized' }, { status: 403 }) } - const accessToken = await refreshAccessTokenIfNeeded( credential, authz.credentialOwnerUserId, requestId ) if (!accessToken) { - logger.error('Failed to get access token', { - credentialId: credential, - userId: authz.credentialOwnerUserId, - }) return NextResponse.json( { error: 'Could not retrieve access token', authRequired: true }, { status: 401 } ) } - - const cloudId = await getJiraCloudId(domain, accessToken) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmApiBaseUrl(cloudIdValidation.sanitized!) - - const { values, lastResponse } = await fetchAllJsmServiceDesks(baseUrl, accessToken) - - if (!lastResponse.ok) { - const errorText = await lastResponse.text() - logger.error('JSM API error:', { - status: lastResponse.status, - statusText: lastResponse.statusText, - error: errorText, - }) - return NextResponse.json( - { - error: parseAtlassianErrorMessage( - lastResponse.status, - lastResponse.statusText, - errorText - ), - }, - { status: lastResponse.status } - ) - } - - const serviceDesks = values.map((sd) => ({ - id: sd.id, - name: sd.projectName, - })) - + const serviceDesks = await listJsmServiceDeskOptions({ domain, accessToken }, request.signal) return NextResponse.json({ serviceDesks }) } catch (error) { + request.signal.throwIfAborted() logger.error('Error listing JSM service desks:', error) return NextResponse.json( - { error: (error as Error).message || 'Internal server error' }, - { status: 500 } + { error: getErrorMessage(error, 'Internal server error') }, + { status: error instanceof JsmOperationError ? error.status : 500 } ) } }) diff --git a/apps/sim/app/api/tools/jsm/servicedesks/route.ts b/apps/sim/app/api/tools/jsm/servicedesks/route.ts deleted file mode 100644 index 3ca85dd9169..00000000000 --- a/apps/sim/app/api/tools/jsm/servicedesks/route.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmServiceDesksContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmServiceDesksAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmServiceDesksContract, request, {}) - if (!parsed.success) return parsed.response - - const { domain, accessToken, cloudId: cloudIdParam, expand, start, limit } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmApiBaseUrl(cloudId) - - const params = new URLSearchParams() - if (expand) params.append('expand', expand) - if (start) params.append('start', start) - if (limit) params.append('limit', limit) - - const url = `${baseUrl}/servicedesk${params.toString() ? `?${params.toString()}` : ''}` - - logger.info('Fetching service desks from:', url) - - const response = await fetch(url, { - method: 'GET', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - serviceDesks: data.values || [], - total: data.size || 0, - isLastPage: data.isLastPage ?? true, - }, - }) - } catch (error) { - logger.error('Error fetching service desks:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/sla/route.ts b/apps/sim/app/api/tools/jsm/sla/route.ts deleted file mode 100644 index 64bd97a035e..00000000000 --- a/apps/sim/app/api/tools/jsm/sla/route.ts +++ /dev/null @@ -1,120 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmSlaContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId, validateJiraIssueKey } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmSlaAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmSlaContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - issueIdOrKey, - start, - limit, - } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!issueIdOrKey) { - logger.error('Missing issueIdOrKey in request') - return NextResponse.json({ error: 'Issue ID or key is required' }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const issueIdOrKeyValidation = validateJiraIssueKey(issueIdOrKey, 'issueIdOrKey') - if (!issueIdOrKeyValidation.isValid) { - return NextResponse.json({ error: issueIdOrKeyValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmApiBaseUrl(cloudId) - - const params = new URLSearchParams() - if (start) params.append('start', start) - if (limit) params.append('limit', limit) - - const url = `${baseUrl}/request/${issueIdOrKey}/sla${params.toString() ? `?${params.toString()}` : ''}` - - logger.info('Fetching SLA info from:', url) - - const response = await fetch(url, { - method: 'GET', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - issueIdOrKey, - slas: data.values || [], - total: data.size || 0, - isLastPage: data.isLastPage ?? true, - }, - }) - } catch (error) { - logger.error('Error fetching SLA info:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/transition/route.ts b/apps/sim/app/api/tools/jsm/transition/route.ts deleted file mode 100644 index 62b2adfa961..00000000000 --- a/apps/sim/app/api/tools/jsm/transition/route.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmTransitionContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - validateAlphanumericId, - validateJiraCloudId, - validateJiraIssueKey, -} from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmTransitionAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmTransitionContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: providedCloudId, - issueIdOrKey, - transitionId, - comment, - } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!issueIdOrKey) { - logger.error('Missing issueIdOrKey in request') - return NextResponse.json({ error: 'Issue ID or key is required' }, { status: 400 }) - } - - if (!transitionId) { - logger.error('Missing transitionId in request') - return NextResponse.json({ error: 'Transition ID is required' }, { status: 400 }) - } - - const cloudId = providedCloudId || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const issueIdOrKeyValidation = validateJiraIssueKey(issueIdOrKey, 'issueIdOrKey') - if (!issueIdOrKeyValidation.isValid) { - return NextResponse.json({ error: issueIdOrKeyValidation.error }, { status: 400 }) - } - - const transitionIdValidation = validateAlphanumericId(transitionId, 'transitionId') - if (!transitionIdValidation.isValid) { - return NextResponse.json({ error: transitionIdValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmApiBaseUrl(cloudId) - - const url = `${baseUrl}/request/${issueIdOrKey}/transition` - - logger.info('Transitioning request at:', url) - - const body: Record = { - id: transitionId, - } - - if (comment) { - body.additionalComment = { - body: comment, - } - } - - const response = await fetch(url, { - method: 'POST', - headers: getJsmHeaders(accessToken), - body: JSON.stringify(body), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - issueIdOrKey, - transitionId, - success: true, - }, - }) - } catch (error) { - logger.error('Error transitioning request:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jsm/transitions/route.ts b/apps/sim/app/api/tools/jsm/transitions/route.ts deleted file mode 100644 index 364c999b08d..00000000000 --- a/apps/sim/app/api/tools/jsm/transitions/route.ts +++ /dev/null @@ -1,120 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jsmTransitionsContract } from '@/lib/api/contracts/selectors/jsm' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateJiraCloudId, validateJiraIssueKey } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' -import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JsmTransitionsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest(jsmTransitionsContract, request, {}) - if (!parsed.success) return parsed.response - - const { - domain, - accessToken, - cloudId: cloudIdParam, - issueIdOrKey, - start, - limit, - } = parsed.data.body - - if (!domain) { - logger.error('Missing domain in request') - return NextResponse.json({ error: 'Domain is required' }, { status: 400 }) - } - - if (!accessToken) { - logger.error('Missing access token in request') - return NextResponse.json({ error: 'Access token is required' }, { status: 400 }) - } - - if (!issueIdOrKey) { - logger.error('Missing issueIdOrKey in request') - return NextResponse.json({ error: 'Issue ID or key is required' }, { status: 400 }) - } - - const cloudId = cloudIdParam || (await getJiraCloudId(domain, accessToken)) - - const cloudIdValidation = validateJiraCloudId(cloudId, 'cloudId') - if (!cloudIdValidation.isValid) { - return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) - } - - const issueIdOrKeyValidation = validateJiraIssueKey(issueIdOrKey, 'issueIdOrKey') - if (!issueIdOrKeyValidation.isValid) { - return NextResponse.json({ error: issueIdOrKeyValidation.error }, { status: 400 }) - } - - const baseUrl = getJsmApiBaseUrl(cloudId) - - const params = new URLSearchParams() - if (start) params.append('start', start) - if (limit) params.append('limit', limit) - - const url = `${baseUrl}/request/${issueIdOrKey}/transition${params.toString() ? `?${params.toString()}` : ''}` - - logger.info('Fetching transitions from:', url) - - const response = await fetch(url, { - method: 'GET', - headers: getJsmHeaders(accessToken), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error('JSM API error:', { - status: response.status, - statusText: response.statusText, - error: errorText, - }) - - return NextResponse.json( - { - error: parseAtlassianErrorMessage(response.status, response.statusText, errorText), - details: errorText, - }, - { status: response.status } - ) - } - - const data = await response.json() - - return NextResponse.json({ - success: true, - output: { - ts: new Date().toISOString(), - issueIdOrKey, - transitions: data.values || [], - total: data.size || 0, - isLastPage: data.isLastPage ?? true, - }, - }) - } catch (error) { - logger.error('Error fetching transitions:', { - error: toError(error).message, - stack: error instanceof Error ? error.stack : undefined, - }) - - return NextResponse.json( - { - error: getErrorMessage(error, 'Internal server error'), - success: false, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jupyter/proxy/route.ts b/apps/sim/app/api/tools/jupyter/proxy/route.ts deleted file mode 100644 index 56118ba424e..00000000000 --- a/apps/sim/app/api/tools/jupyter/proxy/route.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jupyterProxyContract } from '@/lib/api/contracts/tools/jupyter' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - MAX_JSON_API_RESPONSE_BYTES, - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - assertSafeJupyterProxyPath, - buildJupyterAuthHeaders, - normalizeJupyterServerUrl, - UnsafeJupyterPathError, -} from '@/tools/jupyter/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JupyterProxyAPI') - -/** - * Proxies Contents/Kernels/Kernelspecs/Sessions API calls to a self-hosted - * Jupyter server. Self-hosted servers have no fixed public host, so every - * request is server-side (DNS-pinned, http(s) allowed, redirects disabled) - * rather than going through the generic external tool executor, which blocks - * plain-HTTP and private-IP hosts by default. Mirrors the upstream status - * and body verbatim so callers can treat this exactly like a direct fetch. - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Jupyter proxy attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(jupyterProxyContract, request, {}) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - try { - assertSafeJupyterProxyPath(data.path) - } catch (error) { - if (error instanceof UnsafeJupyterPathError) { - return NextResponse.json({ success: false, error: error.message }, { status: 400 }) - } - throw error - } - - const base = normalizeJupyterServerUrl(data.serverUrl) - const url = `${base}/api/${data.path}` - - const urlValidation = await validateUrlWithDNS(url, 'serverUrl', { allowHttp: true }) - if (!urlValidation.isValid || !urlValidation.resolvedIP) { - return NextResponse.json( - { success: false, error: `Invalid Jupyter serverUrl: ${urlValidation.error}` }, - { status: 400 } - ) - } - - const hasBody = data.body !== undefined && data.body !== null - - const upstream = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP, { - method: data.method, - headers: { - ...buildJupyterAuthHeaders(data.token), - ...(hasBody ? { 'Content-Type': 'application/json' } : {}), - }, - body: hasBody ? JSON.stringify(data.body) : undefined, - allowHttp: true, - maxRedirects: 0, - maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, - }) - - const text = await upstream.text() - - return new NextResponse(text.length > 0 ? text : null, { - status: upstream.status, - headers: { 'Content-Type': upstream.headers.get('content-type') || 'application/json' }, - }) - } catch (error) { - logger.error(`[${requestId}] Unexpected error:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/jupyter/upload/route.ts b/apps/sim/app/api/tools/jupyter/upload/route.ts deleted file mode 100644 index 0c4efff228a..00000000000 --- a/apps/sim/app/api/tools/jupyter/upload/route.ts +++ /dev/null @@ -1,171 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { jupyterUploadContract } from '@/lib/api/contracts/storage-transfer' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - MAX_JSON_API_RESPONSE_BYTES, - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { - buildJupyterAuthHeaders, - encodeJupyterPath, - normalizeJupyterServerUrl, - parseJupyterContentModel, - UnsafeJupyterPathError, -} from '@/tools/jupyter/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('JupyterUploadAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Jupyter upload attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(jupyterUploadContract, request, {}) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - let fileBuffer: Buffer - let fileName: string - - if (data.file) { - const userFiles = processFilesToUserFiles([data.file as RawFileInput], requestId, logger) - if (userFiles.length === 0) { - return NextResponse.json({ success: false, error: 'Invalid file input' }, { status: 400 }) - } - const userFile = userFiles[0] - - const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) - if (denied) return denied - - try { - const result = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - }) - fileBuffer = result.buffer - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Failed to download file') }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } - fileName = data.fileName || userFile.name - } else if (data.fileContent) { - fileBuffer = Buffer.from(data.fileContent, 'base64') - fileName = data.fileName || 'file' - } else { - return NextResponse.json({ success: false, error: 'File is required' }, { status: 400 }) - } - - if (/[/\\]/.test(fileName)) { - return NextResponse.json( - { success: false, error: 'File name must not contain path separators' }, - { status: 400 } - ) - } - - const base = normalizeJupyterServerUrl(data.serverUrl) - const destinationDirectory = (data.directory ?? '').replace(/\/+$/, '') - const destinationPath = destinationDirectory ? `${destinationDirectory}/${fileName}` : fileName - - let encodedDestinationPath: string - try { - encodedDestinationPath = encodeJupyterPath(destinationPath) - } catch (error) { - if (error instanceof UnsafeJupyterPathError) { - return NextResponse.json({ success: false, error: error.message }, { status: 400 }) - } - throw error - } - const uploadUrl = `${base}/api/contents/${encodedDestinationPath}` - - const urlValidation = await validateUrlWithDNS(uploadUrl, 'serverUrl', { allowHttp: true }) - if (!urlValidation.isValid || !urlValidation.resolvedIP) { - return NextResponse.json( - { success: false, error: `Invalid Jupyter serverUrl: ${urlValidation.error}` }, - { status: 400 } - ) - } - - const response = await secureFetchWithPinnedIP(uploadUrl, urlValidation.resolvedIP, { - method: 'PUT', - headers: { - ...buildJupyterAuthHeaders(data.token), - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - type: 'file', - format: 'base64', - content: fileBuffer.toString('base64'), - }), - allowHttp: true, - maxRedirects: 0, - maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error(`[${requestId}] Jupyter API error:`, { status: response.status, errorText }) - return NextResponse.json( - { success: false, error: `Jupyter API error: ${response.status} ${errorText}` }, - { status: response.status } - ) - } - - const uploadedValue: unknown = await response.json() - const uploaded = parseJupyterContentModel(uploadedValue) - if (!uploaded) { - logger.error(`[${requestId}] Jupyter returned an invalid upload response`) - return NextResponse.json( - { success: false, error: 'Jupyter returned an invalid upload response' }, - { status: 502 } - ) - } - - const uploadedName = uploaded.name ?? fileName - const uploadedPath = uploaded.path ?? destinationPath - const uploadedSize = uploaded.size ?? fileBuffer.length - const lastModified = uploaded.lastModified ?? null - - logger.info(`[${requestId}] File uploaded to Jupyter: ${uploadedPath}`) - - return NextResponse.json({ - success: true, - output: { - name: uploadedName, - path: uploadedPath, - size: uploadedSize, - lastModified, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Unexpected error:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/latex/route.ts b/apps/sim/app/api/tools/latex/route.ts deleted file mode 100644 index 3890ab2b2fc..00000000000 --- a/apps/sim/app/api/tools/latex/route.ts +++ /dev/null @@ -1,269 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { truncate } from '@sim/utils/string' -import { type NextRequest, NextResponse } from 'next/server' -import { type LatexCompileBody, latexCompileContract } from '@/lib/api/contracts/tools/latex' -import { getValidationErrorMessage, parseRequest, validationErrorResponse } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { - isPayloadSizeLimitError, - readResponseJsonWithLimit, - readResponseToBufferWithLimit, -} from '@/lib/core/utils/stream-limits' -import { getBaseUrl } from '@/lib/core/utils/urls' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -const logger = createLogger('LatexCompileAPI') - -const LATEX_COMPILE_URL = 'https://latex.ytotech.com/builds/sync' -const DEFAULT_COMPILER = 'pdflatex' -const MAX_PDF_BYTES = 25 * 1024 * 1024 -const MAX_ERROR_JSON_BYTES = 4 * 1024 * 1024 -const MAX_ERROR_MESSAGE_CHARS = 4000 -const MAX_ERROR_CODE_CHARS = 100 -/** Leaves headroom within `maxDuration` to store the PDF after compilation. */ -const COMPILE_TIMEOUT_MS = 50_000 - -export const dynamic = 'force-dynamic' -export const maxDuration = 60 - -interface StoredPdfResponse { - pdfFile?: unknown - pdfUrl: string - fileName: string - contentType: string - compiler: string -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest( - latexCompileContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid LaTeX compile request:`, error.issues) - return validationErrorResponse( - error, - getValidationErrorMessage(error, 'Invalid request data') - ) - }, - } - ) - if (!parsed.success) return parsed.response - - const body = parsed.data.body - const compiler = body.compiler || DEFAULT_COMPILER - - logger.info(`[${requestId}] Compiling LaTeX document`, { - compiler, - contentLength: body.content.length, - resourceCount: body.resources?.length ?? 0, - }) - - let upstreamResponse: Response - try { - upstreamResponse = await fetch(LATEX_COMPILE_URL, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - compiler, - resources: [{ main: true, content: body.content }, ...(body.resources ?? [])], - }), - signal: AbortSignal.timeout(COMPILE_TIMEOUT_MS), - }) - } catch (error) { - // The timeout signal is the only abort source on this fetch, so an - // AbortError here is a timeout regardless of which name undici uses. - if ( - error instanceof DOMException && - (error.name === 'TimeoutError' || error.name === 'AbortError') - ) { - logger.error(`[${requestId}] LaTeX compile service timed out`, { - timeoutMs: COMPILE_TIMEOUT_MS, - }) - return NextResponse.json({ error: 'LaTeX compile service timed out' }, { status: 504 }) - } - throw error - } - - const upstreamContentType = upstreamResponse.headers.get('content-type') || '' - if (!upstreamResponse.ok || !upstreamContentType.includes('application/pdf')) { - return await buildCompileErrorResponse(upstreamResponse, requestId) - } - - const pdfBuffer = await readResponseToBufferWithLimit(upstreamResponse, { - maxBytes: MAX_PDF_BYTES, - label: 'compiled PDF', - }) - if (pdfBuffer.length === 0) { - logger.error(`[${requestId}] LaTeX compile service returned an empty PDF`) - return NextResponse.json( - { error: 'LaTeX compile service returned an empty PDF' }, - { status: 502 } - ) - } - - const storedPdf = await storeCompiledPdf(pdfBuffer, body, compiler, authResult.userId) - - logger.info(`[${requestId}] LaTeX compilation completed`, { - compiler, - fileName: storedPdf.fileName, - size: pdfBuffer.length, - }) - - return NextResponse.json(storedPdf) - } catch (error) { - logger.error(`[${requestId}] LaTeX compile route error:`, error) - return NextResponse.json( - { error: getErrorMessage(error, 'LaTeX compilation failed') }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } -}) - -/** - * Builds the output PDF filename: strips any directory components and - * normalizes to a single `.pdf` extension. - */ -function buildPdfFileName(fileName: string | undefined): string { - const base = (fileName || 'document').split(/[/\\]/).pop()?.trim() || 'document' - const withoutExtension = base.toLowerCase().endsWith('.pdf') ? base.slice(0, -4) : base - return `${withoutExtension || 'document'}.pdf` -} - -/** - * Extracts TeX error lines (lines starting with `!`, each with two lines of - * context) from the compiler log files returned by the compile service. - */ -function extractCompilationErrors(logFiles: unknown): string | undefined { - if (typeof logFiles !== 'object' || logFiles === null) return undefined - - const snippets: string[] = [] - for (const log of Object.values(logFiles)) { - if (typeof log !== 'string') continue - const lines = log.split('\n') - for (let i = 0; i < lines.length; i++) { - if (lines[i].startsWith('!')) { - snippets.push(lines.slice(i, i + 3).join('\n')) - } - } - } - - if (snippets.length === 0) return undefined - return truncate([...new Set(snippets)].join('\n\n'), MAX_ERROR_MESSAGE_CHARS) -} - -/** - * Maps a failed compile-service response to a JSON error response: 422 with - * extracted TeX errors for compilation failures, 502 for anything unexpected. - */ -async function buildCompileErrorResponse( - upstreamResponse: Response, - requestId: string -): Promise { - const errorBody = await readResponseJsonWithLimit(upstreamResponse, { - maxBytes: MAX_ERROR_JSON_BYTES, - label: 'LaTeX compile error response', - }).catch(() => undefined) - - const errorRecord = - typeof errorBody === 'object' && errorBody !== null - ? (errorBody as Record) - : undefined - const errorCode = - typeof errorRecord?.error === 'string' - ? truncate(errorRecord.error, MAX_ERROR_CODE_CHARS) - : undefined - const compilationErrors = extractCompilationErrors(errorRecord?.log_files) - const details = compilationErrors ? `:\n${compilationErrors}` : '' - - const isCompilationFailure = - upstreamResponse.status >= 400 && - upstreamResponse.status < 500 && - Boolean(errorCode || compilationErrors) - - if (isCompilationFailure) { - logger.warn(`[${requestId}] LaTeX compilation failed`, { - status: upstreamResponse.status, - errorCode, - }) - return NextResponse.json( - { error: `LaTeX compilation failed (${errorCode || upstreamResponse.status})${details}` }, - { status: 422 } - ) - } - - logger.error(`[${requestId}] LaTeX compile service error`, { - status: upstreamResponse.status, - errorCode, - }) - return NextResponse.json( - { error: `LaTeX compile service error: ${upstreamResponse.status}${details}` }, - { status: 502 } - ) -} - -/** - * Stores the compiled PDF as an execution file when execution context is - * available, falling back to general storage otherwise. - */ -async function storeCompiledPdf( - pdfBuffer: Buffer, - body: LatexCompileBody, - compiler: string, - userId: string -): Promise { - const fileName = buildPdfFileName(body.fileName) - const executionContext = - body.workspaceId && body.workflowId && body.executionId - ? { - workspaceId: body.workspaceId, - workflowId: body.workflowId, - executionId: body.executionId, - } - : null - - if (executionContext) { - const { uploadExecutionFile } = await import('@/lib/uploads/contexts/execution') - const pdfFile = await uploadExecutionFile( - executionContext, - pdfBuffer, - fileName, - 'application/pdf', - userId - ) - - return { - pdfFile, - pdfUrl: pdfFile.url, - fileName, - contentType: 'application/pdf', - compiler, - } - } - - const { StorageService } = await import('@/lib/uploads') - const fileInfo = await StorageService.uploadFile({ - file: pdfBuffer, - fileName, - contentType: 'application/pdf', - context: 'copilot', - }) - - return { - pdfUrl: `${getBaseUrl()}${fileInfo.path}`, - fileName, - contentType: 'application/pdf', - compiler, - } -} diff --git a/apps/sim/app/api/tools/linq/upload/route.ts b/apps/sim/app/api/tools/linq/upload/route.ts deleted file mode 100644 index deb8ffb79d6..00000000000 --- a/apps/sim/app/api/tools/linq/upload/route.ts +++ /dev/null @@ -1,186 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { linqUploadAttachmentContract } from '@/lib/api/contracts/tools/communication/messaging' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { extractLinqError, LINQ_API_BASE, linqHeaders } from '@/tools/linq/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('LinqUploadAttachmentAPI') - -/** Linq pre-upload caps attachments at 100MB. */ -const MAX_SIZE_BYTES = 100 * 1024 * 1024 - -function fileTooLargeError(sizeBytes: number): NextResponse { - return NextResponse.json( - { - success: false, - error: `File exceeds Linq's 100MB attachment limit (${(sizeBytes / (1024 * 1024)).toFixed(2)}MB)`, - }, - { status: 400 } - ) -} - -/** - * Upload a file to Linq as a reusable attachment. - * - * Linq uses a two-step pre-upload flow: register the attachment metadata to - * receive a presigned URL, then PUT the bytes to that URL with the exact - * headers Linq returns. The resulting `attachment_id` can be referenced when - * sending messages or voice memos. - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Linq upload attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(linqUploadAttachmentContract, request, {}) - if (!parsed.success) return parsed.response - const { apiKey, file, fileContent, filename, contentType } = parsed.data.body - - let buffer: Buffer - let resolvedFilename = filename ?? '' - let resolvedContentType = contentType ?? '' - - if (file) { - const userFiles = processFilesToUserFiles([file as RawFileInput], requestId, logger) - if (userFiles.length === 0) { - return NextResponse.json( - { success: false, error: 'No valid file provided' }, - { status: 400 } - ) - } - const userFile = userFiles[0] - const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) - if (denied) return denied - let resolvedContentTypeFromStorage: string - try { - const resolved = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_SIZE_BYTES, - }) - buffer = resolved.buffer - resolvedContentTypeFromStorage = resolved.contentType - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - if (isPayloadSizeLimitError(error)) - return fileTooLargeError(error.observedBytes ?? userFile.size) - logger.error(`[${requestId}] Failed to download Linq attachment file:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error occurred') }, - { status: 500 } - ) - } - if (!resolvedFilename) resolvedFilename = userFile.name - if (!resolvedContentType) - resolvedContentType = - resolvedContentTypeFromStorage || userFile.type || 'application/octet-stream' - } else if (fileContent) { - buffer = Buffer.from(fileContent, 'base64') - if (!resolvedFilename) resolvedFilename = 'file' - if (!resolvedContentType) resolvedContentType = 'application/octet-stream' - } else { - return NextResponse.json( - { success: false, error: 'A file is required to upload an attachment' }, - { status: 400 } - ) - } - - const sizeBytes = buffer.length - if (sizeBytes === 0) { - return NextResponse.json({ success: false, error: 'File is empty' }, { status: 400 }) - } - if (sizeBytes > MAX_SIZE_BYTES) { - return fileTooLargeError(sizeBytes) - } - - logger.info(`[${requestId}] Registering Linq attachment`, { - filename: resolvedFilename, - contentType: resolvedContentType, - sizeBytes, - }) - - const registerResponse = await fetch(`${LINQ_API_BASE}/attachments`, { - method: 'POST', - headers: linqHeaders(apiKey), - body: JSON.stringify({ - filename: resolvedFilename, - content_type: resolvedContentType, - size_bytes: sizeBytes, - }), - }) - const registerData = await registerResponse.json().catch(() => null) - if (!registerResponse.ok) { - return NextResponse.json( - { success: false, error: extractLinqError(registerData, 'Failed to register attachment') }, - { status: registerResponse.status } - ) - } - - const uploadUrl: string | undefined = registerData?.upload_url - const attachmentId: string | undefined = registerData?.attachment_id - if (!uploadUrl || !attachmentId) { - return NextResponse.json( - { success: false, error: 'Linq did not return an upload URL or attachment ID' }, - { status: 502 } - ) - } - - const requiredHeaders: Record = registerData?.required_headers ?? { - 'Content-Type': resolvedContentType, - 'Content-Length': String(sizeBytes), - } - const uploadMethod: string = registerData?.http_method ?? 'PUT' - - logger.info(`[${requestId}] Uploading ${sizeBytes} bytes to presigned URL`) - const uploadResponse = await fetch(uploadUrl, { - method: uploadMethod, - headers: requiredHeaders, - body: new Uint8Array(buffer), - }) - if (!uploadResponse.ok) { - const uploadError = await uploadResponse.text().catch(() => '') - logger.error(`[${requestId}] Presigned upload failed: ${uploadResponse.status}`, uploadError) - return NextResponse.json( - { success: false, error: `Failed to upload file bytes to Linq (${uploadResponse.status})` }, - { status: 502 } - ) - } - - logger.info(`[${requestId}] Attachment uploaded`, { attachmentId }) - return NextResponse.json({ - success: true, - output: { - attachmentId, - downloadUrl: registerData?.download_url ?? null, - filename: resolvedFilename, - contentType: resolvedContentType, - sizeBytes, - status: 'complete', - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error uploading Linq attachment:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error occurred') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/mail/send/route.ts b/apps/sim/app/api/tools/mail/send/route.ts deleted file mode 100644 index 3af8e3c264a..00000000000 --- a/apps/sim/app/api/tools/mail/send/route.ts +++ /dev/null @@ -1,154 +0,0 @@ -import { createLogger } from '@sim/logger' -import { convert } from 'html-to-text' -import { type NextRequest, NextResponse } from 'next/server' -import { Resend } from 'resend' -import { mailSendContract } from '@/lib/api/contracts/tools/mail' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('MailSendAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized mail send attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - message: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated mail request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const parsed = await parseRequest( - mailSendContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - message: getValidationErrorMessage(error, 'Invalid request data'), - errors: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Sending email with user-provided Resend API key`, { - to: validatedData.to, - subject: validatedData.subject, - bodyLength: validatedData.body.length, - from: validatedData.fromAddress, - }) - - const resend = new Resend(validatedData.resendApiKey) - - const contentType = validatedData.contentType || 'text' - const emailBase = { - from: validatedData.fromAddress, - to: validatedData.to, - subject: validatedData.subject, - } - - let emailData: Parameters[0] - if (contentType === 'html') { - emailData = { - ...emailBase, - html: validatedData.body, - text: convert(validatedData.body, { wordwrap: false }), - } - } else { - emailData = { - ...emailBase, - text: validatedData.body, - } - } - - if (validatedData.cc) { - emailData.cc = validatedData.cc - } - - if (validatedData.bcc) { - emailData.bcc = validatedData.bcc - } - - if (validatedData.replyTo) { - emailData.replyTo = validatedData.replyTo - } - - if (validatedData.scheduledAt) { - emailData.scheduledAt = validatedData.scheduledAt - } - - if (validatedData.tags) { - const tagPairs = validatedData.tags.split(',').map((pair) => { - const trimmed = pair.trim() - const colonIndex = trimmed.indexOf(':') - if (colonIndex === -1) return null - const name = trimmed.substring(0, colonIndex).trim() - const value = trimmed.substring(colonIndex + 1).trim() - return { name, value: value || '' } - }) - emailData.tags = tagPairs.filter( - (tag): tag is { name: string; value: string } => tag !== null && !!tag.name - ) - } - - const { data, error } = await resend.emails.send(emailData) - - if (error) { - logger.error(`[${requestId}] Email sending failed:`, error) - return NextResponse.json( - { - success: false, - message: `Failed to send email: ${error.message || 'Unknown error'}`, - }, - { status: 500 } - ) - } - - const result = { - success: true, - message: 'Email sent successfully via Resend', - data, - } - - logger.info(`[${requestId}] Email send result`, { - success: result.success, - message: result.message, - }) - - return NextResponse.json(result) - } catch (error) { - logger.error(`[${requestId}] Error sending email via API:`, error) - - return NextResponse.json( - { - success: false, - message: 'Internal server error while sending email', - data: {}, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/microsoft-dataverse/upload-file/route.ts b/apps/sim/app/api/tools/microsoft-dataverse/upload-file/route.ts deleted file mode 100644 index 826c547df96..00000000000 --- a/apps/sim/app/api/tools/microsoft-dataverse/upload-file/route.ts +++ /dev/null @@ -1,185 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { dataverseUploadFileContract } from '@/lib/api/contracts/tools/microsoft' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { getDataverseBaseUrl } from '@/tools/microsoft_dataverse/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('DataverseUploadFileAPI') - -/** Dataverse Web API's absolute ceiling for a single-request (non-chunked) file column upload. */ -const DATAVERSE_SINGLE_REQUEST_UPLOAD_MAX_BYTES = 128 * 1024 * 1024 - -function uploadTooLargeError(observedBytes: number): NextResponse { - const sizeMB = (observedBytes / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `File size (${sizeMB}MB) exceeds Dataverse's 128MB limit for single-request file column uploads. Split the file and use chunked upload instead.`, - }, - { status: 400 } - ) -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Dataverse upload attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - logger.info( - `[${requestId}] Authenticated Dataverse upload request via ${authResult.authType}`, - { - userId: authResult.userId, - } - ) - - const parsed = await parseRequest(dataverseUploadFileContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Uploading file to Dataverse`, { - entitySetName: validatedData.entitySetName, - recordId: validatedData.recordId, - fileColumn: validatedData.fileColumn, - fileName: validatedData.fileName, - hasFile: !!validatedData.file, - hasFileContent: !!validatedData.fileContent, - }) - - let fileBuffer: Buffer - - if (validatedData.file) { - const rawFile = validatedData.file - logger.info(`[${requestId}] Processing UserFile upload: ${rawFile.name}`) - - let userFile - try { - userFile = processSingleFileToUserFile(rawFile, requestId, logger) - } catch (error) { - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Failed to process file'), - }, - { status: 400 } - ) - } - - const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) - if (denied) return denied - - try { - const servable = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: DATAVERSE_SINGLE_REQUEST_UPLOAD_MAX_BYTES, - }) - fileBuffer = servable.buffer - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - if (isPayloadSizeLimitError(error)) - return uploadTooLargeError(error.observedBytes ?? userFile.size) - logger.error(`[${requestId}] Failed to download file from storage:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Failed to download file') }, - { status: 500 } - ) - } - } else if (validatedData.fileContent) { - fileBuffer = Buffer.from(validatedData.fileContent, 'base64') - } else { - return NextResponse.json( - { success: false, error: 'Either file or fileContent must be provided' }, - { status: 400 } - ) - } - - if (fileBuffer.length > DATAVERSE_SINGLE_REQUEST_UPLOAD_MAX_BYTES) { - const sizeMB = (fileBuffer.length / (1024 * 1024)).toFixed(2) - logger.warn(`[${requestId}] File too large for single-request upload: ${sizeMB}MB`) - return uploadTooLargeError(fileBuffer.length) - } - - const baseUrl = getDataverseBaseUrl(validatedData.environmentUrl) - const uploadUrl = `${baseUrl}/api/data/v9.2/${validatedData.entitySetName.trim()}(${validatedData.recordId.trim()})/${validatedData.fileColumn.trim()}` - - const response = await secureFetchWithValidation( - uploadUrl, - { - method: 'PATCH', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': 'application/octet-stream', - 'OData-MaxVersion': '4.0', - 'OData-Version': '4.0', - 'x-ms-file-name': validatedData.fileName, - }, - body: fileBuffer, - /** - * The tool's own `stripAuthOnRedirect` only covers the hop to this - * route. Dataverse redirects file operations to signed storage hosts, - * so this outbound call has to drop the bearer token itself or the - * redirect target receives a reusable OAuth credential. - */ - stripAuthOnRedirect: true, - }, - 'environmentUrl' - ) - - if (!response.ok) { - const errorData = (await response.json().catch(() => ({}))) as { - error?: { message?: string } - } - const errorMessage = - errorData?.error?.message ?? - `Dataverse API error: ${response.status} ${response.statusText}` - logger.error(`[${requestId}] Dataverse upload file failed`, { - errorData, - status: response.status, - }) - return NextResponse.json({ success: false, error: errorMessage }, { status: response.status }) - } - - logger.info(`[${requestId}] File uploaded to Dataverse successfully`, { - entitySetName: validatedData.entitySetName, - recordId: validatedData.recordId, - fileColumn: validatedData.fileColumn, - }) - - return NextResponse.json({ - success: true, - output: { - recordId: validatedData.recordId, - fileColumn: validatedData.fileColumn, - fileName: validatedData.fileName, - success: true, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error uploading file to Dataverse:`, error) - - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Internal server error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/microsoft_teams/delete_chat_message/route.ts b/apps/sim/app/api/tools/microsoft_teams/delete_chat_message/route.ts deleted file mode 100644 index 113cd533318..00000000000 --- a/apps/sim/app/api/tools/microsoft_teams/delete_chat_message/route.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { teamsDeleteChatMessageContract } from '@/lib/api/contracts/tools/microsoft' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('TeamsDeleteChatMessageAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Teams chat delete attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info( - `[${requestId}] Authenticated Teams chat message delete request via ${authResult.authType}`, - { - userId: authResult.userId, - } - ) - - const parsed = await parseRequest(teamsDeleteChatMessageContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Deleting Teams chat message`, { - chatId: validatedData.chatId, - messageId: validatedData.messageId, - }) - - // First, get the current user's ID (required for chat message deletion endpoint) - const meUrl = 'https://graph.microsoft.com/v1.0/me' - const meResponse = await fetch(meUrl, { - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - }, - }) - - if (!meResponse.ok) { - const errorData = await meResponse.json().catch(() => ({})) - logger.error(`[${requestId}] Failed to get user ID:`, errorData) - return NextResponse.json( - { - success: false, - error: errorData.error?.message || 'Failed to get user information', - }, - { status: meResponse.status } - ) - } - - const userData = await meResponse.json() - const userId = userData.id - - logger.info(`[${requestId}] Retrieved user ID: ${userId}`) - - // Now perform the softDelete operation using the correct endpoint format - const deleteUrl = `https://graph.microsoft.com/v1.0/users/${encodeURIComponent(userId)}/chats/${encodeURIComponent(validatedData.chatId)}/messages/${encodeURIComponent(validatedData.messageId)}/softDelete` - - const deleteResponse = await fetch(deleteUrl, { - method: 'POST', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({}), // softDelete requires an empty JSON body - }) - - if (!deleteResponse.ok) { - const errorData = await deleteResponse.json().catch(() => ({})) - logger.error(`[${requestId}] Microsoft Teams API delete error:`, errorData) - return NextResponse.json( - { - success: false, - error: errorData.error?.message || 'Failed to delete Teams message', - }, - { status: deleteResponse.status } - ) - } - - logger.info(`[${requestId}] Teams message deleted successfully`) - - return NextResponse.json({ - success: true, - output: { - deleted: true, - messageId: validatedData.messageId, - metadata: { - messageId: validatedData.messageId, - chatId: validatedData.chatId, - }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error deleting Teams chat message:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/microsoft_teams/write_channel/route.ts b/apps/sim/app/api/tools/microsoft_teams/write_channel/route.ts deleted file mode 100644 index cdce8f5f38e..00000000000 --- a/apps/sim/app/api/tools/microsoft_teams/write_channel/route.ts +++ /dev/null @@ -1,182 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { teamsWriteChannelContract } from '@/lib/api/contracts/tools/microsoft' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { FileAccessDeniedError } from '@/app/api/files/authorization' -import { uploadFilesForTeamsMessage } from '@/tools/microsoft_teams/server-utils' -import type { GraphApiErrorResponse, GraphChatMessage } from '@/tools/microsoft_teams/types' -import { resolveMentionsForChannel, type TeamsMention } from '@/tools/microsoft_teams/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('TeamsWriteChannelAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Teams channel write attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - const userId = authResult.userId - logger.info( - `[${requestId}] Authenticated Teams channel write request via ${authResult.authType}`, - { - userId, - } - ) - - const parsed = await parseRequest(teamsWriteChannelContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Sending Teams channel message`, { - teamId: validatedData.teamId, - channelId: validatedData.channelId, - hasFiles: !!(validatedData.files && validatedData.files.length > 0), - fileCount: validatedData.files?.length || 0, - }) - - const { attachments, filesOutput } = await uploadFilesForTeamsMessage({ - rawFiles: validatedData.files || [], - accessToken: validatedData.accessToken, - requestId, - logger, - userId, - }) - - let messageContent = validatedData.content - let contentType: 'text' | 'html' = 'text' - const mentionEntities: TeamsMention[] = [] - - try { - const mentionResult = await resolveMentionsForChannel( - validatedData.content, - validatedData.teamId, - validatedData.channelId, - validatedData.accessToken - ) - - if (mentionResult.hasMentions) { - contentType = 'html' - messageContent = mentionResult.updatedContent - mentionEntities.push(...mentionResult.mentions) - logger.info(`[${requestId}] Resolved ${mentionResult.mentions.length} mention(s)`) - } - } catch (error) { - logger.warn(`[${requestId}] Failed to resolve mentions, continuing without them:`, error) - } - - if (attachments.length > 0) { - contentType = 'html' - const attachmentTags = attachments - .map((att) => ``) - .join(' ') - messageContent = `${messageContent}
${attachmentTags}` - } - - const messageBody: { - body: { - contentType: 'text' | 'html' - content: string - } - attachments?: any[] - mentions?: TeamsMention[] - } = { - body: { - contentType, - content: messageContent, - }, - } - - if (attachments.length > 0) { - messageBody.attachments = attachments - } - - if (mentionEntities.length > 0) { - messageBody.mentions = mentionEntities - } - - logger.info(`[${requestId}] Sending message to Teams channel: ${validatedData.channelId}`) - - const teamsUrl = `https://graph.microsoft.com/v1.0/teams/${encodeURIComponent(validatedData.teamId)}/channels/${encodeURIComponent(validatedData.channelId)}/messages` - - const teamsResponse = await secureFetchWithValidation( - teamsUrl, - { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${validatedData.accessToken}`, - }, - body: JSON.stringify(messageBody), - }, - 'teamsUrl' - ) - - if (!teamsResponse.ok) { - const errorData = (await teamsResponse.json().catch(() => ({}))) as GraphApiErrorResponse - logger.error(`[${requestId}] Microsoft Teams API error:`, errorData) - return NextResponse.json( - { - success: false, - error: errorData.error?.message || 'Failed to send Teams channel message', - }, - { status: teamsResponse.status } - ) - } - - const responseData = (await teamsResponse.json()) as GraphChatMessage - logger.info(`[${requestId}] Teams channel message sent successfully`, { - messageId: responseData.id, - attachmentCount: attachments.length, - }) - - return NextResponse.json({ - success: true, - output: { - updatedContent: true, - metadata: { - messageId: responseData.id, - teamId: responseData.channelIdentity?.teamId || validatedData.teamId, - channelId: responseData.channelIdentity?.channelId || validatedData.channelId, - content: responseData.body?.content || validatedData.content, - createdTime: responseData.createdDateTime || new Date().toISOString(), - url: responseData.webUrl || '', - attachmentCount: attachments.length, - }, - files: filesOutput, - }, - }) - } catch (error) { - if (error instanceof FileAccessDeniedError) { - return NextResponse.json({ success: false, error: 'File not found' }, { status: 404 }) - } - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - logger.error(`[${requestId}] Error sending Teams channel message:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/microsoft_teams/write_chat/route.ts b/apps/sim/app/api/tools/microsoft_teams/write_chat/route.ts deleted file mode 100644 index b4af6be45df..00000000000 --- a/apps/sim/app/api/tools/microsoft_teams/write_chat/route.ts +++ /dev/null @@ -1,179 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { teamsWriteChatContract } from '@/lib/api/contracts/tools/microsoft' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { FileAccessDeniedError } from '@/app/api/files/authorization' -import { uploadFilesForTeamsMessage } from '@/tools/microsoft_teams/server-utils' -import type { GraphApiErrorResponse, GraphChatMessage } from '@/tools/microsoft_teams/types' -import { resolveMentionsForChat, type TeamsMention } from '@/tools/microsoft_teams/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('TeamsWriteChatAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Teams chat write attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - const userId = authResult.userId - logger.info( - `[${requestId}] Authenticated Teams chat write request via ${authResult.authType}`, - { - userId, - } - ) - - const parsed = await parseRequest(teamsWriteChatContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Sending Teams chat message`, { - chatId: validatedData.chatId, - hasFiles: !!(validatedData.files && validatedData.files.length > 0), - fileCount: validatedData.files?.length || 0, - }) - - const { attachments, filesOutput } = await uploadFilesForTeamsMessage({ - rawFiles: validatedData.files || [], - accessToken: validatedData.accessToken, - requestId, - logger, - userId, - }) - - let messageContent = validatedData.content - let contentType: 'text' | 'html' = 'text' - const mentionEntities: TeamsMention[] = [] - - try { - const mentionResult = await resolveMentionsForChat( - validatedData.content, - validatedData.chatId, - validatedData.accessToken - ) - - if (mentionResult.hasMentions) { - contentType = 'html' - messageContent = mentionResult.updatedContent - mentionEntities.push(...mentionResult.mentions) - logger.info(`[${requestId}] Resolved ${mentionResult.mentions.length} mention(s)`) - } - } catch (error) { - logger.warn(`[${requestId}] Failed to resolve mentions, continuing without them:`, error) - } - - if (attachments.length > 0) { - contentType = 'html' - const attachmentTags = attachments - .map((att) => ``) - .join(' ') - messageContent = `${messageContent}
${attachmentTags}` - } - - const messageBody: { - body: { - contentType: 'text' | 'html' - content: string - } - attachments?: any[] - mentions?: TeamsMention[] - } = { - body: { - contentType, - content: messageContent, - }, - } - - if (attachments.length > 0) { - messageBody.attachments = attachments - } - - if (mentionEntities.length > 0) { - messageBody.mentions = mentionEntities - } - - logger.info(`[${requestId}] Sending message to Teams chat: ${validatedData.chatId}`) - - const teamsUrl = `https://graph.microsoft.com/v1.0/chats/${encodeURIComponent(validatedData.chatId)}/messages` - - const teamsResponse = await secureFetchWithValidation( - teamsUrl, - { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${validatedData.accessToken}`, - }, - body: JSON.stringify(messageBody), - }, - 'teamsUrl' - ) - - if (!teamsResponse.ok) { - const errorData = (await teamsResponse.json().catch(() => ({}))) as GraphApiErrorResponse - logger.error(`[${requestId}] Microsoft Teams API error:`, errorData) - return NextResponse.json( - { - success: false, - error: errorData.error?.message || 'Failed to send Teams message', - }, - { status: teamsResponse.status } - ) - } - - const responseData = (await teamsResponse.json()) as GraphChatMessage - logger.info(`[${requestId}] Teams message sent successfully`, { - messageId: responseData.id, - attachmentCount: attachments.length, - }) - - return NextResponse.json({ - success: true, - output: { - updatedContent: true, - metadata: { - messageId: responseData.id, - chatId: responseData.chatId || validatedData.chatId, - content: responseData.body?.content || validatedData.content, - createdTime: responseData.createdDateTime || new Date().toISOString(), - url: responseData.webUrl || '', - attachmentCount: attachments.length, - }, - files: filesOutput, - }, - }) - } catch (error) { - if (error instanceof FileAccessDeniedError) { - return NextResponse.json({ success: false, error: 'File not found' }, { status: 404 }) - } - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - logger.error(`[${requestId}] Error sending Teams chat message:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/microsoft_word/append/route.test.ts b/apps/sim/app/api/tools/microsoft_word/append/route.test.ts deleted file mode 100644 index 9030afae97f..00000000000 --- a/apps/sim/app/api/tools/microsoft_word/append/route.test.ts +++ /dev/null @@ -1,251 +0,0 @@ -/** - * @vitest-environment node - */ -import { - createMockRequest, - hybridAuthMockFns, - inputValidationMock, - inputValidationMockFns, -} from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) - -import { buildDocxFromContent } from '@/lib/microsoft-word/document.server' -import { POST } from '@/app/api/tools/microsoft_word/append/route' - -const { mockValidateUrlWithDNS, mockSecureFetchWithPinnedIP } = inputValidationMockFns - -const PINNED_IP = '93.184.216.34' - -const baseBody = { - accessToken: 'token-123', - documentId: 'doc-abc', - content: 'Appended paragraph', -} - -/** A Graph `driveItem` metadata response carrying a content tag. */ -function itemResponse(cTag: string) { - const body = { - id: 'doc-abc', - name: 'notes.docx', - cTag, - file: { mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' }, - } - return { - ok: true, - status: 200, - statusText: '', - headers: new Headers(), - body: null, - text: async () => JSON.stringify(body), - json: async () => body, - arrayBuffer: async () => new ArrayBuffer(0), - } -} - -/** A Graph content response carrying a real `.docx` package. */ -async function docxResponse() { - const buffer = await buildDocxFromContent('Existing paragraph') - return { - ok: true, - status: 200, - statusText: '', - headers: new Headers(), - body: null, - text: async () => '', - json: async () => ({}), - arrayBuffer: async () => - buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength), - } -} - -/** The `createUploadSession` response carrying the preauthenticated upload URL. */ -function uploadSessionResponse(uploadUrl = 'https://sn3302.up.1drv.com/up/session-abc') { - const body = { uploadUrl, expirationDateTime: '2026-01-01T00:00:00Z' } - return { - ok: true, - status: 200, - statusText: '', - headers: new Headers(), - body: null, - text: async () => JSON.stringify(body), - json: async () => body, - arrayBuffer: async () => new ArrayBuffer(0), - } -} - -function preconditionFailedResponse() { - return { - ok: false, - status: 412, - statusText: 'Precondition Failed', - headers: new Headers(), - body: null, - text: async () => '', - json: async () => ({}), - arrayBuffer: async () => new ArrayBuffer(0), - } -} - -beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - mockValidateUrlWithDNS.mockResolvedValue({ - isValid: true, - resolvedIP: PINNED_IP, - originalHostname: 'graph.microsoft.com', - }) -}) - -describe('POST /api/tools/microsoft_word/append', () => { - it('writes through a conditional upload session carrying the content tag', async () => { - mockSecureFetchWithPinnedIP - .mockResolvedValueOnce(itemResponse('tag-1')) - .mockResolvedValueOnce(await docxResponse()) - .mockResolvedValueOnce(uploadSessionResponse()) - .mockResolvedValueOnce(itemResponse('tag-2')) - - const response = await POST(createMockRequest('POST', baseBody)) - - expect(response.status).toBe(200) - const data = (await response.json()) as { - success: boolean - output: { updatedContent: boolean } - } - expect(data.success).toBe(true) - expect(data.output.updatedContent).toBe(true) - - // The precondition rides on the session creation, which Graph documents as - // returning 412 on a mismatch. - const sessionCall = mockSecureFetchWithPinnedIP.mock.calls.find((call) => - String(call[0]).endsWith('/createUploadSession') - ) - expect(sessionCall?.[2]).toMatchObject({ method: 'POST' }) - expect(sessionCall?.[2].headers).toMatchObject({ 'if-match': 'tag-1' }) - - // Bytes go to the preauthenticated URL, and must not carry the bearer token. - const uploadCall = mockSecureFetchWithPinnedIP.mock.calls.at(-1) - expect(uploadCall?.[0]).toBe('https://sn3302.up.1drv.com/up/session-abc') - expect(uploadCall?.[2]).toMatchObject({ method: 'PUT' }) - expect(uploadCall?.[2].headers.Authorization).toBeUndefined() - }) - - it('refuses to overwrite when the service rejects the precondition', async () => { - mockSecureFetchWithPinnedIP - .mockResolvedValueOnce(itemResponse('tag-1')) - .mockResolvedValueOnce(await docxResponse()) - .mockResolvedValueOnce(preconditionFailedResponse()) - - const response = await POST(createMockRequest('POST', baseBody)) - - expect(response.status).toBe(409) - const data = (await response.json()) as { success: boolean; error: string } - expect(data.success).toBe(false) - expect(data.error).toMatch(/no other change was overwritten/) - - // The session was refused, so no bytes were ever sent. - expect(mockSecureFetchWithPinnedIP.mock.calls.some((call) => call[2]?.method === 'PUT')).toBe( - false - ) - }) - - it('maps a malformed document ID to a client error, not a server error', async () => { - const response = await POST(createMockRequest('POST', { ...baseBody, documentId: 'bad/../id' })) - - expect(response.status).toBe(400) - const data = (await response.json()) as { success: boolean; error: string } - expect(data.success).toBe(false) - expect(mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() - }) - - it('refuses to write Word bytes over a drive item that is not a .docx', async () => { - const pdf = { - id: 'doc-abc', - name: 'invoice.pdf', - cTag: 'tag-1', - file: { mimeType: 'application/pdf' }, - } - mockSecureFetchWithPinnedIP.mockResolvedValueOnce({ - ok: true, - status: 200, - statusText: '', - headers: new Headers(), - body: null, - text: async () => JSON.stringify(pdf), - json: async () => pdf, - arrayBuffer: async () => new ArrayBuffer(0), - }) - - const response = await POST(createMockRequest('POST', baseBody)) - - expect(response.status).toBe(400) - const data = (await response.json()) as { error: string } - expect(data.error).toMatch(/not a Word document/) - expect(mockSecureFetchWithPinnedIP.mock.calls.some((call) => call[2]?.method === 'PUT')).toBe( - false - ) - }) - - it('reports a no-op without writing when the content adds no paragraph', async () => { - mockSecureFetchWithPinnedIP - .mockResolvedValueOnce(itemResponse('tag-1')) - .mockResolvedValueOnce(await docxResponse()) - - const response = await POST(createMockRequest('POST', { ...baseBody, content: ' \n \n' })) - - expect(response.status).toBe(200) - const data = (await response.json()) as { output: { updatedContent: boolean } } - expect(data.output.updatedContent).toBe(false) - expect(mockSecureFetchWithPinnedIP.mock.calls.some((call) => call[2]?.method === 'PUT')).toBe( - false - ) - }) - - it('refuses to write when Graph reports no version to compare against', async () => { - const untagged = { - id: 'doc-abc', - name: 'notes.docx', - file: { - mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - }, - } - mockSecureFetchWithPinnedIP.mockResolvedValueOnce({ - ok: true, - status: 200, - statusText: '', - headers: new Headers(), - body: null, - text: async () => JSON.stringify(untagged), - json: async () => untagged, - arrayBuffer: async () => new ArrayBuffer(0), - }) - - const response = await POST(createMockRequest('POST', baseBody)) - - expect(response.status).toBe(409) - const data = (await response.json()) as { error: string } - expect(data.error).toMatch(/did not report a version/) - expect(mockSecureFetchWithPinnedIP.mock.calls.some((call) => call[2]?.method === 'PUT')).toBe( - false - ) - }) - - it('surfaces a conflict raised when the bytes commit', async () => { - mockSecureFetchWithPinnedIP - .mockResolvedValueOnce(itemResponse('tag-1')) - .mockResolvedValueOnce(await docxResponse()) - .mockResolvedValueOnce(uploadSessionResponse()) - .mockResolvedValueOnce(preconditionFailedResponse()) - - const response = await POST(createMockRequest('POST', baseBody)) - - expect(response.status).toBe(409) - const data = (await response.json()) as { error: string } - expect(data.error).toMatch(/no other change was overwritten/) - }) -}) diff --git a/apps/sim/app/api/tools/microsoft_word/append/route.ts b/apps/sim/app/api/tools/microsoft_word/append/route.ts deleted file mode 100644 index cd5962e9aa2..00000000000 --- a/apps/sim/app/api/tools/microsoft_word/append/route.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { microsoftWordAppendContract } from '@/lib/api/contracts/tools/microsoft' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { appendParagraphsToDocx } from '@/lib/microsoft-word/document.server' -import { - downloadDocumentContent, - fetchDocumentItem, - replaceContentIfUnchanged, - requireContentTag, - toDocumentMetadata, -} from '@/lib/microsoft-word/graph.server' -import { microsoftWordErrorResponse } from '@/app/api/tools/microsoft_word/utils' -import { getDocumentBasePath } from '@/tools/microsoft_word/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('MicrosoftWordAppendAPI') - -/** - * Appends paragraphs to a Word document. Graph exposes no document-editing API, - * so the existing `.docx` is downloaded, its body extended in place, and the - * repacked file uploaded back — every other part of the package is preserved. - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Microsoft Word append attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(microsoftWordAppendContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, documentId, content, driveId } = parsed.data.body - - try { - const basePath = getDocumentBasePath(documentId, driveId ?? undefined) - const existingItem = await fetchDocumentItem(basePath, accessToken) - const contentTag = requireContentTag(existingItem) - - const existingBuffer = await downloadDocumentContent(basePath, accessToken) - const { buffer, paragraphsAppended } = await appendParagraphsToDocx(existingBuffer, content) - - if (paragraphsAppended === 0) { - logger.info(`[${requestId}] No paragraphs to append; document left untouched`, { documentId }) - return NextResponse.json({ - success: true, - output: { - updatedContent: false, - metadata: toDocumentMetadata(existingItem, documentId), - }, - }) - } - - const item = await replaceContentIfUnchanged(basePath, accessToken, buffer, contentTag) - - logger.info(`[${requestId}] Appended to Word document`, { - documentId, - paragraphsAppended, - size: item.size, - }) - - return NextResponse.json({ - success: true, - output: { updatedContent: true, metadata: toDocumentMetadata(item, documentId) }, - }) - } catch (error) { - return microsoftWordErrorResponse(error, requestId, logger, 'append') - } -}) diff --git a/apps/sim/app/api/tools/microsoft_word/create-from-template/route.test.ts b/apps/sim/app/api/tools/microsoft_word/create-from-template/route.test.ts deleted file mode 100644 index 8e4bf818b6d..00000000000 --- a/apps/sim/app/api/tools/microsoft_word/create-from-template/route.test.ts +++ /dev/null @@ -1,68 +0,0 @@ -/** - * @vitest-environment node - */ -import { - createMockRequest, - hybridAuthMockFns, - inputValidationMock, - inputValidationMockFns, -} from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) - -import { POST } from '@/app/api/tools/microsoft_word/create-from-template/route' - -const { mockValidateUrlWithDNS, mockSecureFetchWithPinnedIP } = inputValidationMockFns - -const baseBody = { - accessToken: 'token-123', - templateDocumentId: 'template-abc', - name: 'Acme Agreement', -} - -beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - mockValidateUrlWithDNS.mockResolvedValue({ - isValid: true, - resolvedIP: '93.184.216.34', - originalHostname: 'graph.microsoft.com', - }) -}) - -describe('POST /api/tools/microsoft_word/create-from-template', () => { - it('rejects an empty placeholder key rather than failing mid-rewrite', async () => { - // A blank placeholder would match at every position in the document. - const response = await POST( - createMockRequest('POST', { ...baseBody, replacements: { '': 'Acme Corp' } }) - ) - - expect(response.status).toBe(400) - const data = (await response.json()) as { error?: string } - expect(JSON.stringify(data)).toMatch(/placeholder/i) - expect(mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() - }) - - it('rejects an empty placeholder key inside the JSON-string form too', async () => { - const response = await POST( - createMockRequest('POST', { ...baseBody, replacements: '{" ": "Acme Corp"}' }) - ) - - expect(response.status).toBe(400) - expect(mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() - }) - - it('rejects a replacements value that is not an object mapping', async () => { - const response = await POST( - createMockRequest('POST', { ...baseBody, replacements: '["a","b"]' }) - ) - - expect(response.status).toBe(400) - expect(mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/app/api/tools/microsoft_word/create-from-template/route.ts b/apps/sim/app/api/tools/microsoft_word/create-from-template/route.ts deleted file mode 100644 index e98da95f17a..00000000000 --- a/apps/sim/app/api/tools/microsoft_word/create-from-template/route.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { microsoftWordCreateFromTemplateContract } from '@/lib/api/contracts/tools/microsoft' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - DOCX_MIME_TYPE, - parseReplacements, - replaceTextInDocx, -} from '@/lib/microsoft-word/document.server' -import { - downloadDocumentContent, - fetchDocumentItem, - toDocumentMetadata, - uploadDocumentContent, -} from '@/lib/microsoft-word/graph.server' -import { microsoftWordErrorResponse } from '@/app/api/tools/microsoft_word/utils' -import { - buildCreateUploadUrl, - ensureDocxExtension, - getDocumentBasePath, - getDriveBasePath, - getFolderBasePath, -} from '@/tools/microsoft_word/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('MicrosoftWordCreateFromTemplateAPI') - -/** - * Fills a Word template into a new document. The template is downloaded, its - * placeholders substituted in memory, and the result uploaded under a new name — - * so the template itself is never written to, and the copy keeps every style, - * header, footer, and image the template defined. - * - * Graph's `copy` action is deliberately not used: it is asynchronous and only - * answers with a monitor URL, which would force a poll before the new document - * could be filled. - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Microsoft Word template attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(microsoftWordCreateFromTemplateContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, templateDocumentId, name, replacements, matchCase, folderId, driveId } = - parsed.data.body - - try { - const templatePath = getDocumentBasePath(templateDocumentId, driveId ?? undefined) - await fetchDocumentItem(templatePath, accessToken) - - const templateBuffer = await downloadDocumentContent(templatePath, accessToken) - const pairs = parseReplacements(replacements) - - const filled = - pairs.length > 0 - ? await replaceTextInDocx(templateBuffer, pairs, matchCase ?? false) - : { buffer: templateBuffer, occurrencesChanged: 0 } - - const fileName = ensureDocxExtension(name) - const parentPath = folderId?.trim() - ? getFolderBasePath(folderId, driveId ?? undefined) - : `${getDriveBasePath(driveId ?? undefined)}/root` - const uploadUrl = buildCreateUploadUrl(parentPath, fileName) - - const item = await uploadDocumentContent(uploadUrl, accessToken, filled.buffer, DOCX_MIME_TYPE) - - logger.info(`[${requestId}] Created Word document from template`, { - templateDocumentId, - documentId: item.id, - occurrencesChanged: filled.occurrencesChanged, - }) - - return NextResponse.json({ - success: true, - output: { - occurrencesChanged: filled.occurrencesChanged, - metadata: toDocumentMetadata(item, item.id ?? ''), - }, - }) - } catch (error) { - return microsoftWordErrorResponse(error, requestId, logger, 'create-from-template') - } -}) diff --git a/apps/sim/app/api/tools/microsoft_word/create/route.test.ts b/apps/sim/app/api/tools/microsoft_word/create/route.test.ts deleted file mode 100644 index 87920678d95..00000000000 --- a/apps/sim/app/api/tools/microsoft_word/create/route.test.ts +++ /dev/null @@ -1,68 +0,0 @@ -/** - * @vitest-environment node - */ -import { - createMockRequest, - hybridAuthMockFns, - inputValidationMock, - inputValidationMockFns, -} from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) - -import { POST } from '@/app/api/tools/microsoft_word/create/route' - -const { mockValidateUrlWithDNS, mockSecureFetchWithPinnedIP } = inputValidationMockFns - -const baseBody = { - accessToken: 'token-123', - name: 'Q3 Report', - content: 'Hello', -} - -beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - mockValidateUrlWithDNS.mockResolvedValue({ - isValid: true, - resolvedIP: '93.184.216.34', - originalHostname: 'graph.microsoft.com', - }) -}) - -describe('POST /api/tools/microsoft_word/create', () => { - it('rejects a whitespace-only name as a client error, not a server error', async () => { - // The contract's min(1) accepts a space; the name is only known to be - // unusable once the extension helper trims it. - const response = await POST(createMockRequest('POST', { ...baseBody, name: ' ' })) - - expect(response.status).toBe(400) - const data = (await response.json()) as { success: boolean; error: string } - expect(data.success).toBe(false) - expect(data.error).toMatch(/Document name is required/) - expect(mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() - }) - - it('rejects a malformed folder ID as a client error', async () => { - const response = await POST( - createMockRequest('POST', { ...baseBody, folderId: 'not a valid id' }) - ) - - expect(response.status).toBe(400) - const data = (await response.json()) as { success: boolean } - expect(data.success).toBe(false) - expect(mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() - }) - - it('rejects a malformed drive ID as a client error', async () => { - const response = await POST(createMockRequest('POST', { ...baseBody, driveId: 'bad/../drive' })) - - expect(response.status).toBe(400) - expect(mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/app/api/tools/microsoft_word/create/route.ts b/apps/sim/app/api/tools/microsoft_word/create/route.ts deleted file mode 100644 index d80efd91894..00000000000 --- a/apps/sim/app/api/tools/microsoft_word/create/route.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { microsoftWordCreateContract } from '@/lib/api/contracts/tools/microsoft' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { buildDocxFromContent, DOCX_MIME_TYPE } from '@/lib/microsoft-word/document.server' -import { toDocumentMetadata, uploadDocumentContent } from '@/lib/microsoft-word/graph.server' -import { microsoftWordErrorResponse } from '@/app/api/tools/microsoft_word/utils' -import { - buildCreateUploadUrl, - ensureDocxExtension, - getDriveBasePath, - getFolderBasePath, -} from '@/tools/microsoft_word/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('MicrosoftWordCreateAPI') - -/** - * Creates a Word document. Sim generates the `.docx` package itself — Microsoft - * Graph has no document-authoring API — and uploads it with the documented - * path-addressed content upload. - * - * @see https://learn.microsoft.com/en-us/graph/api/driveitem-put-content - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Microsoft Word create attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(microsoftWordCreateContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, name, content, folderId, driveId } = parsed.data.body - - try { - const fileName = ensureDocxExtension(name) - const parentPath = folderId?.trim() - ? getFolderBasePath(folderId, driveId ?? undefined) - : `${getDriveBasePath(driveId ?? undefined)}/root` - const uploadUrl = buildCreateUploadUrl(parentPath, fileName) - - const documentBuffer = await buildDocxFromContent(content ?? '', name) - const item = await uploadDocumentContent(uploadUrl, accessToken, documentBuffer, DOCX_MIME_TYPE) - - logger.info(`[${requestId}] Created Word document`, { documentId: item.id, size: item.size }) - - return NextResponse.json({ - success: true, - output: { metadata: toDocumentMetadata(item, item.id ?? '') }, - }) - } catch (error) { - return microsoftWordErrorResponse(error, requestId, logger, 'create') - } -}) diff --git a/apps/sim/app/api/tools/microsoft_word/export-pdf/route.ts b/apps/sim/app/api/tools/microsoft_word/export-pdf/route.ts deleted file mode 100644 index 2d9bac0630d..00000000000 --- a/apps/sim/app/api/tools/microsoft_word/export-pdf/route.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { microsoftWordExportPdfContract } from '@/lib/api/contracts/tools/microsoft' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { downloadConvertedContent, fetchDocumentItem } from '@/lib/microsoft-word/graph.server' -import { microsoftWordErrorResponse } from '@/app/api/tools/microsoft_word/utils' -import { getDocumentBasePath } from '@/tools/microsoft_word/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('MicrosoftWordExportPdfAPI') - -const PDF_MIME_TYPE = 'application/pdf' - -/** Derives the PDF file name from an explicit override or the source document name. */ -function resolvePdfName(override: string | null | undefined, documentName?: string): string { - const explicit = override?.trim() - if (explicit) { - return explicit.toLowerCase().endsWith('.pdf') ? explicit : `${explicit}.pdf` - } - - const base = documentName?.trim().replace(/\.docx$/i, '') - return base ? `${base}.pdf` : 'document.pdf' -} - -/** - * Exports a Word document as PDF. Graph performs the conversion and answers with - * a redirect to a short-lived preauthenticated download URL, which the shared - * secure fetch follows without forwarding the bearer token. - * - * @see https://learn.microsoft.com/en-us/graph/api/driveitem-get-content-format - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Microsoft Word export attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(microsoftWordExportPdfContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, documentId, fileName, driveId } = parsed.data.body - - try { - const basePath = getDocumentBasePath(documentId, driveId ?? undefined) - const item = await fetchDocumentItem(basePath, accessToken) - const pdfBuffer = await downloadConvertedContent(basePath, accessToken, 'pdf') - - const name = resolvePdfName(fileName, item.name) - - logger.info(`[${requestId}] Exported Word document as PDF`, { - documentId, - name, - size: pdfBuffer.length, - }) - - return NextResponse.json({ - success: true, - output: { - file: { - name, - mimeType: PDF_MIME_TYPE, - data: pdfBuffer.toString('base64'), - size: pdfBuffer.length, - }, - }, - }) - } catch (error) { - return microsoftWordErrorResponse(error, requestId, logger, 'export-pdf') - } -}) diff --git a/apps/sim/app/api/tools/microsoft_word/read/route.ts b/apps/sim/app/api/tools/microsoft_word/read/route.ts deleted file mode 100644 index 177899ef7ec..00000000000 --- a/apps/sim/app/api/tools/microsoft_word/read/route.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { microsoftWordReadContract } from '@/lib/api/contracts/tools/microsoft' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { extractDocxText } from '@/lib/microsoft-word/document.server' -import { - downloadDocumentContent, - fetchDocumentItem, - toDocumentMetadata, -} from '@/lib/microsoft-word/graph.server' -import { microsoftWordErrorResponse } from '@/app/api/tools/microsoft_word/utils' -import { getDocumentBasePath } from '@/tools/microsoft_word/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('MicrosoftWordReadAPI') - -/** - * Reads a Word document's text. Graph serves the raw `.docx` package, so the - * bytes are extracted with Sim's shared DOCX parser rather than by the API. - * - * @see https://learn.microsoft.com/en-us/graph/api/driveitem-get-content - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Microsoft Word read attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(microsoftWordReadContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, documentId, driveId } = parsed.data.body - - try { - const basePath = getDocumentBasePath(documentId, driveId ?? undefined) - const item = await fetchDocumentItem(basePath, accessToken) - const documentBuffer = await downloadDocumentContent(basePath, accessToken) - const content = await extractDocxText(documentBuffer) - - logger.info(`[${requestId}] Read Word document`, { - documentId, - characterCount: content.length, - }) - - return NextResponse.json({ - success: true, - output: { content, metadata: toDocumentMetadata(item, documentId) }, - }) - } catch (error) { - return microsoftWordErrorResponse(error, requestId, logger, 'read') - } -}) diff --git a/apps/sim/app/api/tools/microsoft_word/replace-text/route.ts b/apps/sim/app/api/tools/microsoft_word/replace-text/route.ts deleted file mode 100644 index ede32421b75..00000000000 --- a/apps/sim/app/api/tools/microsoft_word/replace-text/route.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { microsoftWordReplaceTextContract } from '@/lib/api/contracts/tools/microsoft' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { replaceTextInDocx } from '@/lib/microsoft-word/document.server' -import { - downloadDocumentContent, - fetchDocumentItem, - replaceContentIfUnchanged, - requireContentTag, - toDocumentMetadata, -} from '@/lib/microsoft-word/graph.server' -import { microsoftWordErrorResponse } from '@/app/api/tools/microsoft_word/utils' -import { getDocumentBasePath } from '@/tools/microsoft_word/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('MicrosoftWordReplaceTextAPI') - -/** - * Finds and replaces text in a Word document. Graph exposes no document-editing - * API, so the `.docx` is downloaded, its text parts rewritten in place, and the - * repacked file uploaded back. Nothing is uploaded when no occurrence matched, - * which keeps a no-op run from bumping the document's modified time. - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Microsoft Word replace attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(microsoftWordReplaceTextContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, documentId, findText, replaceText, matchCase, driveId } = parsed.data.body - - try { - const basePath = getDocumentBasePath(documentId, driveId ?? undefined) - const existingItem = await fetchDocumentItem(basePath, accessToken) - const contentTag = requireContentTag(existingItem) - - const existingBuffer = await downloadDocumentContent(basePath, accessToken) - const { buffer, occurrencesChanged } = await replaceTextInDocx( - existingBuffer, - [{ find: findText, replace: replaceText ?? '' }], - matchCase ?? false - ) - - if (occurrencesChanged === 0) { - logger.info(`[${requestId}] No occurrences matched; document left untouched`, { documentId }) - return NextResponse.json({ - success: true, - output: { - occurrencesChanged: 0, - metadata: toDocumentMetadata(existingItem, documentId), - }, - }) - } - - const item = await replaceContentIfUnchanged(basePath, accessToken, buffer, contentTag) - - logger.info(`[${requestId}] Replaced text in Word document`, { - documentId, - occurrencesChanged, - }) - - return NextResponse.json({ - success: true, - output: { occurrencesChanged, metadata: toDocumentMetadata(item, documentId) }, - }) - } catch (error) { - return microsoftWordErrorResponse(error, requestId, logger, 'replace-text') - } -}) diff --git a/apps/sim/app/api/tools/microsoft_word/update/route.ts b/apps/sim/app/api/tools/microsoft_word/update/route.ts deleted file mode 100644 index e5067606156..00000000000 --- a/apps/sim/app/api/tools/microsoft_word/update/route.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { microsoftWordUpdateContract } from '@/lib/api/contracts/tools/microsoft' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { buildDocxFromContent, DOCX_MIME_TYPE } from '@/lib/microsoft-word/document.server' -import { - fetchDocumentItem, - toDocumentMetadata, - uploadDocumentContent, -} from '@/lib/microsoft-word/graph.server' -import { microsoftWordErrorResponse } from '@/app/api/tools/microsoft_word/utils' -import { getDocumentBasePath } from '@/tools/microsoft_word/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('MicrosoftWordUpdateAPI') - -/** - * Replaces a Word document's contents with a freshly generated `.docx` package. - * The metadata read first confirms the target is a file rather than a folder, so - * the destructive upload cannot land on the wrong kind of drive item. - * - * @see https://learn.microsoft.com/en-us/graph/api/driveitem-put-content - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Microsoft Word update attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(microsoftWordUpdateContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, documentId, content, driveId } = parsed.data.body - - try { - const basePath = getDocumentBasePath(documentId, driveId ?? undefined) - const existing = await fetchDocumentItem(basePath, accessToken) - - const documentBuffer = await buildDocxFromContent(content, existing.name) - const item = await uploadDocumentContent( - `${basePath}/content`, - accessToken, - documentBuffer, - DOCX_MIME_TYPE - ) - - logger.info(`[${requestId}] Replaced Word document contents`, { - documentId, - size: item.size, - }) - - return NextResponse.json({ - success: true, - output: { updatedContent: true, metadata: toDocumentMetadata(item, documentId) }, - }) - } catch (error) { - return microsoftWordErrorResponse(error, requestId, logger, 'update') - } -}) diff --git a/apps/sim/app/api/tools/microsoft_word/utils.ts b/apps/sim/app/api/tools/microsoft_word/utils.ts deleted file mode 100644 index a2e9cbae402..00000000000 --- a/apps/sim/app/api/tools/microsoft_word/utils.ts +++ /dev/null @@ -1,31 +0,0 @@ -import type { Logger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { NextResponse } from 'next/server' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { GraphRequestError } from '@/lib/microsoft-word/graph.server' -import { MicrosoftWordInputError } from '@/tools/microsoft_word/errors' - -/** - * Projects an error raised while talking to Microsoft Graph — or while building - * the document package — onto the `{ success, error }` envelope the Word tools - * expect, preserving Graph's own status so a 404 or 403 is not reported as a - * Sim-side failure. - */ -export function microsoftWordErrorResponse( - error: unknown, - requestId: string, - logger: Logger, - operation: string -): NextResponse { - const message = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Microsoft Word ${operation} failed`, { error: message }) - - const status = - error instanceof GraphRequestError || error instanceof MicrosoftWordInputError - ? error.status - : isPayloadSizeLimitError(error) - ? 413 - : 500 - - return NextResponse.json({ success: false, error: message }, { status }) -} diff --git a/apps/sim/app/api/tools/mistral/parse/route.test.ts b/apps/sim/app/api/tools/mistral/parse/route.test.ts deleted file mode 100644 index 5dc56bae1b4..00000000000 --- a/apps/sim/app/api/tools/mistral/parse/route.test.ts +++ /dev/null @@ -1,136 +0,0 @@ -/** - * @vitest-environment node - */ -import { - createMockRequest, - hybridAuthMockFns, - inputValidationMock, - inputValidationMockFns, -} from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' -import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { PRIVATE_MODEL_INPUT_PROVENANCE_HEADER } from '@/lib/execution/model-input-provenance' -import { - RESOLVED_SECRET_PROVENANCE_FIELD, - RESOLVED_SECRET_PROVENANCE_METADATA_V1, -} from '@/lib/execution/private-tool-metadata' -import { MISTRAL_OCR_REQUEST_POLICY } from '@/lib/knowledge/documents/ocr-request-policy' - -const { mockDownloadServableFile, mockIsModelSafeWorkspaceFileKey } = vi.hoisted(() => ({ - mockDownloadServableFile: vi.fn(), - mockIsModelSafeWorkspaceFileKey: vi.fn(), -})) - -vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) -vi.mock('@/app/api/files/authorization', () => ({ - assertToolFileAccess: vi.fn().mockResolvedValue(null), -})) -vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ - downloadServableFileFromStorage: mockDownloadServableFile, - resolveInternalFileUrl: vi.fn(), -})) -vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({ - isModelSafeWorkspaceFileKey: mockIsModelSafeWorkspaceFileKey, - MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE: - 'File cannot be sent to a model because its secret provenance is unavailable', -})) - -import { POST } from '@/app/api/tools/mistral/parse/route' - -const PDF_FILE = { - key: 'workspace/workspace-1/document.pdf', - name: 'document.pdf', - size: 3, - type: 'application/pdf', -} - -function createVerifiedRequest() { - return createMockRequest( - 'POST', - { - apiKey: 'mistral-key', - file: PDF_FILE, - [RESOLVED_SECRET_PROVENANCE_FIELD]: { - version: 1, - complete: true, - entries: [], - }, - }, - { [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1 } - ) -} - -describe('POST /api/tools/mistral/parse', () => { - beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ - isValid: true, - resolvedIP: '93.184.216.34', - originalHostname: 'api.mistral.ai', - }) - mockIsModelSafeWorkspaceFileKey.mockResolvedValue(true) - mockDownloadServableFile.mockResolvedValue({ - buffer: Buffer.from('pdf'), - contentType: 'application/pdf', - }) - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValue( - Response.json({ pages: [], usage_info: { pages_processed: 0 } }) - ) - }) - - it('returns 413 when the input file exceeds Mistral request limits', async () => { - mockDownloadServableFile.mockRejectedValueOnce( - new PayloadSizeLimitError({ - label: 'storage file download', - maxBytes: MISTRAL_OCR_REQUEST_POLICY.maxBytes, - observedBytes: MISTRAL_OCR_REQUEST_POLICY.maxBytes + 1, - }) - ) - - const response = await POST(createVerifiedRequest()) - - expect(response.status).toBe(413) - await expect(response.json()).resolves.toEqual({ - success: false, - error: `File exceeds Mistral OCR's ${MISTRAL_OCR_REQUEST_POLICY.maxBytes.toLocaleString()}-byte request limit`, - }) - expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() - }) - - it('returns 502 when Mistral response bytes exceed the secure-fetch cap', async () => { - const responseLimitError = new PayloadSizeLimitError({ - label: 'response body', - maxBytes: 100, - observedBytes: 101, - }) - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce({ - ok: true, - status: 200, - statusText: 'OK', - headers: new Headers(), - body: null, - text: async () => { - throw responseLimitError - }, - json: async () => { - throw responseLimitError - }, - arrayBuffer: async () => { - throw responseLimitError - }, - }) - - const response = await POST(createVerifiedRequest()) - - expect(response.status).toBe(502) - await expect(response.json()).resolves.toEqual({ - success: false, - error: 'Mistral API response exceeded the safe size limit', - }) - }) -}) diff --git a/apps/sim/app/api/tools/mistral/parse/route.ts b/apps/sim/app/api/tools/mistral/parse/route.ts deleted file mode 100644 index 436384ffbc9..00000000000 --- a/apps/sim/app/api/tools/mistral/parse/route.ts +++ /dev/null @@ -1,387 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { mistralParseContract } from '@/lib/api/contracts/tools/media/document-parse' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' -import { decodeDataUriWithinLimit } from '@/lib/file-parsers/data-uri' -import { isFileParserError } from '@/lib/file-parsers/errors' -import { MISTRAL_OCR_REQUEST_POLICY } from '@/lib/knowledge/documents/ocr-request-policy' -import { readBoundedHttpErrorBody } from '@/lib/knowledge/documents/utils' -import { - isModelSafeWorkspaceFileKey, - MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, -} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' -import { - extractStorageKey, - isInternalFileUrl, - processSingleFileToUserFile, -} from '@/lib/uploads/utils/file-utils' -import { - downloadServableFileFromStorage, - resolveInternalFileUrl, - type ServableFile, -} from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('MistralParseAPI') - -function fileSizeLimitResponse() { - return NextResponse.json( - { - success: false, - error: `File exceeds Mistral OCR's ${MISTRAL_OCR_REQUEST_POLICY.maxBytes.toLocaleString()}-byte request limit`, - }, - { status: 413 } - ) -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Mistral parse attempt`, { - error: authResult.error || 'Missing userId', - }) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Unauthorized', - }, - { status: 401 } - ) - } - - const userId = authResult.userId - - const parsed = await parseRequest( - mistralParseContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - - const validatedData = parsed.data.body - const modelInputProvenance = validateOpaqueModelInputProvenance({ - headers: request.headers, - payload: validatedData, - isInternalRequest: true, - }) - if (!modelInputProvenance.success) { - return NextResponse.json( - { success: false, error: modelInputProvenance.error }, - { status: modelInputProvenance.status } - ) - } - - const fileData = validatedData.file || validatedData.fileData - const filePath = typeof fileData === 'string' ? fileData : validatedData.filePath - - if (!fileData && (!filePath || filePath.trim() === '')) { - return NextResponse.json( - { - success: false, - error: 'File input is required', - }, - { status: 400 } - ) - } - - logger.info(`[${requestId}] Mistral parse request`, { - hasFileData: Boolean(fileData), - filePath, - isWorkspaceFile: filePath ? isInternalFileUrl(filePath) : false, - userId, - }) - - const mistralBody: any = { - model: 'mistral-ocr-latest', - } - - if (fileData && typeof fileData === 'object') { - const rawFile = fileData - let userFile - try { - userFile = processSingleFileToUserFile(rawFile, requestId, logger) - } catch (error) { - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Failed to process file'), - }, - { status: 400 } - ) - } - - let mimeType = userFile.type - if (!mimeType || mimeType === 'application/octet-stream') { - const filename = userFile.name?.toLowerCase() || '' - if (filename.endsWith('.pdf')) { - mimeType = 'application/pdf' - } else if (filename.endsWith('.png')) { - mimeType = 'image/png' - } else if (filename.endsWith('.jpg') || filename.endsWith('.jpeg')) { - mimeType = 'image/jpeg' - } else if (filename.endsWith('.gif')) { - mimeType = 'image/gif' - } else if (filename.endsWith('.webp')) { - mimeType = 'image/webp' - } else { - mimeType = 'application/pdf' - } - } - let base64 = userFile.base64 - if (!base64) { - const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) - if (denied) return denied - if (!(await isModelSafeWorkspaceFileKey(userFile.key))) { - return NextResponse.json( - { - success: false, - error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, - }, - { status: 400 } - ) - } - let servableFile: ServableFile - try { - servableFile = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MISTRAL_OCR_REQUEST_POLICY.maxBytes, - }) - } catch (error) { - if (!isPayloadSizeLimitError(error)) throw error - return fileSizeLimitResponse() - } - const { buffer, contentType } = servableFile - base64 = buffer.toString('base64') - if (contentType && contentType !== 'application/octet-stream') { - mimeType = contentType - } - } - - let inlineBytes: number - try { - inlineBytes = base64.startsWith('data:') - ? decodeDataUriWithinLimit(base64, MISTRAL_OCR_REQUEST_POLICY.maxBytes).buffer.length - : Buffer.byteLength(base64, 'base64') - } catch (error) { - const status = isFileParserError(error) && error.code === 'complexity_limit' ? 413 : 400 - return status === 413 - ? fileSizeLimitResponse() - : NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Invalid inline file data'), - }, - { status } - ) - } - if (inlineBytes > MISTRAL_OCR_REQUEST_POLICY.maxBytes) { - return fileSizeLimitResponse() - } - - const base64Payload = base64.startsWith('data:') - ? base64 - : `data:${mimeType};base64,${base64}` - - // Mistral API uses different document types for images vs documents - const isImage = mimeType.startsWith('image/') - if (isImage) { - mistralBody.document = { - type: 'image_url', - image_url: base64Payload, - } - } else { - mistralBody.document = { - type: 'document_url', - document_url: base64Payload, - } - } - } else if (filePath) { - let fileUrl = filePath - - const isInternalFilePath = isInternalFileUrl(filePath) - if (isInternalFilePath) { - const resolution = await resolveInternalFileUrl(filePath, userId, requestId, logger) - if (resolution.error) { - return NextResponse.json( - { - success: false, - error: resolution.error.message, - }, - { status: resolution.error.status } - ) - } - fileUrl = resolution.fileUrl || fileUrl - if (!(await isModelSafeWorkspaceFileKey(extractStorageKey(filePath)))) { - return NextResponse.json( - { - success: false, - error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, - }, - { status: 400 } - ) - } - } else if (filePath.startsWith('/')) { - logger.warn(`[${requestId}] Invalid internal path`, { - userId, - path: filePath.substring(0, 50), - }) - return NextResponse.json( - { - success: false, - error: 'Invalid file path. Only uploaded files are supported for internal paths.', - }, - { status: 400 } - ) - } else { - const urlValidation = await validateUrlWithDNS(fileUrl, 'filePath') - if (!urlValidation.isValid) { - return NextResponse.json( - { - success: false, - error: urlValidation.error, - }, - { status: 400 } - ) - } - } - - const imageExtensions = ['.png', '.jpg', '.jpeg', '.gif', '.webp', '.avif'] - const pathname = new URL(fileUrl).pathname.toLowerCase() - const isImageUrl = imageExtensions.some((ext) => pathname.endsWith(ext)) - - if (isImageUrl) { - mistralBody.document = { - type: 'image_url', - image_url: fileUrl, - } - } else { - mistralBody.document = { - type: 'document_url', - document_url: fileUrl, - } - } - } - - if (validatedData.pages) { - mistralBody.pages = validatedData.pages - } - if (validatedData.includeImageBase64 !== undefined) { - mistralBody.include_image_base64 = validatedData.includeImageBase64 - } - if (validatedData.imageLimit) { - mistralBody.image_limit = validatedData.imageLimit - } - if (validatedData.imageMinSize) { - mistralBody.image_min_size = validatedData.imageMinSize - } - - const mistralEndpoint = 'https://api.mistral.ai/v1/ocr' - const mistralValidation = await validateUrlWithDNS(mistralEndpoint, 'Mistral API URL') - if (!mistralValidation.isValid) { - logger.error(`[${requestId}] Mistral API URL validation failed`, { - error: mistralValidation.error, - }) - return NextResponse.json( - { - success: false, - error: 'Failed to reach Mistral API', - }, - { status: 502 } - ) - } - - const mistralResponse = await secureFetchWithPinnedIP( - mistralEndpoint, - mistralValidation.resolvedIP!, - { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json', - Authorization: `Bearer ${validatedData.apiKey}`, - }, - body: JSON.stringify(mistralBody), - } - ) - - if (!mistralResponse.ok) { - const errorText = await readBoundedHttpErrorBody(mistralResponse) - logger.error(`[${requestId}] Mistral API error`, { - status: mistralResponse.status, - diagnostic: errorText, - }) - return NextResponse.json( - { - success: false, - error: `Mistral API error: ${mistralResponse.statusText}`, - }, - { status: mistralResponse.status } - ) - } - - const mistralData = await mistralResponse.json() - - logger.info(`[${requestId}] Mistral parse successful`) - - return NextResponse.json({ - success: true, - output: mistralData, - }) - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - - if (isPayloadSizeLimitError(error)) { - logger.error(`[${requestId}] Mistral API response exceeded the safe size limit`, { - maxBytes: error.maxBytes, - observedBytes: error.observedBytes, - }) - return NextResponse.json( - { - success: false, - error: 'Mistral API response exceeded the safe size limit', - }, - { status: 502 } - ) - } - - logger.error(`[${requestId}] Error in Mistral parse:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/mongodb/delete/route.ts b/apps/sim/app/api/tools/mongodb/delete/route.ts deleted file mode 100644 index 423f5f5461f..00000000000 --- a/apps/sim/app/api/tools/mongodb/delete/route.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { mongodbDeleteContract } from '@/lib/api/contracts/tools/databases/mongodb' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createMongoDBConnection, - sanitizeCollectionName, - validateFilter, -} from '@/app/api/tools/mongodb/utils' - -const logger = createLogger('MongoDBDeleteAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - let client = null - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized MongoDB delete attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(mongodbDeleteContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Deleting document(s) from ${params.host}:${params.port}/${params.database}.${params.collection} (multi: ${params.multi})` - ) - - const sanitizedCollection = sanitizeCollectionName(params.collection) - - const filterValidation = validateFilter(params.filter) - if (!filterValidation.isValid) { - logger.warn(`[${requestId}] Filter validation failed: ${filterValidation.error}`) - return NextResponse.json( - { error: `Filter validation failed: ${filterValidation.error}` }, - { status: 400 } - ) - } - - let filterDoc - try { - filterDoc = JSON.parse(params.filter) - } catch (error) { - logger.warn(`[${requestId}] Invalid filter JSON: ${params.filter}`) - return NextResponse.json({ error: 'Invalid JSON format in filter' }, { status: 400 }) - } - - client = await createMongoDBConnection({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - authSource: params.authSource, - ssl: params.ssl, - }) - - const db = client.db(params.database) - const coll = db.collection(sanitizedCollection) - - let result - if (params.multi) { - result = await coll.deleteMany(filterDoc) - } else { - result = await coll.deleteOne(filterDoc) - } - - logger.info(`[${requestId}] Delete completed: ${result.deletedCount} documents deleted`) - - return NextResponse.json({ - message: `${result.deletedCount} documents deleted`, - deletedCount: result.deletedCount, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] MongoDB delete failed:`, error) - - return NextResponse.json({ error: `MongoDB delete failed: ${errorMessage}` }, { status: 500 }) - } finally { - if (client) { - await client.close() - } - } -}) diff --git a/apps/sim/app/api/tools/mongodb/execute/route.ts b/apps/sim/app/api/tools/mongodb/execute/route.ts deleted file mode 100644 index bb8f87abdde..00000000000 --- a/apps/sim/app/api/tools/mongodb/execute/route.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { mongodbExecuteContract } from '@/lib/api/contracts/tools/databases/mongodb' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createMongoDBConnection, - sanitizeCollectionName, - validatePipeline, -} from '@/app/api/tools/mongodb/utils' - -const logger = createLogger('MongoDBExecuteAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - let client = null - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized MongoDB execute attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(mongodbExecuteContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Executing aggregation pipeline on ${params.host}:${params.port}/${params.database}.${params.collection}` - ) - - const sanitizedCollection = sanitizeCollectionName(params.collection) - - const pipelineValidation = validatePipeline(params.pipeline) - if (!pipelineValidation.isValid) { - logger.warn(`[${requestId}] Pipeline validation failed: ${pipelineValidation.error}`) - return NextResponse.json( - { error: `Pipeline validation failed: ${pipelineValidation.error}` }, - { status: 400 } - ) - } - - const pipelineDoc = JSON.parse(params.pipeline) - - client = await createMongoDBConnection({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - authSource: params.authSource, - ssl: params.ssl, - }) - - const db = client.db(params.database) - const coll = db.collection(sanitizedCollection) - - const cursor = coll.aggregate(pipelineDoc) - const documents = await cursor.toArray() - - logger.info( - `[${requestId}] Aggregation completed successfully, returned ${documents.length} documents` - ) - - return NextResponse.json({ - message: `Aggregation completed, returned ${documents.length} documents`, - documents, - documentCount: documents.length, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] MongoDB aggregation failed:`, error) - - return NextResponse.json( - { error: `MongoDB aggregation failed: ${errorMessage}` }, - { status: 500 } - ) - } finally { - if (client) { - await client.close() - } - } -}) diff --git a/apps/sim/app/api/tools/mongodb/insert/route.ts b/apps/sim/app/api/tools/mongodb/insert/route.ts deleted file mode 100644 index a5cc1c3b21a..00000000000 --- a/apps/sim/app/api/tools/mongodb/insert/route.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { mongodbInsertContract } from '@/lib/api/contracts/tools/databases/mongodb' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createMongoDBConnection, sanitizeCollectionName } from '@/app/api/tools/mongodb/utils' - -const logger = createLogger('MongoDBInsertAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - let client = null - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized MongoDB insert attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(mongodbInsertContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Inserting ${params.documents.length} document(s) into ${params.host}:${params.port}/${params.database}.${params.collection}` - ) - - const sanitizedCollection = sanitizeCollectionName(params.collection) - client = await createMongoDBConnection({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - authSource: params.authSource, - ssl: params.ssl, - }) - - const db = client.db(params.database) - const coll = db.collection(sanitizedCollection) - - let result - if (params.documents.length === 1) { - result = await coll.insertOne(params.documents[0] as Record) - logger.info(`[${requestId}] Single document inserted successfully`) - return NextResponse.json({ - message: 'Document inserted successfully', - insertedId: result.insertedId.toString(), - documentCount: 1, - }) - } - result = await coll.insertMany(params.documents as Record[]) - const insertedCount = Object.keys(result.insertedIds).length - logger.info(`[${requestId}] ${insertedCount} documents inserted successfully`) - return NextResponse.json({ - message: `${insertedCount} documents inserted successfully`, - insertedIds: Object.values(result.insertedIds).map((id) => id.toString()), - documentCount: insertedCount, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] MongoDB insert failed:`, error) - - return NextResponse.json({ error: `MongoDB insert failed: ${errorMessage}` }, { status: 500 }) - } finally { - if (client) { - await client.close() - } - } -}) diff --git a/apps/sim/app/api/tools/mongodb/introspect/route.ts b/apps/sim/app/api/tools/mongodb/introspect/route.ts deleted file mode 100644 index 61abc03375f..00000000000 --- a/apps/sim/app/api/tools/mongodb/introspect/route.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { mongodbIntrospectContract } from '@/lib/api/contracts/tools/databases/mongodb' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createMongoDBConnection, executeIntrospect } from '@/app/api/tools/mongodb/utils' - -const logger = createLogger('MongoDBIntrospectAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - let client = null - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized MongoDB introspect attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(mongodbIntrospectContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Introspecting MongoDB at ${params.host}:${params.port}${params.database ? `/${params.database}` : ''}` - ) - - client = await createMongoDBConnection({ - host: params.host, - port: params.port, - database: params.database || 'admin', - username: params.username, - password: params.password, - authSource: params.authSource, - ssl: params.ssl, - }) - - const result = await executeIntrospect(client, params.database) - - logger.info( - `[${requestId}] Introspection completed: ${result.databases.length} databases, ${result.collections.length} collections` - ) - - return NextResponse.json({ - message: result.message, - databases: result.databases, - collections: result.collections, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] MongoDB introspect failed:`, error) - - return NextResponse.json( - { error: `MongoDB introspect failed: ${errorMessage}` }, - { status: 500 } - ) - } finally { - if (client) { - await client.close() - } - } -}) diff --git a/apps/sim/app/api/tools/mongodb/query/route.ts b/apps/sim/app/api/tools/mongodb/query/route.ts deleted file mode 100644 index 0b9cfd4ba45..00000000000 --- a/apps/sim/app/api/tools/mongodb/query/route.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { mongodbQueryContract } from '@/lib/api/contracts/tools/databases/mongodb' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createMongoDBConnection, - sanitizeCollectionName, - validateFilter, -} from '@/app/api/tools/mongodb/utils' - -const logger = createLogger('MongoDBQueryAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - let client = null - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized MongoDB query attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(mongodbQueryContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Executing MongoDB query on ${params.host}:${params.port}/${params.database}.${params.collection}` - ) - - const sanitizedCollection = sanitizeCollectionName(params.collection) - - let filter = {} - if (params.query?.trim()) { - const validation = validateFilter(params.query) - if (!validation.isValid) { - logger.warn(`[${requestId}] Filter validation failed: ${validation.error}`) - return NextResponse.json( - { error: `Filter validation failed: ${validation.error}` }, - { status: 400 } - ) - } - filter = JSON.parse(params.query) - } - - let sortCriteria = {} - if (params.sort?.trim()) { - try { - sortCriteria = JSON.parse(params.sort) - } catch (error) { - logger.warn(`[${requestId}] Invalid sort JSON: ${params.sort}`) - return NextResponse.json({ error: 'Invalid JSON format in sort criteria' }, { status: 400 }) - } - } - - client = await createMongoDBConnection({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - authSource: params.authSource, - ssl: params.ssl, - }) - - const db = client.db(params.database) - const coll = db.collection(sanitizedCollection) - - let cursor = coll.find(filter) - - if (Object.keys(sortCriteria).length > 0) { - cursor = cursor.sort(sortCriteria) - } - - const limit = params.limit || 100 - cursor = cursor.limit(limit) - - const documents = await cursor.toArray() - - logger.info( - `[${requestId}] Query executed successfully, returned ${documents.length} documents` - ) - - return NextResponse.json({ - message: `Found ${documents.length} documents`, - documents, - documentCount: documents.length, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] MongoDB query failed:`, error) - - return NextResponse.json({ error: `MongoDB query failed: ${errorMessage}` }, { status: 500 }) - } finally { - if (client) { - await client.close() - } - } -}) diff --git a/apps/sim/app/api/tools/mongodb/update/route.ts b/apps/sim/app/api/tools/mongodb/update/route.ts deleted file mode 100644 index 1163014207d..00000000000 --- a/apps/sim/app/api/tools/mongodb/update/route.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { mongodbUpdateContract } from '@/lib/api/contracts/tools/databases/mongodb' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createMongoDBConnection, - sanitizeCollectionName, - validateFilter, -} from '@/app/api/tools/mongodb/utils' - -const logger = createLogger('MongoDBUpdateAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - let client = null - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized MongoDB update attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(mongodbUpdateContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Updating document(s) in ${params.host}:${params.port}/${params.database}.${params.collection} (multi: ${params.multi}, upsert: ${params.upsert})` - ) - - const sanitizedCollection = sanitizeCollectionName(params.collection) - - const filterValidation = validateFilter(params.filter) - if (!filterValidation.isValid) { - logger.warn(`[${requestId}] Filter validation failed: ${filterValidation.error}`) - return NextResponse.json( - { error: `Filter validation failed: ${filterValidation.error}` }, - { status: 400 } - ) - } - - let filterDoc - let updateDoc - try { - filterDoc = JSON.parse(params.filter) - updateDoc = JSON.parse(params.update) - } catch (error) { - logger.warn(`[${requestId}] Invalid JSON in filter or update`) - return NextResponse.json( - { error: 'Invalid JSON format in filter or update' }, - { status: 400 } - ) - } - - client = await createMongoDBConnection({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - authSource: params.authSource, - ssl: params.ssl, - }) - - const db = client.db(params.database) - const coll = db.collection(sanitizedCollection) - - let result - if (params.multi) { - result = await coll.updateMany(filterDoc, updateDoc, { upsert: params.upsert }) - } else { - result = await coll.updateOne(filterDoc, updateDoc, { upsert: params.upsert }) - } - - logger.info( - `[${requestId}] Update completed: ${result.modifiedCount} modified, ${result.matchedCount} matched${result.upsertedCount ? `, ${result.upsertedCount} upserted` : ''}` - ) - - return NextResponse.json({ - message: `${result.modifiedCount} documents updated${result.upsertedCount ? `, ${result.upsertedCount} documents upserted` : ''}`, - matchedCount: result.matchedCount, - modifiedCount: result.modifiedCount, - documentCount: result.modifiedCount + (result.upsertedCount || 0), - ...(result.upsertedId && { insertedId: result.upsertedId.toString() }), - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] MongoDB update failed:`, error) - - return NextResponse.json({ error: `MongoDB update failed: ${errorMessage}` }, { status: 500 }) - } finally { - if (client) { - await client.close() - } - } -}) diff --git a/apps/sim/app/api/tools/mongodb/utils.ts b/apps/sim/app/api/tools/mongodb/utils.ts deleted file mode 100644 index 7fb17e17424..00000000000 --- a/apps/sim/app/api/tools/mongodb/utils.ts +++ /dev/null @@ -1,197 +0,0 @@ -import { MongoClient } from 'mongodb' -import { - createPinnedLookup, - validateDatabaseHost, -} from '@/lib/core/security/input-validation.server' -import type { MongoDBCollectionInfo, MongoDBConnectionConfig } from '@/tools/mongodb/types' - -export async function createMongoDBConnection(config: MongoDBConnectionConfig) { - const hostValidation = await validateDatabaseHost(config.host, 'host') - if (!hostValidation.isValid) { - throw new Error(hostValidation.error) - } - - const credentials = - config.username && config.password - ? `${encodeURIComponent(config.username)}:${encodeURIComponent(config.password)}@` - : '' - - const queryParams = new URLSearchParams() - - if (config.authSource) { - queryParams.append('authSource', config.authSource) - } - - if (config.ssl === 'required') { - queryParams.append('ssl', 'true') - } - - const queryString = queryParams.toString() - const uri = `mongodb://${credentials}${config.host}:${config.port}/${config.database}${queryString ? `?${queryString}` : ''}` - - const client = new MongoClient(uri, { - connectTimeoutMS: 10000, - socketTimeoutMS: 10000, - maxPoolSize: 1, - lookup: createPinnedLookup(hostValidation.resolvedIP ?? config.host), - }) - - await client.connect() - return client -} - -/** - * Recursively checks an object for dangerous MongoDB operators - * @param obj - The object to check - * @param dangerousOperators - Array of operator names to block - * @returns true if a dangerous operator is found - */ -function containsDangerousOperator(obj: unknown, dangerousOperators: string[]): boolean { - if (typeof obj !== 'object' || obj === null) return false - - for (const key of Object.keys(obj as Record)) { - if (dangerousOperators.includes(key)) return true - if ( - typeof (obj as Record)[key] === 'object' && - containsDangerousOperator((obj as Record)[key], dangerousOperators) - ) { - return true - } - } - return false -} - -export function validateFilter(filter: string): { isValid: boolean; error?: string } { - try { - const parsed = JSON.parse(filter) - - const dangerousOperators = [ - '$where', // Executes arbitrary JavaScript - '$regex', // Can cause ReDoS attacks - '$expr', // Expression evaluation - '$function', // Custom JavaScript functions - '$accumulator', // Custom JavaScript accumulators - '$let', // Variable definitions that could be exploited - ] - - if (containsDangerousOperator(parsed, dangerousOperators)) { - return { - isValid: false, - error: 'Filter contains potentially dangerous operators', - } - } - - return { isValid: true } - } catch (error) { - return { - isValid: false, - error: 'Invalid JSON format in filter', - } - } -} - -export function validatePipeline(pipeline: string): { isValid: boolean; error?: string } { - try { - const parsed = JSON.parse(pipeline) - - if (!Array.isArray(parsed)) { - return { - isValid: false, - error: 'Pipeline must be an array', - } - } - - const dangerousOperators = [ - '$where', // Executes arbitrary JavaScript - '$function', // Custom JavaScript functions - '$accumulator', // Custom JavaScript accumulators - '$let', // Variable definitions that could be exploited - '$merge', // Writes to external collections - '$out', // Writes to external collections - '$currentOp', // Exposes system operation info - '$listSessions', // Exposes session info - '$listLocalSessions', // Exposes local session info - ] - - for (const stage of parsed) { - if (containsDangerousOperator(stage, dangerousOperators)) { - return { - isValid: false, - error: 'Pipeline contains potentially dangerous operators', - } - } - } - - return { isValid: true } - } catch (error) { - return { - isValid: false, - error: 'Invalid JSON format in pipeline', - } - } -} - -export function sanitizeCollectionName(name: string): string { - if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { - throw new Error( - 'Invalid collection name. Must start with letter or underscore and contain only letters, numbers, and underscores.' - ) - } - return name -} - -/** - * Introspect MongoDB to get databases, collections, and indexes - */ -export async function executeIntrospect( - client: MongoClient, - database?: string -): Promise<{ - message: string - databases: string[] - collections: MongoDBCollectionInfo[] -}> { - const databases: string[] = [] - const collections: MongoDBCollectionInfo[] = [] - - if (database) { - databases.push(database) - const db = client.db(database) - const collectionList = await db.listCollections().toArray() - - for (const collInfo of collectionList) { - const coll = db.collection(collInfo.name) - const indexes = await coll.indexes() - const documentCount = await coll.estimatedDocumentCount() - - collections.push({ - name: collInfo.name, - type: collInfo.type || 'collection', - documentCount, - indexes: indexes.map((idx) => ({ - name: idx.name || '', - key: idx.key as Record, - unique: idx.unique || false, - sparse: idx.sparse, - })), - }) - } - } else { - const admin = client.db().admin() - const dbList = await admin.listDatabases() - - for (const dbInfo of dbList.databases) { - databases.push(dbInfo.name) - } - } - - const message = database - ? `Found ${collections.length} collections in database '${database}'` - : `Found ${databases.length} databases` - - return { - message, - databases, - collections, - } -} diff --git a/apps/sim/app/api/tools/mssql/delete/route.ts b/apps/sim/app/api/tools/mssql/delete/route.ts deleted file mode 100644 index 44aab52a61b..00000000000 --- a/apps/sim/app/api/tools/mssql/delete/route.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { mssqlDeleteContract } from '@/lib/api/contracts/tools/databases/mssql' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - buildDeleteQuery, - createMSSQLConnection, - executeQuery, - toRowsResponseBody, -} from '@/app/api/tools/mssql/utils' - -const logger = createLogger('MSSQLDeleteAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized Microsoft SQL Server delete attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(mssqlDeleteContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Deleting data from ${params.table} on ${params.host}:${params.port}/${params.database}` - ) - - /** - * Built before connecting so a rejected WHERE clause or a bad identifier - * costs no TLS+login round trip and answers 400 like the query and execute - * routes, rather than falling through to the catch-all as a 500. - */ - let built: { query: string; values: unknown[] } - try { - built = buildDeleteQuery(params.table, params.where) - } catch (error) { - const message = getErrorMessage(error, 'Invalid statement') - logger.warn(`[${requestId}] Delete statement rejected: ${message}`) - return NextResponse.json( - { error: `Microsoft SQL Server delete failed: ${message}` }, - { status: 400 } - ) - } - - const pool = await createMSSQLConnection(params) - - try { - const result = await executeQuery(pool, built.query, built.values) - - logger.info(`[${requestId}] Delete executed successfully, ${result.rowCount} row(s) deleted`) - - return NextResponse.json( - toRowsResponseBody(result, `Data deleted successfully. ${result.rowCount} row(s) affected.`) - ) - } finally { - await pool.close() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Microsoft SQL Server delete failed:`, error) - - return NextResponse.json( - { error: `Microsoft SQL Server delete failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/mssql/execute/route.ts b/apps/sim/app/api/tools/mssql/execute/route.ts deleted file mode 100644 index 43bf87198b6..00000000000 --- a/apps/sim/app/api/tools/mssql/execute/route.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { mssqlExecuteContract } from '@/lib/api/contracts/tools/databases/mssql' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createMSSQLConnection, - executeQuery, - toRowsResponseBody, - validateQuery, -} from '@/app/api/tools/mssql/utils' - -const logger = createLogger('MSSQLExecuteAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized Microsoft SQL Server execute attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(mssqlExecuteContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Executing raw T-SQL on ${params.host}:${params.port}/${params.database}` - ) - - const validation = validateQuery(params.query) - if (!validation.isValid) { - logger.warn(`[${requestId}] Query validation failed: ${validation.error}`) - return NextResponse.json( - { error: `Query validation failed: ${validation.error}` }, - { status: 400 } - ) - } - - const pool = await createMSSQLConnection(params) - - try { - const result = await executeQuery(pool, params.query) - - logger.info(`[${requestId}] T-SQL executed successfully, ${result.rowCount} row(s) affected`) - - return NextResponse.json( - toRowsResponseBody(result, `SQL executed successfully. ${result.rowCount} row(s) affected.`) - ) - } finally { - await pool.close() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Microsoft SQL Server execute failed:`, error) - - return NextResponse.json( - { error: `Microsoft SQL Server execute failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/mssql/insert/route.ts b/apps/sim/app/api/tools/mssql/insert/route.ts deleted file mode 100644 index 6280ce22e77..00000000000 --- a/apps/sim/app/api/tools/mssql/insert/route.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { mssqlInsertContract } from '@/lib/api/contracts/tools/databases/mssql' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - buildInsertQuery, - createMSSQLConnection, - executeQuery, - toRowsResponseBody, -} from '@/app/api/tools/mssql/utils' - -const logger = createLogger('MSSQLInsertAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized Microsoft SQL Server insert attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(mssqlInsertContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Inserting data into ${params.table} on ${params.host}:${params.port}/${params.database}` - ) - - /** - * Built before connecting so a bad identifier costs no TLS+login round trip - * and answers 400 like the update, delete, query, and execute routes, rather - * than falling through to the catch-all as a 500. - */ - let built: { query: string; values: unknown[] } - try { - built = buildInsertQuery(params.table, params.data) - } catch (error) { - const message = getErrorMessage(error, 'Invalid statement') - logger.warn(`[${requestId}] Insert statement rejected: ${message}`) - return NextResponse.json( - { error: `Microsoft SQL Server insert failed: ${message}` }, - { status: 400 } - ) - } - - const pool = await createMSSQLConnection(params) - - try { - const result = await executeQuery(pool, built.query, built.values) - - logger.info(`[${requestId}] Insert executed successfully, ${result.rowCount} row(s) inserted`) - - return NextResponse.json( - toRowsResponseBody( - result, - `Data inserted successfully. ${result.rowCount} row(s) affected.` - ) - ) - } finally { - await pool.close() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Microsoft SQL Server insert failed:`, error) - - return NextResponse.json( - { error: `Microsoft SQL Server insert failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/mssql/introspect/route.ts b/apps/sim/app/api/tools/mssql/introspect/route.ts deleted file mode 100644 index 7d6788ba57f..00000000000 --- a/apps/sim/app/api/tools/mssql/introspect/route.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { mssqlIntrospectContract } from '@/lib/api/contracts/tools/databases/mssql' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createMSSQLConnection, executeIntrospect } from '@/app/api/tools/mssql/utils' - -const logger = createLogger('MSSQLIntrospectAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized Microsoft SQL Server introspect attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(mssqlIntrospectContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Introspecting Microsoft SQL Server schema on ${params.host}:${params.port}/${params.database}` - ) - - const pool = await createMSSQLConnection(params) - - try { - const result = await executeIntrospect(pool, params.schema) - - logger.info( - `[${requestId}] Introspection completed successfully, found ${result.tables.length} tables` - ) - - return NextResponse.json({ - message: `Schema introspection completed. Found ${result.tables.length} table(s) in schema '${params.schema}'.`, - tables: result.tables, - schemas: result.schemas, - }) - } finally { - await pool.close() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Microsoft SQL Server introspection failed:`, error) - - return NextResponse.json( - { error: `Microsoft SQL Server introspection failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/mssql/query/route.ts b/apps/sim/app/api/tools/mssql/query/route.ts deleted file mode 100644 index ba0a1a1848b..00000000000 --- a/apps/sim/app/api/tools/mssql/query/route.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { mssqlQueryContract } from '@/lib/api/contracts/tools/databases/mssql' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createMSSQLConnection, - executeQuery, - toRowsResponseBody, - validateReadOnlyQuery, -} from '@/app/api/tools/mssql/utils' - -const logger = createLogger('MSSQLQueryAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized Microsoft SQL Server query attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(mssqlQueryContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Executing Microsoft SQL Server query on ${params.host}:${params.port}/${params.database}` - ) - - const validation = validateReadOnlyQuery(params.query) - if (!validation.isValid) { - logger.warn(`[${requestId}] Query validation failed: ${validation.error}`) - return NextResponse.json( - { error: `Query validation failed: ${validation.error}` }, - { status: 400 } - ) - } - - const pool = await createMSSQLConnection(params) - - try { - const result = await executeQuery(pool, params.query) - - logger.info(`[${requestId}] Query executed successfully, returned ${result.rowCount} rows`) - - return NextResponse.json( - toRowsResponseBody( - result, - `Query executed successfully. ${result.rowCount} row(s) returned.` - ) - ) - } finally { - await pool.close() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Microsoft SQL Server query failed:`, error) - - return NextResponse.json( - { error: `Microsoft SQL Server query failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/mssql/route-guards.test.ts b/apps/sim/app/api/tools/mssql/route-guards.test.ts deleted file mode 100644 index 5d57ef4149b..00000000000 --- a/apps/sim/app/api/tools/mssql/route-guards.test.ts +++ /dev/null @@ -1,94 +0,0 @@ -/** - * @vitest-environment node - * - * The insert, update, and delete routes build their statement before opening a - * connection, so a rejected WHERE clause or a bad identifier costs no TLS+login - * round trip and answers 400 like the query and execute routes do. - */ -import { createMockRequest, hybridAuthMockFns } from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { mockResolveHostAddresses, mockConnectionPool, mockQuery } = vi.hoisted(() => { - const query = vi.fn().mockResolvedValue({ recordset: [], rowsAffected: [1] }) - const pool = vi.fn(function ConnectionPool(this: Record) { - this.connect = vi.fn().mockResolvedValue(undefined) - this.close = vi.fn().mockResolvedValue(undefined) - this.request = () => ({ input: vi.fn(), query }) - }) - return { mockResolveHostAddresses: vi.fn(), mockConnectionPool: pool, mockQuery: query } -}) - -vi.mock('mssql', () => ({ - default: { ConnectionPool: mockConnectionPool }, - ConnectionPool: mockConnectionPool, -})) - -vi.mock('@sim/security/dns', () => ({ - resolveHostAddresses: mockResolveHostAddresses, - preferIpv4: (addresses: string[]) => addresses[0], -})) - -import { POST as DELETE_POST } from '@/app/api/tools/mssql/delete/route' -import { POST as INSERT_POST } from '@/app/api/tools/mssql/insert/route' -import { POST as UPDATE_POST } from '@/app/api/tools/mssql/update/route' - -const connection = { - host: 'db.example.com', - port: 1433, - database: 'app', - username: 'app', - password: 'secret', - encrypt: 'enabled', - trustServerCertificate: 'disabled', - connectionTimeout: 15000, -} - -describe('MSSQL insert, update, and delete guards run before connecting', () => { - beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-123', - authType: 'internal_jwt', - }) - mockResolveHostAddresses.mockResolvedValue({ addresses: ['93.184.216.34'], isPrivate: false }) - mockQuery.mockResolvedValue({ recordset: [], rowsAffected: [1] }) - }) - - it.each([ - ['update', UPDATE_POST, { table: 'users', data: { a: 1 }, where: 'id = 1 OR 1=1' }], - ['delete', DELETE_POST, { table: 'users', where: 'id = 1 OR 1=1' }], - ])( - 'answers 400 for a rejected WHERE clause on %s without connecting', - async (_op, handler, body) => { - const response = await handler(createMockRequest('POST', { ...connection, ...body })) - - expect(response.status).toBe(400) - expect(mockConnectionPool).not.toHaveBeenCalled() - } - ) - - it.each([ - ['insert', INSERT_POST, { table: 'users-table', data: { a: 1 } }], - ['insert column', INSERT_POST, { table: 'users', data: { 'bad-col': 1 } }], - ['update', UPDATE_POST, { table: 'users-table', data: { a: 1 }, where: 'id = 1' }], - ['delete', DELETE_POST, { table: 'users-table', where: 'id = 1' }], - ])('answers 400 for a bad identifier on %s without connecting', async (_op, handler, body) => { - const response = await handler(createMockRequest('POST', { ...connection, ...body })) - - expect(response.status).toBe(400) - expect(mockConnectionPool).not.toHaveBeenCalled() - }) - - it.each([ - ['insert', INSERT_POST, { table: 'users', data: { a: 1 } }], - ['update', UPDATE_POST, { table: 'users', data: { a: 1 }, where: 'id = 1' }], - ['delete', DELETE_POST, { table: 'users', where: 'id = 1' }], - ])('still runs an accepted %s statement', async (_op, handler, body) => { - const response = await handler(createMockRequest('POST', { ...connection, ...body })) - - expect(response.status).toBe(200) - expect(mockConnectionPool).toHaveBeenCalledTimes(1) - expect(mockQuery).toHaveBeenCalledTimes(1) - }) -}) diff --git a/apps/sim/app/api/tools/mssql/update/route.ts b/apps/sim/app/api/tools/mssql/update/route.ts deleted file mode 100644 index 392aa3d5d1e..00000000000 --- a/apps/sim/app/api/tools/mssql/update/route.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { mssqlUpdateContract } from '@/lib/api/contracts/tools/databases/mssql' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - buildUpdateQuery, - createMSSQLConnection, - executeQuery, - toRowsResponseBody, -} from '@/app/api/tools/mssql/utils' - -const logger = createLogger('MSSQLUpdateAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized Microsoft SQL Server update attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(mssqlUpdateContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Updating data in ${params.table} on ${params.host}:${params.port}/${params.database}` - ) - - /** - * Built before connecting so a rejected WHERE clause or a bad identifier - * costs no TLS+login round trip and answers 400 like the query and execute - * routes, rather than falling through to the catch-all as a 500. - */ - let built: { query: string; values: unknown[] } - try { - built = buildUpdateQuery(params.table, params.data, params.where) - } catch (error) { - const message = getErrorMessage(error, 'Invalid statement') - logger.warn(`[${requestId}] Update statement rejected: ${message}`) - return NextResponse.json( - { error: `Microsoft SQL Server update failed: ${message}` }, - { status: 400 } - ) - } - - const pool = await createMSSQLConnection(params) - - try { - const result = await executeQuery(pool, built.query, built.values) - - logger.info(`[${requestId}] Update executed successfully, ${result.rowCount} row(s) updated`) - - return NextResponse.json( - toRowsResponseBody(result, `Data updated successfully. ${result.rowCount} row(s) affected.`) - ) - } finally { - await pool.close() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Microsoft SQL Server update failed:`, error) - - return NextResponse.json( - { error: `Microsoft SQL Server update failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/mssql/utils.test.ts b/apps/sim/app/api/tools/mssql/utils.test.ts deleted file mode 100644 index 101bf210dea..00000000000 --- a/apps/sim/app/api/tools/mssql/utils.test.ts +++ /dev/null @@ -1,716 +0,0 @@ -/** - * @vitest-environment node - */ -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { mockResolveHostAddresses, mockConnectionPool, mockConnect, mockClose } = vi.hoisted(() => { - const connect = vi.fn().mockResolvedValue(undefined) - const close = vi.fn().mockResolvedValue(undefined) - const pool = vi.fn(function ConnectionPool(this: Record) { - this.connect = connect - this.close = close - }) - return { - mockResolveHostAddresses: vi.fn(), - mockConnectionPool: pool, - mockConnect: connect, - mockClose: close, - } -}) - -vi.mock('mssql', () => ({ - default: { ConnectionPool: mockConnectionPool }, - ConnectionPool: mockConnectionPool, -})) - -/** - * Only DNS is stubbed. The SSRF guard and the shared WHERE screens stay real, so - * these tests exercise the same masking behavior production does — which is the - * point, since the bypasses below are a property of that masker. - */ -vi.mock('@sim/security/dns', () => ({ - resolveHostAddresses: mockResolveHostAddresses, - preferIpv4: (addresses: string[]) => addresses[0], -})) - -import { - buildDeleteQuery, - buildInsertQuery, - buildUpdateQuery, - createMSSQLConnection, - executeIntrospect, - executeQuery, - type MSSQLConnectionConfig, - toRowsResponseBody, - validateQuery, - validateReadOnlyQuery, -} from '@/app/api/tools/mssql/utils' - -function makeConfig(overrides: Partial = {}): MSSQLConnectionConfig { - return { - host: 'db.example.com', - port: 1433, - database: 'app', - username: 'app', - password: 'secret', - encrypt: 'enabled', - trustServerCertificate: 'disabled', - connectionTimeout: 15000, - ...overrides, - } -} - -describe('validateReadOnlyQuery', () => { - it('accepts an ordinary SELECT and a leading CTE', () => { - expect(validateReadOnlyQuery('SELECT TOP (10) * FROM dbo.users').isValid).toBe(true) - expect( - validateReadOnlyQuery('WITH t AS (SELECT id FROM dbo.users) SELECT * FROM t').isValid - ).toBe(true) - }) - - /** T-SQL does not require whitespace after the opening keyword. */ - it.each(['SELECT*FROM dbo.users', 'SELECT(1)', 'WITH(x) AS (SELECT 1) SELECT * FROM x'])( - 'accepts %s, which has no space after the keyword', - (query) => { - expect(validateReadOnlyQuery(query).isValid).toBe(true) - } - ) - - it('still rejects a keyword that merely starts with SELECT', () => { - expect(validateReadOnlyQuery('SELECTX FROM dbo.users').isValid).toBe(false) - }) - - it('accepts a SELECT whose literal contains a doubled quote', () => { - expect(validateReadOnlyQuery("SELECT * FROM dbo.users WHERE name = 'O''Brien'").isValid).toBe( - true - ) - }) - - it.each([ - ['a bare mutation', 'DELETE FROM dbo.users'], - ['a semicolon batch', 'SELECT 1; DROP TABLE dbo.users'], - ['a semicolon-less batch', 'SELECT 1 DELETE FROM dbo.users'], - ['a CTE-led mutation', 'WITH t AS (SELECT id FROM dbo.users) DELETE FROM t'], - ['a comment', 'SELECT 1 -- DELETE FROM dbo.users'], - ['a stored procedure', 'SELECT 1 FROM dbo.t WHERE x = 1 xp_cmdshell'], - ])('rejects %s', (_label, query) => { - expect(validateReadOnlyQuery(query).isValid).toBe(false) - }) - - /** - * The shared masker treats `\` as a literal escape because it was written for - * the MySQL dialect. T-SQL has no backslash escape, so the server closes the - * literal at the quote the masker swallowed and runs the remainder as code — - * with an even quote count, so a parity check alone does not catch it. - */ - it('rejects a backslash-escaped quote that would hide a mutation from the keyword screen', () => { - const smuggled = String.raw`SELECT * FROM dbo.t WHERE a='x\' DELETE FROM dbo.t WHERE b='y'` - - const result = validateReadOnlyQuery(smuggled) - - expect(result.isValid).toBe(false) - expect(result.error).toMatch(/backslash before a quote/) - }) - - it('rejects a quote inside a bracketed identifier', () => { - expect(validateReadOnlyQuery(`SELECT * FROM dbo.t WHERE [a"] = 1 OR 1=1`).isValid).toBe(false) - expect(validateReadOnlyQuery(`SELECT * FROM dbo.t WHERE [a'] = 1 OR 1=1`).isValid).toBe(false) - }) - - it('rejects an unpaired quote', () => { - expect(validateReadOnlyQuery(`SELECT * FROM dbo.t WHERE a = 'x`).isValid).toBe(false) - }) - - /** Semicolon-less batches that change trigger, session, or transaction state. */ - it.each([ - 'SELECT 1 DISABLE TRIGGER dbo.audit_trigger ON dbo.users', - 'SELECT 1 ENABLE TRIGGER dbo.audit_trigger ON dbo.users', - 'SELECT 1 SET IDENTITY_INSERT dbo.t ON', - 'SELECT 1 BEGIN TRAN', - 'SELECT 1 COMMIT', - 'SELECT 1 ROLLBACK', - ])('rejects the state-changing batch %s', (query) => { - expect(validateReadOnlyQuery(query).isValid).toBe(false) - }) - - /** - * `\bupdate\b` cannot match `UPDATETEXT` — there is no word boundary after - * `update` — so each text statement has to be screened in its own right. - */ - it.each([ - "SELECT 1 UPDATETEXT dbo.t.col @ptr 0 NULL 'x'", - "SELECT 1 WRITETEXT dbo.t.col @ptr 'x'", - 'SELECT 1 READTEXT dbo.t.col @ptr 0 16', - ])('rejects the text statement batch %s', (query) => { - expect(validateReadOnlyQuery(query).isValid).toBe(false) - }) - - /** The same statements reached through the WHERE screen, which shares the list. */ - it.each([ - "id = 1 UPDATETEXT dbo.t.col @ptr 0 NULL 'x'", - "id = 1 WRITETEXT dbo.t.col @ptr 'x'", - 'id = 1 READTEXT dbo.t.col @ptr 0 16', - ])('rejects the text statement %s in a WHERE clause', (where) => { - expect(() => buildDeleteQuery('dbo.users', where)).toThrow() - }) - - /** - * The guard against over-screening. `FETCH` is excluded from the keyword list - * because `OFFSET … FETCH NEXT` is the standard paging clause, and the added - * keywords must not catch ordinary identifiers that merely contain them. - */ - it.each([ - 'SELECT * FROM dbo.users ORDER BY id OFFSET 10 ROWS FETCH NEXT 20 ROWS ONLY', - 'WITH p AS (SELECT id FROM dbo.o) SELECT * FROM p ORDER BY id OFFSET 0 ROWS FETCH NEXT 5 ROWS ONLY', - 'SELECT settled, offset_value, begin_date FROM dbo.t', - 'SELECT updatetext_id, writetext_flag, readtext_offset FROM dbo.t', - 'SELECT TOP (100) id, name FROM dbo.users WHERE is_active = 1', - ])('still accepts the legitimate read %s', (query) => { - expect(validateReadOnlyQuery(query).isValid).toBe(true) - }) -}) - -describe('validateQuery (Execute Raw SQL)', () => { - /** T-SQL needs no space after the keyword; `EXEC(@sql)` is ordinary dynamic SQL. */ - it.each([ - 'EXEC(@sql)', - 'EXECUTE(@sql)', - 'EXEC sp_who', - 'EXECUTE dbo.myproc', - 'SELECT(1)', - 'WITH(x) AS (SELECT 1) SELECT * FROM x', - 'DECLARE @x INT', - ])('accepts %s', (query) => { - expect(validateQuery(query).isValid).toBe(true) - }) - - it.each(['SELECTX 1', 'DROP TABLE dbo.t', 'TRUNCATE TABLE dbo.t'])('rejects %s', (query) => { - expect(validateQuery(query).isValid).toBe(false) - }) -}) - -describe('buildUpdateQuery / buildDeleteQuery WHERE screening', () => { - it('builds a parameterized statement for an ordinary condition', () => { - const { query, values } = buildUpdateQuery('dbo.users', { name: 'Jane' }, 'id = 1') - - expect(query).toBe('UPDATE [dbo].[users] SET [name] = @param1 WHERE id = 1') - expect(values).toEqual(['Jane']) - }) - - /** - * Same masker desynchronisation as above, reached through the mutation path: - * an even quote count, no semicolon, and the tautology invisible to every - * screen that runs over masked text. - */ - it('rejects a backslash-escaped quote that would hide a tautology', () => { - const smuggled = String.raw`id = 'a\' OR 1=1 OR 2>1 AND b = 'x'` - - expect(() => buildDeleteQuery('dbo.users', smuggled)).toThrow(/backslash before a quote/) - expect(() => buildUpdateQuery('dbo.users', { a: 1 }, smuggled)).toThrow( - /backslash before a quote/ - ) - }) - - it('rejects a quote hidden inside a bracketed identifier', () => { - expect(() => buildDeleteQuery('dbo.users', `[a"] = 1 OR 1=1`)).toThrow(/bracketed identifier/) - }) - - /** - * The shared guard sees `OR 1` but not a parenthesised or negated constant. - * These are the specific forms it documents as undetected; the class as a - * whole is not lexically decidable, so this narrows rather than closes it. - */ - it.each([ - 'id = 1 OR (1)', - 'id = 1 OR ((1))', - 'id = 1 OR NOT 0', - 'id = 1 OR NOT (0)', - 'id = 1 OR (TRUE)', - ])('rejects the constant tautology %s', (where) => { - expect(() => buildDeleteQuery('dbo.users', where)).toThrow() - }) - - it.each([ - 'id = 1 OR (priority = 2)', - 'id = 1 OR (1 = priority)', - "status = 'open' OR (retries < 3)", - ])('still accepts the real disjunct %s', (where) => { - expect(() => buildDeleteQuery('dbo.users', where)).not.toThrow() - }) - - it.each([ - ['a semicolon-less batch', "id = 1 DBCC SHRINKDATABASE('app')"], - ['an appended SELECT', 'id = 1 SELECT secret FROM dbo.credentials'], - ['a catalog probe', 'id = 1 AND EXISTS (sys.objects)'], - ['a stored procedure', 'id = 1 AND xp_cmdshell'], - ])('rejects %s', (_label, where) => { - expect(() => buildDeleteQuery('dbo.users', where)).toThrow() - }) -}) - -describe('identifier handling', () => { - it('bracket-quotes every part of a qualified name and binds values', () => { - const { query, values } = buildInsertQuery('dbo.users', { name: 'Jane', age: 30 }) - - expect(query).toBe('INSERT INTO [dbo].[users] ([name], [age]) VALUES (@param1, @param2)') - expect(values).toEqual(['Jane', 30]) - }) - - it('rejects an identifier that is not a plain word', () => { - expect(() => buildInsertQuery('users; DROP TABLE x', { a: 1 })).toThrow(/Invalid identifier/) - expect(() => buildInsertQuery('users', { 'a b': 1 })).toThrow(/Invalid identifier/) - }) - - it('cannot be escaped by pre-closing a bracket', () => { - expect(() => buildInsertQuery('users] DROP TABLE x --[', { a: 1 })).toThrow( - /Invalid identifier/ - ) - }) -}) - -describe('executeQuery parameter binding', () => { - function makePool(recordset: unknown[] = [], rowsAffected: number[] = [0]) { - const input = vi.fn() - const query = vi.fn().mockResolvedValue({ recordset, rowsAffected }) - return { - pool: { request: () => ({ input, query }) } as never, - input, - query, - } - } - - it('binds every value positionally, never interpolating it', async () => { - const { pool, input, query } = makePool() - - await executeQuery(pool, 'INSERT INTO [dbo].[t] ([a]) VALUES (@param1)', ["'; DROP TABLE t --"]) - - expect(query).toHaveBeenCalledWith('INSERT INTO [dbo].[t] ([a]) VALUES (@param1)') - expect(input).toHaveBeenCalledWith('param1', "'; DROP TABLE t --") - }) - - /** - * node-mssql infers NVarChar for an unrecognised object and tedious then - * rejects it with a bare `Invalid string.`, so a nested JSON value has to be - * serialized before it reaches the driver. - */ - it('serializes nested objects and arrays, passing scalars and Dates through', async () => { - const { pool, input } = makePool() - const when = new Date('2020-01-01T00:00:00Z') - - await executeQuery(pool, 'INSERT INTO [dbo].[t] VALUES (@param1, @param2, @param3, @param4)', [ - { nested: true }, - ['a', 'b'], - when, - 42, - ]) - - expect(input).toHaveBeenNthCalledWith(1, 'param1', '{"nested":true}') - expect(input).toHaveBeenNthCalledWith(2, 'param2', '["a","b"]') - expect(input).toHaveBeenNthCalledWith(3, 'param3', when) - expect(input).toHaveBeenNthCalledWith(4, 'param4', 42) - }) - - it('reports affected rows when the statement returns no recordset', async () => { - const { pool } = makePool([], [3]) - - await expect( - executeQuery(pool, 'DELETE FROM [dbo].[t] WHERE id = @param1', [1]) - ).resolves.toEqual({ rows: [], rowCount: 3 }) - }) -}) - -describe('createMSSQLConnection DNS pinning', () => { - beforeEach(() => { - vi.clearAllMocks() - mockConnect.mockResolvedValue(undefined) - mockClose.mockResolvedValue(undefined) - mockResolveHostAddresses.mockResolvedValue({ - addresses: ['93.184.216.34'], - preferred: '93.184.216.34', - }) - }) - - it('never opens a connection when the host cannot be resolved (no SSRF window)', async () => { - mockResolveHostAddresses.mockRejectedValue(new Error('ENOTFOUND')) - - await expect( - createMSSQLConnection(makeConfig({ host: 'rebind.attacker.example' })) - ).rejects.toThrow(/could not be resolved/) - expect(mockConnectionPool).not.toHaveBeenCalled() - }) - - it('keeps the hostname as `server` so TLS SNI and certificate validation still apply', async () => { - await createMSSQLConnection(makeConfig({ host: 'rebind.attacker.example' })) - - expect(mockResolveHostAddresses).toHaveBeenCalledWith('rebind.attacker.example') - const config = mockConnectionPool.mock.calls[0][0] - expect(config.server).toBe('rebind.attacker.example') - }) - - it('routes the socket through a connector bound to the validated IP, not the hostname', async () => { - await createMSSQLConnection(makeConfig({ host: 'rebind.attacker.example' })) - - const config = mockConnectionPool.mock.calls[0][0] - expect(typeof config.options.connector).toBe('function') - expect(config.options.instanceName).toBeUndefined() - }) - - /** - * Only a pool that is handed back reaches the route's `finally`, so a pool - * whose connect rejected has to release itself or a bad credential retried in - * a loop leaks one per attempt. - */ - it('closes the pool and surfaces the original error when connect fails', async () => { - mockConnect.mockRejectedValue(new Error('Login failed for user')) - - await expect(createMSSQLConnection(makeConfig())).rejects.toThrow('Login failed for user') - expect(mockClose).toHaveBeenCalledTimes(1) - }) - - it('does not let a close failure mask the connect error', async () => { - mockConnect.mockRejectedValue(new Error('Login failed for user')) - mockClose.mockRejectedValue(new Error('close blew up')) - - await expect(createMSSQLConnection(makeConfig())).rejects.toThrow('Login failed for user') - }) - - it('leaves the pool open on success so the route controls its lifetime', async () => { - await createMSSQLConnection(makeConfig()) - - expect(mockClose).not.toHaveBeenCalled() - }) - - it('maps the string toggles onto driver booleans without coercing "disabled" to true', async () => { - await createMSSQLConnection( - makeConfig({ encrypt: 'disabled', trustServerCertificate: 'enabled' }) - ) - - const config = mockConnectionPool.mock.calls[0][0] - expect(config.options.encrypt).toBe(false) - expect(config.options.trustServerCertificate).toBe(true) - }) -}) - -describe('read-only screens cover the rest of the session and transaction family', () => { - /** - * Each is a valid semicolon-less second statement, and the file's stated rule - * is that a second statement is rejected structurally rather than by what it - * happens to do. - */ - it.each([ - ['SAVE TRANSACTION', 'SELECT 1 SAVE TRANSACTION sp1'], - ['SAVE TRAN', 'SELECT 1 SAVE TRAN sp1'], - ['OPEN SYMMETRIC KEY', 'SELECT 1 OPEN SYMMETRIC KEY k DECRYPTION BY CERTIFICATE c'], - ['OPEN MASTER KEY', "SELECT 1 OPEN MASTER KEY DECRYPTION BY PASSWORD = 'p'"], - ['CLOSE ALL SYMMETRIC KEYS', 'SELECT 1 CLOSE ALL SYMMETRIC KEYS'], - ['CLOSE MASTER KEY', 'SELECT 1 CLOSE MASTER KEY'], - ['DEALLOCATE', 'SELECT 1 DEALLOCATE cur'], - ['ADD SIGNATURE', 'SELECT 1 ADD SIGNATURE TO dbo.p BY CERTIFICATE c'], - ['RAISERROR WITH LOG', "SELECT 1 RAISERROR ('boom', 16, 1) WITH LOG"], - ])('rejects %s in the Query operation', (_label, query) => { - expect(validateReadOnlyQuery(query).isValid).toBe(false) - }) - - it.each([ - ['SAVE TRANSACTION', 'id = 1 SAVE TRANSACTION sp1'], - ['OPEN SYMMETRIC KEY', 'id = 1 OPEN SYMMETRIC KEY k DECRYPTION BY CERTIFICATE c'], - ['CLOSE ALL SYMMETRIC KEYS', 'id = 1 CLOSE ALL SYMMETRIC KEYS'], - ['DEALLOCATE', 'id = 1 DEALLOCATE cur'], - ['ADD SIGNATURE', 'id = 1 ADD SIGNATURE TO dbo.p BY CERTIFICATE c'], - ['RAISERROR WITH LOG', "id = 1 RAISERROR ('boom', 16, 1) WITH LOG"], - ])('rejects %s in an update or delete WHERE clause', (_label, where) => { - expect(() => buildUpdateQuery('t', { a: 1 }, where)).toThrow() - expect(() => buildDeleteQuery('t', where)).toThrow() - }) - - /** - * The over-screening guard. `open`, `close`, `save`, and `add` are ordinary - * column names (a price table has all four), so a bare-word screen would make - * the plain SELECTs this operation exists to run un-runnable. - */ - it('still accepts ordinary identifiers that start with a screened phrase word', () => { - const allowed = [ - 'SELECT open, close, high, low FROM dbo.prices', - 'SELECT close FROM dbo.prices WHERE open > 10', - 'SELECT save_id, add_on, open_date, close_date FROM dbo.orders', - 'SELECT o.open, o.close FROM dbo.ohlc o ORDER BY o.open DESC', - ] - - for (const query of allowed) { - expect(validateReadOnlyQuery(query)).toEqual({ isValid: true }) - } - - expect(() => buildUpdateQuery('prices', { close: 2 }, 'open > 10')).not.toThrow() - expect(() => buildDeleteQuery('prices', 'close < 1 AND open_date > 0')).not.toThrow() - }) -}) - -describe('read-only screens cover RENAME and the Service Broker statement family', () => { - /** - * `RENAME` is documented T-SQL DDL for Azure Synapse dedicated SQL pools and - * Analytics Platform System, both reachable over TDS with the connection - * fields this block exposes — so a schema change was passing an operation - * advertised as read-only. - */ - it.each([ - ['RENAME OBJECT', 'SELECT 1 RENAME OBJECT dbo.Customer TO Customer1'], - ['RENAME OBJECT COLUMN', 'SELECT 1 RENAME OBJECT dbo.t COLUMN c1 TO c2'], - ['RENAME DATABASE', 'SELECT 1 RENAME DATABASE db1 TO db2'], - ['RECEIVE', 'SELECT 1 RECEIVE TOP(1) * FROM dbo.MyQueue'], - ['END CONVERSATION', "SELECT 1 END CONVERSATION '00000000-0000-0000-0000-000000000000'"], - [ - 'MOVE CONVERSATION', - "SELECT 1 MOVE CONVERSATION '00000000-0000-0000-0000-000000000000' TO '00000000-0000-0000-0000-000000000001'", - ], - ['GET CONVERSATION GROUP', 'SELECT 1 GET CONVERSATION GROUP @g FROM dbo.MyQueue'], - [ - 'SEND ON CONVERSATION', - "SELECT 1 SEND ON CONVERSATION '00000000-0000-0000-0000-000000000000' MESSAGE TYPE [t] ('x')", - ], - ])('rejects %s in the Query operation', (_label, query) => { - expect(validateReadOnlyQuery(query).isValid).toBe(false) - }) - - it.each([ - ['RENAME OBJECT', 'id = 1 RENAME OBJECT dbo.t TO t2'], - ['RECEIVE', 'id = 1 RECEIVE TOP(1) * FROM dbo.MyQueue'], - ['END CONVERSATION', "id = 1 END CONVERSATION '00000000-0000-0000-0000-000000000000'"], - ['GET CONVERSATION GROUP', 'id = 1 GET CONVERSATION GROUP @g FROM dbo.MyQueue'], - ])('rejects %s in an update or delete WHERE clause', (_label, where) => { - expect(() => buildUpdateQuery('t', { a: 1 }, where)).toThrow() - expect(() => buildDeleteQuery('t', where)).toThrow() - }) - - /** - * The over-screening guard. `END` closes every `CASE`, and `rename`/`receive` - * are the stems of ordinary column names, so neither addition may cost the - * plain SELECTs this operation exists to run. - */ - it('still accepts CASE … END and ordinary identifiers built on the new words', () => { - const allowed = [ - "SELECT CASE WHEN status = 1 THEN 'on' ELSE 'off' END FROM dbo.jobs", - "SELECT CASE WHEN a = 1 THEN 'x' END AS conversation_state FROM dbo.t", - 'SELECT renamed_at, rename_log, received_at, receive_queue FROM dbo.audit', - 'SELECT conversation_id, get_flag, move_order, send_at, end_date FROM dbo.t', - ] - - for (const query of allowed) { - expect(validateReadOnlyQuery(query)).toEqual({ isValid: true }) - } - - expect(() => buildUpdateQuery('audit', { a: 1 }, 'renamed_at > 0')).not.toThrow() - expect(() => buildDeleteQuery('audit', 'received_at > 0 AND conversation_id = 3')).not.toThrow() - }) -}) - -describe('executeQuery result caps', () => { - function makeCapPool(recordset: unknown[]) { - return { - request: () => ({ - input: vi.fn(), - query: vi.fn().mockResolvedValue({ recordset, rowsAffected: [0] }), - }), - } as never - } - - it('caps the recordset at the row ceiling and says so', async () => { - const result = await executeQuery( - makeCapPool(Array.from({ length: 10_001 }, (_, i) => ({ i }))), - 'SELECT 1' - ) - - expect(result.rows).toHaveLength(10_000) - expect(result.rowCount).toBe(10_000) - expect(result.truncated).toBe(true) - expect(result.truncationReason).toMatch(/OFFSET/) - }) - - it('caps on bytes even when the row count is small', async () => { - // 20 rows of ~1MB each: well under the row ceiling, well over the byte one. - const fat = Array.from({ length: 20 }, () => ({ blob: 'x'.repeat(1024 * 1024) })) - const result = await executeQuery(makeCapPool(fat), 'SELECT 1') - - expect(result.rows.length).toBeLessThan(20) - expect(result.truncated).toBe(true) - }) - - it('leaves an ordinary result untouched', async () => { - const rows = [{ id: 1 }, { id: 2 }] - const result = await executeQuery(makeCapPool(rows), 'SELECT 1') - - expect(result.rows).toEqual(rows) - expect(result.truncated).toBeUndefined() - expect(result.truncationReason).toBeUndefined() - }) - - it('never serializes past the byte ceiling', async () => { - const fat = Array.from({ length: 20 }, () => ({ blob: 'x'.repeat(1024 * 1024) })) - const result = await executeQuery(makeCapPool(fat), 'SELECT 1') - - expect(JSON.stringify(result.rows).length).toBeLessThanOrEqual(10 * 1024 * 1024) - }) - - it('drops a lone row that is larger than the byte ceiling rather than admitting it', async () => { - const oversized = [{ blob: 'x'.repeat(11 * 1024 * 1024) }] - const result = await executeQuery(makeCapPool(oversized), 'SELECT 1') - - expect(result.rows).toEqual([]) - expect(result.truncated).toBe(true) - expect(result.truncationReason).toMatch(/exceeds the 10 MB response ceiling/) - }) - - /** - * `String.length` counts UTF-16 code units and the response is emitted as - * UTF-8, so a CJK recordset costs three bytes for every unit the old - * accounting charged one for. Measured with `length` these rows fit; measured - * as the bytes that actually go on the wire they are ~3x over. - */ - it('bounds a multibyte recordset by UTF-8 bytes, not UTF-16 code units', async () => { - const cjk = Array.from({ length: 20 }, () => ({ blob: '世'.repeat(1024 * 1024) })) - const result = await executeQuery(makeCapPool(cjk), 'SELECT 1') - - expect(Buffer.byteLength(JSON.stringify(result.rows), 'utf8')).toBeLessThanOrEqual( - 10 * 1024 * 1024 - ) - expect(result.rows.length).toBeGreaterThan(0) - expect(result.truncated).toBe(true) - }) - - /** Emoji are 4 UTF-8 bytes across 2 surrogate code units — a 2:1 undercount. */ - it('bounds an astral-plane recordset by UTF-8 bytes', async () => { - const emoji = Array.from({ length: 20 }, () => ({ blob: '😀'.repeat(1024 * 1024) })) - const result = await executeQuery(makeCapPool(emoji), 'SELECT 1') - - expect(Buffer.byteLength(JSON.stringify(result.rows), 'utf8')).toBeLessThanOrEqual( - 10 * 1024 * 1024 - ) - expect(result.truncated).toBe(true) - }) - - /** - * Rows sized to divide the ceiling exactly, so an accounting that ignores the - * array's commas and the fields around it lands precisely on the limit and the - * body it emits is over by the punctuation and the envelope. - */ - it('keeps the emitted body inside the ceiling once array and envelope overhead is counted', async () => { - const rowPayload = 'x'.repeat(2048 - '{"blob":""}'.length) - const packed = Array.from({ length: 6000 }, () => ({ blob: rowPayload })) - const result = await executeQuery(makeCapPool(packed), 'SELECT 1') - - const body = toRowsResponseBody(result, 'Query executed successfully. rows returned.') - - expect(result.truncated).toBe(true) - expect(Buffer.byteLength(JSON.stringify(body), 'utf8')).toBeLessThanOrEqual(10 * 1024 * 1024) - }) -}) - -describe('toRowsResponseBody truncation disclosure', () => { - it('discloses a truncated result in both the message and machine-readable fields', () => { - const body = toRowsResponseBody( - { - rows: [{ id: 1 }], - rowCount: 1, - truncated: true, - truncationReason: 'Result truncated to 1 row(s): page with OFFSET ... FETCH NEXT.', - }, - 'Query executed successfully. 1 row(s) returned.' - ) - - expect(body.truncated).toBe(true) - expect(body.truncationReason).toMatch(/OFFSET/) - expect(body.message).toBe( - 'Query executed successfully. 1 row(s) returned. Result truncated to 1 row(s): page with OFFSET ... FETCH NEXT.' - ) - }) - - it('leaves a complete result free of truncation fields', () => { - const body = toRowsResponseBody( - { rows: [{ id: 1 }], rowCount: 1 }, - 'Query executed successfully. 1 row(s) returned.' - ) - - expect(body.message).toBe('Query executed successfully. 1 row(s) returned.') - expect(body).not.toHaveProperty('truncated') - expect(body).not.toHaveProperty('truncationReason') - }) -}) - -describe('executeIntrospect issues a fixed number of queries', () => { - const schemas = [{ SCHEMA_NAME: 'dbo' }] - const introspectTables = Array.from({ length: 50 }, (_, i) => ({ - TABLE_NAME: `t${i}`, - TABLE_SCHEMA: 'dbo', - })) - const introspectColumns = introspectTables.flatMap((t) => [ - { - TABLE_NAME: t.TABLE_NAME, - COLUMN_NAME: 'id', - DATA_TYPE: 'int', - IS_NULLABLE: 'NO', - COLUMN_DEFAULT: null, - }, - { - TABLE_NAME: t.TABLE_NAME, - COLUMN_NAME: 'owner_id', - DATA_TYPE: 'int', - IS_NULLABLE: 'YES', - COLUMN_DEFAULT: null, - }, - ]) - const introspectPks = introspectTables.map((t) => ({ - TABLE_NAME: t.TABLE_NAME, - COLUMN_NAME: 'id', - })) - const introspectFks = introspectTables.map((t) => ({ - TABLE_NAME: t.TABLE_NAME, - COLUMN_NAME: 'owner_id', - REFERENCED_TABLE_SCHEMA: 'dbo', - REFERENCED_TABLE_NAME: 'owners', - REFERENCED_COLUMN_NAME: 'id', - })) - const introspectIndexes = introspectTables.map((t) => ({ - TABLE_NAME: t.TABLE_NAME, - INDEX_NAME: `ix_${t.TABLE_NAME}_owner`, - COLUMN_NAME: 'owner_id', - IS_UNIQUE: 0, - })) - - function makeIntrospectPool() { - const query = vi.fn(async (text: string) => { - if (text.includes('FROM sys.schemas s')) return { recordset: schemas } - if (text.includes('INFORMATION_SCHEMA.TABLES')) return { recordset: introspectTables } - if (text.includes('INFORMATION_SCHEMA.COLUMNS')) return { recordset: introspectColumns } - if (text.includes('PRIMARY KEY')) return { recordset: introspectPks } - if (text.includes('sys.foreign_keys')) return { recordset: introspectFks } - if (text.includes('sys.index_columns')) return { recordset: introspectIndexes } - throw new Error(`unexpected query: ${text}`) - }) - return { pool: { request: () => ({ input: vi.fn().mockReturnThis(), query }) } as never, query } - } - - it('does not scale its round trips with the table count', async () => { - // Previously 4 queries per table plus 2: 50 tables meant 202 sequential - // round trips, each under its own request timeout. - const { pool, query } = makeIntrospectPool() - - const result = await executeIntrospect(pool, 'dbo') - - expect(result.tables).toHaveLength(50) - expect(query.mock.calls.length).toBeLessThanOrEqual(6) - }) - - it('still attributes columns, keys, and indexes to the right table', async () => { - const { pool } = makeIntrospectPool() - - const result = await executeIntrospect(pool, 'dbo') - const table = result.tables.find((t) => t.name === 't7')! - - expect(table.schema).toBe('dbo') - expect(table.columns.map((c) => c.name)).toEqual(['id', 'owner_id']) - expect(table.primaryKey).toEqual(['id']) - expect(table.columns[0].isPrimaryKey).toBe(true) - expect(table.columns[1].isForeignKey).toBe(true) - expect(table.columns[1].references).toEqual({ schema: 'dbo', table: 'owners', column: 'id' }) - expect(table.indexes).toEqual([{ name: 'ix_t7_owner', columns: ['owner_id'], unique: false }]) - }) -}) diff --git a/apps/sim/app/api/tools/mssql/utils.ts b/apps/sim/app/api/tools/mssql/utils.ts deleted file mode 100644 index 8cadafcc9cb..00000000000 --- a/apps/sim/app/api/tools/mssql/utils.ts +++ /dev/null @@ -1,1030 +0,0 @@ -import net from 'node:net' -import sql from 'mssql' -import { - maskSqlStringLiterals, - validateDatabaseHost, - validateSqlWhereClause, -} from '@/lib/core/security/input-validation.server' - -export interface MSSQLConnectionConfig { - host: string - port: number - database: string - username: string - password: string - encrypt: 'enabled' | 'disabled' - trustServerCertificate: 'enabled' | 'disabled' - connectionTimeout: number -} - -/** - * Opens a TCP socket to an already-validated IP address. - * - * Tedious calls the `connector` instead of resolving and connecting itself, so - * this is what keeps the connection pinned to the address the SSRF guard - * approved rather than to whatever DNS answers a second time. - * @see https://tediousjs.github.io/tedious/api-connection.html - */ -function connectToPinnedAddress(address: string, port: number, timeoutMs: number) { - return new Promise((resolve, reject) => { - const socket = net.connect({ host: address, port }) - socket.setNoDelay(true) - socket.setTimeout(timeoutMs) - - const fail = (error: Error) => { - socket.destroy() - reject(error) - } - - socket.once('connect', () => { - socket.setTimeout(0) - resolve(socket) - }) - socket.once('timeout', () => fail(new Error(`Connection to ${address}:${port} timed out`))) - socket.once('error', fail) - }) -} - -/** - * Opens a single-connection `mssql` pool against the SSRF-validated address. - * - * `options.connector` supplies a socket already connected to the resolved IP, - * which closes the DNS-rebinding window the way the PostgreSQL and MySQL tools - * do. `server` stays the original hostname because tedious derives the TLS - * `servername` from it independently of the connector, so SNI and certificate - * validation are unaffected by the pin. - * - * Named instances are deliberately unsupported: tedious resolves them with a - * UDP SQL Server Browser lookup issued against the hostname *outside* the - * connector, and node-mssql deletes `port` whenever `instanceName` is set, so - * there is no configuration in which a named instance stays pinned. Connect to - * a named instance by giving it a static TCP port instead. - * - * An Azure SQL `Redirect` routing response is unsupported for the same reason - * and fails the same safe way. tedious reconnects on a LOGIN7 routing envchange, - * but a zero-argument connector ignores the redirect target and reconnects to - * the pinned address — which is the behavior we want, since the redirect target - * is chosen by the server and honoring it would be the rebinding the pin exists - * to stop. Reach Azure SQL through a `Proxy`-policy connection. - * - * `requestTimeout` intentionally tracks `connectionTimeout` off the single knob - * the block exposes, which the field labels as covering both. tedious governs - * the whole login handshake (prelogin, TLS, LOGIN7) with `connectTimeout` - * regardless of the connector, so the socket's own timeout is cleared once it is - * connected rather than left to fire during a slow login. - * @see https://tediousjs.github.io/tedious/api-connection.html - * @see https://github.com/tediousjs/node-mssql#general-same-for-all-drivers - */ -export async function createMSSQLConnection( - config: MSSQLConnectionConfig -): Promise { - const hostValidation = await validateDatabaseHost(config.host, 'host') - if (!hostValidation.isValid) { - throw new Error(hostValidation.error) - } - - const pinnedAddress = hostValidation.resolvedIP ?? config.host - - const pool = new sql.ConnectionPool({ - server: config.host, - port: config.port, - database: config.database, - user: config.username, - password: config.password, - connectionTimeout: config.connectionTimeout, - requestTimeout: config.connectionTimeout, - pool: { - max: 1, - min: 0, - idleTimeoutMillis: 20000, - }, - options: { - encrypt: config.encrypt === 'enabled', - trustServerCertificate: config.trustServerCertificate === 'enabled', - connector: () => connectToPinnedAddress(pinnedAddress, config.port, config.connectionTimeout), - }, - }) - try { - await pool.connect() - } catch (error) { - /** - * Only a pool that was handed back gets closed by the route's `finally`, so - * a pool that failed to connect has to release its own tarn resources here - * or a bad credential retried in a loop leaks one every attempt. `close()` - * on a pool that never connected is a no-op rather than an error, and its - * own failure must not mask the connect error the caller needs to see. - */ - await pool.close().catch(() => {}) - throw error - } - - return pool -} - -export interface MSSQLQueryResult { - rows: unknown[] - rowCount: number - /** Set when the recordset hit a row or byte ceiling and rows were dropped. */ - truncated?: boolean - /** Human-readable explanation of the ceiling that was hit. */ - truncationReason?: string -} - -/** - * Prepares a JSON-sourced value for `request.input()`. - * - * With no explicit type, node-mssql infers one from the value — and its object - * branch only recognises `String`, `Number`, `Boolean`, `Date`, `Buffer`, and - * `Table`. Anything else (a nested object, an array) is inferred as `NVarChar` - * while staying an object, and tedious's `NVarChar.validate` then throws a bare - * `Invalid string.` with nothing naming the column. Since `data` arrives as - * arbitrary JSON, serializing those to JSON text is both the only sound binding - * and what a caller writing into an `nvarchar`/JSON column means. - * @see https://github.com/tediousjs/node-mssql#data-types - */ -function toBindableValue(value: unknown): unknown { - if (value === null || value === undefined) return value - if (typeof value !== 'object') return value - if (value instanceof Date || Buffer.isBuffer(value)) return value - return JSON.stringify(value) -} - -/** - * Ceilings on what a single statement may materialize into the response. - * - * The driver buffers the whole recordset before `request.query` resolves, and - * the route then serializes it into a JSON body, so an unbounded `SELECT` over a - * large table is held in memory twice. A caller who wants more pages it with - * `OFFSET ... FETCH NEXT`. The byte ceiling exists because row count alone does - * not bound size — 1,000 rows of `nvarchar(max)` is not a small result. - */ -const MSSQL_MAX_RESULT_ROWS = 10_000 -const MSSQL_MAX_RESULT_BYTES = 10 * 1024 * 1024 - -/** - * Bytes held back from {@link MSSQL_MAX_RESULT_BYTES} for the part of the - * response body that is not a row. - * - * {@link toRowsResponseBody} wraps `rows` in `message`, `rowCount`, and — when - * the recordset was capped — `truncated` and `truncationReason`, none of which - * the per-row accounting can see. Those are a few hundred bytes at their - * longest (the truncation prose is the bulk of it), so the reserve is set an - * order of magnitude above the worst case and costs 0.04% of the ceiling. The - * alternative, serializing the assembled body to check it, would re-serialize - * the whole recordset a second time for no useful precision. - */ -const MSSQL_RESPONSE_ENVELOPE_BYTES = 4096 - -/** What the serialized `rows` array itself may occupy. */ -const MSSQL_MAX_ROWS_BYTES = MSSQL_MAX_RESULT_BYTES - MSSQL_RESPONSE_ENVELOPE_BYTES - -/** - * Truncates a recordset to the row and byte ceilings. - * - * Measures each row with `JSON.stringify` because that is what the route will do - * anyway, so the number bounds the response the caller actually receives rather - * than an in-memory estimate that does not correspond to it. Each row is - * serialized exactly once and its cost accumulated, rather than re-serializing - * the growing array per row, which would be quadratic on a large recordset. - * - * The size is `Buffer.byteLength(..., 'utf8')`, not `String.length`. `length` - * counts UTF-16 code units while `NextResponse.json` emits UTF-8, and every - * character above U+007F costs more bytes than code units — worst case 3:1, for - * the U+0800–U+FFFF range that holds CJK, so a recordset of Chinese text passed - * a 10 MB `length` budget while serializing to nearly 30 MB. (Astral characters - * such as emoji are only 2:1: 4 bytes across 2 surrogate code units.) - * - * The array's own punctuation is counted too — one byte per row covers the - * opening `[` for the first row and the separating `,` for each one after it, - * with the leading byte standing in for the closing `]` — and - * {@link MSSQL_RESPONSE_ENVELOPE_BYTES} covers the fields around it. Without - * both, a result packed exactly to the ceiling still emitted a body over it. - * - * A row is admitted only when it still fits, so a single row larger than the - * byte ceiling is dropped rather than admitted as a lone exception — otherwise - * `SELECT` of one `nvarchar(max)` value would serialize an unbounded body and - * the ceiling would bound everything except the case it exists for. The drop is - * disclosed through {@link MSSQLQueryResult.truncationReason}, so an empty - * recordset is never mistaken for an empty table. - */ -function capRecordset(rows: unknown[]): { rows: unknown[]; truncated: boolean } { - if (rows.length === 0) return { rows, truncated: false } - - const capped: unknown[] = [] - /** The closing `]`; each row below pays for its own `[` or `,`. */ - let bytes = 1 - - for (const row of rows) { - if (capped.length >= MSSQL_MAX_RESULT_ROWS) break - const serialized = JSON.stringify(row) - /** - * `JSON.stringify` answers `undefined` for a value it cannot represent, but - * an array element in that position serializes as the four bytes of `null`. - */ - const rowBytes = serialized === undefined ? 4 : Buffer.byteLength(serialized, 'utf8') - if (bytes + rowBytes + 1 > MSSQL_MAX_ROWS_BYTES) break - bytes += rowBytes + 1 - capped.push(row) - } - - return { rows: capped, truncated: capped.length < rows.length } -} - -/** - * Runs a statement with positional values bound as `@param1`, `@param2`, … . - * - * `recordset` holds the first result set and `rowsAffected` holds one count per - * statement, so a SELECT reports its row count and a DML statement reports the - * summed affected rows. - * @see https://github.com/tediousjs/node-mssql#request - */ -export async function executeQuery( - pool: sql.ConnectionPool, - query: string, - values: unknown[] = [] -): Promise { - const request = pool.request() - values.forEach((value, index) => { - request.input(`param${index + 1}`, toBindableValue(value)) - }) - - const result = await request.query(query) - const { rows, truncated } = capRecordset(result.recordset ?? []) - const affected = (result.rowsAffected ?? []).reduce( - (total: number, count: number) => total + count, - 0 - ) - - return { - rows, - rowCount: rows.length > 0 ? rows.length : affected, - ...(truncated && { - truncated: true, - truncationReason: - rows.length === 0 - ? `No rows returned: the first row alone exceeds the ${MSSQL_MAX_RESULT_BYTES / (1024 * 1024)} MB response ceiling. Select fewer columns, or slice large values with SUBSTRING.` - : `Result truncated to ${rows.length} row(s): a single statement returns at most ${MSSQL_MAX_RESULT_ROWS} rows or ${MSSQL_MAX_RESULT_BYTES / (1024 * 1024)} MB. Page with OFFSET ... FETCH NEXT to read the rest.`, - }), - } -} - -/** - * Builds the success body every statement route returns. - * - * A truncated recordset is disclosed twice on purpose: folded into `message`, so - * an agent that reads only the status line still learns rows were dropped, and - * as `truncated`/`truncationReason`, so a caller can branch on it without - * parsing prose. Without this the route reported a capped result as a complete - * one and paging looked unnecessary. - */ -export function toRowsResponseBody(result: MSSQLQueryResult, message: string) { - return { - message: result.truncationReason ? `${message} ${result.truncationReason}` : message, - rows: result.rows, - rowCount: result.rowCount, - ...(result.truncated && { - truncated: true, - truncationReason: result.truncationReason, - }), - } -} - -/** - * Every T-SQL keyword that introduces a statement with an effect — DML, DDL, - * permissions, and the administrative commands (`DBCC`, `BACKUP`, `RESTORE`, - * `SHUTDOWN`, `KILL`, …) that are easy to forget precisely because they are not - * DML. One list serves both the read-only query screen and the WHERE screen so - * the two cannot drift apart; a gap in either is a gap in both, which is how - * `DBCC` slipped past a per-site list. - * - * `DISABLE`/`ENABLE` are here because `SELECT 1 DISABLE TRIGGER dbo.audit ON - * dbo.users` is a valid semicolon-less batch that turns auditing off, and - * `SET`/`BEGIN`/`COMMIT`/`ROLLBACK` because session and transaction state are - * changed the same way (`SET IDENTITY_INSERT`, `SET ANSI_NULLS`). The rest of - * that family — `SAVE TRANSACTION`, the symmetric/master key statements, - * `ADD SIGNATURE`, and `RAISERROR ... WITH LOG` — opens with a word that is also - * an ordinary identifier, so it is screened as a two-token phrase in - * {@link MSSQL_STATEMENT_PHRASES} instead. - * - * The text statements `UPDATETEXT`, `WRITETEXT`, and `READTEXT` are listed in - * their own right rather than left to `update`: there is no word boundary after - * `update` in `UPDATETEXT`, so `\bupdate\b` never matches it and - * `SELECT 1 UPDATETEXT dbo.t.col @ptr 0 NULL 'x'` would otherwise pass every - * screen on the advertised read-only path. `READTEXT` reads rather than writes, - * but it introduces a second statement in exactly the same semicolon-less way, - * which is what this list exists to reject. - * - * `RENAME` is documented T-SQL DDL — it applies to Azure Synapse Analytics - * dedicated SQL pools and Analytics Platform System, both of which speak TDS on - * port 1433 and are reachable with exactly the connection fields this block - * exposes. `SELECT 1 RENAME OBJECT dbo.Customer TO Customer1` is a valid - * semicolon-less batch that changes schema through an operation advertised as - * read-only, and `RENAME DATABASE` and `RENAME OBJECT … COLUMN … TO …` reach it - * the same way. - * - * `RECEIVE` is the Service Broker read that *removes* the messages it returns, - * so it is a write in everything but name. Its siblings — `END`/`MOVE`/`GET` - * `CONVERSATION` and `SEND ON CONVERSATION` — open with words that are ordinary - * identifiers (`END` closes every `CASE`), so they are screened as phrases in - * {@link MSSQL_STATEMENT_PHRASES} instead. - * - * `FETCH` is deliberately **absent**: `OFFSET … FETCH NEXT` is the standard - * T-SQL paging clause, so screening it would reject the ordinary paged SELECT - * this operation exists to run. Word boundaries keep the additions off ordinary - * identifiers — `settled`, `offset_value`, and `begin_date` all match nothing. - * @see https://learn.microsoft.com/en-us/sql/t-sql/statements/statements - */ -const MSSQL_STATEMENT_KEYWORDS = - /\b(?:insert|update|updatetext|writetext|readtext|delete|merge|drop|create|alter|truncate|rename|receive|disable|enable|set|begin|commit|rollback|grant|revoke|deny|exec|execute|backup|restore|shutdown|reconfigure|dbcc|kill|checkpoint|use|bulk|revert|setuser|openrowset|opendatasource|openquery|openxml|waitfor|into|deallocate)\b/i - -/** - * The remaining session, transaction, cursor, and key-management statements, - * every one of which is a valid semicolon-less second statement the single-word - * list above cannot carry. - * - * Each is matched as a **two-token** phrase rather than a bare word, because the - * leading words are ordinary identifiers: `open` and `close` are columns in any - * price table, `save` and `add` are common verbs, and `END` closes every `CASE`. - * Screening those bare would reject the plain SELECTs this operation exists to - * run. `DEALLOCATE` is the one exception and lives in the word list above — it - * has no ordinary-identifier reading. - * - * Most of these write neither table data nor schema, which is why they were - * missed; they are screened because the file's stated rule is that a second - * statement is rejected structurally, not by what it happens to do. - * `RAISERROR ... WITH LOG` writes to the error log and the Windows application - * log, so it is not inert. The Service Broker conversation statements are not - * inert either: `END CONVERSATION ... WITH CLEANUP` drops every message in a - * conversation, `MOVE CONVERSATION` reassigns it, and `SEND ON CONVERSATION` - * enqueues a message — and the handles they need are enumerable through this - * same path, because the catalog screen applies only to WHERE clauses. - * @see https://learn.microsoft.com/en-us/sql/t-sql/statements/end-conversation-transact-sql - * @see https://learn.microsoft.com/en-us/sql/t-sql/statements/statements - */ -const MSSQL_STATEMENT_PHRASES: readonly RegExp[] = [ - /\bsave\s+tran(?:saction)?\b/i, - /\bopen\s+(?:symmetric|master)\s+key\b/i, - /\bclose\s+(?:all\s+symmetric\s+keys|master\s+key|symmetric\s+key)\b/i, - /\badd\s+signature\b/i, - /\braiserror[\s\S]*?\bwith\s+log\b/i, - /\b(?:end|move|get)\s+conversation\b/i, - /\bsend\s+on\s+conversation\b/i, -] - -/** Matches the first screened statement phrase, or `null`. */ -function matchStatementPhrase(masked: string): string | null { - for (const pattern of MSSQL_STATEMENT_PHRASES) { - const match = pattern.exec(masked) - if (match) return match[0] - } - return null -} - -/** Extended, OLE-automation, and system stored procedures, called with or without `EXEC`. */ -const MSSQL_PROCEDURE_PATTERN = /\b(?:xp_|sp_)\w+/i - -/** - * Rejects a second statement in a batch. - * - * T-SQL treats the semicolon as optional, so this catches only the explicit - * form; the keyword screen above is what catches the semicolon-less one. Run on - * literal-masked text so a semicolon inside a quoted value is not a statement. - */ -const MSSQL_STACKED_STATEMENT = /;\s*\S/ - -/** - * Rejects SQL comments in the read-only path. - * - * A block comment placed inside a keyword splits it as far as a lexical scan is - * concerned, so no amount of keyword coverage helps if the server rejoins the - * halves into one token. Rather than model how the server's tokenizer treats an - * interior comment — which cannot be settled without a live instance — the - * read-only path refuses comments outright. A SELECT submitted through this - * operation has no need for one, and Execute Raw SQL still accepts them. - * - * {@link maskSqlStringLiterals} deliberately leaves comment markers intact, so - * this still fires after masking while `'-- not a comment'` inside a literal - * does not trip it. - */ -const MSSQL_COMMENT = /--|\/\*|\*\// - -/** - * A quote character {@link maskSqlStringLiterals} treats as opening a literal. - * Backtick is included because the masker honours it even though T-SQL does not. - */ -const MSSQL_MASKER_QUOTES = ["'", '"', '`'] as const - -/** A backslash directly before one of {@link MSSQL_MASKER_QUOTES}. */ -const MSSQL_BACKSLASH_ESCAPED_QUOTE = /\\['"`]/ - -/** Any quote character inside a bracket-quoted identifier. */ -const MSSQL_QUOTE_IN_BRACKETS = /\[[^\]]*['"`]/ - -/** - * Rejects input whose quoting would make literal masking unreliable. - * - * Every screen below runs over {@link maskSqlStringLiterals} output, so anything - * the masker hides is invisible to all of them — and the masker is written for - * the ANSI/MySQL dialect, not T-SQL. Three ways to desynchronise it, each of - * which lets real SQL hide inside what the masker believes is a string: - * - * 1. **An unpaired quote.** The masker runs to the end of the input looking for - * a partner, masking everything on the way. - * 2. **A quote inside a bracketed identifier.** Brackets are SQL Server's other - * quoting form and the masker does not track them, so `[a"] = 1 OR 1=1` - * reads as one unterminated literal. - * 3. **A backslash before a quote.** The masker treats `\` as an escape and - * swallows the quote that follows — but **T-SQL has no backslash escape**, so - * the server closes the literal there and executes the rest as code. This is - * the dangerous one, because it survives an even quote count: - * `a='x\' DELETE FROM dbo.t WHERE b='y'` holds four quotes and masks to - * `a= y`, hiding the DELETE from the keyword screen entirely. - * - * All three are rejected rather than modelled. T-SQL escapes a quote by doubling - * it and has no use for a backslash escape or a backtick, so nothing correct is - * turned away except a value ending in a literal backslash (`'C:\'`) — which the - * Execute Raw SQL operation still accepts. Failing closed here is what lets - * every screen below trust that what the mask left visible is all the code there - * is. - * @see https://learn.microsoft.com/en-us/sql/relational-databases/databases/database-identifiers - * @see https://learn.microsoft.com/en-us/sql/t-sql/data-types/constants-transact-sql - */ -function hasUnreliableQuoting(value: string): boolean { - for (const quote of MSSQL_MASKER_QUOTES) { - let count = 0 - for (const char of value) { - if (char === quote) count++ - } - if (count % 2 !== 0) return true - } - - return MSSQL_BACKSLASH_ESCAPED_QUOTE.test(value) || MSSQL_QUOTE_IN_BRACKETS.test(value) -} - -/** - * Restricts the Query operation to statements that only read. - * - * The block label, the tool description, and the docs all present this - * operation as SELECT-only, so it must not double as a second path to DML — - * `mssql_execute` is the operation that accepts mutations. Without this an - * agent choosing `mssql_query` because "it is only a SELECT" could delete rows. - * - * Four screens, deliberately layered so no single one has to be exhaustive: the - * statement must open with `SELECT` (or a leading `WITH`, since a CTE is the - * normal way to write a non-trivial SELECT); it may not contain a comment, which - * would otherwise let a keyword be split in half; it may not contain a second - * statement after a semicolon, which rejects `SELECT 1; ` structurally - * rather than by naming the anything; and it may not mention a - * statement-introducing keyword or a stored procedure, which covers both the - * semicolon-less batch and the `WITH x AS (...) DELETE FROM x` form that a - * leading-token check alone would miss. - * - * All screening runs over literal-masked text, so ordinary prose in a WHERE - * clause does not trip it. The screens are lexical rather than a parser, so a - * query using a keyword as a bare identifier is rejected too — it fails closed, - * and the Execute Raw SQL operation is the escape hatch. - */ -export function validateReadOnlyQuery(query: string): { isValid: boolean; error?: string } { - const trimmedQuery = query.trim() - - /** - * `\b` rather than `\s`: T-SQL does not require whitespace after the keyword, - * so `SELECT*FROM dbo.users` and `SELECT(1)` are valid reads that a - * whitespace-anchored check would push to Execute Raw SQL for no reason. The - * boundary still refuses `SELECTX`, and it cannot loosen the screen overall — - * the keyword and batch checks below run on the whole statement regardless of - * how it opens. - */ - if (!/^(?:select|with)\b/i.test(trimmedQuery)) { - return { - isValid: false, - error: - 'The Query operation only accepts SELECT statements, optionally led by a WITH clause. Use the Execute Raw SQL operation to run anything else.', - } - } - - if (hasUnreliableQuoting(trimmedQuery)) { - return { - isValid: false, - error: - 'The Query operation could not read this statement reliably: it has an unpaired quote, a quote inside a bracketed identifier, or a backslash before a quote. T-SQL escapes a quote by doubling it.', - } - } - - const masked = maskSqlStringLiterals(trimmedQuery) - - if (MSSQL_COMMENT.test(masked)) { - return { - isValid: false, - error: - 'The Query operation does not accept SQL comments, because a comment can split a keyword. Use the Execute Raw SQL operation if the statement needs one.', - } - } - - if (MSSQL_STACKED_STATEMENT.test(masked)) { - return { - isValid: false, - error: - 'The Query operation runs a single SELECT statement. Use the Execute Raw SQL operation to run a batch.', - } - } - - const disallowed = - MSSQL_STATEMENT_KEYWORDS.exec(masked)?.[0] ?? - MSSQL_PROCEDURE_PATTERN.exec(masked)?.[0] ?? - matchStatementPhrase(masked) - if (disallowed) { - return { - isValid: false, - error: `The Query operation cannot run ${disallowed.toUpperCase()}. Use the Execute Raw SQL operation for statements that modify data, schema, or server state.`, - } - } - - return { isValid: true } -} - -/** - * Restricts Execute Raw SQL to a statement kind the operation advertises. - * - * The anchor is `\b` rather than `\s`, matching the read-only screen, because - * T-SQL does not require whitespace after the opening keyword: `EXEC(@sql)` is - * the ordinary way to run dynamic SQL and `SELECT(1)` is a valid read, both of - * which a whitespace anchor refused on the one operation meant to accept them. - * `EXECUTE x` still matches — the alternation backtracks from `exec` to - * `execute` when the boundary fails — and `SELECTX` is still refused. - * - * This is a statement-kind check, not a security screen. Execute Raw SQL is the - * deliberate escape hatch: the caller supplies their own credentials, so the - * boundary is what those credentials may do, not what this pattern matches. - */ -export function validateQuery(query: string): { isValid: boolean; error?: string } { - const trimmedQuery = query.trim() - - const allowedStatements = /^(select|insert|update|delete|with|merge|exec|execute|declare)\b/i - if (!allowedStatements.test(trimmedQuery)) { - return { - isValid: false, - error: - 'Only SELECT, INSERT, UPDATE, DELETE, WITH, MERGE, EXEC, EXECUTE, and DECLARE statements are allowed', - } - } - - return { isValid: true } -} - -export function buildInsertQuery(table: string, data: Record) { - const sanitizedTable = sanitizeIdentifier(table) - const columns = Object.keys(data) - const values = Object.values(data) - const placeholders = columns.map((_, index) => `@param${index + 1}`).join(', ') - - const query = `INSERT INTO ${sanitizedTable} (${columns.map(sanitizeIdentifier).join(', ')}) VALUES (${placeholders})` - - return { query, values } -} - -export function buildUpdateQuery(table: string, data: Record, where: string) { - validateWhereClause(where) - - const sanitizedTable = sanitizeIdentifier(table) - const columns = Object.keys(data) - const values = Object.values(data) - - const setClause = columns - .map((col, index) => `${sanitizeIdentifier(col)} = @param${index + 1}`) - .join(', ') - const query = `UPDATE ${sanitizedTable} SET ${setClause} WHERE ${where}` - - return { query, values } -} - -export function buildDeleteQuery(table: string, where: string) { - validateWhereClause(where) - - const sanitizedTable = sanitizeIdentifier(table) - const query = `DELETE FROM ${sanitizedTable} WHERE ${where}` - - return { query, values: [] as unknown[] } -} - -/** - * Rejects `SELECT` inside an update or delete WHERE clause. - * - * `SELECT` cannot live in the shared {@link MSSQL_STATEMENT_KEYWORDS} list, since - * the read-only query screen exists to *permit* it — but appended to a WHERE it - * is an exfiltration channel: `id = 1 SELECT secret FROM dbo.credentials` runs - * as a second statement and the route hands back its recordset as though it were - * the mutation's result. - * - * The cost is that a subquery condition (`id IN (SELECT ...)`) is refused here. - * That is a deliberate trade — the WHERE text is interpolated rather than bound, - * so the screen cannot tell a subquery from an appended statement, and the - * Execute Raw SQL operation covers a subquery-driven update. - */ -const MSSQL_WHERE_SELECT = /\bselect\b/i - -/** - * SQL Server catalog surfaces a WHERE clause has no business reading: the - * information schema, the `sys.*` catalog views, and the legacy compatibility - * views still reachable as `master..sysobjects`. - * @see https://learn.microsoft.com/en-us/sql/relational-databases/system-catalog-views/catalog-views-transact-sql - */ -/** - * Constant tautologies the shared guard's `OR ` rule cannot see because - * a parenthesis or a `NOT` sits between the operator and the constant — - * `OR (1)`, `OR ((1))`, `OR NOT 0`, `OR NOT (FALSE)`. - * - * Both patterns require the constant to be the *whole* parenthesised term, so a - * real disjunct is untouched: `OR (1 = priority)` does not match, because a - * closing paren does not follow the digit. - * - * This narrows the gap; it does not close it, and it is not meant to. An - * always-true expression cannot be recognised lexically in general — - * `OR 2 > 1`, `OR LEN(x) >= 0`, and `OR id IS NOT NULL` all survive any pattern - * list — which is why {@link validateWhereClause} is documented as - * defense-in-depth rather than a boundary. - */ -const MSSQL_WHERE_CONSTANT_TAUTOLOGY: readonly RegExp[] = [ - /\bor\s+(?:not\s+)*\(+\s*(?:\d+(?:\.\d+)?|true|false)\s*\)+/i, - /\bor\s+not\s+(?:\d+(?:\.\d+)?|true|false)\b/i, -] - -const MSSQL_CATALOG_PATTERNS: readonly RegExp[] = [ - /information_schema/i, - /\bsys\./i, - /\.\.\s*sys\w*/i, - /\bsys(?:objects|columns|databases|users|indexes|comments)\b/i, -] - -/** - * Rejects WHERE clauses containing injection or always-true tautology patterns - * so a user-supplied condition cannot broaden an update or delete to every row. - * - * Delegates the shared checks to {@link validateSqlWhereClause} — which masks - * string literals before scanning, so prose inside a quoted value cannot trip a - * structural pattern — then adds the two things it cannot know about. - * - * The first is the one that does not generalize: **T-SQL does not require a - * statement terminator**, so `id = 1 DBCC SHRINKDATABASE('db')` is a valid - * two-statement batch and every semicolon-anchored stacked-query check, the - * shared guard's included, reads straight past it. That is why the screen is - * {@link MSSQL_STATEMENT_KEYWORDS} — the same list the read-only query screen - * uses, so a keyword can never be covered in one place and missed in the other. - * Word boundaries keep ordinary column names (`updated_at`, `deleted_at`, - * `created_by`) matching nothing; a column whose name *is* a bare keyword has - * to be reached through the Execute Raw SQL operation. The second is the - * catalog surface above, plus the `xp_`/`sp_` procedures. - * - * As the shared guard's own documentation states, this is defense-in-depth - * rather than a security boundary: the caller supplies their own database - * credentials and can run equivalent SQL through the Execute Raw SQL operation. - * It stops the easy ways an injected condition escalates, nothing more. - * @throws {Error} If the WHERE clause matches any screened pattern - */ -function validateWhereClause(where: string): void { - if (hasUnreliableQuoting(where)) { - throw new Error( - 'WHERE clause has an unpaired quote, a quote inside a bracketed identifier, or a backslash before a quote. T-SQL escapes a quote by doubling it.' - ) - } - - const shared = validateSqlWhereClause(where, 'WHERE clause') - if (!shared.isValid) { - throw new Error(shared.error) - } - - const masked = maskSqlStringLiterals(where) - if ( - MSSQL_STATEMENT_KEYWORDS.test(masked) || - matchStatementPhrase(masked) !== null || - MSSQL_PROCEDURE_PATTERN.test(masked) || - MSSQL_WHERE_SELECT.test(masked) || - MSSQL_WHERE_CONSTANT_TAUTOLOGY.some((pattern) => pattern.test(masked)) || - MSSQL_CATALOG_PATTERNS.some((pattern) => pattern.test(masked)) - ) { - throw new Error('WHERE clause contains potentially dangerous operation') - } -} - -export function sanitizeIdentifier(identifier: string): string { - if (identifier.includes('.')) { - const parts = identifier.split('.') - return parts.map((part) => sanitizeSingleIdentifier(part)).join('.') - } - - return sanitizeSingleIdentifier(identifier) -} - -function sanitizeSingleIdentifier(identifier: string): string { - const cleaned = identifier.replace(/[[\]]/g, '') - - if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(cleaned)) { - throw new Error( - `Invalid identifier: ${identifier}. Identifiers must start with a letter or underscore and contain only letters, numbers, and underscores.` - ) - } - - return `[${cleaned}]` -} - -export interface MSSQLIntrospectionResult { - tables: Array<{ - name: string - schema: string - columns: Array<{ - name: string - type: string - nullable: boolean - default: string | null - isPrimaryKey: boolean - isForeignKey: boolean - references?: { - schema: string - table: string - column: string - } - }> - primaryKey: string[] - foreignKeys: Array<{ - column: string - referencesSchema: string - referencesTable: string - referencesColumn: string - }> - indexes: Array<{ - name: string - columns: string[] - unique: boolean - }> - }> - schemas: string[] -} - -interface SchemaRow { - SCHEMA_NAME: string -} - -interface TableRow { - TABLE_NAME: string - TABLE_SCHEMA: string -} - -interface ColumnRow { - TABLE_NAME: string - COLUMN_NAME: string - DATA_TYPE: string - IS_NULLABLE: string - COLUMN_DEFAULT: string | null -} - -interface KeyColumnRow { - TABLE_NAME: string - COLUMN_NAME: string -} - -interface ForeignKeyRow { - TABLE_NAME: string - COLUMN_NAME: string - REFERENCED_TABLE_SCHEMA: string - REFERENCED_TABLE_NAME: string - REFERENCED_COLUMN_NAME: string -} - -interface IndexRow { - TABLE_NAME: string - INDEX_NAME: string - COLUMN_NAME: string - IS_UNIQUE: boolean | number -} - -/** - * Reads table, column, key, and index metadata for a schema. - * - * Every view read here except `sys.schemas` is metadata-visibility filtered — - * "limited to securables that a user either owns, or on which the user was - * granted some permission" — so a low-privilege login gets a silently partial - * result rather than an error. - * @see https://learn.microsoft.com/en-us/sql/relational-databases/system-information-schema-views/system-information-schema-views-transact-sql - * @see https://learn.microsoft.com/en-us/sql/relational-databases/security/metadata-visibility-configuration - */ -export async function executeIntrospect( - pool: sql.ConnectionPool, - schemaName: string -): Promise { - /** - * `sys.schemas` rather than `INFORMATION_SCHEMA.SCHEMATA` because it needs - * only membership in `public` and carries no metadata-visibility caveat. - * @see https://learn.microsoft.com/en-us/sql/relational-databases/system-catalog-views/schemas-catalog-views-sys-schemas - */ - const schemasResult = await pool.request().query( - `SELECT s.name AS SCHEMA_NAME - FROM sys.schemas s - WHERE s.name NOT IN ('sys', 'INFORMATION_SCHEMA', 'guest', - 'db_accessadmin', 'db_backupoperator', 'db_datareader', 'db_datawriter', - 'db_ddladmin', 'db_denydatareader', 'db_denydatawriter', 'db_owner', 'db_securityadmin') - ORDER BY s.name` - ) - const schemas = schemasResult.recordset.map((row: SchemaRow) => row.SCHEMA_NAME) - - const tablesResult = await pool - .request() - .input('schema', schemaName) - .query( - `SELECT TABLE_NAME, TABLE_SCHEMA - FROM INFORMATION_SCHEMA.TABLES - WHERE TABLE_SCHEMA = @schema AND TABLE_TYPE = 'BASE TABLE' - ORDER BY TABLE_NAME` - ) - - const tableRows = tablesResult.recordset as TableRow[] - if (tableRows.length === 0) return { tables: [], schemas } - - /** - * The column, primary key, foreign key, and index reads below are filtered by - * schema and grouped in memory, rather than run once per table. Per-table they - * were four round trips each — a 500-table schema meant ~2,000 sequential - * queries, every one under its own connection timeout. - */ - const columnsResult = await pool - .request() - .input('schema', schemaName) - .query( - `SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE, IS_NULLABLE, COLUMN_DEFAULT - FROM INFORMATION_SCHEMA.COLUMNS - WHERE TABLE_SCHEMA = @schema - ORDER BY TABLE_NAME, ORDINAL_POSITION` - ) - - const pkResult = await pool - .request() - .input('schema', schemaName) - .query( - `SELECT tc.TABLE_NAME, kcu.COLUMN_NAME - FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc - JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu - ON tc.CONSTRAINT_NAME = kcu.CONSTRAINT_NAME - AND tc.CONSTRAINT_SCHEMA = kcu.CONSTRAINT_SCHEMA - WHERE tc.CONSTRAINT_TYPE = 'PRIMARY KEY' - AND tc.TABLE_SCHEMA = @schema - ORDER BY tc.TABLE_NAME, kcu.ORDINAL_POSITION` - ) - - const fkResult = await pool - .request() - .input('schema', schemaName) - .query( - /** - * Resolved through the catalog views rather than - * `INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS`, which reaches the - * referenced side by joining `TABLE_CONSTRAINTS` — a view that returns - * "one row for each table constraint" and so has no row at all when a - * foreign key references a unique *index*, silently dropping the key. - * The catalog views resolve the referenced table and column by ID. - * @see https://learn.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-foreign-key-columns-transact-sql - * @see https://learn.microsoft.com/en-us/sql/relational-databases/system-information-schema-views/table-constraints-transact-sql - */ - `SELECT - pt.name AS TABLE_NAME, - pc.name AS COLUMN_NAME, - rs.name AS REFERENCED_TABLE_SCHEMA, - rt.name AS REFERENCED_TABLE_NAME, - rc.name AS REFERENCED_COLUMN_NAME - FROM sys.foreign_keys fk - JOIN sys.foreign_key_columns fkc ON fkc.constraint_object_id = fk.object_id - JOIN sys.tables pt ON pt.object_id = fk.parent_object_id - JOIN sys.schemas ps ON ps.schema_id = pt.schema_id - JOIN sys.columns pc - ON pc.object_id = fkc.parent_object_id AND pc.column_id = fkc.parent_column_id - JOIN sys.tables rt ON rt.object_id = fkc.referenced_object_id - JOIN sys.schemas rs ON rs.schema_id = rt.schema_id - JOIN sys.columns rc - ON rc.object_id = fkc.referenced_object_id AND rc.column_id = fkc.referenced_column_id - WHERE ps.name = @schema - ORDER BY pt.name, fk.name, fkc.constraint_column_id` - ) - - const indexResult = await pool - .request() - .input('schema', schemaName) - .query( - /** - * `key_ordinal > 0` restricts the result to key columns: it is the - * "ordinal (1-based) within set of key-columns", and `0` marks INCLUDEd - * non-key columns, partitioning columns, **and every column of an XML, - * spatial, columnstore, or JSON index**. The partitioning columns are - * why `is_included_column` alone is not enough — those report `0` for it - * too. The index families are the cost of the filter: they contribute no - * key column, so they are absent from the result rather than listed with - * an empty column set. Rowstore keys, which is what a query planner - * reader is after, are reported in full. - * - * `is_hypothetical = 0` drops the statistics-only indexes the Database - * Engine Tuning Advisor leaves behind ("can't be used directly as a data - * access path"), and `is_disabled = 0` drops indexes that exist but are - * not maintained. Reporting either as a live index misleads. - * - * `is_primary_key = 0` keeps the primary key out, since `primaryKey` - * carries it already. A UNIQUE *constraint* is deliberately left in: it - * is a unique index and nothing else in the result reports it. - * @see https://learn.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-index-columns-transact-sql - * @see https://learn.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-indexes-transact-sql - */ - `SELECT t.name AS TABLE_NAME, i.name AS INDEX_NAME, c.name AS COLUMN_NAME, - i.is_unique AS IS_UNIQUE - FROM sys.indexes i - JOIN sys.index_columns ic - ON i.object_id = ic.object_id AND i.index_id = ic.index_id - JOIN sys.columns c - ON ic.object_id = c.object_id AND ic.column_id = c.column_id - JOIN sys.tables t ON i.object_id = t.object_id - JOIN sys.schemas s ON t.schema_id = s.schema_id - WHERE s.name = @schema - AND i.is_primary_key = 0 - AND i.is_hypothetical = 0 - AND i.is_disabled = 0 - AND i.name IS NOT NULL - AND ic.key_ordinal > 0 - ORDER BY t.name, i.name, ic.key_ordinal` - ) - - /** Groups rows by their `TABLE_NAME`, preserving each group's server order. */ - function groupByTable(rows: TRow[]): Map { - const grouped = new Map() - for (const row of rows) { - const existing = grouped.get(row.TABLE_NAME) - if (existing) existing.push(row) - else grouped.set(row.TABLE_NAME, [row]) - } - return grouped - } - - const columnsByTable = groupByTable(columnsResult.recordset as ColumnRow[]) - const pkByTable = groupByTable(pkResult.recordset as KeyColumnRow[]) - const fkByTable = groupByTable(fkResult.recordset as ForeignKeyRow[]) - const indexRowsByTable = groupByTable(indexResult.recordset as IndexRow[]) - - const tables: MSSQLIntrospectionResult['tables'] = [] - - for (const tableRow of tableRows) { - const tableName = tableRow.TABLE_NAME - const tableSchema = tableRow.TABLE_SCHEMA - - const primaryKeyColumns = (pkByTable.get(tableName) ?? []).map((row) => row.COLUMN_NAME) - - const foreignKeys = (fkByTable.get(tableName) ?? []).map((row) => ({ - column: row.COLUMN_NAME, - referencesSchema: row.REFERENCED_TABLE_SCHEMA, - referencesTable: row.REFERENCED_TABLE_NAME, - referencesColumn: row.REFERENCED_COLUMN_NAME, - })) - - const fkByColumn = new Map() - for (const fk of foreignKeys) { - if (!fkByColumn.has(fk.column)) fkByColumn.set(fk.column, fk) - } - - const indexMap = new Map() - for (const row of indexRowsByTable.get(tableName) ?? []) { - const indexName = row.INDEX_NAME - if (!indexMap.has(indexName)) { - indexMap.set(indexName, { name: indexName, columns: [], unique: Boolean(row.IS_UNIQUE) }) - } - indexMap.get(indexName)!.columns.push(row.COLUMN_NAME) - } - const indexes = Array.from(indexMap.values()) - - const primaryKeySet = new Set(primaryKeyColumns) - - const columns = (columnsByTable.get(tableName) ?? []).map((col) => { - const columnName = col.COLUMN_NAME - const fk = fkByColumn.get(columnName) - - return { - name: columnName, - type: col.DATA_TYPE, - nullable: col.IS_NULLABLE === 'YES', - default: col.COLUMN_DEFAULT ?? null, - isPrimaryKey: primaryKeySet.has(columnName), - isForeignKey: fk !== undefined, - ...(fk && { - references: { - schema: fk.referencesSchema, - table: fk.referencesTable, - column: fk.referencesColumn, - }, - }), - } - }) - - tables.push({ - name: tableName, - schema: tableSchema, - columns, - primaryKey: primaryKeyColumns, - foreignKeys, - indexes, - }) - } - - return { tables, schemas } -} diff --git a/apps/sim/app/api/tools/mysql/delete/route.ts b/apps/sim/app/api/tools/mysql/delete/route.ts deleted file mode 100644 index 5fedac77ff9..00000000000 --- a/apps/sim/app/api/tools/mysql/delete/route.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { mysqlDeleteContract } from '@/lib/api/contracts/tools/databases/mysql' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { buildDeleteQuery, createMySQLConnection, executeQuery } from '@/app/api/tools/mysql/utils' - -const logger = createLogger('MySQLDeleteAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized MySQL delete attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(mysqlDeleteContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Deleting data from ${params.table} on ${params.host}:${params.port}/${params.database}` - ) - - const connection = await createMySQLConnection({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - ssl: params.ssl, - }) - - try { - const { query, values } = buildDeleteQuery(params.table, params.where) - const result = await executeQuery(connection, query, values) - - logger.info(`[${requestId}] Delete executed successfully, ${result.rowCount} row(s) deleted`) - - return NextResponse.json({ - message: `Data deleted successfully. ${result.rowCount} row(s) affected.`, - rows: result.rows, - rowCount: result.rowCount, - }) - } finally { - await connection.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] MySQL delete failed:`, error) - - return NextResponse.json({ error: `MySQL delete failed: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/mysql/execute/route.ts b/apps/sim/app/api/tools/mysql/execute/route.ts deleted file mode 100644 index cdebfeb8107..00000000000 --- a/apps/sim/app/api/tools/mysql/execute/route.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { mysqlExecuteContract } from '@/lib/api/contracts/tools/databases/mysql' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createMySQLConnection, executeQuery, validateQuery } from '@/app/api/tools/mysql/utils' - -const logger = createLogger('MySQLExecuteAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized MySQL execute attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(mysqlExecuteContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Executing raw SQL on ${params.host}:${params.port}/${params.database}` - ) - - const validation = validateQuery(params.query) - if (!validation.isValid) { - logger.warn(`[${requestId}] Query validation failed: ${validation.error}`) - return NextResponse.json( - { error: `Query validation failed: ${validation.error}` }, - { status: 400 } - ) - } - - const connection = await createMySQLConnection({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - ssl: params.ssl, - }) - - try { - const result = await executeQuery(connection, params.query) - - logger.info(`[${requestId}] SQL executed successfully, ${result.rowCount} row(s) affected`) - - return NextResponse.json({ - message: `SQL executed successfully. ${result.rowCount} row(s) affected.`, - rows: result.rows, - rowCount: result.rowCount, - }) - } finally { - await connection.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] MySQL execute failed:`, error) - - return NextResponse.json({ error: `MySQL execute failed: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/mysql/insert/route.ts b/apps/sim/app/api/tools/mysql/insert/route.ts deleted file mode 100644 index 5c2e81f278b..00000000000 --- a/apps/sim/app/api/tools/mysql/insert/route.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { mysqlInsertContract } from '@/lib/api/contracts/tools/databases/mysql' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { buildInsertQuery, createMySQLConnection, executeQuery } from '@/app/api/tools/mysql/utils' - -const logger = createLogger('MySQLInsertAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized MySQL insert attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(mysqlInsertContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Inserting data into ${params.table} on ${params.host}:${params.port}/${params.database}` - ) - - const connection = await createMySQLConnection({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - ssl: params.ssl, - }) - - try { - const { query, values } = buildInsertQuery(params.table, params.data) - const result = await executeQuery(connection, query, values) - - logger.info(`[${requestId}] Insert executed successfully, ${result.rowCount} row(s) inserted`) - - return NextResponse.json({ - message: `Data inserted successfully. ${result.rowCount} row(s) affected.`, - rows: result.rows, - rowCount: result.rowCount, - }) - } finally { - await connection.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] MySQL insert failed:`, error) - - return NextResponse.json({ error: `MySQL insert failed: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/mysql/introspect/route.ts b/apps/sim/app/api/tools/mysql/introspect/route.ts deleted file mode 100644 index 6f4cad144a5..00000000000 --- a/apps/sim/app/api/tools/mysql/introspect/route.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { mysqlIntrospectContract } from '@/lib/api/contracts/tools/databases/mysql' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createMySQLConnection, executeIntrospect } from '@/app/api/tools/mysql/utils' - -const logger = createLogger('MySQLIntrospectAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized MySQL introspect attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(mysqlIntrospectContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Introspecting MySQL schema on ${params.host}:${params.port}/${params.database}` - ) - - const connection = await createMySQLConnection({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - ssl: params.ssl, - }) - - try { - const result = await executeIntrospect(connection, params.database) - - logger.info( - `[${requestId}] Introspection completed successfully, found ${result.tables.length} tables` - ) - - return NextResponse.json({ - message: `Schema introspection completed. Found ${result.tables.length} table(s) in database '${params.database}'.`, - tables: result.tables, - databases: result.databases, - }) - } finally { - await connection.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] MySQL introspection failed:`, error) - - return NextResponse.json( - { error: `MySQL introspection failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/mysql/query/route.ts b/apps/sim/app/api/tools/mysql/query/route.ts deleted file mode 100644 index e1ff6c039ef..00000000000 --- a/apps/sim/app/api/tools/mysql/query/route.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { mysqlQueryContract } from '@/lib/api/contracts/tools/databases/mysql' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createMySQLConnection, executeQuery, validateQuery } from '@/app/api/tools/mysql/utils' - -const logger = createLogger('MySQLQueryAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized MySQL query attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(mysqlQueryContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Executing MySQL query on ${params.host}:${params.port}/${params.database}` - ) - - const validation = validateQuery(params.query) - if (!validation.isValid) { - logger.warn(`[${requestId}] Query validation failed: ${validation.error}`) - return NextResponse.json( - { error: `Query validation failed: ${validation.error}` }, - { status: 400 } - ) - } - - const connection = await createMySQLConnection({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - ssl: params.ssl, - }) - - try { - const result = await executeQuery(connection, params.query) - - logger.info(`[${requestId}] Query executed successfully, returned ${result.rowCount} rows`) - - return NextResponse.json({ - message: `Query executed successfully. ${result.rowCount} row(s) returned.`, - rows: result.rows, - rowCount: result.rowCount, - }) - } finally { - await connection.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] MySQL query failed:`, error) - - return NextResponse.json({ error: `MySQL query failed: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/mysql/update/route.ts b/apps/sim/app/api/tools/mysql/update/route.ts deleted file mode 100644 index 5237c2a4b58..00000000000 --- a/apps/sim/app/api/tools/mysql/update/route.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { mysqlUpdateContract } from '@/lib/api/contracts/tools/databases/mysql' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { buildUpdateQuery, createMySQLConnection, executeQuery } from '@/app/api/tools/mysql/utils' - -const logger = createLogger('MySQLUpdateAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized MySQL update attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(mysqlUpdateContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Updating data in ${params.table} on ${params.host}:${params.port}/${params.database}` - ) - - const connection = await createMySQLConnection({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - ssl: params.ssl, - }) - - try { - const { query, values } = buildUpdateQuery(params.table, params.data, params.where) - const result = await executeQuery(connection, query, values) - - logger.info(`[${requestId}] Update executed successfully, ${result.rowCount} row(s) updated`) - - return NextResponse.json({ - message: `Data updated successfully. ${result.rowCount} row(s) affected.`, - rows: result.rows, - rowCount: result.rowCount, - }) - } finally { - await connection.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] MySQL update failed:`, error) - - return NextResponse.json({ error: `MySQL update failed: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/mysql/utils.ts b/apps/sim/app/api/tools/mysql/utils.ts deleted file mode 100644 index 971bc31ba21..00000000000 --- a/apps/sim/app/api/tools/mysql/utils.ts +++ /dev/null @@ -1,325 +0,0 @@ -import net from 'node:net' -import mysql from 'mysql2/promise' -import { validateDatabaseHost } from '@/lib/core/security/input-validation.server' - -export interface MySQLConnectionConfig { - host: string - port: number - database: string - username: string - password: string - ssl?: 'disabled' | 'required' | 'preferred' -} - -export async function createMySQLConnection(config: MySQLConnectionConfig) { - const hostValidation = await validateDatabaseHost(config.host, 'host') - if (!hostValidation.isValid) { - throw new Error(hostValidation.error) - } - - const resolvedIP = hostValidation.resolvedIP ?? config.host - - const connectionConfig: mysql.ConnectionOptions = { - host: config.host, - port: config.port, - database: config.database, - user: config.username, - password: config.password, - stream: () => { - const socket = net.connect({ host: resolvedIP, port: config.port, timeout: 10000 }) - socket.setNoDelay(true) - return socket - }, - } - - if (config.ssl === 'disabled') { - } else if (config.ssl === 'required') { - connectionConfig.ssl = { rejectUnauthorized: true } - } else if (config.ssl === 'preferred') { - connectionConfig.ssl = { rejectUnauthorized: false } - } - - return mysql.createConnection(connectionConfig) -} - -export async function executeQuery( - connection: mysql.Connection, - query: string, - values?: unknown[] -) { - const [rows, fields] = await connection.execute(query, values) - - if (Array.isArray(rows)) { - return { - rows: rows as unknown[], - rowCount: rows.length, - fields, - } - } - - return { - rows: [], - rowCount: (rows as mysql.ResultSetHeader).affectedRows || 0, - fields, - } -} - -export function validateQuery(query: string): { isValid: boolean; error?: string } { - const trimmedQuery = query.trim().toLowerCase() - - const allowedStatements = /^(select|insert|update|delete|with|show|describe|explain)\s+/i - if (!allowedStatements.test(trimmedQuery)) { - return { - isValid: false, - error: - 'Only SELECT, INSERT, UPDATE, DELETE, WITH, SHOW, DESCRIBE, and EXPLAIN statements are allowed', - } - } - - return { isValid: true } -} - -export function buildInsertQuery(table: string, data: Record) { - const sanitizedTable = sanitizeIdentifier(table) - const columns = Object.keys(data) - const values = Object.values(data) - const placeholders = columns.map(() => '?').join(', ') - - const query = `INSERT INTO ${sanitizedTable} (${columns.map(sanitizeIdentifier).join(', ')}) VALUES (${placeholders})` - - return { query, values } -} - -export function buildUpdateQuery(table: string, data: Record, where: string) { - validateWhereClause(where) - - const sanitizedTable = sanitizeIdentifier(table) - const columns = Object.keys(data) - const values = Object.values(data) - - const setClause = columns.map((col) => `${sanitizeIdentifier(col)} = ?`).join(', ') - const query = `UPDATE ${sanitizedTable} SET ${setClause} WHERE ${where}` - - return { query, values } -} - -export function buildDeleteQuery(table: string, where: string) { - validateWhereClause(where) - - const sanitizedTable = sanitizeIdentifier(table) - const query = `DELETE FROM ${sanitizedTable} WHERE ${where}` - - return { query, values: [] } -} - -/** - * Validates a WHERE clause to prevent SQL injection attacks - * @param where - The WHERE clause string to validate - * @throws {Error} If the WHERE clause contains potentially dangerous patterns - */ -function validateWhereClause(where: string): void { - const dangerousPatterns = [ - // DDL and DML injection via stacked queries - /;\s*(drop|delete|insert|update|create|alter|grant|revoke)/i, - // Union-based injection - /union\s+(all\s+)?select/i, - // File operations - /into\s+outfile/i, - /into\s+dumpfile/i, - /load_file\s*\(/i, - // Comment-based injection (can truncate query) - /--/, - /\/\*/, - /\*\//, - // Tautologies - always true/false conditions using backreferences - // Matches OR 'x'='x' or OR x=x (same value both sides) but NOT OR col='value' - /\bor\s+(['"]?)(\w+)\1\s*=\s*\1\2\1/i, - /\bor\s+true\b/i, - /\bor\s+false\b/i, - // AND tautologies (less common but still used in attacks) - /\band\s+(['"]?)(\w+)\1\s*=\s*\1\2\1/i, - /\band\s+true\b/i, - /\band\s+false\b/i, - // Time-based blind injection - /\bsleep\s*\(/i, - /\bbenchmark\s*\(/i, - /\bwaitfor\s+delay/i, - // Stacked queries (any statement after semicolon) - /;\s*\w+/, - // Information schema queries - /information_schema/i, - /mysql\./i, - // System functions and procedures - /\bxp_cmdshell/i, - ] - - for (const pattern of dangerousPatterns) { - if (pattern.test(where)) { - throw new Error('WHERE clause contains potentially dangerous operation') - } - } -} - -export function sanitizeIdentifier(identifier: string): string { - if (identifier.includes('.')) { - const parts = identifier.split('.') - return parts.map((part) => sanitizeSingleIdentifier(part)).join('.') - } - - return sanitizeSingleIdentifier(identifier) -} - -function sanitizeSingleIdentifier(identifier: string): string { - const cleaned = identifier.replace(/`/g, '') - - if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(cleaned)) { - throw new Error( - `Invalid identifier: ${identifier}. Identifiers must start with a letter or underscore and contain only letters, numbers, and underscores.` - ) - } - - return `\`${cleaned}\`` -} - -export interface MySQLIntrospectionResult { - tables: Array<{ - name: string - database: string - columns: Array<{ - name: string - type: string - nullable: boolean - default: string | null - isPrimaryKey: boolean - isForeignKey: boolean - autoIncrement: boolean - references?: { - table: string - column: string - } - }> - primaryKey: string[] - foreignKeys: Array<{ - column: string - referencesTable: string - referencesColumn: string - }> - indexes: Array<{ - name: string - columns: string[] - unique: boolean - }> - }> - databases: string[] -} - -export async function executeIntrospect( - connection: mysql.Connection, - databaseName: string -): Promise { - const [databasesRows] = await connection.execute( - `SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA - WHERE SCHEMA_NAME NOT IN ('mysql', 'information_schema', 'performance_schema', 'sys') - ORDER BY SCHEMA_NAME` - ) - const databases = databasesRows.map((row) => row.SCHEMA_NAME) - - const [tablesRows] = await connection.execute( - `SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES - WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = 'BASE TABLE' - ORDER BY TABLE_NAME`, - [databaseName] - ) - - const tables = [] - - for (const tableRow of tablesRows) { - const tableName = tableRow.TABLE_NAME - - const [columnsRows] = await connection.execute( - `SELECT COLUMN_NAME, DATA_TYPE, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, EXTRA - FROM INFORMATION_SCHEMA.COLUMNS - WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? - ORDER BY ORDINAL_POSITION`, - [databaseName, tableName] - ) - - const [pkRows] = await connection.execute( - `SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE - WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND CONSTRAINT_NAME = 'PRIMARY' - ORDER BY ORDINAL_POSITION`, - [databaseName, tableName] - ) - const primaryKeyColumns = pkRows.map((row) => row.COLUMN_NAME) - - const [fkRows] = await connection.execute( - `SELECT kcu.COLUMN_NAME, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME - FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu - WHERE kcu.TABLE_SCHEMA = ? AND kcu.TABLE_NAME = ? AND kcu.REFERENCED_TABLE_NAME IS NOT NULL`, - [databaseName, tableName] - ) - - const foreignKeys = fkRows.map((row) => ({ - column: row.COLUMN_NAME, - referencesTable: row.REFERENCED_TABLE_NAME, - referencesColumn: row.REFERENCED_COLUMN_NAME, - })) - - const fkColumnSet = new Set(foreignKeys.map((fk) => fk.column)) - - const [indexRows] = await connection.execute( - `SELECT INDEX_NAME, COLUMN_NAME, SEQ_IN_INDEX, NON_UNIQUE - FROM INFORMATION_SCHEMA.STATISTICS - WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND INDEX_NAME != 'PRIMARY' - ORDER BY INDEX_NAME, SEQ_IN_INDEX`, - [databaseName, tableName] - ) - - const indexMap = new Map() - for (const row of indexRows) { - const indexName = row.INDEX_NAME - if (!indexMap.has(indexName)) { - indexMap.set(indexName, { - name: indexName, - columns: [], - unique: row.NON_UNIQUE === 0, - }) - } - indexMap.get(indexName)!.columns.push(row.COLUMN_NAME) - } - const indexes = Array.from(indexMap.values()) - - const columns = columnsRows.map((col) => { - const columnName = col.COLUMN_NAME - const fk = foreignKeys.find((f) => f.column === columnName) - const isAutoIncrement = col.EXTRA?.toLowerCase().includes('auto_increment') || false - - return { - name: columnName, - type: col.COLUMN_TYPE || col.DATA_TYPE, - nullable: col.IS_NULLABLE === 'YES', - default: col.COLUMN_DEFAULT, - isPrimaryKey: primaryKeyColumns.includes(columnName), - isForeignKey: fkColumnSet.has(columnName), - autoIncrement: isAutoIncrement, - ...(fk && { - references: { - table: fk.referencesTable, - column: fk.referencesColumn, - }, - }), - } - }) - - tables.push({ - name: tableName, - database: databaseName, - columns, - primaryKey: primaryKeyColumns, - foreignKeys, - indexes, - }) - } - - return { tables, databases } -} diff --git a/apps/sim/app/api/tools/neo4j/create/route.ts b/apps/sim/app/api/tools/neo4j/create/route.ts deleted file mode 100644 index edd32617837..00000000000 --- a/apps/sim/app/api/tools/neo4j/create/route.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { neo4jCreateContract } from '@/lib/api/contracts/tools/databases/neo4j' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - convertNeo4jTypesToJSON, - createNeo4jDriver, - validateCypherQuery, -} from '@/app/api/tools/neo4j/utils' - -const logger = createLogger('Neo4jCreateAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - let driver = null - let session = null - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized Neo4j create attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(neo4jCreateContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Executing Neo4j create on ${params.host}:${params.port}/${params.database}` - ) - - const validation = validateCypherQuery(params.cypherQuery) - if (!validation.isValid) { - logger.warn(`[${requestId}] Cypher query validation failed: ${validation.error}`) - return NextResponse.json( - { error: `Query validation failed: ${validation.error}` }, - { status: 400 } - ) - } - - driver = await createNeo4jDriver({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - encryption: params.encryption, - }) - - session = driver.session({ database: params.database }) - - const result = await session.run(params.cypherQuery, params.parameters) - - const records = result.records.map((record) => { - const obj: Record = {} - record.keys.forEach((key) => { - if (typeof key === 'string') { - obj[key] = convertNeo4jTypesToJSON(record.get(key)) - } - }) - return obj - }) - - const summary = { - resultAvailableAfter: result.summary.resultAvailableAfter.toNumber(), - resultConsumedAfter: result.summary.resultConsumedAfter.toNumber(), - counters: { - nodesCreated: result.summary.counters.updates().nodesCreated, - nodesDeleted: result.summary.counters.updates().nodesDeleted, - relationshipsCreated: result.summary.counters.updates().relationshipsCreated, - relationshipsDeleted: result.summary.counters.updates().relationshipsDeleted, - propertiesSet: result.summary.counters.updates().propertiesSet, - labelsAdded: result.summary.counters.updates().labelsAdded, - labelsRemoved: result.summary.counters.updates().labelsRemoved, - indexesAdded: result.summary.counters.updates().indexesAdded, - indexesRemoved: result.summary.counters.updates().indexesRemoved, - constraintsAdded: result.summary.counters.updates().constraintsAdded, - constraintsRemoved: result.summary.counters.updates().constraintsRemoved, - }, - } - - logger.info( - `[${requestId}] Create executed successfully, created ${summary.counters.nodesCreated} nodes and ${summary.counters.relationshipsCreated} relationships, returned ${records.length} records` - ) - - return NextResponse.json({ - message: `Created ${summary.counters.nodesCreated} nodes and ${summary.counters.relationshipsCreated} relationships`, - records, - recordCount: records.length, - summary, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Neo4j create failed:`, error) - - return NextResponse.json({ error: `Neo4j create failed: ${errorMessage}` }, { status: 500 }) - } finally { - if (session) { - await session.close() - } - if (driver) { - await driver.close() - } - } -}) diff --git a/apps/sim/app/api/tools/neo4j/delete/route.ts b/apps/sim/app/api/tools/neo4j/delete/route.ts deleted file mode 100644 index 449d0122e33..00000000000 --- a/apps/sim/app/api/tools/neo4j/delete/route.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { neo4jDeleteContract } from '@/lib/api/contracts/tools/databases/neo4j' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createNeo4jDriver, validateCypherQuery } from '@/app/api/tools/neo4j/utils' - -const logger = createLogger('Neo4jDeleteAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - let driver = null - let session = null - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized Neo4j delete attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(neo4jDeleteContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Executing Neo4j delete on ${params.host}:${params.port}/${params.database}` - ) - - const validation = validateCypherQuery(params.cypherQuery) - if (!validation.isValid) { - logger.warn(`[${requestId}] Cypher query validation failed: ${validation.error}`) - return NextResponse.json( - { error: `Query validation failed: ${validation.error}` }, - { status: 400 } - ) - } - - driver = await createNeo4jDriver({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - encryption: params.encryption, - }) - - session = driver.session({ database: params.database }) - - const result = await session.run(params.cypherQuery, params.parameters) - - const summary = { - resultAvailableAfter: result.summary.resultAvailableAfter.toNumber(), - resultConsumedAfter: result.summary.resultConsumedAfter.toNumber(), - counters: { - nodesCreated: result.summary.counters.updates().nodesCreated, - nodesDeleted: result.summary.counters.updates().nodesDeleted, - relationshipsCreated: result.summary.counters.updates().relationshipsCreated, - relationshipsDeleted: result.summary.counters.updates().relationshipsDeleted, - propertiesSet: result.summary.counters.updates().propertiesSet, - labelsAdded: result.summary.counters.updates().labelsAdded, - labelsRemoved: result.summary.counters.updates().labelsRemoved, - indexesAdded: result.summary.counters.updates().indexesAdded, - indexesRemoved: result.summary.counters.updates().indexesRemoved, - constraintsAdded: result.summary.counters.updates().constraintsAdded, - constraintsRemoved: result.summary.counters.updates().constraintsRemoved, - }, - } - - logger.info( - `[${requestId}] Delete executed successfully, deleted ${summary.counters.nodesDeleted} nodes and ${summary.counters.relationshipsDeleted} relationships` - ) - - return NextResponse.json({ - message: `Deleted ${summary.counters.nodesDeleted} nodes and ${summary.counters.relationshipsDeleted} relationships`, - summary, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Neo4j delete failed:`, error) - - return NextResponse.json({ error: `Neo4j delete failed: ${errorMessage}` }, { status: 500 }) - } finally { - if (session) { - await session.close() - } - if (driver) { - await driver.close() - } - } -}) diff --git a/apps/sim/app/api/tools/neo4j/execute/route.ts b/apps/sim/app/api/tools/neo4j/execute/route.ts deleted file mode 100644 index bf36ca56f08..00000000000 --- a/apps/sim/app/api/tools/neo4j/execute/route.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { neo4jExecuteContract } from '@/lib/api/contracts/tools/databases/neo4j' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - convertNeo4jTypesToJSON, - createNeo4jDriver, - validateCypherQuery, -} from '@/app/api/tools/neo4j/utils' - -const logger = createLogger('Neo4jExecuteAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - let driver = null - let session = null - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized Neo4j execute attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(neo4jExecuteContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Executing Neo4j query on ${params.host}:${params.port}/${params.database}` - ) - - const validation = validateCypherQuery(params.cypherQuery) - if (!validation.isValid) { - logger.warn(`[${requestId}] Cypher query validation failed: ${validation.error}`) - return NextResponse.json( - { error: `Query validation failed: ${validation.error}` }, - { status: 400 } - ) - } - - driver = await createNeo4jDriver({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - encryption: params.encryption, - }) - - session = driver.session({ database: params.database }) - - const result = await session.run(params.cypherQuery, params.parameters) - - const records = result.records.map((record) => { - const obj: Record = {} - record.keys.forEach((key) => { - if (typeof key === 'string') { - obj[key] = convertNeo4jTypesToJSON(record.get(key)) - } - }) - return obj - }) - - const summary = { - resultAvailableAfter: result.summary.resultAvailableAfter.toNumber(), - resultConsumedAfter: result.summary.resultConsumedAfter.toNumber(), - counters: { - nodesCreated: result.summary.counters.updates().nodesCreated, - nodesDeleted: result.summary.counters.updates().nodesDeleted, - relationshipsCreated: result.summary.counters.updates().relationshipsCreated, - relationshipsDeleted: result.summary.counters.updates().relationshipsDeleted, - propertiesSet: result.summary.counters.updates().propertiesSet, - labelsAdded: result.summary.counters.updates().labelsAdded, - labelsRemoved: result.summary.counters.updates().labelsRemoved, - indexesAdded: result.summary.counters.updates().indexesAdded, - indexesRemoved: result.summary.counters.updates().indexesRemoved, - constraintsAdded: result.summary.counters.updates().constraintsAdded, - constraintsRemoved: result.summary.counters.updates().constraintsRemoved, - }, - } - - logger.info(`[${requestId}] Query executed successfully, returned ${records.length} records`) - - return NextResponse.json({ - message: `Query executed successfully, returned ${records.length} records`, - records, - recordCount: records.length, - summary, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Neo4j execute failed:`, error) - - return NextResponse.json({ error: `Neo4j execute failed: ${errorMessage}` }, { status: 500 }) - } finally { - if (session) { - await session.close() - } - if (driver) { - await driver.close() - } - } -}) diff --git a/apps/sim/app/api/tools/neo4j/introspect/route.ts b/apps/sim/app/api/tools/neo4j/introspect/route.ts deleted file mode 100644 index 8d4fccc4569..00000000000 --- a/apps/sim/app/api/tools/neo4j/introspect/route.ts +++ /dev/null @@ -1,193 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { neo4jIntrospectContract } from '@/lib/api/contracts/tools/databases/neo4j' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createNeo4jDriver } from '@/app/api/tools/neo4j/utils' -import type { Neo4jNodeSchema, Neo4jRelationshipSchema } from '@/tools/neo4j/types' - -const logger = createLogger('Neo4jIntrospectAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - let driver = null - let session = null - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized Neo4j introspect attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(neo4jIntrospectContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Introspecting Neo4j database at ${params.host}:${params.port}/${params.database}` - ) - - driver = await createNeo4jDriver({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - encryption: params.encryption, - }) - - session = driver.session({ database: params.database }) - - const labelsResult = await session.run( - 'CALL db.labels() YIELD label RETURN label ORDER BY label' - ) - const labels: string[] = labelsResult.records.map((record) => record.get('label') as string) - - const relationshipTypesResult = await session.run( - 'CALL db.relationshipTypes() YIELD relationshipType RETURN relationshipType ORDER BY relationshipType' - ) - const relationshipTypes: string[] = relationshipTypesResult.records.map( - (record) => record.get('relationshipType') as string - ) - - const nodeSchemas: Neo4jNodeSchema[] = [] - try { - const nodePropertiesResult = await session.run( - 'CALL db.schema.nodeTypeProperties() YIELD nodeLabels, propertyName, propertyTypes RETURN nodeLabels, propertyName, propertyTypes' - ) - - const nodePropertiesMap = new Map>() - - for (const record of nodePropertiesResult.records) { - const nodeLabels = record.get('nodeLabels') as string[] - const propertyName = record.get('propertyName') as string - const propertyTypes = record.get('propertyTypes') as string[] - - const labelKey = nodeLabels.join(':') - if (!nodePropertiesMap.has(labelKey)) { - nodePropertiesMap.set(labelKey, []) - } - nodePropertiesMap.get(labelKey)!.push({ name: propertyName, types: propertyTypes }) - } - - for (const [labelKey, properties] of nodePropertiesMap) { - nodeSchemas.push({ - label: labelKey, - properties, - }) - } - } catch (nodePropsError) { - logger.warn( - `[${requestId}] Could not fetch node properties (may not be supported in this Neo4j version): ${nodePropsError}` - ) - } - - const relationshipSchemas: Neo4jRelationshipSchema[] = [] - try { - const relPropertiesResult = await session.run( - 'CALL db.schema.relTypeProperties() YIELD relationshipType, propertyName, propertyTypes RETURN relationshipType, propertyName, propertyTypes' - ) - - const relPropertiesMap = new Map>() - - for (const record of relPropertiesResult.records) { - const relType = record.get('relationshipType') as string - const propertyName = record.get('propertyName') as string | null - const propertyTypes = record.get('propertyTypes') as string[] - - if (!relPropertiesMap.has(relType)) { - relPropertiesMap.set(relType, []) - } - if (propertyName) { - relPropertiesMap.get(relType)!.push({ name: propertyName, types: propertyTypes }) - } - } - - for (const [relType, properties] of relPropertiesMap) { - relationshipSchemas.push({ - type: relType, - properties, - }) - } - } catch (relPropsError) { - logger.warn( - `[${requestId}] Could not fetch relationship properties (may not be supported in this Neo4j version): ${relPropsError}` - ) - } - - const constraints: Array<{ - name: string - type: string - entityType: string - properties: string[] - }> = [] - try { - const constraintsResult = await session.run('SHOW CONSTRAINTS') - - for (const record of constraintsResult.records) { - const name = record.get('name') as string - const type = record.get('type') as string - const entityType = record.get('entityType') as string - const properties = (record.get('properties') as string[]) || [] - - constraints.push({ name, type, entityType, properties }) - } - } catch (constraintsError) { - logger.warn( - `[${requestId}] Could not fetch constraints (may not be supported in this Neo4j version): ${constraintsError}` - ) - } - - const indexes: Array<{ name: string; type: string; entityType: string; properties: string[] }> = - [] - try { - const indexesResult = await session.run('SHOW INDEXES') - - for (const record of indexesResult.records) { - const name = record.get('name') as string - const type = record.get('type') as string - const entityType = record.get('entityType') as string - const properties = (record.get('properties') as string[]) || [] - - indexes.push({ name, type, entityType, properties }) - } - } catch (indexesError) { - logger.warn( - `[${requestId}] Could not fetch indexes (may not be supported in this Neo4j version): ${indexesError}` - ) - } - - logger.info( - `[${requestId}] Introspection completed: ${labels.length} labels, ${relationshipTypes.length} relationship types, ${constraints.length} constraints, ${indexes.length} indexes` - ) - - return NextResponse.json({ - message: `Database introspection completed: found ${labels.length} labels, ${relationshipTypes.length} relationship types, ${nodeSchemas.length} node schemas, ${relationshipSchemas.length} relationship schemas, ${constraints.length} constraints, ${indexes.length} indexes`, - labels, - relationshipTypes, - nodeSchemas, - relationshipSchemas, - constraints, - indexes, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Neo4j introspection failed:`, error) - - return NextResponse.json( - { error: `Neo4j introspection failed: ${errorMessage}` }, - { status: 500 } - ) - } finally { - if (session) { - await session.close() - } - if (driver) { - await driver.close() - } - } -}) diff --git a/apps/sim/app/api/tools/neo4j/merge/route.ts b/apps/sim/app/api/tools/neo4j/merge/route.ts deleted file mode 100644 index 032d897f8ba..00000000000 --- a/apps/sim/app/api/tools/neo4j/merge/route.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { neo4jMergeContract } from '@/lib/api/contracts/tools/databases/neo4j' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - convertNeo4jTypesToJSON, - createNeo4jDriver, - validateCypherQuery, -} from '@/app/api/tools/neo4j/utils' - -const logger = createLogger('Neo4jMergeAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - let driver = null - let session = null - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized Neo4j merge attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(neo4jMergeContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Executing Neo4j merge on ${params.host}:${params.port}/${params.database}` - ) - - const validation = validateCypherQuery(params.cypherQuery) - if (!validation.isValid) { - logger.warn(`[${requestId}] Cypher query validation failed: ${validation.error}`) - return NextResponse.json( - { error: `Query validation failed: ${validation.error}` }, - { status: 400 } - ) - } - - driver = await createNeo4jDriver({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - encryption: params.encryption, - }) - - session = driver.session({ database: params.database }) - - const result = await session.run(params.cypherQuery, params.parameters) - - const records = result.records.map((record) => { - const obj: Record = {} - record.keys.forEach((key) => { - if (typeof key === 'string') { - obj[key] = convertNeo4jTypesToJSON(record.get(key)) - } - }) - return obj - }) - - const summary = { - resultAvailableAfter: result.summary.resultAvailableAfter.toNumber(), - resultConsumedAfter: result.summary.resultConsumedAfter.toNumber(), - counters: { - nodesCreated: result.summary.counters.updates().nodesCreated, - nodesDeleted: result.summary.counters.updates().nodesDeleted, - relationshipsCreated: result.summary.counters.updates().relationshipsCreated, - relationshipsDeleted: result.summary.counters.updates().relationshipsDeleted, - propertiesSet: result.summary.counters.updates().propertiesSet, - labelsAdded: result.summary.counters.updates().labelsAdded, - labelsRemoved: result.summary.counters.updates().labelsRemoved, - indexesAdded: result.summary.counters.updates().indexesAdded, - indexesRemoved: result.summary.counters.updates().indexesRemoved, - constraintsAdded: result.summary.counters.updates().constraintsAdded, - constraintsRemoved: result.summary.counters.updates().constraintsRemoved, - }, - } - - logger.info( - `[${requestId}] Merge executed successfully, created ${summary.counters.nodesCreated} nodes, ${summary.counters.relationshipsCreated} relationships, returned ${records.length} records` - ) - - return NextResponse.json({ - message: `Merge completed: ${summary.counters.nodesCreated} nodes created, ${summary.counters.relationshipsCreated} relationships created`, - records, - recordCount: records.length, - summary, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Neo4j merge failed:`, error) - - return NextResponse.json({ error: `Neo4j merge failed: ${errorMessage}` }, { status: 500 }) - } finally { - if (session) { - await session.close() - } - if (driver) { - await driver.close() - } - } -}) diff --git a/apps/sim/app/api/tools/neo4j/query/route.ts b/apps/sim/app/api/tools/neo4j/query/route.ts deleted file mode 100644 index 31550d4499b..00000000000 --- a/apps/sim/app/api/tools/neo4j/query/route.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { neo4jQueryContract } from '@/lib/api/contracts/tools/databases/neo4j' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - convertNeo4jTypesToJSON, - createNeo4jDriver, - validateCypherQuery, -} from '@/app/api/tools/neo4j/utils' - -const logger = createLogger('Neo4jQueryAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - let driver = null - let session = null - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized Neo4j query attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(neo4jQueryContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Executing Neo4j query on ${params.host}:${params.port}/${params.database}` - ) - - const validation = validateCypherQuery(params.cypherQuery) - if (!validation.isValid) { - logger.warn(`[${requestId}] Cypher query validation failed: ${validation.error}`) - return NextResponse.json( - { error: `Query validation failed: ${validation.error}` }, - { status: 400 } - ) - } - - driver = await createNeo4jDriver({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - encryption: params.encryption, - }) - - session = driver.session({ database: params.database }) - - const result = await session.run(params.cypherQuery, params.parameters) - - const records = result.records.map((record) => { - const obj: Record = {} - record.keys.forEach((key) => { - if (typeof key === 'string') { - obj[key] = convertNeo4jTypesToJSON(record.get(key)) - } - }) - return obj - }) - - const summary = { - resultAvailableAfter: result.summary.resultAvailableAfter.toNumber(), - resultConsumedAfter: result.summary.resultConsumedAfter.toNumber(), - counters: { - nodesCreated: result.summary.counters.updates().nodesCreated, - nodesDeleted: result.summary.counters.updates().nodesDeleted, - relationshipsCreated: result.summary.counters.updates().relationshipsCreated, - relationshipsDeleted: result.summary.counters.updates().relationshipsDeleted, - propertiesSet: result.summary.counters.updates().propertiesSet, - labelsAdded: result.summary.counters.updates().labelsAdded, - labelsRemoved: result.summary.counters.updates().labelsRemoved, - indexesAdded: result.summary.counters.updates().indexesAdded, - indexesRemoved: result.summary.counters.updates().indexesRemoved, - constraintsAdded: result.summary.counters.updates().constraintsAdded, - constraintsRemoved: result.summary.counters.updates().constraintsRemoved, - }, - } - - logger.info(`[${requestId}] Query executed successfully, returned ${records.length} records`) - - return NextResponse.json({ - message: `Found ${records.length} records`, - records, - recordCount: records.length, - summary, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Neo4j query failed:`, error) - - return NextResponse.json({ error: `Neo4j query failed: ${errorMessage}` }, { status: 500 }) - } finally { - if (session) { - await session.close() - } - if (driver) { - await driver.close() - } - } -}) diff --git a/apps/sim/app/api/tools/neo4j/update/route.ts b/apps/sim/app/api/tools/neo4j/update/route.ts deleted file mode 100644 index f680eedbd8a..00000000000 --- a/apps/sim/app/api/tools/neo4j/update/route.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { neo4jUpdateContract } from '@/lib/api/contracts/tools/databases/neo4j' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - convertNeo4jTypesToJSON, - createNeo4jDriver, - validateCypherQuery, -} from '@/app/api/tools/neo4j/utils' - -const logger = createLogger('Neo4jUpdateAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - let driver = null - let session = null - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized Neo4j update attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(neo4jUpdateContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Executing Neo4j update on ${params.host}:${params.port}/${params.database}` - ) - - const validation = validateCypherQuery(params.cypherQuery) - if (!validation.isValid) { - logger.warn(`[${requestId}] Cypher query validation failed: ${validation.error}`) - return NextResponse.json( - { error: `Query validation failed: ${validation.error}` }, - { status: 400 } - ) - } - - driver = await createNeo4jDriver({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - encryption: params.encryption, - }) - - session = driver.session({ database: params.database }) - - const result = await session.run(params.cypherQuery, params.parameters) - - const records = result.records.map((record) => { - const obj: Record = {} - record.keys.forEach((key) => { - if (typeof key === 'string') { - obj[key] = convertNeo4jTypesToJSON(record.get(key)) - } - }) - return obj - }) - - const summary = { - resultAvailableAfter: result.summary.resultAvailableAfter.toNumber(), - resultConsumedAfter: result.summary.resultConsumedAfter.toNumber(), - counters: { - nodesCreated: result.summary.counters.updates().nodesCreated, - nodesDeleted: result.summary.counters.updates().nodesDeleted, - relationshipsCreated: result.summary.counters.updates().relationshipsCreated, - relationshipsDeleted: result.summary.counters.updates().relationshipsDeleted, - propertiesSet: result.summary.counters.updates().propertiesSet, - labelsAdded: result.summary.counters.updates().labelsAdded, - labelsRemoved: result.summary.counters.updates().labelsRemoved, - indexesAdded: result.summary.counters.updates().indexesAdded, - indexesRemoved: result.summary.counters.updates().indexesRemoved, - constraintsAdded: result.summary.counters.updates().constraintsAdded, - constraintsRemoved: result.summary.counters.updates().constraintsRemoved, - }, - } - - logger.info( - `[${requestId}] Update executed successfully, ${summary.counters.propertiesSet} properties set, returned ${records.length} records` - ) - - return NextResponse.json({ - message: `Updated ${summary.counters.propertiesSet} properties`, - records, - recordCount: records.length, - summary, - }) - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Neo4j update failed:`, error) - - return NextResponse.json({ error: `Neo4j update failed: ${errorMessage}` }, { status: 500 }) - } finally { - if (session) { - await session.close() - } - if (driver) { - await driver.close() - } - } -}) diff --git a/apps/sim/app/api/tools/neo4j/utils.ts b/apps/sim/app/api/tools/neo4j/utils.ts deleted file mode 100644 index 75df20798d6..00000000000 --- a/apps/sim/app/api/tools/neo4j/utils.ts +++ /dev/null @@ -1,120 +0,0 @@ -import neo4j from 'neo4j-driver' -import { validateDatabaseHost } from '@/lib/core/security/input-validation.server' -import type { Neo4jConnectionConfig } from '@/tools/neo4j/types' - -export async function createNeo4jDriver(config: Neo4jConnectionConfig) { - const hostValidation = await validateDatabaseHost(config.host, 'host') - if (!hostValidation.isValid) { - throw new Error(hostValidation.error) - } - - const isAuraHost = - config.host === 'databases.neo4j.io' || config.host.endsWith('.databases.neo4j.io') - - let protocol: string - if (isAuraHost) { - protocol = 'neo4j+s' - } else { - protocol = config.encryption === 'enabled' ? 'bolt+s' : 'bolt' - } - - const useIPPinning = !protocol.endsWith('+s') - const resolvedIP = hostValidation.resolvedIP ?? config.host - const uriHost = useIPPinning - ? resolvedIP.includes(':') - ? `[${resolvedIP}]` - : resolvedIP - : config.host - const uri = `${protocol}://${uriHost}:${config.port}` - - const driverConfig: any = { - maxConnectionPoolSize: 1, - connectionTimeout: 10000, - } - - if (!protocol.endsWith('+s')) { - driverConfig.encrypted = config.encryption === 'enabled' ? 'ENCRYPTION_ON' : 'ENCRYPTION_OFF' - } - - const driver = neo4j.driver(uri, neo4j.auth.basic(config.username, config.password), driverConfig) - - await driver.verifyConnectivity() - - return driver -} - -export function validateCypherQuery(query: string): { isValid: boolean; error?: string } { - if (!query || typeof query !== 'string') { - return { - isValid: false, - error: 'Query must be a non-empty string', - } - } - - const trimmedQuery = query.trim() - if (trimmedQuery.length === 0) { - return { - isValid: false, - error: 'Query cannot be empty', - } - } - - return { isValid: true } -} - -export function convertNeo4jTypesToJSON(value: unknown): unknown { - if (value === null || value === undefined) { - return value - } - - if (typeof value === 'object' && value !== null && 'toNumber' in value) { - return (value as any).toNumber() - } - - if (Array.isArray(value)) { - return value.map(convertNeo4jTypesToJSON) - } - - if (typeof value === 'object') { - const obj = value as any - - if (obj.labels && obj.properties && obj.identity) { - return { - identity: obj.identity.toNumber ? obj.identity.toNumber() : obj.identity, - labels: obj.labels, - properties: convertNeo4jTypesToJSON(obj.properties), - } - } - - if (obj.type && obj.properties && obj.identity && obj.start && obj.end) { - return { - identity: obj.identity.toNumber ? obj.identity.toNumber() : obj.identity, - start: obj.start.toNumber ? obj.start.toNumber() : obj.start, - end: obj.end.toNumber ? obj.end.toNumber() : obj.end, - type: obj.type, - properties: convertNeo4jTypesToJSON(obj.properties), - } - } - - if (obj.start && obj.end && obj.segments) { - return { - start: convertNeo4jTypesToJSON(obj.start), - end: convertNeo4jTypesToJSON(obj.end), - segments: obj.segments.map((seg: any) => ({ - start: convertNeo4jTypesToJSON(seg.start), - relationship: convertNeo4jTypesToJSON(seg.relationship), - end: convertNeo4jTypesToJSON(seg.end), - })), - length: obj.length, - } - } - - const result: Record = {} - for (const [key, val] of Object.entries(obj)) { - result[key] = convertNeo4jTypesToJSON(val) - } - return result - } - - return value -} diff --git a/apps/sim/app/api/tools/onedrive/download/route.test.ts b/apps/sim/app/api/tools/onedrive/download/route.test.ts deleted file mode 100644 index ea93ea79eba..00000000000 --- a/apps/sim/app/api/tools/onedrive/download/route.test.ts +++ /dev/null @@ -1,107 +0,0 @@ -/** - * @vitest-environment node - */ -import { - createMockRequest, - hybridAuthMockFns, - inputValidationMock, - inputValidationMockFns, -} from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' -import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' - -vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) - -import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { POST } from '@/app/api/tools/onedrive/download/route' - -const { mockValidateUrlWithDNS, mockSecureFetchWithPinnedIP } = inputValidationMockFns - -const PINNED_IP = '93.184.216.34' - -const baseBody = { - accessToken: 'token-123', - fileId: 'file-abc', -} - -function jsonResponse(body: unknown, ok = true) { - return { - ok, - status: ok ? 200 : 400, - statusText: '', - headers: new Headers(), - body: null, - text: async () => JSON.stringify(body), - json: async () => body, - arrayBuffer: async () => new ArrayBuffer(0), - } -} - -function fileResponse(bytes: number) { - return { - ok: true, - status: 200, - statusText: '', - headers: new Headers(), - body: null, - text: async () => '', - json: async () => ({}), - arrayBuffer: async () => new ArrayBuffer(bytes), - } -} - -beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - mockValidateUrlWithDNS.mockResolvedValue({ - isValid: true, - resolvedIP: PINNED_IP, - originalHostname: 'graph.microsoft.com', - }) -}) - -describe('POST /api/tools/onedrive/download', () => { - it('downloads a normal file under the size cap', async () => { - mockSecureFetchWithPinnedIP - .mockResolvedValueOnce( - jsonResponse({ id: 'file-abc', name: 'report.pdf', file: { mimeType: 'application/pdf' } }) - ) - .mockResolvedValueOnce(fileResponse(1024)) - - const response = await POST(createMockRequest('POST', baseBody)) - expect(response.status).toBe(200) - const data = (await response.json()) as { success: boolean; output: { file: { size: number } } } - expect(data.success).toBe(true) - expect(data.output.file.size).toBe(1024) - - const downloadCall = mockSecureFetchWithPinnedIP.mock.calls[1] - expect(downloadCall[2]).toMatchObject({ maxResponseBytes: MAX_FILE_SIZE }) - }) - - it('surfaces a clean 413 when the streamed content exceeds the cap', async () => { - mockSecureFetchWithPinnedIP - .mockResolvedValueOnce( - jsonResponse({ - id: 'file-abc', - name: 'huge.bin', - file: { mimeType: 'application/octet-stream' }, - }) - ) - .mockRejectedValueOnce( - new PayloadSizeLimitError({ - label: 'response body', - maxBytes: MAX_FILE_SIZE, - observedBytes: MAX_FILE_SIZE + 1, - }) - ) - - const response = await POST(createMockRequest('POST', baseBody)) - expect(response.status).toBe(413) - const data = (await response.json()) as { success: boolean } - expect(data.success).toBe(false) - }) -}) diff --git a/apps/sim/app/api/tools/onedrive/download/route.ts b/apps/sim/app/api/tools/onedrive/download/route.ts deleted file mode 100644 index d1c314397c3..00000000000 --- a/apps/sim/app/api/tools/onedrive/download/route.ts +++ /dev/null @@ -1,176 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { onedriveDownloadContract } from '@/lib/api/contracts/tools/microsoft' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' - -export const dynamic = 'force-dynamic' - -/** Microsoft Graph API error response structure */ -interface GraphApiError { - error?: { - code?: string - message?: string - } -} - -/** Microsoft Graph API drive item metadata response */ -interface DriveItemMetadata { - id?: string - name?: string - folder?: Record - file?: { - mimeType?: string - } -} - -const logger = createLogger('OneDriveDownloadAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized OneDrive download attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - const parsed = await parseRequest(onedriveDownloadContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, fileId, fileName } = parsed.data.body - const authHeader = `Bearer ${accessToken}` - - logger.info(`[${requestId}] Getting file metadata from OneDrive`, { fileId }) - - const metadataUrl = `https://graph.microsoft.com/v1.0/me/drive/items/${fileId}` - const metadataUrlValidation = await validateUrlWithDNS(metadataUrl, 'metadataUrl') - if (!metadataUrlValidation.isValid) { - return NextResponse.json( - { success: false, error: metadataUrlValidation.error }, - { status: 400 } - ) - } - - const metadataResponse = await secureFetchWithPinnedIP( - metadataUrl, - metadataUrlValidation.resolvedIP!, - { - headers: { Authorization: authHeader }, - } - ) - - if (!metadataResponse.ok) { - const errorDetails = (await metadataResponse.json().catch(() => ({}))) as GraphApiError - logger.error(`[${requestId}] Failed to get file metadata`, { - status: metadataResponse.status, - error: errorDetails, - }) - return NextResponse.json( - { success: false, error: errorDetails.error?.message || 'Failed to get file metadata' }, - { status: 400 } - ) - } - - const metadata = (await metadataResponse.json()) as DriveItemMetadata - - if (metadata.folder && !metadata.file) { - logger.error(`[${requestId}] Attempted to download a folder`, { - itemId: metadata.id, - itemName: metadata.name, - }) - return NextResponse.json( - { - success: false, - error: `Cannot download folder "${metadata.name}". Please select a file instead.`, - }, - { status: 400 } - ) - } - - const mimeType = metadata.file?.mimeType || 'application/octet-stream' - - logger.info(`[${requestId}] Downloading file from OneDrive`, { fileId, mimeType }) - - const downloadUrl = `https://graph.microsoft.com/v1.0/me/drive/items/${fileId}/content` - const downloadUrlValidation = await validateUrlWithDNS(downloadUrl, 'downloadUrl') - if (!downloadUrlValidation.isValid) { - return NextResponse.json( - { success: false, error: downloadUrlValidation.error }, - { status: 400 } - ) - } - - const downloadResponse = await secureFetchWithPinnedIP( - downloadUrl, - downloadUrlValidation.resolvedIP!, - { - headers: { Authorization: authHeader }, - maxResponseBytes: MAX_FILE_SIZE, - } - ) - - if (!downloadResponse.ok) { - const downloadError = (await downloadResponse.json().catch(() => ({}))) as GraphApiError - logger.error(`[${requestId}] Failed to download file`, { - status: downloadResponse.status, - error: downloadError, - }) - return NextResponse.json( - { success: false, error: downloadError.error?.message || 'Failed to download file' }, - { status: 400 } - ) - } - - const arrayBuffer = await downloadResponse.arrayBuffer() - const fileBuffer = Buffer.from(arrayBuffer) - - const resolvedName = fileName || metadata.name || 'download' - - logger.info(`[${requestId}] File downloaded successfully`, { - fileId, - name: resolvedName, - size: fileBuffer.length, - mimeType, - }) - - const base64Data = fileBuffer.toString('base64') - - return NextResponse.json({ - success: true, - output: { - file: { - name: resolvedName, - mimeType, - data: base64Data, - size: fileBuffer.length, - }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error downloading OneDrive file:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/onedrive/folder/route.ts b/apps/sim/app/api/tools/onedrive/folder/route.ts deleted file mode 100644 index df3d192ad9c..00000000000 --- a/apps/sim/app/api/tools/onedrive/folder/route.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { createLogger } from '@sim/logger' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { onedriveFolderQuerySchema } from '@/lib/api/contracts/selectors/microsoft' -import { getValidationErrorMessage } from '@/lib/api/server' -import { authorizeCredentialUse } from '@/lib/auth/credential-access' -import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('OneDriveFolderAPI') - -export const GET = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const { searchParams } = new URL(request.url) - const validation = onedriveFolderQuerySchema.safeParse({ - credentialId: searchParams.get('credentialId') ?? '', - fileId: searchParams.get('fileId') ?? '', - }) - if (!validation.success) { - return NextResponse.json( - { error: getValidationErrorMessage(validation.error, 'Invalid request') }, - { status: 400 } - ) - } - const { credentialId, fileId } = validation.data - - const fileIdValidation = validateMicrosoftGraphId(fileId, 'fileId') - if (!fileIdValidation.isValid) { - return NextResponse.json({ error: fileIdValidation.error }, { status: 400 }) - } - - const credAccess = await authorizeCredentialUse(request, { - credentialId, - requireWorkflowIdForInternal: false, - }) - if (!credAccess.ok || !credAccess.credentialOwnerUserId) { - logger.warn(`[${requestId}] Credential access denied`, { error: credAccess.error }) - return NextResponse.json({ error: credAccess.error || 'Unauthorized' }, { status: 401 }) - } - - const accessToken = await refreshAccessTokenIfNeeded( - credentialId, - credAccess.credentialOwnerUserId, - requestId - ) - if (!accessToken) { - return NextResponse.json({ error: 'Failed to obtain valid access token' }, { status: 401 }) - } - - const response = await fetch( - `https://graph.microsoft.com/v1.0/me/drive/items/${fileId}?$select=id,name,folder,webUrl,createdDateTime,lastModifiedDateTime`, - { - headers: { - Authorization: `Bearer ${accessToken}`, - }, - } - ) - - if (!response.ok) { - const errorData = await response.json().catch(() => ({ error: { message: 'Unknown error' } })) - return NextResponse.json( - { error: errorData.error?.message || 'Failed to fetch folder from OneDrive' }, - { status: response.status } - ) - } - - const folder = await response.json() - - const transformedFolder = { - id: folder.id, - name: folder.name, - mimeType: 'application/vnd.microsoft.graph.folder', - webViewLink: folder.webUrl, - createdTime: folder.createdDateTime, - modifiedTime: folder.lastModifiedDateTime, - } - - return NextResponse.json({ file: transformedFolder }, { status: 200 }) - } catch (error) { - logger.error(`[${requestId}] Error fetching folder from OneDrive`, error) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/onedrive/upload/route.ts b/apps/sim/app/api/tools/onedrive/upload/route.ts deleted file mode 100644 index 2e7ff49837d..00000000000 --- a/apps/sim/app/api/tools/onedrive/upload/route.ts +++ /dev/null @@ -1,434 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import * as XLSX from 'xlsx' -import { onedriveUploadContract } from '@/lib/api/contracts/tools/microsoft' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation' -import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - getExtensionFromMimeType, - processSingleFileToUserFile, -} from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { normalizeExcelValues } from '@/tools/onedrive/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('OneDriveUploadAPI') - -const MICROSOFT_GRAPH_BASE = 'https://graph.microsoft.com/v1.0' - -/** Microsoft Graph's ceiling for a simple (non-chunked) drive-item upload. */ -const MAX_SIMPLE_UPLOAD_BYTES = 250 * 1024 * 1024 - -function fileTooLargeError(observedBytes: number): NextResponse { - const sizeMB = (observedBytes / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `File size (${sizeMB}MB) exceeds OneDrive's limit of 250MB for simple uploads. Use chunked upload for larger files.`, - }, - { status: 400 } - ) -} - -/** Microsoft Graph DriveItem response */ -interface OneDriveFileData { - id: string - name: string - size: number - webUrl: string - createdDateTime: string - lastModifiedDateTime: string - file?: { mimeType: string } - parentReference?: { id: string; path: string } - '@microsoft.graph.downloadUrl'?: string -} - -/** Microsoft Graph Excel range response */ -interface ExcelRangeData { - address?: string - addressLocal?: string - values?: unknown[][] -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized OneDrive upload attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated OneDrive upload request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const parsed = await parseRequest(onedriveUploadContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - const excelValues = normalizeExcelValues(validatedData.values) - - let fileBuffer: Buffer - let mimeType: string - - const isExcelCreation = - validatedData.mimeType === - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' && !validatedData.file - - if (isExcelCreation) { - const workbook = XLSX.utils.book_new() - const worksheet = XLSX.utils.aoa_to_sheet([[]]) - XLSX.utils.book_append_sheet(workbook, worksheet, 'Sheet1') - - const xlsxBuffer = XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' }) - fileBuffer = Buffer.from(xlsxBuffer) - mimeType = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' - } else { - const rawFile = validatedData.file - - if (!rawFile) { - return NextResponse.json( - { - success: false, - error: 'No file provided', - }, - { status: 400 } - ) - } - - let userFile - try { - userFile = processSingleFileToUserFile(rawFile, requestId, logger) - } catch (error) { - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Failed to process file'), - }, - { status: 400 } - ) - } - - const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) - if (denied) return denied - - try { - const result = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_SIMPLE_UPLOAD_BYTES, - }) - fileBuffer = result.buffer - mimeType = result.contentType || userFile.type || 'application/octet-stream' - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - if (isPayloadSizeLimitError(error)) { - return fileTooLargeError(error.observedBytes ?? userFile.size) - } - logger.error(`[${requestId}] Failed to download file from storage:`, error) - return NextResponse.json( - { - success: false, - error: `Failed to download file: ${getErrorMessage(error, 'Unknown error')}`, - }, - { status: 500 } - ) - } - } - - if (fileBuffer.length > MAX_SIMPLE_UPLOAD_BYTES) { - logger.warn( - `[${requestId}] File too large: ${(fileBuffer.length / (1024 * 1024)).toFixed(2)}MB` - ) - return fileTooLargeError(fileBuffer.length) - } - - let fileName = validatedData.fileName - const hasExtension = fileName.includes('.') && fileName.lastIndexOf('.') > 0 - - if (!hasExtension) { - const extension = getExtensionFromMimeType(mimeType) - if (extension) { - fileName = `${fileName}.${extension}` - logger.info(`[${requestId}] Added extension to filename: ${fileName}`) - } - } else if (isExcelCreation && !fileName.endsWith('.xlsx')) { - fileName = `${fileName.replace(/\.[^.]*$/, '')}.xlsx` - } - - let uploadUrl: string - const folderId = validatedData.folderId?.trim() - - if (folderId && folderId !== '') { - const folderIdValidation = validateMicrosoftGraphId(folderId, 'folderId') - if (!folderIdValidation.isValid) { - logger.warn(`[${requestId}] Invalid folder ID`, { error: folderIdValidation.error }) - return NextResponse.json( - { - success: false, - error: folderIdValidation.error, - }, - { status: 400 } - ) - } - uploadUrl = `${MICROSOFT_GRAPH_BASE}/me/drive/items/${encodeURIComponent(folderId)}:/${encodeURIComponent(fileName)}:/content` - } else { - uploadUrl = `${MICROSOFT_GRAPH_BASE}/me/drive/root:/${encodeURIComponent(fileName)}:/content` - } - - // Add conflict behavior if specified (defaults to replace by Microsoft Graph API) - if (validatedData.conflictBehavior) { - uploadUrl += `?@microsoft.graph.conflictBehavior=${validatedData.conflictBehavior}` - } - - const uploadResponse = await secureFetchWithValidation( - uploadUrl, - { - method: 'PUT', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': mimeType, - }, - body: fileBuffer, - }, - 'uploadUrl' - ) - - if (!uploadResponse.ok) { - const errorText = await uploadResponse.text() - return NextResponse.json( - { - success: false, - error: `OneDrive upload failed: ${uploadResponse.statusText}`, - details: errorText, - }, - { status: uploadResponse.status } - ) - } - - const fileData = (await uploadResponse.json()) as OneDriveFileData - - let excelWriteResult: any | undefined - const shouldWriteExcelContent = - isExcelCreation && Array.isArray(excelValues) && excelValues.length > 0 - - if (shouldWriteExcelContent) { - try { - let workbookSessionId: string | undefined - const sessionUrl = `${MICROSOFT_GRAPH_BASE}/me/drive/items/${encodeURIComponent( - fileData.id - )}/workbook/createSession` - const sessionResp = await secureFetchWithValidation( - sessionUrl, - { - method: 'POST', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ persistChanges: true }), - }, - 'sessionUrl' - ) - - if (sessionResp.ok) { - const sessionData = (await sessionResp.json()) as { id?: string } - workbookSessionId = sessionData?.id - } - - let sheetName = 'Sheet1' - try { - const listUrl = `${MICROSOFT_GRAPH_BASE}/me/drive/items/${encodeURIComponent( - fileData.id - )}/workbook/worksheets?$select=name&$orderby=position&$top=1` - const listResp = await secureFetchWithValidation( - listUrl, - { - method: 'GET', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - ...(workbookSessionId ? { 'workbook-session-id': workbookSessionId } : {}), - }, - }, - 'listUrl' - ) - if (listResp.ok) { - const listData = (await listResp.json()) as { value?: Array<{ name?: string }> } - const firstSheetName = listData?.value?.[0]?.name - if (firstSheetName) { - sheetName = firstSheetName - } - } else { - const listErr = await listResp.text() - logger.warn(`[${requestId}] Failed to list worksheets, using default Sheet1`, { - status: listResp.status, - error: listErr, - }) - } - } catch (listError) { - logger.warn(`[${requestId}] Error listing worksheets, using default Sheet1`, listError) - } - - let processedValues: any = excelValues || [] - - if ( - Array.isArray(processedValues) && - processedValues.length > 0 && - typeof processedValues[0] === 'object' && - !Array.isArray(processedValues[0]) - ) { - const ws = XLSX.utils.json_to_sheet(processedValues) - processedValues = XLSX.utils.sheet_to_json(ws, { header: 1, defval: '' }) - } - - const rowsCount = processedValues.length - const colsCount = Math.max(...processedValues.map((row: any[]) => row.length), 0) - processedValues = processedValues.map((row: any[]) => { - const paddedRow = [...row] - while (paddedRow.length < colsCount) paddedRow.push('') - return paddedRow - }) - - const indexToColLetters = (index: number): string => { - let n = index - let s = '' - while (n > 0) { - const rem = (n - 1) % 26 - s = String.fromCharCode(65 + rem) + s - n = Math.floor((n - 1) / 26) - } - return s - } - - const endColLetters = colsCount > 0 ? indexToColLetters(colsCount) : 'A' - const endRow = rowsCount > 0 ? rowsCount : 1 - const computedRangeAddress = `A1:${endColLetters}${endRow}` - - const url = new URL( - `${MICROSOFT_GRAPH_BASE}/me/drive/items/${encodeURIComponent( - fileData.id - )}/workbook/worksheets('${encodeURIComponent( - sheetName - )}')/range(address='${encodeURIComponent(computedRangeAddress)}')` - ) - - const excelWriteResponse = await secureFetchWithValidation( - url.toString(), - { - method: 'PATCH', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': 'application/json', - ...(workbookSessionId ? { 'workbook-session-id': workbookSessionId } : {}), - }, - body: JSON.stringify({ values: processedValues }), - }, - 'excelWriteUrl' - ) - - if (!excelWriteResponse || !excelWriteResponse.ok) { - const errorText = excelWriteResponse ? await excelWriteResponse.text() : 'no response' - logger.error(`[${requestId}] Excel content write failed`, { - status: excelWriteResponse?.status, - statusText: excelWriteResponse?.statusText, - error: errorText, - }) - excelWriteResult = { - success: false, - error: `Excel write failed: ${excelWriteResponse?.statusText || 'unknown'}`, - details: errorText, - } - } else { - const writeData = (await excelWriteResponse.json()) as ExcelRangeData - const addr = writeData.address || writeData.addressLocal - const v = writeData.values || [] - excelWriteResult = { - success: true, - updatedRange: addr, - updatedRows: Array.isArray(v) ? v.length : undefined, - updatedColumns: Array.isArray(v) && v[0] ? v[0].length : undefined, - updatedCells: Array.isArray(v) && v[0] ? v.length * v[0].length : undefined, - } - } - - if (workbookSessionId) { - try { - const closeUrl = `${MICROSOFT_GRAPH_BASE}/me/drive/items/${encodeURIComponent( - fileData.id - )}/workbook/closeSession` - const closeResp = await secureFetchWithValidation( - closeUrl, - { - method: 'POST', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'workbook-session-id': workbookSessionId, - }, - }, - 'closeSessionUrl' - ) - if (!closeResp.ok) { - const closeText = await closeResp.text() - logger.warn(`[${requestId}] Failed to close Excel session`, { - status: closeResp.status, - error: closeText, - }) - } - } catch (closeErr) { - logger.warn(`[${requestId}] Error closing Excel session`, closeErr) - } - } - } catch (err) { - logger.error(`[${requestId}] Exception during Excel content write`, err) - excelWriteResult = { - success: false, - error: getErrorMessage(err, 'Unknown error during Excel write'), - } - } - } - - return NextResponse.json({ - success: true, - output: { - file: { - id: fileData.id, - name: fileData.name, - mimeType: fileData.file?.mimeType || mimeType, - webViewLink: fileData.webUrl, - webContentLink: fileData['@microsoft.graph.downloadUrl'], - size: fileData.size, - createdTime: fileData.createdDateTime, - modifiedTime: fileData.lastModifiedDateTime, - parentReference: fileData.parentReference, - }, - ...(excelWriteResult ? { excelWriteResult } : {}), - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error uploading file to OneDrive:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/onepassword/create-item/route.ts b/apps/sim/app/api/tools/onepassword/create-item/route.ts deleted file mode 100644 index 7785a02f3e9..00000000000 --- a/apps/sim/app/api/tools/onepassword/create-item/route.ts +++ /dev/null @@ -1,106 +0,0 @@ -import type { ItemCreateParams } from '@1password/sdk' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { onePasswordCreateItemContract } from '@/lib/api/contracts/tools/onepassword' -import { parseRequest, validationErrorResponse } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - connectRequest, - createOnePasswordClient, - normalizeSdkItem, - resolveCredentials, - toSdkCategory, - toSdkFieldType, -} from '../utils' - -const logger = createLogger('OnePasswordCreateItemAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized 1Password create-item attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest( - onePasswordCreateItemContract, - request, - {}, - { - validationErrorResponse: (error) => validationErrorResponse(error, 'Invalid request data'), - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - const creds = resolveCredentials(params) - - logger.info(`[${requestId}] Creating item in vault ${params.vaultId} (${creds.mode} mode)`) - - if (creds.mode === 'service_account') { - const client = await createOnePasswordClient(creds.serviceAccountToken!) - - const parsedTags = params.tags - ? params.tags - .split(',') - .map((t) => t.trim()) - .filter(Boolean) - : undefined - - const parsedFields = params.fields - ? (JSON.parse(params.fields) as Array>).map((f) => ({ - id: f.id || generateId().slice(0, 8), - title: f.label || f.title || '', - fieldType: toSdkFieldType(f.type || 'STRING'), - value: f.value || '', - sectionId: f.section?.id ?? f.sectionId, - })) - : undefined - - const item = await client.items.create({ - vaultId: params.vaultId, - category: toSdkCategory(params.category), - title: params.title || '', - tags: parsedTags, - fields: parsedFields, - } as ItemCreateParams) - - return NextResponse.json(normalizeSdkItem(item)) - } - - const connectBody: Record = { - vault: { id: params.vaultId }, - category: params.category, - } - if (params.title) connectBody.title = params.title - if (params.tags) connectBody.tags = params.tags.split(',').map((t) => t.trim()) - if (params.fields) connectBody.fields = JSON.parse(params.fields) - - const response = await connectRequest({ - serverUrl: creds.serverUrl!, - apiKey: creds.apiKey!, - path: `/v1/vaults/${params.vaultId}/items`, - method: 'POST', - body: connectBody, - }) - - const data = await response.json() - if (!response.ok) { - return NextResponse.json( - { error: data.message || 'Failed to create item' }, - { status: response.status } - ) - } - - return NextResponse.json(data) - } catch (error) { - const message = getErrorMessage(error, 'Unknown error') - logger.error(`[${requestId}] Create item failed:`, error) - return NextResponse.json({ error: `Failed to create item: ${message}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/onepassword/delete-item/route.ts b/apps/sim/app/api/tools/onepassword/delete-item/route.ts deleted file mode 100644 index 716e7a97fc8..00000000000 --- a/apps/sim/app/api/tools/onepassword/delete-item/route.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { onePasswordDeleteItemContract } from '@/lib/api/contracts/tools/onepassword' -import { parseRequest, validationErrorResponse } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { connectRequest, createOnePasswordClient, resolveCredentials } from '../utils' - -const logger = createLogger('OnePasswordDeleteItemAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized 1Password delete-item attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest( - onePasswordDeleteItemContract, - request, - {}, - { - validationErrorResponse: (error) => validationErrorResponse(error, 'Invalid request data'), - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - const creds = resolveCredentials(params) - - logger.info( - `[${requestId}] Deleting item ${params.itemId} from vault ${params.vaultId} (${creds.mode} mode)` - ) - - if (creds.mode === 'service_account') { - const client = await createOnePasswordClient(creds.serviceAccountToken!) - await client.items.delete(params.vaultId, params.itemId) - return NextResponse.json({ success: true }) - } - - const response = await connectRequest({ - serverUrl: creds.serverUrl!, - apiKey: creds.apiKey!, - path: `/v1/vaults/${params.vaultId}/items/${params.itemId}`, - method: 'DELETE', - }) - - if (!response.ok) { - const data = await response.json().catch(() => ({})) - return NextResponse.json( - { error: (data as Record).message || 'Failed to delete item' }, - { status: response.status } - ) - } - - return NextResponse.json({ success: true }) - } catch (error) { - const message = getErrorMessage(error, 'Unknown error') - logger.error(`[${requestId}] Delete item failed:`, error) - return NextResponse.json({ error: `Failed to delete item: ${message}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/onepassword/get-item-file/route.ts b/apps/sim/app/api/tools/onepassword/get-item-file/route.ts deleted file mode 100644 index 2018b75ade1..00000000000 --- a/apps/sim/app/api/tools/onepassword/get-item-file/route.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { onePasswordGetItemFileContract } from '@/lib/api/contracts/tools/onepassword' -import { parseRequest, validationErrorResponse } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' -import { - connectRequest, - createOnePasswordClient, - findItemFileAttributes, - resolveCredentials, -} from '../utils' - -const logger = createLogger('OnePasswordGetItemFileAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized 1Password get-item-file attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest( - onePasswordGetItemFileContract, - request, - {}, - { - validationErrorResponse: (error) => validationErrorResponse(error, 'Invalid request data'), - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - const creds = resolveCredentials(params) - - logger.info( - `[${requestId}] Downloading file ${params.fileId} from item ${params.itemId} (${creds.mode} mode)` - ) - - if (creds.mode === 'service_account') { - const client = await createOnePasswordClient(creds.serviceAccountToken!) - const item = await client.items.get(params.vaultId, params.itemId) - const attr = findItemFileAttributes(item, params.fileId) - if (!attr) { - return NextResponse.json({ error: 'File not found on item' }, { status: 404 }) - } - - const content = await client.items.files.read(params.vaultId, params.itemId, attr) - return NextResponse.json({ - file: { - name: attr.name, - mimeType: 'application/octet-stream', - data: Buffer.from(content).toString('base64'), - size: attr.size, - }, - }) - } - - const metaResponse = await connectRequest({ - serverUrl: creds.serverUrl!, - apiKey: creds.apiKey!, - path: `/v1/vaults/${params.vaultId}/items/${params.itemId}/files/${params.fileId}`, - method: 'GET', - }) - if (!metaResponse.ok) { - const metaData = await metaResponse.json().catch(() => ({})) - return NextResponse.json( - { error: metaData.message || 'Failed to get file metadata' }, - { status: metaResponse.status } - ) - } - const meta = await metaResponse.json() - - const contentResponse = await connectRequest({ - serverUrl: creds.serverUrl!, - apiKey: creds.apiKey!, - path: `/v1/vaults/${params.vaultId}/items/${params.itemId}/files/${params.fileId}/content`, - method: 'GET', - maxResponseBytes: MAX_FILE_SIZE, - }) - if (!contentResponse.ok) { - const errorData = await contentResponse.json().catch(() => ({})) - return NextResponse.json( - { error: errorData.message || 'Failed to download file content' }, - { status: contentResponse.status } - ) - } - - const buffer = Buffer.from(await contentResponse.arrayBuffer()) - return NextResponse.json({ - file: { - name: meta.name ?? 'attachment', - mimeType: contentResponse.headers.get('content-type') || 'application/octet-stream', - data: buffer.toString('base64'), - size: meta.size ?? buffer.length, - }, - }) - } catch (error) { - const message = getErrorMessage(error, 'Unknown error') - logger.error(`[${requestId}] Get item file failed:`, error) - return NextResponse.json({ error: `Failed to get item file: ${message}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/onepassword/get-item/route.ts b/apps/sim/app/api/tools/onepassword/get-item/route.ts deleted file mode 100644 index 9693a65d6f9..00000000000 --- a/apps/sim/app/api/tools/onepassword/get-item/route.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { onePasswordGetItemContract } from '@/lib/api/contracts/tools/onepassword' -import { parseRequest, validationErrorResponse } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - connectRequest, - createOnePasswordClient, - normalizeSdkItem, - resolveCredentials, -} from '../utils' - -const logger = createLogger('OnePasswordGetItemAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized 1Password get-item attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest( - onePasswordGetItemContract, - request, - {}, - { - validationErrorResponse: (error) => validationErrorResponse(error, 'Invalid request data'), - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - const creds = resolveCredentials(params) - - logger.info( - `[${requestId}] Getting item ${params.itemId} from vault ${params.vaultId} (${creds.mode} mode)` - ) - - if (creds.mode === 'service_account') { - const client = await createOnePasswordClient(creds.serviceAccountToken!) - const item = await client.items.get(params.vaultId, params.itemId) - return NextResponse.json(normalizeSdkItem(item)) - } - - const response = await connectRequest({ - serverUrl: creds.serverUrl!, - apiKey: creds.apiKey!, - path: `/v1/vaults/${params.vaultId}/items/${params.itemId}`, - method: 'GET', - }) - - const data = await response.json() - if (!response.ok) { - return NextResponse.json( - { error: data.message || 'Failed to get item' }, - { status: response.status } - ) - } - - return NextResponse.json(data) - } catch (error) { - const message = getErrorMessage(error, 'Unknown error') - logger.error(`[${requestId}] Get item failed:`, error) - return NextResponse.json({ error: `Failed to get item: ${message}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/onepassword/get-vault/route.ts b/apps/sim/app/api/tools/onepassword/get-vault/route.ts deleted file mode 100644 index 474f7fc2a78..00000000000 --- a/apps/sim/app/api/tools/onepassword/get-vault/route.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { onePasswordGetVaultContract } from '@/lib/api/contracts/tools/onepassword' -import { parseRequest, validationErrorResponse } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - connectRequest, - createOnePasswordClient, - normalizeSdkVault, - resolveCredentials, -} from '../utils' - -const logger = createLogger('OnePasswordGetVaultAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized 1Password get-vault attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest( - onePasswordGetVaultContract, - request, - {}, - { - validationErrorResponse: (error) => validationErrorResponse(error, 'Invalid request data'), - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - const creds = resolveCredentials(params) - - logger.info(`[${requestId}] Getting 1Password vault ${params.vaultId} (${creds.mode} mode)`) - - if (creds.mode === 'service_account') { - const client = await createOnePasswordClient(creds.serviceAccountToken!) - const vaults = await client.vaults.list() - const vault = vaults.find((v) => v.id === params.vaultId) - - if (!vault) { - return NextResponse.json({ error: 'Vault not found' }, { status: 404 }) - } - - return NextResponse.json(normalizeSdkVault(vault)) - } - - const response = await connectRequest({ - serverUrl: creds.serverUrl!, - apiKey: creds.apiKey!, - path: `/v1/vaults/${params.vaultId}`, - method: 'GET', - }) - - const data = await response.json() - if (!response.ok) { - return NextResponse.json( - { error: data.message || 'Failed to get vault' }, - { status: response.status } - ) - } - - return NextResponse.json(data) - } catch (error) { - const message = getErrorMessage(error, 'Unknown error') - logger.error(`[${requestId}] Get vault failed:`, error) - return NextResponse.json({ error: `Failed to get vault: ${message}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/onepassword/list-items/route.ts b/apps/sim/app/api/tools/onepassword/list-items/route.ts deleted file mode 100644 index 395d0955ced..00000000000 --- a/apps/sim/app/api/tools/onepassword/list-items/route.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { onePasswordListItemsContract } from '@/lib/api/contracts/tools/onepassword' -import { parseRequest, validationErrorResponse } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - connectRequest, - createOnePasswordClient, - matchesFilter, - normalizeSdkItemOverview, - resolveCredentials, -} from '../utils' - -const logger = createLogger('OnePasswordListItemsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized 1Password list-items attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest( - onePasswordListItemsContract, - request, - {}, - { - validationErrorResponse: (error) => validationErrorResponse(error, 'Invalid request data'), - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - const creds = resolveCredentials(params) - - logger.info(`[${requestId}] Listing items in vault ${params.vaultId} (${creds.mode} mode)`) - - if (creds.mode === 'service_account') { - const client = await createOnePasswordClient(creds.serviceAccountToken!) - const items = await client.items.list(params.vaultId) - const normalized = items.map(normalizeSdkItemOverview) - - if (params.filter) { - const filter = params.filter - const filtered = normalized.filter((item) => - matchesFilter(item.title ?? '', item.id ?? '', filter) - ) - return NextResponse.json(filtered) - } - - return NextResponse.json(normalized) - } - - const query = params.filter ? `filter=${encodeURIComponent(params.filter)}` : undefined - const response = await connectRequest({ - serverUrl: creds.serverUrl!, - apiKey: creds.apiKey!, - path: `/v1/vaults/${params.vaultId}/items`, - method: 'GET', - query, - }) - - const data = await response.json() - if (!response.ok) { - return NextResponse.json( - { error: data.message || 'Failed to list items' }, - { status: response.status } - ) - } - - return NextResponse.json(data) - } catch (error) { - const message = getErrorMessage(error, 'Unknown error') - logger.error(`[${requestId}] List items failed:`, error) - return NextResponse.json({ error: `Failed to list items: ${message}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/onepassword/list-vaults/route.ts b/apps/sim/app/api/tools/onepassword/list-vaults/route.ts deleted file mode 100644 index 3638db1c2d7..00000000000 --- a/apps/sim/app/api/tools/onepassword/list-vaults/route.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { onePasswordListVaultsContract } from '@/lib/api/contracts/tools/onepassword' -import { parseRequest, validationErrorResponse } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - connectRequest, - createOnePasswordClient, - matchesFilter, - normalizeSdkVault, - resolveCredentials, -} from '../utils' - -const logger = createLogger('OnePasswordListVaultsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized 1Password list-vaults attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest( - onePasswordListVaultsContract, - request, - {}, - { - validationErrorResponse: (error) => validationErrorResponse(error, 'Invalid request data'), - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - const creds = resolveCredentials(params) - - logger.info(`[${requestId}] Listing 1Password vaults (${creds.mode} mode)`) - - if (creds.mode === 'service_account') { - const client = await createOnePasswordClient(creds.serviceAccountToken!) - const vaults = await client.vaults.list() - const normalized = vaults.map(normalizeSdkVault) - - if (params.filter) { - const filter = params.filter - const filtered = normalized.filter((v) => matchesFilter(v.name ?? '', v.id ?? '', filter)) - return NextResponse.json(filtered) - } - - return NextResponse.json(normalized) - } - - const query = params.filter ? `filter=${encodeURIComponent(params.filter)}` : undefined - const response = await connectRequest({ - serverUrl: creds.serverUrl!, - apiKey: creds.apiKey!, - path: '/v1/vaults', - method: 'GET', - query, - }) - - const data = await response.json() - if (!response.ok) { - return NextResponse.json( - { error: data.message || 'Failed to list vaults' }, - { status: response.status } - ) - } - - return NextResponse.json(data) - } catch (error) { - const message = getErrorMessage(error, 'Unknown error') - logger.error(`[${requestId}] List vaults failed:`, error) - return NextResponse.json({ error: `Failed to list vaults: ${message}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/onepassword/replace-item/route.ts b/apps/sim/app/api/tools/onepassword/replace-item/route.ts deleted file mode 100644 index 67d7a7b10f8..00000000000 --- a/apps/sim/app/api/tools/onepassword/replace-item/route.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { onePasswordReplaceItemContract } from '@/lib/api/contracts/tools/onepassword' -import { parseRequest, validationErrorResponse } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - connectItemToSdkItem, - connectRequest, - createOnePasswordClient, - normalizeSdkItem, - resolveCredentials, -} from '../utils' - -const logger = createLogger('OnePasswordReplaceItemAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized 1Password replace-item attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest( - onePasswordReplaceItemContract, - request, - {}, - { - validationErrorResponse: (error) => validationErrorResponse(error, 'Invalid request data'), - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - const creds = resolveCredentials(params) - const itemData = JSON.parse(params.item) - - logger.info( - `[${requestId}] Replacing item ${params.itemId} in vault ${params.vaultId} (${creds.mode} mode)` - ) - - if (creds.mode === 'service_account') { - const client = await createOnePasswordClient(creds.serviceAccountToken!) - - const existing = await client.items.get(params.vaultId, params.itemId) - const sdkItem = connectItemToSdkItem(itemData, existing) - const result = await client.items.put(sdkItem) - return NextResponse.json(normalizeSdkItem(result)) - } - - const response = await connectRequest({ - serverUrl: creds.serverUrl!, - apiKey: creds.apiKey!, - path: `/v1/vaults/${params.vaultId}/items/${params.itemId}`, - method: 'PUT', - body: itemData, - }) - - const data = await response.json() - if (!response.ok) { - return NextResponse.json( - { error: data.message || 'Failed to replace item' }, - { status: response.status } - ) - } - - return NextResponse.json(data) - } catch (error) { - const message = getErrorMessage(error, 'Unknown error') - logger.error(`[${requestId}] Replace item failed:`, error) - return NextResponse.json({ error: `Failed to replace item: ${message}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/onepassword/resolve-secret/route.ts b/apps/sim/app/api/tools/onepassword/resolve-secret/route.ts deleted file mode 100644 index 798b10b9795..00000000000 --- a/apps/sim/app/api/tools/onepassword/resolve-secret/route.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { onePasswordResolveSecretContract } from '@/lib/api/contracts/tools/onepassword' -import { parseRequest, validationErrorResponse } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createOnePasswordClient, resolveCredentials } from '../utils' - -const logger = createLogger('OnePasswordResolveSecretAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized 1Password resolve-secret attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest( - onePasswordResolveSecretContract, - request, - {}, - { - validationErrorResponse: (error) => validationErrorResponse(error, 'Invalid request data'), - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - const creds = resolveCredentials(params) - - if (creds.mode !== 'service_account') { - return NextResponse.json( - { error: 'Resolve Secret is only available in Service Account mode' }, - { status: 400 } - ) - } - - logger.info(`[${requestId}] Resolving secret reference (service_account mode)`) - - const client = await createOnePasswordClient(creds.serviceAccountToken!) - const secret = await client.secrets.resolve(params.secretReference) - - return NextResponse.json({ - value: secret, - reference: params.secretReference, - }) - } catch (error) { - const message = getErrorMessage(error, 'Unknown error') - logger.error(`[${requestId}] Resolve secret failed:`, error) - return NextResponse.json({ error: `Failed to resolve secret: ${message}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/onepassword/update-item/route.ts b/apps/sim/app/api/tools/onepassword/update-item/route.ts deleted file mode 100644 index 70fa927b992..00000000000 --- a/apps/sim/app/api/tools/onepassword/update-item/route.ts +++ /dev/null @@ -1,162 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { onePasswordUpdateItemContract } from '@/lib/api/contracts/tools/onepassword' -import { parseRequest, validationErrorResponse } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - connectItemToSdkItem, - connectRequest, - createOnePasswordClient, - normalizeSdkItem, - resolveCredentials, -} from '../utils' - -const logger = createLogger('OnePasswordUpdateItemAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized 1Password update-item attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest( - onePasswordUpdateItemContract, - request, - {}, - { - validationErrorResponse: (error) => validationErrorResponse(error, 'Invalid request data'), - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - const creds = resolveCredentials(params) - const ops = JSON.parse(params.operations) as JsonPatchOperation[] - - logger.info( - `[${requestId}] Updating item ${params.itemId} in vault ${params.vaultId} (${creds.mode} mode)` - ) - - if (creds.mode === 'service_account') { - const client = await createOnePasswordClient(creds.serviceAccountToken!) - - const existing = await client.items.get(params.vaultId, params.itemId) - - // Patch operations are documented and typed against the Connect-shaped - // vocabulary (label/type/section.id) that get_item/create_item/replace_item - // return — apply them to that normalized view, then convert back to the - // SDK's vocabulary (title/fieldType/sectionId) before writing. Patching the - // raw SDK item directly would silently no-op most field/category writes. - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const connectItem = normalizeSdkItem(existing) as Record - for (const op of ops) { - applyPatch(connectItem, op) - } - - const sdkItem = connectItemToSdkItem(connectItem, existing) - const result = await client.items.put(sdkItem) - return NextResponse.json(normalizeSdkItem(result)) - } - - const response = await connectRequest({ - serverUrl: creds.serverUrl!, - apiKey: creds.apiKey!, - path: `/v1/vaults/${params.vaultId}/items/${params.itemId}`, - method: 'PATCH', - body: ops, - }) - - const data = await response.json() - if (!response.ok) { - return NextResponse.json( - { error: data.message || 'Failed to update item' }, - { status: response.status } - ) - } - - return NextResponse.json(data) - } catch (error) { - const message = getErrorMessage(error, 'Unknown error') - logger.error(`[${requestId}] Update item failed:`, error) - return NextResponse.json({ error: `Failed to update item: ${message}` }, { status: 500 }) - } -}) - -interface JsonPatchOperation { - op: 'add' | 'remove' | 'replace' - path: string - value?: unknown -} - -/** Apply a single RFC6902 JSON Patch operation to a mutable object. */ -// eslint-disable-next-line @typescript-eslint/no-explicit-any -function applyPatch(item: Record, op: JsonPatchOperation) { - const segments = op.path.split('/').filter(Boolean) - - if (segments.length === 1) { - const key = segments[0] - if (op.op === 'replace' || op.op === 'add') { - item[key] = op.value - } else if (op.op === 'remove') { - delete item[key] - } - return - } - - let target = item - for (let i = 0; i < segments.length - 1; i++) { - const seg = segments[i] - if (Array.isArray(target)) { - target = arrayElementForSegment(target, seg) - } else { - target = target[seg] - } - if (target === undefined || target === null) return - } - - const lastSeg = segments[segments.length - 1] - - if (op.op === 'replace' || op.op === 'add') { - if (Array.isArray(target) && lastSeg === '-') { - target.push(op.value) - } else if (Array.isArray(target)) { - const index = arrayIndexForSegment(target, lastSeg) - if (index !== -1) target[index] = op.value - } else { - target[lastSeg] = op.value - } - } else if (op.op === 'remove') { - if (Array.isArray(target)) { - const index = arrayIndexForSegment(target, lastSeg) - if (index !== -1) target.splice(index, 1) - } else { - delete target[lastSeg] - } - } -} - -/** - * Resolves an array element for a JSON Patch path segment. 1Password's PATCH API - * addresses items in the `fields`/`sections` arrays by their `id`, not by numeric - * array index (e.g. `/fields/{fieldId}/value`), so a numeric-looking segment is - * only treated as a literal index when no element's `id` matches it. - */ -// eslint-disable-next-line @typescript-eslint/no-explicit-any -function arrayIndexForSegment(target: any[], segment: string): number { - const byId = target.findIndex((el) => el && typeof el === 'object' && el.id === segment) - if (byId !== -1) return byId - const index = Number(segment) - return Number.isInteger(index) && index >= 0 && index < target.length ? index : -1 -} - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -function arrayElementForSegment(target: any[], segment: string): any { - const index = arrayIndexForSegment(target, segment) - return index === -1 ? undefined : target[index] -} diff --git a/apps/sim/app/api/tools/onepassword/utils.test.ts b/apps/sim/app/api/tools/onepassword/utils.test.ts deleted file mode 100644 index 160bbe69b7f..00000000000 --- a/apps/sim/app/api/tools/onepassword/utils.test.ts +++ /dev/null @@ -1,121 +0,0 @@ -/** - * @vitest-environment node - */ -import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' -import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' - -const { mockDnsLookup } = vi.hoisted(() => ({ - mockDnsLookup: vi.fn(), -})) - -vi.mock('dns/promises', () => ({ - default: { lookup: mockDnsLookup }, -})) - -import { validateConnectServerUrl } from '@/app/api/tools/onepassword/utils' - -afterAll(resetEnvFlagsMock) - -describe('validateConnectServerUrl', () => { - beforeEach(() => { - vi.clearAllMocks() - setEnvFlags({ isHosted: false }) - }) - - it('rejects a non-URL string', async () => { - await expect(validateConnectServerUrl('not a url')).rejects.toThrow('is not a valid URL') - }) - - describe('hosted deployment', () => { - beforeEach(() => { - setEnvFlags({ isHosted: true }) - }) - - it.each([ - ['loopback', 'http://127.0.0.1:8080'], - ['RFC1918 10.x', 'http://10.0.0.5'], - ['RFC1918 192.168.x', 'http://192.168.1.1:8443'], - ['RFC1918 172.16.x', 'http://172.16.0.9'], - ['link-local metadata', 'http://169.254.169.254'], - ['IPv4-mapped IPv6 private', 'http://[::ffff:10.0.0.1]'], - ['IPv6 loopback', 'http://[::1]'], - ])('blocks %s', async (_label, url) => { - await expect(validateConnectServerUrl(url)).rejects.toThrow( - 'cannot point to a private or reserved IP address' - ) - }) - - it('allows a public IP literal', async () => { - await expect(validateConnectServerUrl('https://8.8.8.8')).resolves.toBe('8.8.8.8') - }) - - it('blocks a hostname that resolves to a private IP', async () => { - mockDnsLookup.mockResolvedValue([{ address: '10.1.2.3', family: 4 }]) - await expect(validateConnectServerUrl('https://connect.internal')).rejects.toThrow( - 'cannot point to a private or reserved IP address' - ) - }) - - it('allows a hostname that resolves to a public IP', async () => { - mockDnsLookup.mockResolvedValue([{ address: '93.184.216.34', family: 4 }]) - await expect(validateConnectServerUrl('https://connect.example.com')).resolves.toBe( - '93.184.216.34' - ) - }) - - it('prefers the IPv4 address for a dual-stack host (avoids unreachable IPv6 pin)', async () => { - mockDnsLookup.mockResolvedValue([ - { address: '2606:4700::6810:85e5', family: 6 }, - { address: '93.184.216.34', family: 4 }, - ]) - await expect(validateConnectServerUrl('https://connect.example.com')).resolves.toBe( - '93.184.216.34' - ) - }) - - it('pins the sole IPv6 address for an IPv6-only host', async () => { - mockDnsLookup.mockResolvedValue([{ address: '2606:4700::6810:85e5', family: 6 }]) - await expect(validateConnectServerUrl('https://connect.example.com')).resolves.toBe( - '2606:4700::6810:85e5' - ) - }) - }) - - describe('self-hosted deployment', () => { - beforeEach(() => { - setEnvFlags({ isHosted: false }) - }) - - it.each([ - ['loopback', 'http://127.0.0.1:8080', '127.0.0.1'], - ['RFC1918 10.x', 'http://10.0.0.5', '10.0.0.5'], - ['RFC1918 192.168.x', 'http://192.168.1.1:8443', '192.168.1.1'], - ])('allows %s (private Connect server)', async (_label, url, expected) => { - await expect(validateConnectServerUrl(url)).resolves.toBe(expected) - }) - - it('still blocks link-local metadata', async () => { - await expect(validateConnectServerUrl('http://169.254.169.254')).rejects.toThrow( - 'cannot point to a link-local address' - ) - }) - - it('still blocks IPv6 link-local', async () => { - await expect(validateConnectServerUrl('http://[fe80::1]')).rejects.toThrow( - 'cannot point to a link-local address' - ) - }) - - it('allows a hostname that resolves to a private IP', async () => { - mockDnsLookup.mockResolvedValue([{ address: '10.1.2.3', family: 4 }]) - await expect(validateConnectServerUrl('https://connect.internal')).resolves.toBe('10.1.2.3') - }) - }) - - it('rejects when DNS resolution fails', async () => { - mockDnsLookup.mockRejectedValue(new Error('ENOTFOUND')) - await expect(validateConnectServerUrl('https://nope.invalid')).rejects.toThrow( - 'could not be resolved' - ) - }) -}) diff --git a/apps/sim/app/api/tools/onepassword/utils.ts b/apps/sim/app/api/tools/onepassword/utils.ts deleted file mode 100644 index 1b84548a59b..00000000000 --- a/apps/sim/app/api/tools/onepassword/utils.ts +++ /dev/null @@ -1,587 +0,0 @@ -import type { - FileAttributes, - Item, - ItemCategory, - ItemField, - ItemFieldType, - ItemFile, - ItemOverview, - ItemSection, - VaultOverview, - Website, -} from '@1password/sdk' -import { createLogger } from '@sim/logger' -import { resolveHostAddresses } from '@sim/security/dns' -import { isPrivateIp, unwrapIpv6Brackets } from '@sim/security/ssrf' -import { toError } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import * as ipaddr from 'ipaddr.js' -import { isHosted } from '@/lib/core/config/env-flags' -import { - MAX_JSON_API_RESPONSE_BYTES, - secureFetchWithPinnedIP, -} from '@/lib/core/security/input-validation.server' - -/** Connect-format field type strings returned by normalization. */ -type ConnectFieldType = - | 'STRING' - | 'CONCEALED' - | 'EMAIL' - | 'URL' - | 'OTP' - | 'PHONE' - | 'DATE' - | 'MONTH_YEAR' - | 'MENU' - | 'ADDRESS' - | 'REFERENCE' - | 'SSHKEY' - | 'CREDIT_CARD_NUMBER' - | 'CREDIT_CARD_TYPE' - -/** Connect-format category strings returned by normalization. */ -type ConnectCategory = - | 'LOGIN' - | 'PASSWORD' - | 'API_CREDENTIAL' - | 'SECURE_NOTE' - | 'SERVER' - | 'DATABASE' - | 'CREDIT_CARD' - | 'IDENTITY' - | 'SSH_KEY' - | 'DOCUMENT' - | 'SOFTWARE_LICENSE' - | 'EMAIL_ACCOUNT' - | 'MEMBERSHIP' - | 'PASSPORT' - | 'REWARD_PROGRAM' - | 'DRIVER_LICENSE' - | 'BANK_ACCOUNT' - | 'MEDICAL_RECORD' - | 'OUTDOOR_LICENSE' - | 'WIRELESS_ROUTER' - | 'SOCIAL_SECURITY_NUMBER' - | 'CUSTOM' - -/** Normalized vault shape matching the Connect API response. */ -export interface NormalizedVault { - id: string - name: string - description: null - attributeVersion: number - contentVersion: number - items: number - type: string - createdAt: string | null - updatedAt: string | null -} - -/** Normalized item overview shape matching the Connect API response. */ -export interface NormalizedItemOverview { - id: string - title: string - vault: { id: string } - category: ConnectCategory - urls: Array<{ href: string; label: string | null; primary: boolean }> - favorite: boolean - tags: string[] - version: number - state: string | null - createdAt: string | null - updatedAt: string | null - lastEditedBy: null -} - -/** Normalized field shape matching the Connect API response. */ -interface NormalizedField { - id: string - label: string - type: ConnectFieldType - purpose: string - value: string | null - section: { id: string } | null - generate: boolean - recipe: null - entropy: null -} - -/** Normalized attached-file metadata shape matching the Connect API response. */ -export interface NormalizedItemFile { - id: string - name: string - size: number - section: { id: string } | null -} - -/** Normalized full item shape matching the Connect API response. */ -export interface NormalizedItem extends NormalizedItemOverview { - fields: NormalizedField[] - sections: Array<{ id: string; label: string }> - files: NormalizedItemFile[] -} - -/** - * SDK field type string values → Connect field type mapping. - * Uses string literals instead of enum imports to avoid loading the WASM module at build time. - */ -const SDK_TO_CONNECT_FIELD_TYPE: Record = { - Text: 'STRING', - Concealed: 'CONCEALED', - Email: 'EMAIL', - Url: 'URL', - Totp: 'OTP', - Phone: 'PHONE', - Date: 'DATE', - MonthYear: 'MONTH_YEAR', - Menu: 'MENU', - Address: 'ADDRESS', - Reference: 'REFERENCE', - SshKey: 'SSHKEY', - CreditCardNumber: 'CREDIT_CARD_NUMBER', - CreditCardType: 'CREDIT_CARD_TYPE', -} - -/** SDK category string values → Connect category mapping. */ -const SDK_TO_CONNECT_CATEGORY: Record = { - Login: 'LOGIN', - Password: 'PASSWORD', - ApiCredentials: 'API_CREDENTIAL', - SecureNote: 'SECURE_NOTE', - Server: 'SERVER', - Database: 'DATABASE', - CreditCard: 'CREDIT_CARD', - Identity: 'IDENTITY', - SshKey: 'SSH_KEY', - Document: 'DOCUMENT', - SoftwareLicense: 'SOFTWARE_LICENSE', - Email: 'EMAIL_ACCOUNT', - Membership: 'MEMBERSHIP', - Passport: 'PASSPORT', - Rewards: 'REWARD_PROGRAM', - DriverLicense: 'DRIVER_LICENSE', - BankAccount: 'BANK_ACCOUNT', - MedicalRecord: 'MEDICAL_RECORD', - OutdoorLicense: 'OUTDOOR_LICENSE', - Router: 'WIRELESS_ROUTER', - SocialSecurityNumber: 'SOCIAL_SECURITY_NUMBER', - CryptoWallet: 'CUSTOM', - Person: 'CUSTOM', - Unsupported: 'CUSTOM', -} - -/** Connect category → SDK category string mapping. */ -const CONNECT_TO_SDK_CATEGORY: Record = { - LOGIN: 'Login', - PASSWORD: 'Password', - API_CREDENTIAL: 'ApiCredentials', - SECURE_NOTE: 'SecureNote', - SERVER: 'Server', - DATABASE: 'Database', - CREDIT_CARD: 'CreditCard', - IDENTITY: 'Identity', - SSH_KEY: 'SshKey', - DOCUMENT: 'Document', - SOFTWARE_LICENSE: 'SoftwareLicense', - EMAIL_ACCOUNT: 'Email', - MEMBERSHIP: 'Membership', - PASSPORT: 'Passport', - REWARD_PROGRAM: 'Rewards', - DRIVER_LICENSE: 'DriverLicense', - BANK_ACCOUNT: 'BankAccount', - MEDICAL_RECORD: 'MedicalRecord', - OUTDOOR_LICENSE: 'OutdoorLicense', - WIRELESS_ROUTER: 'Router', - SOCIAL_SECURITY_NUMBER: 'SocialSecurityNumber', -} - -/** Connect field type → SDK field type string mapping. */ -const CONNECT_TO_SDK_FIELD_TYPE: Record = { - STRING: 'Text', - CONCEALED: 'Concealed', - EMAIL: 'Email', - URL: 'Url', - OTP: 'Totp', - TOTP: 'Totp', - PHONE: 'Phone', - DATE: 'Date', - MONTH_YEAR: 'MonthYear', - MENU: 'Menu', - ADDRESS: 'Address', - REFERENCE: 'Reference', - SSHKEY: 'SshKey', - CREDIT_CARD_NUMBER: 'CreditCardNumber', - CREDIT_CARD_TYPE: 'CreditCardType', -} - -export type ConnectionMode = 'service_account' | 'connect' - -export interface CredentialParams { - connectionMode?: ConnectionMode | null - serviceAccountToken?: string | null - serverUrl?: string | null - apiKey?: string | null -} - -export interface ResolvedCredentials { - mode: ConnectionMode - serviceAccountToken?: string - serverUrl?: string - apiKey?: string -} - -/** Determine which backend to use based on provided credentials. */ -export function resolveCredentials(params: CredentialParams): ResolvedCredentials { - const mode = params.connectionMode ?? (params.serviceAccountToken ? 'service_account' : 'connect') - - if (mode === 'service_account') { - if (!params.serviceAccountToken) { - throw new Error('Service Account token is required for Service Account mode') - } - return { mode, serviceAccountToken: params.serviceAccountToken } - } - - if (!params.serverUrl || !params.apiKey) { - throw new Error('Server URL and Connect token are required for Connect Server mode') - } - return { mode, serverUrl: params.serverUrl, apiKey: params.apiKey } -} - -/** - * Create a 1Password SDK client from a service account token. - * Uses dynamic import to avoid loading the WASM module at build time. - */ -export async function createOnePasswordClient(serviceAccountToken: string) { - const { createClient } = await import('@1password/sdk') - return createClient({ - auth: serviceAccountToken, - integrationName: 'Sim Studio', - integrationVersion: '1.0.0', - }) -} - -const connectLogger = createLogger('OnePasswordConnect') - -/** - * Enforces the SSRF policy for a resolved Connect server IP. - * - * On the hosted service, all private and reserved IPs are blocked — a tenant has - * no legitimate reason to point Connect at the platform's internal network. On - * self-hosted deployments only link-local (cloud metadata) is blocked, since the - * operator controls both the workflows and the network and Connect servers - * legitimately live on private (RFC1918) addresses. - * - * @throws Error if the IP is not permitted under the active policy. - */ -function assertConnectIpAllowed(ip: string, hostname: string): void { - if (isHosted) { - if (isPrivateIp(ip)) { - connectLogger.warn('1Password Connect server URL resolves to a private or reserved IP', { - hostname, - resolvedIP: ip, - }) - throw new Error('1Password server URL cannot point to a private or reserved IP address') - } - return - } - - if (ipaddr.isValid(ip) && ipaddr.process(ip).range() === 'linkLocal') { - connectLogger.warn('1Password Connect server URL resolves to a link-local IP', { - hostname, - resolvedIP: ip, - }) - throw new Error('1Password server URL cannot point to a link-local address') - } -} - -/** - * Validates a Connect server URL against the SSRF policy and returns the resolved - * IP for DNS pinning to prevent TOCTOU rebinding. See {@link assertConnectIpAllowed} - * for the hosted vs. self-hosted policy. - * @throws Error if the URL is invalid, fails the IP policy, or DNS fails. - */ -export async function validateConnectServerUrl(serverUrl: string): Promise { - let hostname: string - try { - hostname = new URL(serverUrl).hostname - } catch { - throw new Error('1Password server URL is not a valid URL') - } - - const clean = unwrapIpv6Brackets(hostname) - - if (ipaddr.isValid(clean)) { - assertConnectIpAllowed(clean, clean) - return clean - } - - let addresses: string[] - let address: string - try { - const resolved = await resolveHostAddresses(clean) - addresses = resolved.addresses - address = resolved.preferred - } catch (error) { - connectLogger.warn('DNS lookup failed for 1Password Connect server URL', { - hostname: clean, - error: toError(error).message, - }) - throw new Error('1Password server URL hostname could not be resolved') - } - - for (const candidate of addresses) { - assertConnectIpAllowed(candidate, clean) - } - return address -} - -/** Minimal response shape used by all connectRequest callers. */ -export interface ConnectResponse { - ok: boolean - status: number - statusText: string - headers: { get: (name: string) => string | null } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - json: () => Promise - text: () => Promise - arrayBuffer: () => Promise -} - -/** - * Proxy a request to the 1Password Connect Server. - * - * The Connect server is self-hosted at a user-supplied `serverUrl`, so the response body - * is always capped. JSON endpoints use {@link MAX_JSON_API_RESPONSE_BYTES}; callers - * downloading file content pass a larger `maxResponseBytes` explicitly. - */ -export async function connectRequest(options: { - serverUrl: string - apiKey: string - path: string - method: string - body?: unknown - query?: string - maxResponseBytes?: number -}): Promise { - const resolvedIP = await validateConnectServerUrl(options.serverUrl) - - const base = options.serverUrl.replace(/\/$/, '') - const queryStr = options.query ? `?${options.query}` : '' - const url = `${base}${options.path}${queryStr}` - - const headers: Record = { - Authorization: `Bearer ${options.apiKey}`, - } - - if (options.body) { - headers['Content-Type'] = 'application/json' - } - - return secureFetchWithPinnedIP(url, resolvedIP, { - method: options.method, - headers, - body: options.body ? JSON.stringify(options.body) : undefined, - allowHttp: true, - maxResponseBytes: options.maxResponseBytes ?? MAX_JSON_API_RESPONSE_BYTES, - }) -} - -/** Normalize an SDK VaultOverview to match Connect API vault shape. */ -export function normalizeSdkVault(vault: VaultOverview): NormalizedVault { - return { - id: vault.id, - name: vault.title, - description: null, - attributeVersion: 0, - contentVersion: 0, - items: 0, - type: 'USER_CREATED', - createdAt: - vault.createdAt instanceof Date ? vault.createdAt.toISOString() : (vault.createdAt ?? null), - updatedAt: - vault.updatedAt instanceof Date ? vault.updatedAt.toISOString() : (vault.updatedAt ?? null), - } -} - -/** Normalize an SDK ItemOverview to match Connect API item summary shape. */ -export function normalizeSdkItemOverview(item: ItemOverview): NormalizedItemOverview { - return { - id: item.id, - title: item.title, - vault: { id: item.vaultId }, - category: SDK_TO_CONNECT_CATEGORY[item.category] ?? 'CUSTOM', - urls: (item.websites ?? []).map((w: Website) => ({ - href: w.url, - label: w.label ?? null, - primary: false, - })), - favorite: false, - tags: item.tags ?? [], - version: 0, - state: item.state === 'archived' ? 'ARCHIVED' : null, - createdAt: - item.createdAt instanceof Date ? item.createdAt.toISOString() : (item.createdAt ?? null), - updatedAt: - item.updatedAt instanceof Date ? item.updatedAt.toISOString() : (item.updatedAt ?? null), - lastEditedBy: null, - } -} - -/** Normalize a full SDK Item to match Connect API FullItem shape. */ -export function normalizeSdkItem(item: Item): NormalizedItem { - return { - id: item.id, - title: item.title, - vault: { id: item.vaultId }, - category: SDK_TO_CONNECT_CATEGORY[item.category] ?? 'CUSTOM', - urls: (item.websites ?? []).map((w: Website) => ({ - href: w.url, - label: w.label ?? null, - primary: false, - })), - favorite: false, - tags: item.tags ?? [], - version: item.version ?? 0, - state: null, - fields: (item.fields ?? []).map((field: ItemField) => ({ - id: field.id, - label: field.title, - type: SDK_TO_CONNECT_FIELD_TYPE[field.fieldType] ?? 'STRING', - purpose: '', - value: field.value ?? null, - section: field.sectionId ? { id: field.sectionId } : null, - generate: false, - recipe: null, - entropy: null, - })), - sections: (item.sections ?? []).map((section: ItemSection) => ({ - id: section.id, - label: section.title, - })), - files: [ - ...(item.files ?? []).map((file: ItemFile) => ({ - id: file.attributes.id, - name: file.attributes.name, - size: file.attributes.size, - section: file.sectionId ? { id: file.sectionId } : null, - })), - ...(item.document - ? [ - { - id: item.document.id, - name: item.document.name, - size: item.document.size, - section: null, - }, - ] - : []), - ], - createdAt: - item.createdAt instanceof Date ? item.createdAt.toISOString() : (item.createdAt ?? null), - updatedAt: - item.updatedAt instanceof Date ? item.updatedAt.toISOString() : (item.updatedAt ?? null), - lastEditedBy: null, - } -} - -/** - * Find an attached file's SDK {@link FileAttributes} on an item by file ID. - * Checks both the `files` array and the single `document` attribute that - * Document-category items carry instead of a `files` entry. - */ -export function findItemFileAttributes(item: Item, fileId: string): FileAttributes | undefined { - if (item.document?.id === fileId) return item.document - return item.files?.find((file) => file.attributes.id === fileId)?.attributes -} - -/** - * Convert a Connect-shaped item (the vocabulary `normalizeSdkItem` produces and - * this integration's tools document — `label`/`type`/`section: {id}`) back into - * an SDK-compatible {@link Item} for `client.items.put()`. Falls back to `existing` - * for any array the caller didn't provide, so partial input (e.g. Replace Item's - * optional fields) is preserved. - * - * Service Account mode must always convert through this function before calling - * `put()` — never apply a Connect-shaped JSON Patch directly onto a raw SDK - * {@link Item}, since SDK field/category vocabulary differs from Connect's - * (`title` vs `label`, `fieldType` vs `type`, `sectionId` vs `section.id`, SDK - * category enum strings vs Connect's SCREAMING_SNAKE_CASE) and silently no-ops or - * corrupts the write otherwise. - */ -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export function connectItemToSdkItem(connectItem: Record, existing: Item): Item { - const existingFieldsById = new Map((existing.fields ?? []).map((f) => [f.id, f])) - const existingSectionsById = new Map((existing.sections ?? []).map((s) => [s.id, s])) - - return { - ...existing, - id: existing.id, - vaultId: existing.vaultId, - title: connectItem.title || existing.title, - category: connectItem.category ? toSdkCategory(connectItem.category) : existing.category, - fields: Array.isArray(connectItem.fields) - ? // eslint-disable-next-line @typescript-eslint/no-explicit-any - connectItem.fields.map((f: Record) => ({ - // Preserve any SDK-only metadata (e.g. password-generation `details`) - // on fields that already existed — only brand-new fields start bare. - ...(f.id ? existingFieldsById.get(f.id) : undefined), - id: f.id || generateId().slice(0, 8), - title: f.label || f.title || '', - fieldType: toSdkFieldType(f.type || 'STRING'), - value: f.value || '', - sectionId: f.section?.id ?? f.sectionId, - })) - : existing.fields, - sections: Array.isArray(connectItem.sections) - ? // eslint-disable-next-line @typescript-eslint/no-explicit-any - connectItem.sections.map((s: Record) => ({ - ...(s.id ? existingSectionsById.get(s.id) : undefined), - id: s.id || '', - title: s.label || s.title || '', - })) - : existing.sections, - notes: connectItem.notes ?? existing.notes, - tags: connectItem.tags ?? existing.tags, - websites: Array.isArray(connectItem.urls ?? connectItem.websites) - ? (connectItem.urls ?? connectItem.websites).map( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (u: Record) => ({ - url: u.href || u.url || '', - label: u.label || '', - autofillBehavior: 'AnywhereOnWebsite' as const, - }) - ) - : existing.websites, - } as Item -} - -/** - * Best-effort SCIM `eq` filter matcher for Service Account mode, which has no - * server-side filtering (unlike Connect, whose `filter` query param is forwarded - * verbatim and evaluated by the Connect server). Recognizes `attribute eq "value"` - * (quotes optional) as an exact, case-insensitive match against the named attribute - * — `id` compares against the id, anything else (name/title/etc.) against the - * display value; anything that doesn't parse as `eq` falls back to a - * case-insensitive substring match against both so the field remains useful for - * free-text search. - */ -export function matchesFilter(value: string, id: string, filter: string): boolean { - const eqMatch = filter.match(/^\s*(\S+)\s+eq\s+"?([^"]*)"?\s*$/i) - if (eqMatch) { - const [, attribute, needle] = eqMatch - const target = attribute.toLowerCase() === 'id' ? id : value - return target.toLowerCase() === needle.toLowerCase() - } - const needle = filter.toLowerCase() - return value.toLowerCase().includes(needle) || id.toLowerCase().includes(needle) -} - -/** Convert a Connect-style category string to the SDK category string. */ -export function toSdkCategory(category: string): `${ItemCategory}` { - return CONNECT_TO_SDK_CATEGORY[category] ?? 'Login' -} - -/** Convert a Connect-style field type string to the SDK field type string. */ -export function toSdkFieldType(type: string): `${ItemFieldType}` { - return CONNECT_TO_SDK_FIELD_TYPE[type] ?? 'Text' -} diff --git a/apps/sim/app/api/tools/outlook/copy/route.ts b/apps/sim/app/api/tools/outlook/copy/route.ts deleted file mode 100644 index 406231899fb..00000000000 --- a/apps/sim/app/api/tools/outlook/copy/route.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { outlookCopyContract } from '@/lib/api/contracts/tools/microsoft' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('OutlookCopyAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Outlook copy attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated Outlook copy request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const parsed = await parseRequest(outlookCopyContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Copying Outlook email`, { - messageId: validatedData.messageId, - destinationId: validatedData.destinationId, - }) - - const graphEndpoint = `https://graph.microsoft.com/v1.0/me/messages/${validatedData.messageId}/copy` - - logger.info(`[${requestId}] Sending to Microsoft Graph API: ${graphEndpoint}`) - - const graphResponse = await fetch(graphEndpoint, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${validatedData.accessToken}`, - }, - body: JSON.stringify({ - destinationId: validatedData.destinationId, - }), - }) - - if (!graphResponse.ok) { - const errorData = await graphResponse.json().catch(() => ({})) - logger.error(`[${requestId}] Microsoft Graph API error:`, errorData) - return NextResponse.json( - { - success: false, - error: errorData.error?.message || 'Failed to copy email', - }, - { status: graphResponse.status } - ) - } - - const responseData = await graphResponse.json() - - logger.info(`[${requestId}] Email copied successfully`, { - originalMessageId: validatedData.messageId, - copiedMessageId: responseData.id, - destinationFolderId: responseData.parentFolderId, - }) - - return NextResponse.json({ - success: true, - output: { - message: 'Email copied successfully', - originalMessageId: validatedData.messageId, - copiedMessageId: responseData.id, - destinationFolderId: responseData.parentFolderId, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error copying Outlook email:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/outlook/delete/route.ts b/apps/sim/app/api/tools/outlook/delete/route.ts deleted file mode 100644 index 1f3e4c1b5c5..00000000000 --- a/apps/sim/app/api/tools/outlook/delete/route.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { outlookDeleteContract } from '@/lib/api/contracts/tools/microsoft' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('OutlookDeleteAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Outlook delete attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated Outlook delete request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const parsed = await parseRequest(outlookDeleteContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Deleting Outlook email`, { - messageId: validatedData.messageId, - }) - - const graphEndpoint = `https://graph.microsoft.com/v1.0/me/messages/${validatedData.messageId}` - - logger.info(`[${requestId}] Sending to Microsoft Graph API: ${graphEndpoint}`) - - const graphResponse = await fetch(graphEndpoint, { - method: 'DELETE', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - }, - }) - - if (!graphResponse.ok) { - const errorData = await graphResponse.json().catch(() => ({})) - logger.error(`[${requestId}] Microsoft Graph API error:`, errorData) - return NextResponse.json( - { - success: false, - error: errorData.error?.message || 'Failed to delete email', - }, - { status: graphResponse.status } - ) - } - - logger.info(`[${requestId}] Email deleted successfully`, { - messageId: validatedData.messageId, - }) - - return NextResponse.json({ - success: true, - output: { - message: 'Email moved to Deleted Items successfully', - messageId: validatedData.messageId, - status: 'deleted', - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error deleting Outlook email:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/outlook/draft/route.ts b/apps/sim/app/api/tools/outlook/draft/route.ts deleted file mode 100644 index 52b114bed2a..00000000000 --- a/apps/sim/app/api/tools/outlook/draft/route.ts +++ /dev/null @@ -1,200 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { outlookDraftContract } from '@/lib/api/contracts/tools/microsoft' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('OutlookDraftAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Outlook draft attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - const userId = authResult.userId - logger.info(`[${requestId}] Authenticated Outlook draft request via ${authResult.authType}`, { - userId, - }) - - const parsed = await parseRequest(outlookDraftContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Creating Outlook draft`, { - to: validatedData.to, - subject: validatedData.subject, - hasAttachments: !!(validatedData.attachments && validatedData.attachments.length > 0), - attachmentCount: validatedData.attachments?.length || 0, - }) - - const toRecipients = validatedData.to.split(',').map((email) => ({ - emailAddress: { address: email.trim() }, - })) - - const ccRecipients = validatedData.cc - ? validatedData.cc.split(',').map((email) => ({ - emailAddress: { address: email.trim() }, - })) - : undefined - - const bccRecipients = validatedData.bcc - ? validatedData.bcc.split(',').map((email) => ({ - emailAddress: { address: email.trim() }, - })) - : undefined - - const message: any = { - subject: validatedData.subject, - body: { - contentType: validatedData.contentType || 'text', - content: validatedData.body, - }, - toRecipients, - } - - if (ccRecipients) { - message.ccRecipients = ccRecipients - } - - if (bccRecipients) { - message.bccRecipients = bccRecipients - } - - if (validatedData.attachments && validatedData.attachments.length > 0) { - const rawAttachments = validatedData.attachments - logger.info(`[${requestId}] Processing ${rawAttachments.length} attachment(s)`) - - const attachments = processFilesToUserFiles(rawAttachments, requestId, logger) - - if (attachments.length > 0) { - const totalSize = attachments.reduce((sum, file) => sum + file.size, 0) - const maxSize = 4 * 1024 * 1024 // 4MB - - if (totalSize > maxSize) { - const sizeMB = (totalSize / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds Outlook's limit of 4MB per request`, - }, - { status: 400 } - ) - } - - const accessResults = await Promise.all( - attachments.map((file) => assertToolFileAccess(file.key, userId, requestId, logger)) - ) - const denied = accessResults.find((r) => r !== null) - if (denied) return denied - - let resolved: Array<{ buffer: Buffer; contentType: string }> - try { - resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, { - totalMaxBytes: maxSize, - label: 'Total attachment size', - }) - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - if (isPayloadSizeLimitError(error)) { - const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds Outlook's limit of 4MB per request`, - }, - { status: 400 } - ) - } - logger.error(`[${requestId}] Failed to download an attachment:`, error) - return NextResponse.json( - { - success: false, - error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`, - }, - { status: 500 } - ) - } - - const attachmentObjects = attachments.map((file, i) => ({ - '@odata.type': '#microsoft.graph.fileAttachment', - name: file.name, - contentType: resolved[i].contentType || file.type || 'application/octet-stream', - contentBytes: resolved[i].buffer.toString('base64'), - })) - - logger.info(`[${requestId}] Converted ${attachmentObjects.length} attachments to base64`) - message.attachments = attachmentObjects - } - } - - const graphEndpoint = 'https://graph.microsoft.com/v1.0/me/messages' - - logger.info(`[${requestId}] Creating draft via Microsoft Graph API`) - - const graphResponse = await fetch(graphEndpoint, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${validatedData.accessToken}`, - }, - body: JSON.stringify(message), - }) - - if (!graphResponse.ok) { - const errorData = await graphResponse.json().catch(() => ({})) - logger.error(`[${requestId}] Microsoft Graph API error:`, errorData) - return NextResponse.json( - { - success: false, - error: errorData.error?.message || 'Failed to create draft', - }, - { status: graphResponse.status } - ) - } - - const responseData = await graphResponse.json() - logger.info(`[${requestId}] Draft created successfully, ID: ${responseData.id}`) - - return NextResponse.json({ - success: true, - output: { - message: 'Draft created successfully', - messageId: responseData.id, - subject: responseData.subject, - attachmentCount: message.attachments?.length || 0, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error creating Outlook draft:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/outlook/mark-read/route.ts b/apps/sim/app/api/tools/outlook/mark-read/route.ts deleted file mode 100644 index faf2ae20c59..00000000000 --- a/apps/sim/app/api/tools/outlook/mark-read/route.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { outlookMarkReadContract } from '@/lib/api/contracts/tools/microsoft' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('OutlookMarkReadAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Outlook mark read attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info( - `[${requestId}] Authenticated Outlook mark read request via ${authResult.authType}`, - { - userId: authResult.userId, - } - ) - - const parsed = await parseRequest(outlookMarkReadContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Marking Outlook email as read`, { - messageId: validatedData.messageId, - }) - - const graphEndpoint = `https://graph.microsoft.com/v1.0/me/messages/${validatedData.messageId}` - - logger.info(`[${requestId}] Sending to Microsoft Graph API: ${graphEndpoint}`) - - const graphResponse = await fetch(graphEndpoint, { - method: 'PATCH', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${validatedData.accessToken}`, - }, - body: JSON.stringify({ - isRead: true, - }), - }) - - if (!graphResponse.ok) { - const errorData = await graphResponse.json().catch(() => ({})) - logger.error(`[${requestId}] Microsoft Graph API error:`, errorData) - return NextResponse.json( - { - success: false, - error: errorData.error?.message || 'Failed to mark email as read', - }, - { status: graphResponse.status } - ) - } - - const responseData = await graphResponse.json() - - logger.info(`[${requestId}] Email marked as read successfully`, { - messageId: responseData.id, - isRead: responseData.isRead, - }) - - return NextResponse.json({ - success: true, - output: { - message: 'Email marked as read successfully', - messageId: responseData.id, - isRead: responseData.isRead, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error marking Outlook email as read:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/outlook/mark-unread/route.ts b/apps/sim/app/api/tools/outlook/mark-unread/route.ts deleted file mode 100644 index e08caffa219..00000000000 --- a/apps/sim/app/api/tools/outlook/mark-unread/route.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { outlookMarkUnreadContract } from '@/lib/api/contracts/tools/microsoft' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('OutlookMarkUnreadAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Outlook mark unread attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info( - `[${requestId}] Authenticated Outlook mark unread request via ${authResult.authType}`, - { - userId: authResult.userId, - } - ) - - const parsed = await parseRequest(outlookMarkUnreadContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Marking Outlook email as unread`, { - messageId: validatedData.messageId, - }) - - const graphEndpoint = `https://graph.microsoft.com/v1.0/me/messages/${validatedData.messageId}` - - logger.info(`[${requestId}] Sending to Microsoft Graph API: ${graphEndpoint}`) - - const graphResponse = await fetch(graphEndpoint, { - method: 'PATCH', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${validatedData.accessToken}`, - }, - body: JSON.stringify({ - isRead: false, - }), - }) - - if (!graphResponse.ok) { - const errorData = await graphResponse.json().catch(() => ({})) - logger.error(`[${requestId}] Microsoft Graph API error:`, errorData) - return NextResponse.json( - { - success: false, - error: errorData.error?.message || 'Failed to mark email as unread', - }, - { status: graphResponse.status } - ) - } - - const responseData = await graphResponse.json() - - logger.info(`[${requestId}] Email marked as unread successfully`, { - messageId: responseData.id, - isRead: responseData.isRead, - }) - - return NextResponse.json({ - success: true, - output: { - message: 'Email marked as unread successfully', - messageId: responseData.id, - isRead: responseData.isRead, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error marking Outlook email as unread:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/outlook/move/route.ts b/apps/sim/app/api/tools/outlook/move/route.ts deleted file mode 100644 index d27440ffe80..00000000000 --- a/apps/sim/app/api/tools/outlook/move/route.ts +++ /dev/null @@ -1,96 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { outlookMoveContract } from '@/lib/api/contracts/tools/microsoft' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('OutlookMoveAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Outlook move attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated Outlook move request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const parsed = await parseRequest(outlookMoveContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Moving Outlook email`, { - messageId: validatedData.messageId, - destinationId: validatedData.destinationId, - }) - - const graphEndpoint = `https://graph.microsoft.com/v1.0/me/messages/${validatedData.messageId}/move` - - logger.info(`[${requestId}] Sending to Microsoft Graph API: ${graphEndpoint}`) - - const graphResponse = await fetch(graphEndpoint, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${validatedData.accessToken}`, - }, - body: JSON.stringify({ - destinationId: validatedData.destinationId, - }), - }) - - if (!graphResponse.ok) { - const errorData = await graphResponse.json().catch(() => ({})) - logger.error(`[${requestId}] Microsoft Graph API error:`, errorData) - return NextResponse.json( - { - success: false, - error: errorData.error?.message || 'Failed to move email', - }, - { status: graphResponse.status } - ) - } - - const responseData = await graphResponse.json() - - logger.info(`[${requestId}] Email moved successfully`, { - messageId: responseData.id, - parentFolderId: responseData.parentFolderId, - }) - - return NextResponse.json({ - success: true, - output: { - message: 'Email moved successfully', - messageId: responseData.id, - newFolderId: responseData.parentFolderId, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error moving Outlook email:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/outlook/send/route.ts b/apps/sim/app/api/tools/outlook/send/route.ts deleted file mode 100644 index 82dd3e4474a..00000000000 --- a/apps/sim/app/api/tools/outlook/send/route.ts +++ /dev/null @@ -1,211 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { outlookSendContract } from '@/lib/api/contracts/tools/microsoft' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('OutlookSendAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Outlook send attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - const userId = authResult.userId - logger.info(`[${requestId}] Authenticated Outlook send request via ${authResult.authType}`, { - userId, - }) - - const parsed = await parseRequest(outlookSendContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Sending Outlook email`, { - to: validatedData.to, - subject: validatedData.subject, - hasAttachments: !!(validatedData.attachments && validatedData.attachments.length > 0), - attachmentCount: validatedData.attachments?.length || 0, - }) - - const toRecipients = validatedData.to.split(',').map((email) => ({ - emailAddress: { address: email.trim() }, - })) - - const ccRecipients = validatedData.cc - ? validatedData.cc.split(',').map((email) => ({ - emailAddress: { address: email.trim() }, - })) - : undefined - - const bccRecipients = validatedData.bcc - ? validatedData.bcc.split(',').map((email) => ({ - emailAddress: { address: email.trim() }, - })) - : undefined - - const message: any = { - subject: validatedData.subject, - body: { - contentType: validatedData.contentType || 'text', - content: validatedData.body, - }, - toRecipients, - } - - if (ccRecipients) { - message.ccRecipients = ccRecipients - } - - if (bccRecipients) { - message.bccRecipients = bccRecipients - } - - if (validatedData.attachments && validatedData.attachments.length > 0) { - const rawAttachments = validatedData.attachments - logger.info(`[${requestId}] Processing ${rawAttachments.length} attachment(s)`) - - const attachments = processFilesToUserFiles(rawAttachments, requestId, logger) - - if (attachments.length > 0) { - const totalSize = attachments.reduce((sum, file) => sum + file.size, 0) - const maxSize = 3 * 1024 * 1024 // 3MB - Microsoft Graph API limit for inline attachments - - if (totalSize > maxSize) { - const sizeMB = (totalSize / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds Microsoft Graph API limit of 3MB per request`, - }, - { status: 400 } - ) - } - - const accessResults = await Promise.all( - attachments.map((file) => assertToolFileAccess(file.key, userId, requestId, logger)) - ) - const denied = accessResults.find((r) => r !== null) - if (denied) return denied - - let resolved: Array<{ buffer: Buffer; contentType: string }> - try { - resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, { - totalMaxBytes: maxSize, - label: 'Total attachment size', - }) - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - if (isPayloadSizeLimitError(error)) { - const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds Microsoft Graph API limit of 3MB per request`, - }, - { status: 400 } - ) - } - logger.error(`[${requestId}] Failed to download an attachment:`, error) - return NextResponse.json( - { - success: false, - error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`, - }, - { status: 500 } - ) - } - - const attachmentObjects = attachments.map((file, i) => ({ - '@odata.type': '#microsoft.graph.fileAttachment', - name: file.name, - contentType: resolved[i].contentType || file.type || 'application/octet-stream', - contentBytes: resolved[i].buffer.toString('base64'), - })) - - logger.info(`[${requestId}] Converted ${attachmentObjects.length} attachments to base64`) - message.attachments = attachmentObjects - } - } - - const graphEndpoint = validatedData.replyToMessageId - ? `https://graph.microsoft.com/v1.0/me/messages/${validatedData.replyToMessageId}/reply` - : 'https://graph.microsoft.com/v1.0/me/sendMail' - - logger.info(`[${requestId}] Sending to Microsoft Graph API: ${graphEndpoint}`) - - const graphResponse = await fetch(graphEndpoint, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${validatedData.accessToken}`, - }, - body: JSON.stringify( - validatedData.replyToMessageId - ? { - comment: validatedData.body, - message: message, - } - : { - message: message, - saveToSentItems: true, - } - ), - }) - - if (!graphResponse.ok) { - const errorData = await graphResponse.json().catch(() => ({})) - logger.error(`[${requestId}] Microsoft Graph API error:`, errorData) - return NextResponse.json( - { - success: false, - error: errorData.error?.message || 'Failed to send email', - }, - { status: graphResponse.status } - ) - } - - logger.info(`[${requestId}] Email sent successfully`) - - return NextResponse.json({ - success: true, - output: { - message: 'Email sent successfully', - status: 'sent', - timestamp: new Date().toISOString(), - attachmentCount: message.attachments?.length || 0, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error sending Outlook email:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/persona/import-accounts/route.ts b/apps/sim/app/api/tools/persona/import-accounts/route.ts deleted file mode 100644 index ca59a41b053..00000000000 --- a/apps/sim/app/api/tools/persona/import-accounts/route.ts +++ /dev/null @@ -1,147 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { personaImportAccountsContract } from '@/lib/api/contracts/tools/persona' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess, FileAccessDeniedError } from '@/app/api/files/authorization' -import { - buildPersonaHeaders, - extractPersonaErrorMessage, - mapImporter, - PERSONA_API_BASE, -} from '@/tools/persona/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('PersonaImportAccountsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Persona import attempt`, { - error: authResult.error || 'Missing userId', - }) - return NextResponse.json( - { success: false, error: authResult.error || 'Unauthorized' }, - { status: 401 } - ) - } - - const userId = authResult.userId - - const parsed = await parseRequest(personaImportAccountsContract, request, {}) - if (!parsed.success) return parsed.response - - const { apiKey, file } = parsed.data.body - - const userFiles = processFilesToUserFiles([file], requestId, logger) - if (userFiles.length === 0) { - return NextResponse.json( - { success: false, error: 'Invalid file input: a stored CSV file is required' }, - { status: 400 } - ) - } - - const userFile = userFiles[0] - const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) - if (denied) return denied - - let buffer: Buffer - try { - const resolved = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - }) - buffer = resolved.buffer - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - logger.error(`[${requestId}] Failed to download Persona import file:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Internal server error') }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } - - logger.info(`[${requestId}] Importing accounts into Persona`, { - fileName: userFile.name, - fileSize: buffer.length, - userId, - }) - - const personaResponse = await fetch(`${PERSONA_API_BASE}/importer/accounts`, { - method: 'POST', - headers: buildPersonaHeaders(apiKey), - body: JSON.stringify({ - data: { - attributes: { - file: { - data: buffer.toString('base64'), - filename: userFile.name, - }, - }, - }, - }), - }) - - const personaData = await personaResponse.json().catch(() => null) - - if (!personaResponse.ok) { - const personaError = extractPersonaErrorMessage( - personaData, - `Persona API error: ${personaResponse.statusText}` - ) - logger.error(`[${requestId}] Persona import accounts failed`, { - status: personaResponse.status, - error: personaError, - }) - return NextResponse.json( - { success: false, error: personaError }, - { status: personaResponse.status } - ) - } - - const importer = mapImporter(personaData?.data ?? {}) - if (!importer.id) { - logger.error(`[${requestId}] Persona import accounts returned an unexpected response body`, { - status: personaResponse.status, - }) - return NextResponse.json( - { success: false, error: 'Persona returned an unexpected response for the account import' }, - { status: 502 } - ) - } - - logger.info(`[${requestId}] Persona account import created`, { - importerId: importer.id, - }) - - return NextResponse.json({ - success: true, - output: { - importer, - }, - }) - } catch (error) { - if (error instanceof FileAccessDeniedError) { - return NextResponse.json({ success: false, error: 'File not found' }, { status: 404 }) - } - - logger.error(`[${requestId}] Error importing accounts into Persona:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Internal server error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/pipedrive/get-files/route.ts b/apps/sim/app/api/tools/pipedrive/get-files/route.ts deleted file mode 100644 index 9249cb76e69..00000000000 --- a/apps/sim/app/api/tools/pipedrive/get-files/route.ts +++ /dev/null @@ -1,187 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { pipedriveGetFilesContract } from '@/lib/api/contracts/tools/pipedrive' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getFileExtension, getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils' -import { getPipedriveAuthHeaders } from '@/tools/pipedrive/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('PipedriveGetFilesAPI') - -interface PipedriveFile { - id?: number - name?: string - url?: string -} - -/** - * Whether a download URL belongs to Pipedrive. The workspace credential is - * only attached to Pipedrive-owned hosts — if the API ever returns an - * external/CDN download URL, it is fetched without credentials. - */ -function isPipedriveHost(url: string): boolean { - try { - const hostname = new URL(url).hostname.toLowerCase() - return hostname === 'pipedrive.com' || hostname.endsWith('.pipedrive.com') - } catch { - return false - } -} - -interface PipedriveApiResponse { - success: boolean - data?: PipedriveFile[] - additional_data?: { - pagination?: { - more_items_in_collection: boolean - next_start: number - } - } - error?: string -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Pipedrive get files attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - const parsed = await parseRequest(pipedriveGetFilesContract, request, {}) - if (!parsed.success) return parsed.response - - const { accessToken, authStyle, sort, limit, start, downloadFiles } = parsed.data.body - const authHeaders = getPipedriveAuthHeaders({ accessToken, authStyle }) - - const baseUrl = 'https://api.pipedrive.com/v1/files' - const queryParams = new URLSearchParams() - - if (sort) queryParams.append('sort', sort) - if (limit) queryParams.append('limit', limit) - if (start) queryParams.append('start', start) - - const queryString = queryParams.toString() - const apiUrl = queryString ? `${baseUrl}?${queryString}` : baseUrl - - logger.info(`[${requestId}] Fetching files from Pipedrive`) - - const urlValidation = await validateUrlWithDNS(apiUrl, 'apiUrl') - if (!urlValidation.isValid) { - return NextResponse.json({ success: false, error: urlValidation.error }, { status: 400 }) - } - - const response = await secureFetchWithPinnedIP(apiUrl, urlValidation.resolvedIP!, { - method: 'GET', - headers: authHeaders, - }) - - const data = (await response.json()) as PipedriveApiResponse - - if (!data.success) { - logger.error(`[${requestId}] Pipedrive API request failed`, { data }) - return NextResponse.json( - { success: false, error: data.error || 'Failed to fetch files from Pipedrive' }, - { status: 400 } - ) - } - - const files = data.data || [] - const hasMore = data.additional_data?.pagination?.more_items_in_collection || false - const nextStart = data.additional_data?.pagination?.next_start ?? null - const downloadedFiles: Array<{ - name: string - mimeType: string - data: string - size: number - }> = [] - - if (downloadFiles) { - // Bare auth headers for byte downloads — no Accept: application/json. - const downloadAuthHeaders: Record = - authStyle === 'x-api-token' - ? { 'x-api-token': accessToken } - : { Authorization: `Bearer ${accessToken}` } - - for (const file of files) { - if (!file?.url) continue - - try { - const fileUrlValidation = await validateUrlWithDNS(file.url, 'fileUrl') - if (!fileUrlValidation.isValid) continue - - const downloadResponse = await secureFetchWithPinnedIP( - file.url, - fileUrlValidation.resolvedIP!, - { - method: 'GET', - headers: isPipedriveHost(file.url) ? downloadAuthHeaders : {}, - } - ) - - if (!downloadResponse.ok) continue - - const arrayBuffer = await downloadResponse.arrayBuffer() - const buffer = Buffer.from(arrayBuffer) - const extension = getFileExtension(file.name || '') - const mimeType = - downloadResponse.headers.get('content-type') || getMimeTypeFromExtension(extension) - const fileName = file.name || `pipedrive-file-${file.id || Date.now()}` - - downloadedFiles.push({ - name: fileName, - mimeType, - data: buffer.toString('base64'), - size: buffer.length, - }) - } catch (error) { - logger.warn(`[${requestId}] Failed to download file ${file.id}:`, error) - } - } - } - - logger.info(`[${requestId}] Pipedrive files fetched successfully`, { - fileCount: files.length, - downloadedCount: downloadedFiles.length, - }) - - return NextResponse.json({ - success: true, - output: { - files, - downloadedFiles: downloadedFiles.length > 0 ? downloadedFiles : undefined, - total_items: files.length, - has_more: hasMore, - next_start: nextStart, - success: true, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error fetching Pipedrive files:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/postgresql/delete/route.ts b/apps/sim/app/api/tools/postgresql/delete/route.ts deleted file mode 100644 index bdf9fcb605e..00000000000 --- a/apps/sim/app/api/tools/postgresql/delete/route.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { postgresqlDeleteContract } from '@/lib/api/contracts/tools/databases/postgresql' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createPostgresConnection, executeDelete } from '@/app/api/tools/postgresql/utils' - -const logger = createLogger('PostgreSQLDeleteAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized PostgreSQL delete attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(postgresqlDeleteContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Deleting data from ${params.table} on ${params.host}:${params.port}/${params.database}` - ) - - const sql = await createPostgresConnection({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - ssl: params.ssl, - }) - - try { - const result = await executeDelete(sql, params.table, params.where) - - logger.info(`[${requestId}] Delete executed successfully, ${result.rowCount} row(s) deleted`) - - return NextResponse.json({ - message: `Data deleted successfully. ${result.rowCount} row(s) affected.`, - rows: result.rows, - rowCount: result.rowCount, - }) - } finally { - await sql.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] PostgreSQL delete failed:`, error) - - return NextResponse.json( - { error: `PostgreSQL delete failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/postgresql/execute/route.ts b/apps/sim/app/api/tools/postgresql/execute/route.ts deleted file mode 100644 index 3a5bbd5388b..00000000000 --- a/apps/sim/app/api/tools/postgresql/execute/route.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { postgresqlExecuteContract } from '@/lib/api/contracts/tools/databases/postgresql' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createPostgresConnection, - executeQuery, - validateQuery, -} from '@/app/api/tools/postgresql/utils' - -const logger = createLogger('PostgreSQLExecuteAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized PostgreSQL execute attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(postgresqlExecuteContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Executing raw SQL on ${params.host}:${params.port}/${params.database}` - ) - - const validation = validateQuery(params.query) - if (!validation.isValid) { - logger.warn(`[${requestId}] Query validation failed: ${validation.error}`) - return NextResponse.json( - { error: `Query validation failed: ${validation.error}` }, - { status: 400 } - ) - } - - const sql = await createPostgresConnection({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - ssl: params.ssl, - }) - - try { - const result = await executeQuery(sql, params.query) - - logger.info(`[${requestId}] SQL executed successfully, ${result.rowCount} row(s) affected`) - - return NextResponse.json({ - message: `SQL executed successfully. ${result.rowCount} row(s) affected.`, - rows: result.rows, - rowCount: result.rowCount, - }) - } finally { - await sql.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] PostgreSQL execute failed:`, error) - - return NextResponse.json( - { error: `PostgreSQL execute failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/postgresql/insert/route.ts b/apps/sim/app/api/tools/postgresql/insert/route.ts deleted file mode 100644 index 760d26e1be2..00000000000 --- a/apps/sim/app/api/tools/postgresql/insert/route.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { postgresqlInsertContract } from '@/lib/api/contracts/tools/databases/postgresql' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createPostgresConnection, executeInsert } from '@/app/api/tools/postgresql/utils' - -const logger = createLogger('PostgreSQLInsertAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized PostgreSQL insert attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(postgresqlInsertContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Inserting data into ${params.table} on ${params.host}:${params.port}/${params.database}` - ) - - const sql = await createPostgresConnection({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - ssl: params.ssl, - }) - - try { - const result = await executeInsert(sql, params.table, params.data) - - logger.info(`[${requestId}] Insert executed successfully, ${result.rowCount} row(s) inserted`) - - return NextResponse.json({ - message: `Data inserted successfully. ${result.rowCount} row(s) affected.`, - rows: result.rows, - rowCount: result.rowCount, - }) - } finally { - await sql.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] PostgreSQL insert failed:`, error) - - return NextResponse.json( - { error: `PostgreSQL insert failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/postgresql/introspect/route.ts b/apps/sim/app/api/tools/postgresql/introspect/route.ts deleted file mode 100644 index 0a295f93b9b..00000000000 --- a/apps/sim/app/api/tools/postgresql/introspect/route.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { postgresqlIntrospectContract } from '@/lib/api/contracts/tools/databases/postgresql' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createPostgresConnection, executeIntrospect } from '@/app/api/tools/postgresql/utils' - -const logger = createLogger('PostgreSQLIntrospectAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized PostgreSQL introspect attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(postgresqlIntrospectContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Introspecting PostgreSQL schema on ${params.host}:${params.port}/${params.database}` - ) - - const sql = await createPostgresConnection({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - ssl: params.ssl, - }) - - try { - const result = await executeIntrospect(sql, params.schema) - - logger.info( - `[${requestId}] Introspection completed successfully, found ${result.tables.length} tables` - ) - - return NextResponse.json({ - message: `Schema introspection completed. Found ${result.tables.length} table(s) in schema '${params.schema}'.`, - tables: result.tables, - schemas: result.schemas, - }) - } finally { - await sql.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] PostgreSQL introspection failed:`, error) - - return NextResponse.json( - { error: `PostgreSQL introspection failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/postgresql/query/route.ts b/apps/sim/app/api/tools/postgresql/query/route.ts deleted file mode 100644 index d47ad84189f..00000000000 --- a/apps/sim/app/api/tools/postgresql/query/route.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { postgresqlQueryContract } from '@/lib/api/contracts/tools/databases/postgresql' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createPostgresConnection, executeQuery } from '@/app/api/tools/postgresql/utils' - -const logger = createLogger('PostgreSQLQueryAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized PostgreSQL query attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(postgresqlQueryContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Executing PostgreSQL query on ${params.host}:${params.port}/${params.database}` - ) - - const sql = await createPostgresConnection({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - ssl: params.ssl, - }) - - try { - const result = await executeQuery(sql, params.query) - - logger.info(`[${requestId}] Query executed successfully, returned ${result.rowCount} rows`) - - return NextResponse.json({ - message: `Query executed successfully. ${result.rowCount} row(s) returned.`, - rows: result.rows, - rowCount: result.rowCount, - }) - } finally { - await sql.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] PostgreSQL query failed:`, error) - - return NextResponse.json({ error: `PostgreSQL query failed: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/postgresql/update/route.ts b/apps/sim/app/api/tools/postgresql/update/route.ts deleted file mode 100644 index 6533b47f5f4..00000000000 --- a/apps/sim/app/api/tools/postgresql/update/route.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { postgresqlUpdateContract } from '@/lib/api/contracts/tools/databases/postgresql' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createPostgresConnection, executeUpdate } from '@/app/api/tools/postgresql/utils' - -const logger = createLogger('PostgreSQLUpdateAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized PostgreSQL update attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(postgresqlUpdateContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Updating data in ${params.table} on ${params.host}:${params.port}/${params.database}` - ) - - const sql = await createPostgresConnection({ - host: params.host, - port: params.port, - database: params.database, - username: params.username, - password: params.password, - ssl: params.ssl, - }) - - try { - const result = await executeUpdate(sql, params.table, params.data, params.where) - - logger.info(`[${requestId}] Update executed successfully, ${result.rowCount} row(s) updated`) - - return NextResponse.json({ - message: `Data updated successfully. ${result.rowCount} row(s) affected.`, - rows: result.rows, - rowCount: result.rowCount, - }) - } finally { - await sql.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] PostgreSQL update failed:`, error) - - return NextResponse.json( - { error: `PostgreSQL update failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/postgresql/utils.test.ts b/apps/sim/app/api/tools/postgresql/utils.test.ts deleted file mode 100644 index a02f96b950f..00000000000 --- a/apps/sim/app/api/tools/postgresql/utils.test.ts +++ /dev/null @@ -1,73 +0,0 @@ -/** - * @vitest-environment node - */ -import { beforeEach, describe, expect, it, vi } from 'vitest' -import type { PostgresConnectionConfig } from '@/tools/postgresql/types' - -const { mockValidateDatabaseHost, mockPostgres } = vi.hoisted(() => ({ - mockValidateDatabaseHost: vi.fn(), - mockPostgres: vi.fn(() => ({})), -})) - -vi.mock('postgres', () => ({ default: mockPostgres })) - -vi.mock('@/lib/core/security/input-validation.server', () => ({ - validateDatabaseHost: mockValidateDatabaseHost, -})) - -import { createPostgresConnection } from '@/app/api/tools/postgresql/utils' - -function makeConfig(overrides: Partial = {}): PostgresConnectionConfig { - return { - host: 'db.example.com', - port: 5432, - database: 'app', - username: 'app', - password: 'secret', - ssl: 'required', - ...overrides, - } -} - -describe('createPostgresConnection DNS pinning', () => { - beforeEach(() => { - vi.clearAllMocks() - mockValidateDatabaseHost.mockResolvedValue({ - isValid: true, - resolvedIP: '93.184.216.34', - originalHostname: 'db.example.com', - }) - }) - - it('never opens a connection when host validation fails (no SSRF window)', async () => { - mockValidateDatabaseHost.mockResolvedValue({ - isValid: false, - error: 'host resolves to a blocked IP address', - }) - - await expect( - createPostgresConnection(makeConfig({ host: 'rebind.attacker.example' })) - ).rejects.toThrow('host resolves to a blocked IP address') - expect(mockPostgres).not.toHaveBeenCalled() - }) - - it.each(['disabled', 'required', 'preferred'] as const)( - 'connects to the validated IP for ssl=%s (hostname never re-resolved)', - async (ssl) => { - await createPostgresConnection(makeConfig({ host: 'rebind.attacker.example', ssl })) - - expect(mockValidateDatabaseHost).toHaveBeenCalledWith('rebind.attacker.example', 'host') - const options = mockPostgres.mock.calls[0][0] - // The TCP target is always the validated IP — re-resolution can never happen. - expect(options.host).toBe('93.184.216.34') - } - ) - - it('preserves the hostname as the TLS servername for verifying ssl modes', async () => { - await createPostgresConnection(makeConfig({ host: 'db.example.com', ssl: 'required' })) - - const options = mockPostgres.mock.calls[0][0] - expect(options.host).toBe('93.184.216.34') - expect(options.ssl).toMatchObject({ servername: 'db.example.com' }) - }) -}) diff --git a/apps/sim/app/api/tools/postgresql/utils.ts b/apps/sim/app/api/tools/postgresql/utils.ts deleted file mode 100644 index 983f983288c..00000000000 --- a/apps/sim/app/api/tools/postgresql/utils.ts +++ /dev/null @@ -1,377 +0,0 @@ -import postgres from 'postgres' -import { validateDatabaseHost } from '@/lib/core/security/input-validation.server' -import type { PostgresConnectionConfig } from '@/tools/postgresql/types' - -export async function createPostgresConnection(config: PostgresConnectionConfig) { - const hostValidation = await validateDatabaseHost(config.host, 'host') - if (!hostValidation.isValid) { - throw new Error(hostValidation.error) - } - - const resolvedHost = hostValidation.resolvedIP ?? config.host - - const sslConfig: boolean | 'prefer' | { rejectUnauthorized: boolean; servername?: string } = - config.ssl === 'disabled' - ? false - : config.ssl === 'preferred' - ? 'prefer' - : { rejectUnauthorized: false, servername: config.host } - - const sql = postgres({ - // Pin the validated IP (never the hostname) to prevent DNS rebinding; SNI stays the hostname above. - host: resolvedHost, - port: config.port, - database: config.database, - username: config.username, - password: config.password, - ssl: sslConfig, - connect_timeout: 10, // 10 seconds - idle_timeout: 20, // 20 seconds - max_lifetime: 60 * 30, // 30 minutes - max: 1, // Single connection for tool usage - }) - - return sql -} - -export async function executeQuery( - sql: any, - query: string, - params: unknown[] = [] -): Promise<{ rows: unknown[]; rowCount: number }> { - const result = await sql.unsafe(query, params) - const rowCount = result.count ?? result.length ?? 0 - return { - rows: Array.isArray(result) ? result : [result], - rowCount, - } -} - -export function validateQuery(query: string): { isValid: boolean; error?: string } { - const trimmedQuery = query.trim().toLowerCase() - - const allowedStatements = /^(select|insert|update|delete|with|explain|analyze|show)\s+/i - if (!allowedStatements.test(trimmedQuery)) { - return { - isValid: false, - error: - 'Only SELECT, INSERT, UPDATE, DELETE, WITH, EXPLAIN, ANALYZE, and SHOW statements are allowed', - } - } - - return { isValid: true } -} - -export function sanitizeIdentifier(identifier: string): string { - if (identifier.includes('.')) { - const parts = identifier.split('.') - return parts.map((part) => sanitizeSingleIdentifier(part)).join('.') - } - - return sanitizeSingleIdentifier(identifier) -} - -/** - * Validates a WHERE clause to prevent SQL injection attacks - * @param where - The WHERE clause string to validate - * @throws {Error} If the WHERE clause contains potentially dangerous patterns - */ -function validateWhereClause(where: string): void { - const dangerousPatterns = [ - // DDL and DML injection via stacked queries - /;\s*(drop|delete|insert|update|create|alter|grant|revoke)/i, - // Union-based injection - /union\s+(all\s+)?select/i, - // File operations - /into\s+outfile/i, - /load_file\s*\(/i, - /pg_read_file/i, - // Comment-based injection (can truncate query) - /--/, - /\/\*/, - /\*\//, - // Tautologies - always true/false conditions using backreferences - // Matches OR 'x'='x' or OR x=x (same value both sides) but NOT OR col='value' - /\bor\s+(['"]?)(\w+)\1\s*=\s*\1\2\1/i, - /\bor\s+true\b/i, - /\bor\s+false\b/i, - // AND tautologies (less common but still used in attacks) - /\band\s+(['"]?)(\w+)\1\s*=\s*\1\2\1/i, - /\band\s+true\b/i, - /\band\s+false\b/i, - // Time-based blind injection - /\bsleep\s*\(/i, - /\bwaitfor\s+delay/i, - /\bpg_sleep\s*\(/i, - /\bbenchmark\s*\(/i, - // Stacked queries (any statement after semicolon) - /;\s*\w+/, - // Information schema / system catalog queries - /information_schema/i, - /pg_catalog/i, - // System functions and procedures - /\bxp_cmdshell/i, - ] - - for (const pattern of dangerousPatterns) { - if (pattern.test(where)) { - throw new Error('WHERE clause contains potentially dangerous operation') - } - } -} - -function sanitizeSingleIdentifier(identifier: string): string { - const cleaned = identifier.replace(/"/g, '') - - if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(cleaned)) { - throw new Error( - `Invalid identifier: ${identifier}. Identifiers must start with a letter or underscore and contain only letters, numbers, and underscores.` - ) - } - - return `"${cleaned}"` -} - -export async function executeInsert( - sql: any, - table: string, - data: Record -): Promise<{ rows: unknown[]; rowCount: number }> { - const sanitizedTable = sanitizeIdentifier(table) - const columns = Object.keys(data) - const sanitizedColumns = columns.map((col) => sanitizeIdentifier(col)) - const placeholders = columns.map((_, index) => `$${index + 1}`) - const values = columns.map((col) => data[col]) - - const query = `INSERT INTO ${sanitizedTable} (${sanitizedColumns.join(', ')}) VALUES (${placeholders.join(', ')}) RETURNING *` - const result = await sql.unsafe(query, values) - - const rowCount = result.count ?? result.length ?? 0 - return { - rows: Array.isArray(result) ? result : [result], - rowCount, - } -} - -export async function executeUpdate( - sql: any, - table: string, - data: Record, - where: string -): Promise<{ rows: unknown[]; rowCount: number }> { - validateWhereClause(where) - - const sanitizedTable = sanitizeIdentifier(table) - const columns = Object.keys(data) - const sanitizedColumns = columns.map((col) => sanitizeIdentifier(col)) - const setClause = sanitizedColumns.map((col, index) => `${col} = $${index + 1}`).join(', ') - const values = columns.map((col) => data[col]) - - const query = `UPDATE ${sanitizedTable} SET ${setClause} WHERE ${where} RETURNING *` - const result = await sql.unsafe(query, values) - - const rowCount = result.count ?? result.length ?? 0 - return { - rows: Array.isArray(result) ? result : [result], - rowCount, - } -} - -export async function executeDelete( - sql: any, - table: string, - where: string -): Promise<{ rows: unknown[]; rowCount: number }> { - validateWhereClause(where) - - const sanitizedTable = sanitizeIdentifier(table) - const query = `DELETE FROM ${sanitizedTable} WHERE ${where} RETURNING *` - const result = await sql.unsafe(query, []) - - const rowCount = result.count ?? result.length ?? 0 - return { - rows: Array.isArray(result) ? result : [result], - rowCount, - } -} - -export interface IntrospectionResult { - tables: Array<{ - name: string - schema: string - columns: Array<{ - name: string - type: string - nullable: boolean - default: string | null - isPrimaryKey: boolean - isForeignKey: boolean - references?: { - table: string - column: string - } - }> - primaryKey: string[] - foreignKeys: Array<{ - column: string - referencesTable: string - referencesColumn: string - }> - indexes: Array<{ - name: string - columns: string[] - unique: boolean - }> - }> - schemas: string[] -} - -export async function executeIntrospect( - sql: any, - schemaName = 'public' -): Promise { - const schemasResult = await sql` - SELECT schema_name - FROM information_schema.schemata - WHERE schema_name NOT IN ('pg_catalog', 'information_schema', 'pg_toast') - ORDER BY schema_name - ` - const schemas = schemasResult.map((row: { schema_name: string }) => row.schema_name) - - const tablesResult = await sql` - SELECT table_name, table_schema - FROM information_schema.tables - WHERE table_schema = ${schemaName} - AND table_type = 'BASE TABLE' - ORDER BY table_name - ` - - const tables = [] - - for (const tableRow of tablesResult) { - const tableName = tableRow.table_name - const tableSchema = tableRow.table_schema - - const columnsResult = await sql` - SELECT - c.column_name, - c.data_type, - c.is_nullable, - c.column_default, - c.udt_name - FROM information_schema.columns c - WHERE c.table_schema = ${tableSchema} - AND c.table_name = ${tableName} - ORDER BY c.ordinal_position - ` - - const pkResult = await sql` - SELECT kcu.column_name - FROM information_schema.table_constraints tc - JOIN information_schema.key_column_usage kcu - ON tc.constraint_name = kcu.constraint_name - AND tc.table_schema = kcu.table_schema - WHERE tc.constraint_type = 'PRIMARY KEY' - AND tc.table_schema = ${tableSchema} - AND tc.table_name = ${tableName} - ` - const primaryKeyColumns = pkResult.map((row: { column_name: string }) => row.column_name) - - const fkResult = await sql` - SELECT - kcu.column_name, - ccu.table_name AS foreign_table_name, - ccu.column_name AS foreign_column_name - FROM information_schema.table_constraints tc - JOIN information_schema.key_column_usage kcu - ON tc.constraint_name = kcu.constraint_name - AND tc.table_schema = kcu.table_schema - JOIN information_schema.constraint_column_usage ccu - ON ccu.constraint_name = tc.constraint_name - AND ccu.table_schema = tc.table_schema - WHERE tc.constraint_type = 'FOREIGN KEY' - AND tc.table_schema = ${tableSchema} - AND tc.table_name = ${tableName} - ` - - const foreignKeys = fkResult.map( - (row: { column_name: string; foreign_table_name: string; foreign_column_name: string }) => ({ - column: row.column_name, - referencesTable: row.foreign_table_name, - referencesColumn: row.foreign_column_name, - }) - ) - - const fkColumnSet = new Set(foreignKeys.map((fk: { column: string }) => fk.column)) - - const indexesResult = await sql` - SELECT - i.relname AS index_name, - a.attname AS column_name, - ix.indisunique AS is_unique - FROM pg_class t - JOIN pg_index ix ON t.oid = ix.indrelid - JOIN pg_class i ON i.oid = ix.indexrelid - JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(ix.indkey) - JOIN pg_namespace n ON n.oid = t.relnamespace - WHERE t.relkind = 'r' - AND n.nspname = ${tableSchema} - AND t.relname = ${tableName} - AND NOT ix.indisprimary - ORDER BY i.relname, a.attnum - ` - - const indexMap = new Map() - for (const row of indexesResult) { - const indexName = row.index_name - if (!indexMap.has(indexName)) { - indexMap.set(indexName, { - name: indexName, - columns: [], - unique: row.is_unique, - }) - } - indexMap.get(indexName)!.columns.push(row.column_name) - } - const indexes = Array.from(indexMap.values()) - - const columns = columnsResult.map( - (col: { - column_name: string - data_type: string - is_nullable: string - column_default: string | null - udt_name: string - }) => { - const columnName = col.column_name - const fk = foreignKeys.find((f: { column: string }) => f.column === columnName) - - return { - name: columnName, - type: col.data_type === 'USER-DEFINED' ? col.udt_name : col.data_type, - nullable: col.is_nullable === 'YES', - default: col.column_default, - isPrimaryKey: primaryKeyColumns.includes(columnName), - isForeignKey: fkColumnSet.has(columnName), - ...(fk && { - references: { - table: fk.referencesTable, - column: fk.referencesColumn, - }, - }), - } - } - ) - - tables.push({ - name: tableName, - schema: tableSchema, - columns, - primaryKey: primaryKeyColumns, - foreignKeys, - indexes, - }) - } - - return { tables, schemas } -} diff --git a/apps/sim/app/api/tools/pulse/parse/route.ts b/apps/sim/app/api/tools/pulse/parse/route.ts deleted file mode 100644 index a7c93c218cd..00000000000 --- a/apps/sim/app/api/tools/pulse/parse/route.ts +++ /dev/null @@ -1,185 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { pulseParseContract } from '@/lib/api/contracts/tools/media/document-parse' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' -import { resolveFileInputToUrl } from '@/lib/uploads/utils/file-utils.server' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('PulseParseAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Pulse parse attempt`, { - error: authResult.error || 'Missing userId', - }) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Unauthorized', - }, - { status: 401 } - ) - } - - const userId = authResult.userId - - const parsed = await parseRequest( - pulseParseContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - - const validatedData = parsed.data.body - const modelInputProvenance = validateOpaqueModelInputProvenance({ - headers: request.headers, - payload: validatedData, - isInternalRequest: true, - }) - if (!modelInputProvenance.success) { - return NextResponse.json( - { success: false, error: modelInputProvenance.error }, - { status: modelInputProvenance.status } - ) - } - - logger.info(`[${requestId}] Pulse parse request`, { - hasInlineFile: Boolean(validatedData.file), - hasFilePath: Boolean(validatedData.filePath), - userId, - }) - - const resolution = await resolveFileInputToUrl({ - file: validatedData.file, - filePath: validatedData.filePath, - userId, - requestId, - logger, - modelEgress: true, - }) - - if (resolution.error) { - return NextResponse.json( - { success: false, error: resolution.error.message }, - { status: resolution.error.status } - ) - } - - const fileUrl = resolution.fileUrl - if (!fileUrl) { - return NextResponse.json({ success: false, error: 'File input is required' }, { status: 400 }) - } - - const formData = new FormData() - formData.append('file_url', fileUrl) - - if (validatedData.pages) { - formData.append('pages', validatedData.pages) - } - if (validatedData.extractFigure !== undefined) { - formData.append('extract_figure', String(validatedData.extractFigure)) - } - if (validatedData.figureDescription !== undefined) { - formData.append('figure_description', String(validatedData.figureDescription)) - } - if (validatedData.returnHtml !== undefined) { - formData.append('return_html', String(validatedData.returnHtml)) - } - if (validatedData.chunking) { - formData.append('chunking', validatedData.chunking) - } - if (validatedData.chunkSize !== undefined) { - formData.append('chunk_size', String(validatedData.chunkSize)) - } - - const pulseEndpoint = 'https://api.runpulse.com/extract' - const pulseValidation = await validateUrlWithDNS(pulseEndpoint, 'Pulse API URL') - if (!pulseValidation.isValid) { - logger.error(`[${requestId}] Pulse API URL validation failed`, { - error: pulseValidation.error, - }) - return NextResponse.json( - { - success: false, - error: 'Failed to reach Pulse API', - }, - { status: 502 } - ) - } - - const pulsePayload = new Response(formData) - const contentType = pulsePayload.headers.get('content-type') || 'multipart/form-data' - const bodyBuffer = Buffer.from(await pulsePayload.arrayBuffer()) - const pulseResponse = await secureFetchWithPinnedIP( - pulseEndpoint, - pulseValidation.resolvedIP!, - { - method: 'POST', - headers: { - 'x-api-key': validatedData.apiKey, - 'Content-Type': contentType, - }, - body: bodyBuffer, - } - ) - - if (!pulseResponse.ok) { - const errorText = await pulseResponse.text() - logger.error(`[${requestId}] Pulse API error:`, errorText) - return NextResponse.json( - { - success: false, - error: `Pulse API error: ${pulseResponse.statusText}`, - }, - { status: pulseResponse.status } - ) - } - - const pulseData = await pulseResponse.json() - - logger.info(`[${requestId}] Pulse parse successful`) - - return NextResponse.json({ - success: true, - output: pulseData, - }) - } catch (error) { - logger.error(`[${requestId}] Error in Pulse parse:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/quiver/image-to-svg/route.ts b/apps/sim/app/api/tools/quiver/image-to-svg/route.ts deleted file mode 100644 index faf79554834..00000000000 --- a/apps/sim/app/api/tools/quiver/image-to-svg/route.ts +++ /dev/null @@ -1,202 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { quiverImageToSvgContract } from '@/lib/api/contracts/tools/quiver' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' -import { - isModelSafeWorkspaceFileKey, - MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, -} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' -import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { assertToolFileAccess } from '@/app/api/files/authorization' - -const logger = createLogger('QuiverImageToSvgAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseRequest( - quiverImageToSvgContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ), - } - ) - if (!parsed.success) return parsed.response - const data = parsed.data.body - const modelInputProvenance = validateOpaqueModelInputProvenance({ - headers: request.headers, - payload: data, - isInternalRequest: true, - }) - if (!modelInputProvenance.success) { - return NextResponse.json( - { success: false, error: modelInputProvenance.error }, - { status: modelInputProvenance.status } - ) - } - - let apiImage: { url: string } | { base64: string } - - if (typeof data.image === 'string') { - try { - const parsed = JSON.parse(data.image) - if (parsed && typeof parsed === 'object') { - const userFiles = processFilesToUserFiles([parsed as RawFileInput], requestId, logger) - if (userFiles.length > 0) { - const denied = await assertToolFileAccess( - userFiles[0].key, - authResult.userId, - requestId, - logger - ) - if (denied) return denied - if (!(await isModelSafeWorkspaceFileKey(userFiles[0].key))) { - return NextResponse.json( - { - success: false, - error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, - }, - { status: 400 } - ) - } - const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger, { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - }) - apiImage = { base64: buffer.toString('base64') } - } else { - return NextResponse.json( - { success: false, error: 'Invalid file input' }, - { status: 400 } - ) - } - } else { - apiImage = { url: data.image } - } - } catch { - apiImage = { url: data.image } - } - } else if (typeof data.image === 'object' && data.image !== null) { - const userFiles = processFilesToUserFiles([data.image as RawFileInput], requestId, logger) - if (userFiles.length > 0) { - const denied = await assertToolFileAccess( - userFiles[0].key, - authResult.userId, - requestId, - logger - ) - if (denied) return denied - if (!(await isModelSafeWorkspaceFileKey(userFiles[0].key))) { - return NextResponse.json( - { - success: false, - error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, - }, - { status: 400 } - ) - } - const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger, { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - }) - apiImage = { base64: buffer.toString('base64') } - } else { - return NextResponse.json({ success: false, error: 'Invalid file input' }, { status: 400 }) - } - } else { - return NextResponse.json({ success: false, error: 'Image is required' }, { status: 400 }) - } - - const apiBody: Record = { - model: data.model, - image: apiImage, - } - - if (data.temperature != null) apiBody.temperature = data.temperature - if (data.top_p != null) apiBody.top_p = data.top_p - if (data.max_output_tokens != null) apiBody.max_output_tokens = data.max_output_tokens - if (data.presence_penalty != null) apiBody.presence_penalty = data.presence_penalty - if (data.auto_crop != null) apiBody.auto_crop = data.auto_crop - if (data.target_size != null) apiBody.target_size = data.target_size - - logger.info(`[${requestId}] Calling Quiver vectorization API with model: ${data.model}`) - - const response = await fetch('https://api.quiver.ai/v1/svgs/vectorizations', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${data.apiKey}`, - }, - body: JSON.stringify(apiBody), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error(`[${requestId}] Quiver API error: ${response.status} - ${errorText}`) - return NextResponse.json( - { success: false, error: `Quiver API error: ${response.status} - ${errorText}` }, - { status: response.status } - ) - } - - const result = await response.json() - - if (!result.data || result.data.length === 0) { - return NextResponse.json( - { success: false, error: 'No SVG data returned from Quiver API' }, - { status: 500 } - ) - } - - const svgContent = result.data[0].svg - const svgBuffer = Buffer.from(svgContent, 'utf-8') - const file = { - name: 'vectorized.svg', - mimeType: 'image/svg+xml', - data: svgBuffer.toString('base64'), - size: svgBuffer.length, - } - - return NextResponse.json({ - success: true, - output: { - file, - files: [file], - svgContent, - id: result.id ?? null, - usage: result.usage - ? { - totalTokens: result.usage.total_tokens ?? 0, - inputTokens: result.usage.input_tokens ?? 0, - outputTokens: result.usage.output_tokens ?? 0, - } - : null, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error in Quiver image-to-svg:`, error) - const message = getErrorMessage(error, 'Unknown error') - return NextResponse.json({ success: false, error: message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/quiver/text-to-svg/route.ts b/apps/sim/app/api/tools/quiver/text-to-svg/route.ts deleted file mode 100644 index eb40c76997d..00000000000 --- a/apps/sim/app/api/tools/quiver/text-to-svg/route.ts +++ /dev/null @@ -1,200 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { quiverTextToSvgContract } from '@/lib/api/contracts/tools/quiver' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' -import { - isModelSafeWorkspaceFileKey, - MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, -} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' -import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { assertToolFileAccess } from '@/app/api/files/authorization' - -const logger = createLogger('QuiverTextToSvgAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - const userId = authResult.userId - - try { - const parsed = await parseRequest( - quiverTextToSvgContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ), - } - ) - if (!parsed.success) return parsed.response - const data = parsed.data.body - const modelInputProvenance = validateOpaqueModelInputProvenance({ - headers: request.headers, - payload: data, - isInternalRequest: true, - }) - if (!modelInputProvenance.success) { - return NextResponse.json( - { success: false, error: modelInputProvenance.error }, - { status: modelInputProvenance.status } - ) - } - - const apiReferences: Array<{ url: string } | { base64: string }> = [] - // Every reference is buffered and base64'd before the list is sliced to 4, so the - // budget has to span the whole loop rather than bound each file on its own. - let referenceBudget = MAX_BUFFERED_TRANSFER_BYTES - - if (data.references) { - const rawRefs = Array.isArray(data.references) ? data.references : [data.references] - - for (const ref of rawRefs) { - if (typeof ref === 'string') { - try { - const parsed = JSON.parse(ref) - if (parsed && typeof parsed === 'object') { - const userFiles = processFilesToUserFiles([parsed as RawFileInput], requestId, logger) - if (userFiles.length > 0) { - const denied = await assertToolFileAccess( - userFiles[0].key, - userId, - requestId, - logger - ) - if (denied) return denied - if (!(await isModelSafeWorkspaceFileKey(userFiles[0].key))) { - return NextResponse.json( - { - success: false, - error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, - }, - { status: 400 } - ) - } - const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger, { - maxBytes: referenceBudget, - }) - referenceBudget -= buffer.length - apiReferences.push({ base64: buffer.toString('base64') }) - } - } - } catch { - apiReferences.push({ url: ref }) - } - } else if (typeof ref === 'object' && ref !== null) { - const userFiles = processFilesToUserFiles([ref as RawFileInput], requestId, logger) - if (userFiles.length > 0) { - const denied = await assertToolFileAccess(userFiles[0].key, userId, requestId, logger) - if (denied) return denied - if (!(await isModelSafeWorkspaceFileKey(userFiles[0].key))) { - return NextResponse.json( - { - success: false, - error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, - }, - { status: 400 } - ) - } - const buffer = await downloadFileFromStorage(userFiles[0], requestId, logger, { - maxBytes: referenceBudget, - }) - referenceBudget -= buffer.length - apiReferences.push({ base64: buffer.toString('base64') }) - } - } - } - } - - const apiBody: Record = { - model: data.model, - prompt: data.prompt, - } - - if (data.instructions) apiBody.instructions = data.instructions - if (apiReferences.length > 0) apiBody.references = apiReferences.slice(0, 4) - if (data.n != null) apiBody.n = data.n - if (data.temperature != null) apiBody.temperature = data.temperature - if (data.top_p != null) apiBody.top_p = data.top_p - if (data.max_output_tokens != null) apiBody.max_output_tokens = data.max_output_tokens - if (data.presence_penalty != null) apiBody.presence_penalty = data.presence_penalty - - logger.info(`[${requestId}] Calling Quiver API with model: ${data.model}`) - - const response = await fetch('https://api.quiver.ai/v1/svgs/generations', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${data.apiKey}`, - }, - body: JSON.stringify(apiBody), - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error(`[${requestId}] Quiver API error: ${response.status} - ${errorText}`) - return NextResponse.json( - { success: false, error: `Quiver API error: ${response.status} - ${errorText}` }, - { status: response.status } - ) - } - - const result = await response.json() - - if (!result.data || result.data.length === 0) { - return NextResponse.json( - { success: false, error: 'No SVG data returned from Quiver API' }, - { status: 500 } - ) - } - - const files = result.data.map((entry: { svg: string }, index: number) => { - const buffer = Buffer.from(entry.svg, 'utf-8') - return { - name: result.data.length > 1 ? `generated-${index + 1}.svg` : 'generated.svg', - mimeType: 'image/svg+xml', - data: buffer.toString('base64'), - size: buffer.length, - } - }) - - return NextResponse.json({ - success: true, - output: { - file: files[0], - files, - svgContent: result.data[0].svg, - id: result.id ?? null, - usage: result.usage - ? { - totalTokens: result.usage.total_tokens ?? 0, - inputTokens: result.usage.input_tokens ?? 0, - outputTokens: result.usage.output_tokens ?? 0, - } - : null, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error in Quiver text-to-svg:`, error) - const message = getErrorMessage(error, 'Unknown error') - return NextResponse.json({ success: false, error: message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/rds/delete/route.ts b/apps/sim/app/api/tools/rds/delete/route.ts deleted file mode 100644 index a83e5959c1d..00000000000 --- a/apps/sim/app/api/tools/rds/delete/route.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { rdsDeleteContract } from '@/lib/api/contracts/tools/databases/rds' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createRdsClient, executeDelete } from '@/app/api/tools/rds/utils' - -const logger = createLogger('RDSDeleteAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(rdsDeleteContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Deleting from RDS table ${params.table} in ${params.database}`) - - const client = createRdsClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - resourceArn: params.resourceArn, - secretArn: params.secretArn, - database: params.database, - }) - - try { - const result = await executeDelete( - client, - params.resourceArn, - params.secretArn, - params.database, - params.table, - params.conditions - ) - - logger.info(`[${requestId}] Delete executed successfully, affected ${result.rowCount} rows`) - - return NextResponse.json({ - message: `Delete executed successfully. ${result.rowCount} row(s) deleted.`, - rows: result.rows, - rowCount: result.rowCount, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] RDS delete failed:`, error) - - return NextResponse.json({ error: `RDS delete failed: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/rds/execute/route.ts b/apps/sim/app/api/tools/rds/execute/route.ts deleted file mode 100644 index 408c5bfe29f..00000000000 --- a/apps/sim/app/api/tools/rds/execute/route.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { rdsExecuteContract } from '@/lib/api/contracts/tools/databases/rds' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createRdsClient, executeStatement } from '@/app/api/tools/rds/utils' - -const logger = createLogger('RDSExecuteAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(rdsExecuteContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Executing raw SQL on RDS database ${params.database}`) - - const client = createRdsClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - resourceArn: params.resourceArn, - secretArn: params.secretArn, - database: params.database, - }) - - try { - const result = await executeStatement( - client, - params.resourceArn, - params.secretArn, - params.database, - params.query - ) - - logger.info(`[${requestId}] Execute completed successfully, affected ${result.rowCount} rows`) - - return NextResponse.json({ - message: `Query executed successfully. ${result.rowCount} row(s) affected.`, - rows: result.rows, - rowCount: result.rowCount, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] RDS execute failed:`, error) - - return NextResponse.json({ error: `RDS execute failed: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/rds/insert/route.ts b/apps/sim/app/api/tools/rds/insert/route.ts deleted file mode 100644 index ae7751adf67..00000000000 --- a/apps/sim/app/api/tools/rds/insert/route.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { rdsInsertContract } from '@/lib/api/contracts/tools/databases/rds' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createRdsClient, executeInsert } from '@/app/api/tools/rds/utils' - -const logger = createLogger('RDSInsertAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(rdsInsertContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Inserting into RDS table ${params.table} in ${params.database}`) - - const client = createRdsClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - resourceArn: params.resourceArn, - secretArn: params.secretArn, - database: params.database, - }) - - try { - const result = await executeInsert( - client, - params.resourceArn, - params.secretArn, - params.database, - params.table, - params.data - ) - - logger.info(`[${requestId}] Insert executed successfully, affected ${result.rowCount} rows`) - - return NextResponse.json({ - message: `Insert executed successfully. ${result.rowCount} row(s) inserted.`, - rows: result.rows, - rowCount: result.rowCount, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] RDS insert failed:`, error) - - return NextResponse.json({ error: `RDS insert failed: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/rds/introspect/route.ts b/apps/sim/app/api/tools/rds/introspect/route.ts deleted file mode 100644 index 32e3df10595..00000000000 --- a/apps/sim/app/api/tools/rds/introspect/route.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { rdsIntrospectContract } from '@/lib/api/contracts/tools/databases/rds' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createRdsClient, executeIntrospect, type RdsEngine } from '@/app/api/tools/rds/utils' - -const logger = createLogger('RDSIntrospectAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(rdsIntrospectContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Introspecting RDS Aurora database${params.database ? ` (${params.database})` : ''}` - ) - - const client = createRdsClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - resourceArn: params.resourceArn, - secretArn: params.secretArn, - database: params.database, - }) - - try { - const result = await executeIntrospect( - client, - params.resourceArn, - params.secretArn, - params.database, - params.schema, - params.engine as RdsEngine | undefined - ) - - logger.info( - `[${requestId}] Introspection completed successfully. Engine: ${result.engine}, found ${result.tables.length} tables` - ) - - return NextResponse.json({ - message: `Schema introspection completed. Engine: ${result.engine}. Found ${result.tables.length} table(s).`, - engine: result.engine, - tables: result.tables, - schemas: result.schemas, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] RDS introspection failed:`, error) - - return NextResponse.json( - { error: `RDS introspection failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/rds/query/route.ts b/apps/sim/app/api/tools/rds/query/route.ts deleted file mode 100644 index cc511122847..00000000000 --- a/apps/sim/app/api/tools/rds/query/route.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { rdsQueryContract } from '@/lib/api/contracts/tools/databases/rds' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createRdsClient, executeStatement, validateQuery } from '@/app/api/tools/rds/utils' - -const logger = createLogger('RDSQueryAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(rdsQueryContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Executing RDS query on ${params.database}`) - - const validation = validateQuery(params.query) - if (!validation.isValid) { - logger.warn(`[${requestId}] Query validation failed: ${validation.error}`) - return NextResponse.json({ error: validation.error }, { status: 400 }) - } - - const client = createRdsClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - resourceArn: params.resourceArn, - secretArn: params.secretArn, - database: params.database, - }) - - try { - const result = await executeStatement( - client, - params.resourceArn, - params.secretArn, - params.database, - params.query - ) - - logger.info(`[${requestId}] Query executed successfully, returned ${result.rowCount} rows`) - - return NextResponse.json({ - message: `Query executed successfully. ${result.rowCount} row(s) returned.`, - rows: result.rows, - rowCount: result.rowCount, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] RDS query failed:`, error) - - return NextResponse.json({ error: `RDS query failed: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/rds/update/route.ts b/apps/sim/app/api/tools/rds/update/route.ts deleted file mode 100644 index 6ea2df5c10e..00000000000 --- a/apps/sim/app/api/tools/rds/update/route.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { rdsUpdateContract } from '@/lib/api/contracts/tools/databases/rds' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createRdsClient, executeUpdate } from '@/app/api/tools/rds/utils' - -const logger = createLogger('RDSUpdateAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(rdsUpdateContract, request, { logger }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Updating RDS table ${params.table} in ${params.database}`) - - const client = createRdsClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - resourceArn: params.resourceArn, - secretArn: params.secretArn, - database: params.database, - }) - - try { - const result = await executeUpdate( - client, - params.resourceArn, - params.secretArn, - params.database, - params.table, - params.data, - params.conditions - ) - - logger.info(`[${requestId}] Update executed successfully, affected ${result.rowCount} rows`) - - return NextResponse.json({ - message: `Update executed successfully. ${result.rowCount} row(s) updated.`, - rows: result.rows, - rowCount: result.rowCount, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] RDS update failed:`, error) - - return NextResponse.json({ error: `RDS update failed: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/rds/utils.ts b/apps/sim/app/api/tools/rds/utils.ts deleted file mode 100644 index ea76642a380..00000000000 --- a/apps/sim/app/api/tools/rds/utils.ts +++ /dev/null @@ -1,727 +0,0 @@ -import { - ExecuteStatementCommand, - type ExecuteStatementCommandOutput, - type Field, - RDSDataClient, - type SqlParameter, -} from '@aws-sdk/client-rds-data' -import type { RdsConnectionConfig } from '@/tools/rds/types' - -export function createRdsClient(config: RdsConnectionConfig): RDSDataClient { - return new RDSDataClient({ - region: config.region, - credentials: { - accessKeyId: config.accessKeyId, - secretAccessKey: config.secretAccessKey, - }, - }) -} - -export async function executeStatement( - client: RDSDataClient, - resourceArn: string, - secretArn: string, - database: string | undefined, - sql: string, - parameters?: SqlParameter[] -): Promise<{ rows: Record[]; rowCount: number }> { - const command = new ExecuteStatementCommand({ - resourceArn, - secretArn, - ...(database && { database }), - sql, - ...(parameters && parameters.length > 0 && { parameters }), - includeResultMetadata: true, - }) - - const response = await client.send(command) - const rows = parseRdsResponse(response) - - return { - rows, - rowCount: response.numberOfRecordsUpdated ?? rows.length, - } -} - -function parseRdsResponse(response: ExecuteStatementCommandOutput): Record[] { - if (!response.records || !response.columnMetadata) { - return [] - } - - const columnNames = response.columnMetadata.map((col) => col.name || col.label || 'unknown') - - return response.records.map((record) => { - const row: Record = {} - record.forEach((field, index) => { - const columnName = columnNames[index] || `column_${index}` - row[columnName] = parseFieldValue(field) - }) - return row - }) -} - -function parseFieldValue(field: Field): unknown { - if (field.isNull) return null - if (field.stringValue !== undefined) return field.stringValue - if (field.longValue !== undefined) return field.longValue - if (field.doubleValue !== undefined) return field.doubleValue - if (field.booleanValue !== undefined) return field.booleanValue - if (field.blobValue !== undefined) return Buffer.from(field.blobValue).toString('base64') - if (field.arrayValue !== undefined) { - const arr = field.arrayValue - if (arr.stringValues) return arr.stringValues - if (arr.longValues) return arr.longValues - if (arr.doubleValues) return arr.doubleValues - if (arr.booleanValues) return arr.booleanValues - if (arr.arrayValues) return arr.arrayValues.map((f) => parseFieldValue({ arrayValue: f })) - return [] - } - return null -} - -export function validateQuery(query: string): { isValid: boolean; error?: string } { - const trimmedQuery = query.trim().toLowerCase() - - const allowedStatements = /^(select|insert|update|delete|with|explain|show)\s+/i - if (!allowedStatements.test(trimmedQuery)) { - return { - isValid: false, - error: 'Only SELECT, INSERT, UPDATE, DELETE, WITH, EXPLAIN, and SHOW statements are allowed', - } - } - - return { isValid: true } -} - -export function sanitizeIdentifier(identifier: string): string { - if (identifier.includes('.')) { - const parts = identifier.split('.') - return parts.map((part) => sanitizeSingleIdentifier(part)).join('.') - } - - return sanitizeSingleIdentifier(identifier) -} - -function sanitizeSingleIdentifier(identifier: string): string { - const cleaned = identifier.replace(/`/g, '').replace(/"/g, '') - - if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(cleaned)) { - throw new Error( - `Invalid identifier: ${identifier}. Identifiers must start with a letter or underscore and contain only letters, numbers, and underscores.` - ) - } - - return cleaned -} - -/** - * Convert a JS value to an RDS Data API SqlParameter value - */ -function toSqlParameterValue(value: unknown): SqlParameter['value'] { - if (value === null || value === undefined) { - return { isNull: true } - } - if (typeof value === 'boolean') { - return { booleanValue: value } - } - if (typeof value === 'number') { - if (Number.isInteger(value)) { - return { longValue: value } - } - return { doubleValue: value } - } - if (typeof value === 'string') { - return { stringValue: value } - } - if (value instanceof Uint8Array || Buffer.isBuffer(value)) { - return { blobValue: value } - } - // Objects/arrays as JSON strings - return { stringValue: JSON.stringify(value) } -} - -/** - * Build parameterized INSERT query - */ -export async function executeInsert( - client: RDSDataClient, - resourceArn: string, - secretArn: string, - database: string | undefined, - table: string, - data: Record -): Promise<{ rows: Record[]; rowCount: number }> { - const sanitizedTable = sanitizeIdentifier(table) - const columns = Object.keys(data) - const sanitizedColumns = columns.map((col) => sanitizeIdentifier(col)) - - const placeholders = columns.map((col) => `:${col}`) - const parameters: SqlParameter[] = columns.map((col) => ({ - name: col, - value: toSqlParameterValue(data[col]), - })) - - const sql = `INSERT INTO ${sanitizedTable} (${sanitizedColumns.join(', ')}) VALUES (${placeholders.join(', ')})` - - return executeStatement(client, resourceArn, secretArn, database, sql, parameters) -} - -/** - * Build parameterized UPDATE query with conditions - */ -export async function executeUpdate( - client: RDSDataClient, - resourceArn: string, - secretArn: string, - database: string | undefined, - table: string, - data: Record, - conditions: Record -): Promise<{ rows: Record[]; rowCount: number }> { - const sanitizedTable = sanitizeIdentifier(table) - - // Build SET clause with parameters - const dataColumns = Object.keys(data) - const setClause = dataColumns.map((col) => `${sanitizeIdentifier(col)} = :set_${col}`).join(', ') - - // Build WHERE clause with parameters - const conditionColumns = Object.keys(conditions) - if (conditionColumns.length === 0) { - throw new Error('At least one condition is required for UPDATE operations') - } - const whereClause = conditionColumns - .map((col) => `${sanitizeIdentifier(col)} = :where_${col}`) - .join(' AND ') - - // Build parameters array (prefixed to avoid name collisions) - const parameters: SqlParameter[] = [ - ...dataColumns.map((col) => ({ - name: `set_${col}`, - value: toSqlParameterValue(data[col]), - })), - ...conditionColumns.map((col) => ({ - name: `where_${col}`, - value: toSqlParameterValue(conditions[col]), - })), - ] - - const sql = `UPDATE ${sanitizedTable} SET ${setClause} WHERE ${whereClause}` - - return executeStatement(client, resourceArn, secretArn, database, sql, parameters) -} - -/** - * Build parameterized DELETE query with conditions - */ -export async function executeDelete( - client: RDSDataClient, - resourceArn: string, - secretArn: string, - database: string | undefined, - table: string, - conditions: Record -): Promise<{ rows: Record[]; rowCount: number }> { - const sanitizedTable = sanitizeIdentifier(table) - - // Build WHERE clause with parameters - const conditionColumns = Object.keys(conditions) - if (conditionColumns.length === 0) { - throw new Error('At least one condition is required for DELETE operations') - } - const whereClause = conditionColumns - .map((col) => `${sanitizeIdentifier(col)} = :${col}`) - .join(' AND ') - - const parameters: SqlParameter[] = conditionColumns.map((col) => ({ - name: col, - value: toSqlParameterValue(conditions[col]), - })) - - const sql = `DELETE FROM ${sanitizedTable} WHERE ${whereClause}` - - return executeStatement(client, resourceArn, secretArn, database, sql, parameters) -} - -export type RdsEngine = 'aurora-postgresql' | 'aurora-mysql' - -export interface RdsIntrospectionResult { - engine: RdsEngine - tables: Array<{ - name: string - schema: string - columns: Array<{ - name: string - type: string - nullable: boolean - default: string | null - isPrimaryKey: boolean - isForeignKey: boolean - references?: { - table: string - column: string - } - }> - primaryKey: string[] - foreignKeys: Array<{ - column: string - referencesTable: string - referencesColumn: string - }> - indexes: Array<{ - name: string - columns: string[] - unique: boolean - }> - }> - schemas: string[] -} - -/** - * Detects the database engine by querying SELECT VERSION() - */ -async function detectEngine( - client: RDSDataClient, - resourceArn: string, - secretArn: string, - database: string | undefined -): Promise { - const result = await executeStatement( - client, - resourceArn, - secretArn, - database, - 'SELECT VERSION()' - ) - - if (result.rows.length > 0) { - const versionRow = result.rows[0] as Record - const versionValue = Object.values(versionRow)[0] - const versionString = String(versionValue).toLowerCase() - - if (versionString.includes('postgresql') || versionString.includes('postgres')) { - return 'aurora-postgresql' - } - if (versionString.includes('mysql') || versionString.includes('mariadb')) { - return 'aurora-mysql' - } - } - - throw new Error('Unable to detect database engine. Please specify the engine parameter.') -} - -/** - * Introspects PostgreSQL schema using INFORMATION_SCHEMA - */ -async function introspectPostgresql( - client: RDSDataClient, - resourceArn: string, - secretArn: string, - database: string | undefined, - schemaName: string -): Promise { - const schemasResult = await executeStatement( - client, - resourceArn, - secretArn, - database, - `SELECT schema_name FROM information_schema.schemata - WHERE schema_name NOT IN ('pg_catalog', 'information_schema', 'pg_toast') - ORDER BY schema_name` - ) - const schemas = schemasResult.rows.map((row) => (row as { schema_name: string }).schema_name) - - const tablesResult = await executeStatement( - client, - resourceArn, - secretArn, - database, - `SELECT table_name, table_schema - FROM information_schema.tables - WHERE table_schema = :schemaName - AND table_type = 'BASE TABLE' - ORDER BY table_name`, - [{ name: 'schemaName', value: { stringValue: schemaName } }] - ) - - const tables = [] - - for (const tableRow of tablesResult.rows) { - const row = tableRow as { table_name: string; table_schema: string } - const tableName = row.table_name - const tableSchema = row.table_schema - - const columnsResult = await executeStatement( - client, - resourceArn, - secretArn, - database, - `SELECT - c.column_name, - c.data_type, - c.is_nullable, - c.column_default, - c.udt_name - FROM information_schema.columns c - WHERE c.table_schema = :tableSchema - AND c.table_name = :tableName - ORDER BY c.ordinal_position`, - [ - { name: 'tableSchema', value: { stringValue: tableSchema } }, - { name: 'tableName', value: { stringValue: tableName } }, - ] - ) - - const pkResult = await executeStatement( - client, - resourceArn, - secretArn, - database, - `SELECT kcu.column_name - FROM information_schema.table_constraints tc - JOIN information_schema.key_column_usage kcu - ON tc.constraint_name = kcu.constraint_name - AND tc.table_schema = kcu.table_schema - WHERE tc.constraint_type = 'PRIMARY KEY' - AND tc.table_schema = :tableSchema - AND tc.table_name = :tableName`, - [ - { name: 'tableSchema', value: { stringValue: tableSchema } }, - { name: 'tableName', value: { stringValue: tableName } }, - ] - ) - const primaryKeyColumns = pkResult.rows.map((r) => (r as { column_name: string }).column_name) - - const fkResult = await executeStatement( - client, - resourceArn, - secretArn, - database, - `SELECT - kcu.column_name, - ccu.table_name AS foreign_table_name, - ccu.column_name AS foreign_column_name - FROM information_schema.table_constraints tc - JOIN information_schema.key_column_usage kcu - ON tc.constraint_name = kcu.constraint_name - AND tc.table_schema = kcu.table_schema - JOIN information_schema.constraint_column_usage ccu - ON ccu.constraint_name = tc.constraint_name - AND ccu.table_schema = tc.table_schema - WHERE tc.constraint_type = 'FOREIGN KEY' - AND tc.table_schema = :tableSchema - AND tc.table_name = :tableName`, - [ - { name: 'tableSchema', value: { stringValue: tableSchema } }, - { name: 'tableName', value: { stringValue: tableName } }, - ] - ) - - const foreignKeys = fkResult.rows.map((r) => { - const fkRow = r as { - column_name: string - foreign_table_name: string - foreign_column_name: string - } - return { - column: fkRow.column_name, - referencesTable: fkRow.foreign_table_name, - referencesColumn: fkRow.foreign_column_name, - } - }) - - const fkColumnSet = new Set(foreignKeys.map((fk) => fk.column)) - - const indexesResult = await executeStatement( - client, - resourceArn, - secretArn, - database, - `SELECT - i.relname AS index_name, - a.attname AS column_name, - ix.indisunique AS is_unique - FROM pg_class t - JOIN pg_index ix ON t.oid = ix.indrelid - JOIN pg_class i ON i.oid = ix.indexrelid - JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(ix.indkey) - JOIN pg_namespace n ON n.oid = t.relnamespace - WHERE t.relkind = 'r' - AND n.nspname = :tableSchema - AND t.relname = :tableName - AND NOT ix.indisprimary - ORDER BY i.relname, a.attnum`, - [ - { name: 'tableSchema', value: { stringValue: tableSchema } }, - { name: 'tableName', value: { stringValue: tableName } }, - ] - ) - - const indexMap = new Map() - for (const idxRow of indexesResult.rows) { - const idx = idxRow as { index_name: string; column_name: string; is_unique: boolean } - const indexName = idx.index_name - if (!indexMap.has(indexName)) { - indexMap.set(indexName, { - name: indexName, - columns: [], - unique: idx.is_unique, - }) - } - indexMap.get(indexName)!.columns.push(idx.column_name) - } - const indexes = Array.from(indexMap.values()) - - const columns = columnsResult.rows.map((colRow) => { - const col = colRow as { - column_name: string - data_type: string - is_nullable: string - column_default: string | null - udt_name: string - } - const columnName = col.column_name - const fk = foreignKeys.find((f) => f.column === columnName) - - return { - name: columnName, - type: col.data_type === 'USER-DEFINED' ? col.udt_name : col.data_type, - nullable: col.is_nullable === 'YES', - default: col.column_default, - isPrimaryKey: primaryKeyColumns.includes(columnName), - isForeignKey: fkColumnSet.has(columnName), - ...(fk && { - references: { - table: fk.referencesTable, - column: fk.referencesColumn, - }, - }), - } - }) - - tables.push({ - name: tableName, - schema: tableSchema, - columns, - primaryKey: primaryKeyColumns, - foreignKeys, - indexes, - }) - } - - return { engine: 'aurora-postgresql', tables, schemas } -} - -/** - * Introspects MySQL schema using INFORMATION_SCHEMA - */ -async function introspectMysql( - client: RDSDataClient, - resourceArn: string, - secretArn: string, - database: string | undefined, - schemaName: string -): Promise { - const schemasResult = await executeStatement( - client, - resourceArn, - secretArn, - database, - `SELECT SCHEMA_NAME as schema_name FROM information_schema.SCHEMATA - WHERE SCHEMA_NAME NOT IN ('mysql', 'information_schema', 'performance_schema', 'sys') - ORDER BY SCHEMA_NAME` - ) - const schemas = schemasResult.rows.map((row) => (row as { schema_name: string }).schema_name) - - const tablesResult = await executeStatement( - client, - resourceArn, - secretArn, - database, - `SELECT TABLE_NAME as table_name, TABLE_SCHEMA as table_schema - FROM information_schema.TABLES - WHERE TABLE_SCHEMA = :schemaName - AND TABLE_TYPE = 'BASE TABLE' - ORDER BY TABLE_NAME`, - [{ name: 'schemaName', value: { stringValue: schemaName } }] - ) - - const tables = [] - - for (const tableRow of tablesResult.rows) { - const row = tableRow as { table_name: string; table_schema: string } - const tableName = row.table_name - const tableSchema = row.table_schema - - const columnsResult = await executeStatement( - client, - resourceArn, - secretArn, - database, - `SELECT - COLUMN_NAME as column_name, - DATA_TYPE as data_type, - IS_NULLABLE as is_nullable, - COLUMN_DEFAULT as column_default, - COLUMN_TYPE as column_type, - COLUMN_KEY as column_key - FROM information_schema.COLUMNS - WHERE TABLE_SCHEMA = :tableSchema - AND TABLE_NAME = :tableName - ORDER BY ORDINAL_POSITION`, - [ - { name: 'tableSchema', value: { stringValue: tableSchema } }, - { name: 'tableName', value: { stringValue: tableName } }, - ] - ) - - const pkResult = await executeStatement( - client, - resourceArn, - secretArn, - database, - `SELECT COLUMN_NAME as column_name - FROM information_schema.KEY_COLUMN_USAGE - WHERE TABLE_SCHEMA = :tableSchema - AND TABLE_NAME = :tableName - AND CONSTRAINT_NAME = 'PRIMARY' - ORDER BY ORDINAL_POSITION`, - [ - { name: 'tableSchema', value: { stringValue: tableSchema } }, - { name: 'tableName', value: { stringValue: tableName } }, - ] - ) - const primaryKeyColumns = pkResult.rows.map((r) => (r as { column_name: string }).column_name) - - const fkResult = await executeStatement( - client, - resourceArn, - secretArn, - database, - `SELECT - kcu.COLUMN_NAME as column_name, - kcu.REFERENCED_TABLE_NAME as foreign_table_name, - kcu.REFERENCED_COLUMN_NAME as foreign_column_name - FROM information_schema.KEY_COLUMN_USAGE kcu - WHERE kcu.TABLE_SCHEMA = :tableSchema - AND kcu.TABLE_NAME = :tableName - AND kcu.REFERENCED_TABLE_NAME IS NOT NULL`, - [ - { name: 'tableSchema', value: { stringValue: tableSchema } }, - { name: 'tableName', value: { stringValue: tableName } }, - ] - ) - - const foreignKeys = fkResult.rows.map((r) => { - const fkRow = r as { - column_name: string - foreign_table_name: string - foreign_column_name: string - } - return { - column: fkRow.column_name, - referencesTable: fkRow.foreign_table_name, - referencesColumn: fkRow.foreign_column_name, - } - }) - - const fkColumnSet = new Set(foreignKeys.map((fk) => fk.column)) - - const indexesResult = await executeStatement( - client, - resourceArn, - secretArn, - database, - `SELECT - INDEX_NAME as index_name, - COLUMN_NAME as column_name, - NON_UNIQUE as non_unique - FROM information_schema.STATISTICS - WHERE TABLE_SCHEMA = :tableSchema - AND TABLE_NAME = :tableName - AND INDEX_NAME != 'PRIMARY' - ORDER BY INDEX_NAME, SEQ_IN_INDEX`, - [ - { name: 'tableSchema', value: { stringValue: tableSchema } }, - { name: 'tableName', value: { stringValue: tableName } }, - ] - ) - - const indexMap = new Map() - for (const idxRow of indexesResult.rows) { - const idx = idxRow as { index_name: string; column_name: string; non_unique: number } - const indexName = idx.index_name - if (!indexMap.has(indexName)) { - indexMap.set(indexName, { - name: indexName, - columns: [], - unique: idx.non_unique === 0, - }) - } - indexMap.get(indexName)!.columns.push(idx.column_name) - } - const indexes = Array.from(indexMap.values()) - - const columns = columnsResult.rows.map((colRow) => { - const col = colRow as { - column_name: string - data_type: string - is_nullable: string - column_default: string | null - column_type: string - column_key: string - } - const columnName = col.column_name - const fk = foreignKeys.find((f) => f.column === columnName) - - return { - name: columnName, - type: col.column_type || col.data_type, - nullable: col.is_nullable === 'YES', - default: col.column_default, - isPrimaryKey: col.column_key === 'PRI', - isForeignKey: fkColumnSet.has(columnName), - ...(fk && { - references: { - table: fk.referencesTable, - column: fk.referencesColumn, - }, - }), - } - }) - - tables.push({ - name: tableName, - schema: tableSchema, - columns, - primaryKey: primaryKeyColumns, - foreignKeys, - indexes, - }) - } - - return { engine: 'aurora-mysql', tables, schemas } -} - -/** - * Introspects RDS Aurora database schema with auto-detection of engine type - */ -export async function executeIntrospect( - client: RDSDataClient, - resourceArn: string, - secretArn: string, - database: string | undefined, - schemaName?: string, - engine?: RdsEngine -): Promise { - const detectedEngine = engine || (await detectEngine(client, resourceArn, secretArn, database)) - - if (detectedEngine === 'aurora-postgresql') { - const schema = schemaName || 'public' - return introspectPostgresql(client, resourceArn, secretArn, database, schema) - } - const schema = schemaName || database || '' - if (!schema) { - throw new Error('Schema or database name is required for MySQL introspection') - } - return introspectMysql(client, resourceArn, secretArn, database, schema) -} diff --git a/apps/sim/app/api/tools/redis/execute/route.ts b/apps/sim/app/api/tools/redis/execute/route.ts deleted file mode 100644 index 7a38c676b95..00000000000 --- a/apps/sim/app/api/tools/redis/execute/route.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import Redis from 'ioredis' -import { type NextRequest, NextResponse } from 'next/server' -import { redisExecuteContract } from '@/lib/api/contracts/tools/databases/redis' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateDatabaseHost } from '@/lib/core/security/input-validation.server' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -const logger = createLogger('RedisAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - let client: Redis | null = null - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseToolRequest(redisExecuteContract, request, { - errorFormat: 'firstError', - logger, - }) - if (!parsed.success) return parsed.response - const { url, command, args } = parsed.data.body - - const parsedUrl = new URL(url) - const hostname = - parsedUrl.hostname.startsWith('[') && parsedUrl.hostname.endsWith(']') - ? parsedUrl.hostname.slice(1, -1) - : parsedUrl.hostname - const hostValidation = await validateDatabaseHost(hostname, 'host') - if (!hostValidation.isValid) { - return NextResponse.json({ error: hostValidation.error }, { status: 400 }) - } - - const resolvedIP = hostValidation.resolvedIP ?? hostname - const tlsEnabled = parsedUrl.protocol === 'rediss:' - const port = parsedUrl.port ? Number(parsedUrl.port) : 6379 - const username = parsedUrl.username ? decodeURIComponent(parsedUrl.username) : undefined - const password = parsedUrl.password ? decodeURIComponent(parsedUrl.password) : undefined - - let db = 0 - if (parsedUrl.pathname && parsedUrl.pathname.length > 1) { - const dbSegment = parsedUrl.pathname.slice(1) - const parsedDb = Number.parseInt(dbSegment, 10) - if (!Number.isFinite(parsedDb) || String(parsedDb) !== dbSegment) { - return NextResponse.json( - { error: `Invalid Redis database index in URL path: '${dbSegment}'` }, - { status: 400 } - ) - } - db = parsedDb - } - - client = new Redis({ - host: resolvedIP, - port, - username, - password, - db, - family: resolvedIP.includes(':') ? 6 : 4, - tls: tlsEnabled ? { servername: hostname } : undefined, - connectTimeout: 10000, - commandTimeout: 10000, - maxRetriesPerRequest: 1, - lazyConnect: true, - }) - - await client.connect() - - const cmd = command.toUpperCase() - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const result = await (client as any).call(cmd, ...args) - - await client.quit() - client = null - - return NextResponse.json({ result }) - } catch (error) { - logger.error('Redis command failed', { error }) - const errorMessage = getErrorMessage(error, 'Redis command failed') - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } finally { - if (client) { - try { - await client.quit() - } catch { - client.disconnect() - } - } - } -}) diff --git a/apps/sim/app/api/tools/reducto/parse/route.ts b/apps/sim/app/api/tools/reducto/parse/route.ts deleted file mode 100644 index 6eef3ad6066..00000000000 --- a/apps/sim/app/api/tools/reducto/parse/route.ts +++ /dev/null @@ -1,182 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { reductoParseContract } from '@/lib/api/contracts/tools/media/document-parse' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' -import { resolveFileInputToUrl } from '@/lib/uploads/utils/file-utils.server' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('ReductoParseAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Reducto parse attempt`, { - error: authResult.error || 'Missing userId', - }) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Unauthorized', - }, - { status: 401 } - ) - } - - const userId = authResult.userId - - const parsed = await parseRequest( - reductoParseContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - - const validatedData = parsed.data.body - const modelInputProvenance = validateOpaqueModelInputProvenance({ - headers: request.headers, - payload: validatedData, - isInternalRequest: true, - }) - if (!modelInputProvenance.success) { - return NextResponse.json( - { success: false, error: modelInputProvenance.error }, - { status: modelInputProvenance.status } - ) - } - - logger.info(`[${requestId}] Reducto parse request`, { - hasInlineFile: Boolean(validatedData.file), - hasFilePath: Boolean(validatedData.filePath), - userId, - }) - - const resolution = await resolveFileInputToUrl({ - file: validatedData.file, - filePath: validatedData.filePath, - userId, - requestId, - logger, - modelEgress: true, - }) - - if (resolution.error) { - return NextResponse.json( - { success: false, error: resolution.error.message }, - { status: resolution.error.status } - ) - } - - const fileUrl = resolution.fileUrl - if (!fileUrl) { - return NextResponse.json({ success: false, error: 'File input is required' }, { status: 400 }) - } - - const reductoBody: Record = { - input: fileUrl, - } - - if (validatedData.pages && validatedData.pages.length > 0) { - // Reducto API expects page_range as an object with start/end, not an array - const pages = validatedData.pages - reductoBody.settings = { - page_range: { - start: Math.min(...pages), - end: Math.max(...pages), - }, - } - } - - if (validatedData.tableOutputFormat) { - reductoBody.formatting = { - table_output_format: validatedData.tableOutputFormat, - } - } - - const reductoEndpoint = 'https://platform.reducto.ai/parse' - const reductoValidation = await validateUrlWithDNS(reductoEndpoint, 'Reducto API URL') - if (!reductoValidation.isValid) { - logger.error(`[${requestId}] Reducto API URL validation failed`, { - error: reductoValidation.error, - }) - return NextResponse.json( - { - success: false, - error: 'Failed to reach Reducto API', - }, - { status: 502 } - ) - } - - const reductoResponse = await secureFetchWithPinnedIP( - reductoEndpoint, - reductoValidation.resolvedIP!, - { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json', - Authorization: `Bearer ${validatedData.apiKey}`, - }, - body: JSON.stringify(reductoBody), - } - ) - - if (!reductoResponse.ok) { - const errorText = await reductoResponse.text() - logger.error(`[${requestId}] Reducto API error:`, errorText) - return NextResponse.json( - { - success: false, - error: `Reducto API error: ${reductoResponse.statusText}`, - }, - { status: reductoResponse.status } - ) - } - - const reductoData = await reductoResponse.json() - - logger.info(`[${requestId}] Reducto parse successful`) - - return NextResponse.json({ - success: true, - output: reductoData, - }) - } catch (error) { - logger.error(`[${requestId}] Error in Reducto parse:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/s3/copy-object/route.ts b/apps/sim/app/api/tools/s3/copy-object/route.ts deleted file mode 100644 index 28a71cf12d6..00000000000 --- a/apps/sim/app/api/tools/s3/copy-object/route.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { CopyObjectCommand, type ObjectCannedACL, S3Client } from '@aws-sdk/client-s3' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsS3CopyObjectContract } from '@/lib/api/contracts/tools/aws/s3-copy-object' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('S3CopyObjectAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized S3 copy object attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated S3 copy object request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const parsed = await parseToolRequest(awsS3CopyObjectContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Copying S3 object`, { - source: `${validatedData.sourceBucket}/${validatedData.sourceKey}`, - destination: `${validatedData.destinationBucket}/${validatedData.destinationKey}`, - }) - - // Initialize S3 client - const s3Client = new S3Client({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - // Copy object (properly encode the source key for CopySource parameter) - const encodedSourceKey = validatedData.sourceKey.split('/').map(encodeURIComponent).join('/') - const copySource = `${validatedData.sourceBucket}/${encodedSourceKey}` - const copyCommand = new CopyObjectCommand({ - Bucket: validatedData.destinationBucket, - Key: validatedData.destinationKey, - CopySource: copySource, - ACL: validatedData.acl as ObjectCannedACL | undefined, - }) - - const result = await s3Client.send(copyCommand) - - logger.info(`[${requestId}] Object copied successfully`, { - source: copySource, - destination: `${validatedData.destinationBucket}/${validatedData.destinationKey}`, - etag: result.CopyObjectResult?.ETag, - }) - - // Generate public URL for destination (properly encode the destination key) - const encodedDestKey = validatedData.destinationKey.split('/').map(encodeURIComponent).join('/') - const url = `https://${validatedData.destinationBucket}.s3.${validatedData.region}.amazonaws.com/${encodedDestKey}` - const uri = `s3://${validatedData.destinationBucket}/${validatedData.destinationKey}` - - return NextResponse.json({ - success: true, - output: { - url, - uri, - copySourceVersionId: result.CopySourceVersionId, - versionId: result.VersionId, - etag: result.CopyObjectResult?.ETag, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error copying S3 object:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/s3/create-bucket/route.ts b/apps/sim/app/api/tools/s3/create-bucket/route.ts deleted file mode 100644 index 055ee8bf509..00000000000 --- a/apps/sim/app/api/tools/s3/create-bucket/route.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { - type BucketCannedACL, - type BucketLocationConstraint, - CreateBucketCommand, - S3Client, -} from '@aws-sdk/client-s3' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsS3CreateBucketContract } from '@/lib/api/contracts/tools/aws/s3-create-bucket' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('S3CreateBucketAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized S3 create bucket attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info( - `[${requestId}] Authenticated S3 create bucket request via ${authResult.authType}`, - { - userId: authResult.userId, - } - ) - - const parsed = await parseToolRequest(awsS3CreateBucketContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Creating S3 bucket`, { - bucket: validatedData.bucketName, - region: validatedData.region, - }) - - const s3Client = new S3Client({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - const createCommand = new CreateBucketCommand({ - Bucket: validatedData.bucketName, - ACL: (validatedData.acl as BucketCannedACL | undefined) || undefined, - CreateBucketConfiguration: - validatedData.region === 'us-east-1' - ? undefined - : { LocationConstraint: validatedData.region as BucketLocationConstraint }, - }) - - const result = await s3Client.send(createCommand) - - logger.info(`[${requestId}] Bucket created successfully`, { - bucket: validatedData.bucketName, - location: result.Location, - }) - - return NextResponse.json({ - success: true, - output: { - bucket: validatedData.bucketName, - location: result.Location ?? null, - bucketArn: result.BucketArn ?? null, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error creating S3 bucket:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/s3/delete-bucket/route.ts b/apps/sim/app/api/tools/s3/delete-bucket/route.ts deleted file mode 100644 index c3d89e92cd2..00000000000 --- a/apps/sim/app/api/tools/s3/delete-bucket/route.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { DeleteBucketCommand, S3Client } from '@aws-sdk/client-s3' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsS3DeleteBucketContract } from '@/lib/api/contracts/tools/aws/s3-delete-bucket' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('S3DeleteBucketAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized S3 delete bucket attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info( - `[${requestId}] Authenticated S3 delete bucket request via ${authResult.authType}`, - { - userId: authResult.userId, - } - ) - - const parsed = await parseToolRequest(awsS3DeleteBucketContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Deleting S3 bucket`, { - bucket: validatedData.bucketName, - }) - - const s3Client = new S3Client({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - const deleteCommand = new DeleteBucketCommand({ - Bucket: validatedData.bucketName, - }) - - await s3Client.send(deleteCommand) - - logger.info(`[${requestId}] Bucket deleted successfully`, { - bucket: validatedData.bucketName, - }) - - return NextResponse.json({ - success: true, - output: { - deleted: true, - bucket: validatedData.bucketName, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error deleting S3 bucket:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/s3/delete-object/route.ts b/apps/sim/app/api/tools/s3/delete-object/route.ts deleted file mode 100644 index 29a5778b6e4..00000000000 --- a/apps/sim/app/api/tools/s3/delete-object/route.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { DeleteObjectCommand, S3Client } from '@aws-sdk/client-s3' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsS3DeleteObjectContract } from '@/lib/api/contracts/tools/aws/s3-delete-object' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('S3DeleteObjectAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized S3 delete object attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info( - `[${requestId}] Authenticated S3 delete object request via ${authResult.authType}`, - { - userId: authResult.userId, - } - ) - - const parsed = await parseToolRequest(awsS3DeleteObjectContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Deleting S3 object`, { - bucket: validatedData.bucketName, - key: validatedData.objectKey, - }) - - // Initialize S3 client - const s3Client = new S3Client({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - // Delete object - const deleteCommand = new DeleteObjectCommand({ - Bucket: validatedData.bucketName, - Key: validatedData.objectKey, - }) - - const result = await s3Client.send(deleteCommand) - - logger.info(`[${requestId}] Object deleted successfully`, { - bucket: validatedData.bucketName, - key: validatedData.objectKey, - deleteMarker: result.DeleteMarker, - }) - - return NextResponse.json({ - success: true, - output: { - key: validatedData.objectKey, - deleteMarker: result.DeleteMarker, - versionId: result.VersionId, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error deleting S3 object:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/s3/delete-objects/route.ts b/apps/sim/app/api/tools/s3/delete-objects/route.ts deleted file mode 100644 index f8b74278427..00000000000 --- a/apps/sim/app/api/tools/s3/delete-objects/route.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { DeleteObjectsCommand, S3Client } from '@aws-sdk/client-s3' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsS3DeleteObjectsContract } from '@/lib/api/contracts/tools/aws/s3-delete-objects' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('S3DeleteObjectsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized S3 delete objects attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info( - `[${requestId}] Authenticated S3 delete objects request via ${authResult.authType}`, - { - userId: authResult.userId, - } - ) - - const parsed = await parseToolRequest(awsS3DeleteObjectsContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Deleting S3 objects`, { - bucket: validatedData.bucketName, - count: validatedData.keys.length, - }) - - const s3Client = new S3Client({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - const deleteCommand = new DeleteObjectsCommand({ - Bucket: validatedData.bucketName, - Delete: { - Objects: validatedData.keys.map((key) => ({ Key: key })), - Quiet: validatedData.quiet ?? false, - }, - }) - - const result = await s3Client.send(deleteCommand) - - const deleted = (result.Deleted || []).map((obj) => ({ - key: obj.Key ?? null, - versionId: obj.VersionId ?? null, - deleteMarker: obj.DeleteMarker ?? null, - })) - - const errors = (result.Errors || []).map((err) => ({ - key: err.Key ?? null, - code: err.Code ?? null, - message: err.Message ?? null, - })) - - logger.info(`[${requestId}] Delete objects completed`, { - bucket: validatedData.bucketName, - deleted: deleted.length, - errors: errors.length, - }) - - return NextResponse.json({ - success: true, - output: { - deleted, - errors, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error deleting S3 objects:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/s3/head-object/route.ts b/apps/sim/app/api/tools/s3/head-object/route.ts deleted file mode 100644 index 898a29c7167..00000000000 --- a/apps/sim/app/api/tools/s3/head-object/route.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { HeadObjectCommand, S3Client } from '@aws-sdk/client-s3' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsS3HeadObjectContract } from '@/lib/api/contracts/tools/aws/s3-head-object' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('S3HeadObjectAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized S3 head object attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated S3 head object request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const parsed = await parseToolRequest(awsS3HeadObjectContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Fetching S3 object metadata`, { - bucket: validatedData.bucketName, - key: validatedData.objectKey, - }) - - const s3Client = new S3Client({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - const headCommand = new HeadObjectCommand({ - Bucket: validatedData.bucketName, - Key: validatedData.objectKey, - VersionId: validatedData.versionId || undefined, - }) - - const result = await s3Client.send(headCommand) - - logger.info(`[${requestId}] Object metadata retrieved`, { - bucket: validatedData.bucketName, - key: validatedData.objectKey, - }) - - return NextResponse.json({ - success: true, - output: { - exists: true, - contentLength: result.ContentLength ?? null, - contentType: result.ContentType ?? null, - etag: result.ETag ?? null, - lastModified: result.LastModified?.toISOString() ?? null, - versionId: result.VersionId ?? null, - storageClass: result.StorageClass ?? null, - serverSideEncryption: result.ServerSideEncryption ?? null, - deleteMarker: result.DeleteMarker ?? null, - metadata: result.Metadata ?? {}, - }, - }) - } catch (error) { - const metadata = error as { name?: string; $metadata?: { httpStatusCode?: number } } - if (metadata?.name === 'NotFound' || metadata?.$metadata?.httpStatusCode === 404) { - return NextResponse.json({ - success: true, - output: { - exists: false, - contentLength: null, - contentType: null, - etag: null, - lastModified: null, - versionId: null, - storageClass: null, - serverSideEncryption: null, - deleteMarker: null, - metadata: {}, - }, - }) - } - - logger.error(`[${requestId}] Error fetching S3 object metadata:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/s3/list-buckets/route.ts b/apps/sim/app/api/tools/s3/list-buckets/route.ts deleted file mode 100644 index 1a951bf5632..00000000000 --- a/apps/sim/app/api/tools/s3/list-buckets/route.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { ListBucketsCommand, S3Client } from '@aws-sdk/client-s3' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsS3ListBucketsContract } from '@/lib/api/contracts/tools/aws/s3-list-buckets' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('S3ListBucketsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized S3 list buckets attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated S3 list buckets request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const parsed = await parseToolRequest(awsS3ListBucketsContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Listing S3 buckets`, { - prefix: validatedData.prefix || '(none)', - maxBuckets: validatedData.maxBuckets || '(all)', - }) - - const s3Client = new S3Client({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - const listCommand = new ListBucketsCommand({ - Prefix: validatedData.prefix || undefined, - MaxBuckets: validatedData.maxBuckets || undefined, - ContinuationToken: validatedData.continuationToken || undefined, - }) - - const result = await s3Client.send(listCommand) - - const buckets = (result.Buckets || []).map((bucket) => ({ - name: bucket.Name || '', - creationDate: bucket.CreationDate?.toISOString() ?? null, - region: bucket.BucketRegion ?? null, - })) - - logger.info(`[${requestId}] Listed ${buckets.length} buckets`) - - return NextResponse.json({ - success: true, - output: { - buckets, - owner: result.Owner - ? { - displayName: result.Owner.DisplayName ?? null, - id: result.Owner.ID ?? null, - } - : null, - continuationToken: result.ContinuationToken ?? null, - prefix: result.Prefix ?? null, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error listing S3 buckets:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/s3/list-objects/route.ts b/apps/sim/app/api/tools/s3/list-objects/route.ts deleted file mode 100644 index 0f2ad914679..00000000000 --- a/apps/sim/app/api/tools/s3/list-objects/route.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { ListObjectsV2Command, S3Client } from '@aws-sdk/client-s3' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsS3ListObjectsContract } from '@/lib/api/contracts/tools/aws/s3-list-objects' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('S3ListObjectsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized S3 list objects attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated S3 list objects request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const parsed = await parseToolRequest(awsS3ListObjectsContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Listing S3 objects`, { - bucket: validatedData.bucketName, - prefix: validatedData.prefix || '(none)', - maxKeys: validatedData.maxKeys || 1000, - }) - - // Initialize S3 client - const s3Client = new S3Client({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - // List objects - const listCommand = new ListObjectsV2Command({ - Bucket: validatedData.bucketName, - Prefix: validatedData.prefix || undefined, - MaxKeys: validatedData.maxKeys || undefined, - ContinuationToken: validatedData.continuationToken || undefined, - }) - - const result = await s3Client.send(listCommand) - - const objects = (result.Contents || []).map((obj) => ({ - key: obj.Key || '', - size: obj.Size || 0, - lastModified: obj.LastModified?.toISOString() || '', - etag: obj.ETag || '', - })) - - logger.info(`[${requestId}] Listed ${objects.length} objects`, { - bucket: validatedData.bucketName, - isTruncated: result.IsTruncated, - }) - - return NextResponse.json({ - success: true, - output: { - objects, - isTruncated: result.IsTruncated, - nextContinuationToken: result.NextContinuationToken, - keyCount: result.KeyCount, - prefix: validatedData.prefix, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error listing S3 objects:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/s3/presigned-url/route.ts b/apps/sim/app/api/tools/s3/presigned-url/route.ts deleted file mode 100644 index 72b443d974d..00000000000 --- a/apps/sim/app/api/tools/s3/presigned-url/route.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3' -import { getSignedUrl } from '@aws-sdk/s3-request-presigner' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsS3PresignedUrlContract } from '@/lib/api/contracts/tools/aws/s3-presigned-url' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('S3PresignedUrlAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized S3 presigned URL attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info( - `[${requestId}] Authenticated S3 presigned URL request via ${authResult.authType}`, - { - userId: authResult.userId, - } - ) - - const parsed = await parseToolRequest(awsS3PresignedUrlContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Generating S3 presigned URL`, { - bucket: validatedData.bucketName, - key: validatedData.objectKey, - method: validatedData.method, - expiresIn: validatedData.expiresIn, - }) - - const s3Client = new S3Client({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - const command = - validatedData.method === 'put' - ? new PutObjectCommand({ - Bucket: validatedData.bucketName, - Key: validatedData.objectKey, - ContentType: validatedData.contentType || undefined, - }) - : new GetObjectCommand({ - Bucket: validatedData.bucketName, - Key: validatedData.objectKey, - }) - - const url = await getSignedUrl(s3Client, command, { - expiresIn: validatedData.expiresIn, - }) - - const expiresAt = new Date(Date.now() + validatedData.expiresIn * 1000).toISOString() - - logger.info(`[${requestId}] Presigned URL generated`, { - bucket: validatedData.bucketName, - key: validatedData.objectKey, - }) - - return NextResponse.json({ - success: true, - output: { - url, - method: validatedData.method, - expiresIn: validatedData.expiresIn, - expiresAt, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error generating S3 presigned URL:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/s3/put-object/route.ts b/apps/sim/app/api/tools/s3/put-object/route.ts deleted file mode 100644 index 026713914fb..00000000000 --- a/apps/sim/app/api/tools/s3/put-object/route.ts +++ /dev/null @@ -1,163 +0,0 @@ -import { type ObjectCannedACL, PutObjectCommand, S3Client } from '@aws-sdk/client-s3' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsS3PutObjectContract } from '@/lib/api/contracts/tools/aws/s3-put-object' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('S3PutObjectAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized S3 put object attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated S3 put object request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const parsed = await parseToolRequest(awsS3PutObjectContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Uploading to S3`, { - bucket: validatedData.bucketName, - key: validatedData.objectKey, - hasFile: !!validatedData.file, - hasContent: !!validatedData.content, - }) - - const s3Client = new S3Client({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - let uploadBody: Buffer | string - let uploadContentType: string | undefined - - if (validatedData.file) { - const rawFile = validatedData.file - logger.info(`[${requestId}] Processing file upload: ${rawFile.name}`) - - let userFile - try { - userFile = processSingleFileToUserFile(rawFile, requestId, logger) - } catch (error) { - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Failed to process file'), - }, - { status: 400 } - ) - } - - const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) - if (denied) return denied - - let downloadedContentType = '' - try { - const result = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - }) - uploadBody = result.buffer - downloadedContentType = result.contentType - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Failed to download file') }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } - - uploadContentType = - validatedData.contentType || - downloadedContentType || - userFile.type || - 'application/octet-stream' - } else if (validatedData.content) { - uploadBody = Buffer.from(validatedData.content, 'utf-8') - uploadContentType = validatedData.contentType || 'text/plain' - } else { - return NextResponse.json( - { - success: false, - error: 'Either file or content must be provided', - }, - { status: 400 } - ) - } - - const putCommand = new PutObjectCommand({ - Bucket: validatedData.bucketName, - Key: validatedData.objectKey, - Body: uploadBody, - ContentType: uploadContentType, - ACL: validatedData.acl as ObjectCannedACL | undefined, - }) - - const result = await s3Client.send(putCommand) - - logger.info(`[${requestId}] File uploaded successfully`, { - etag: result.ETag, - bucket: validatedData.bucketName, - key: validatedData.objectKey, - }) - - const encodedKey = validatedData.objectKey.split('/').map(encodeURIComponent).join('/') - const url = `https://${validatedData.bucketName}.s3.${validatedData.region}.amazonaws.com/${encodedKey}` - const uri = `s3://${validatedData.bucketName}/${validatedData.objectKey}` - - return NextResponse.json({ - success: true, - output: { - url, - uri, - etag: result.ETag, - location: url, - key: validatedData.objectKey, - bucket: validatedData.bucketName, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error uploading to S3:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/sap_concur/proxy/route.ts b/apps/sim/app/api/tools/sap_concur/proxy/route.ts deleted file mode 100644 index efb207ab48f..00000000000 --- a/apps/sim/app/api/tools/sap_concur/proxy/route.ts +++ /dev/null @@ -1,194 +0,0 @@ -import { createLogger } from '@sim/logger' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { getValidationErrorMessage, isZodError } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - MAX_JSON_API_RESPONSE_BYTES, - secureFetchWithValidation, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - assertSafeExternalUrl, - describeSapConcurFetchError, - extractSapConcurError, - fetchSapConcurAccessToken, - forwardedSapConcurHeaders, - SAP_CONCUR_OUTBOUND_FETCH_TIMEOUT_MS, - type SapConcurProxyRequest, - SapConcurProxyRequestSchema, -} from '@/app/api/tools/sap_concur/shared' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('SapConcurProxyAPI') - -type ProxyRequest = SapConcurProxyRequest - -function buildApiUrl(geolocation: string, req: ProxyRequest): string { - const base = geolocation.replace(/\/+$/, '') - const subPath = req.path.startsWith('/') ? req.path : `/${req.path}` - const url = `${base}${subPath}` - - if (!req.query || Object.keys(req.query).length === 0) { - return url - } - const search = new URLSearchParams() - for (const [key, value] of Object.entries(req.query)) { - if (value === undefined || value === null) continue - search.append(key, String(value)) - } - const queryString = search.toString() - if (!queryString) return url - return url.includes('?') ? `${url}&${queryString}` : `${url}?${queryString}` -} - -/** - * Map a non-2xx Concur status that cannot be re-emitted as an error status onto 502. - * - * With `maxRedirects: 0` a 3xx carrying a `Location` never reaches here — it rejects with - * "Too many redirects" and is handled in the outer catch. What does reach here is a 3xx - * *without* a `Location`, and a 304, which is excluded from the redirect handling - * upstream. Neither is a usable error status to return to the caller. - */ -function clampErrorStatus(status: number): number { - return status >= 400 ? status : 502 -} - -interface Invocation { - status: number - body: unknown - raw: string - /** Concur response headers forwarded onto this route's response. */ - headers: Record -} - -/** - * Invoke a Concur API endpoint with the bearer token. - * - * `concur-correlationid` is a support/tracing header expected to be a fresh RFC 4122 - * UUID per request; it does not scope a request to a company. Redirects are refused so - * the Authorization header is never forwarded to another origin. - * - * `stripAuthOnRedirect` is unreachable while `maxRedirects` is 0 — no redirect is ever - * followed for it to act on. It is kept as defense-in-depth so raising `maxRedirects` - * later cannot silently start forwarding the bearer token; do not remove it as dead code. - */ -/** - * Read a Concur response body, keeping the upstream status meaningful. - * - * On a success status the body is the result, so a stream failure is a real - * error and must propagate. On an error status the body only supplies the - * message, and throwing would turn Concur's 4xx into a Sim 500 — the status is - * preserved instead and the message falls back to the generic HTTP-status form. - */ -export async function readConcurProxyBody(response: { - status: number - text: () => Promise -}): Promise { - const read = response.text() - if (response.status >= 200 && response.status < 300) return read - return read.catch(() => '') -} - -async function callConcur( - req: ProxyRequest, - accessToken: string, - geolocation: string -): Promise { - const url = assertSafeExternalUrl(buildApiUrl(geolocation, req), 'apiUrl').toString() - const hasBody = req.body !== undefined && req.body !== null - const headers: Record = { - Authorization: `Bearer ${accessToken}`, - Accept: req.accept ?? 'application/json', - } - if (hasBody) headers['Content-Type'] = req.contentType ?? 'application/json' - headers['concur-correlationid'] = generateId() - - const response = await secureFetchWithValidation( - url, - { - method: req.method, - headers, - body: hasBody - ? typeof req.body === 'string' - ? req.body - : JSON.stringify(req.body) - : undefined, - timeout: SAP_CONCUR_OUTBOUND_FETCH_TIMEOUT_MS, - maxRedirects: 0, - stripAuthOnRedirect: true, - maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, - }, - 'apiUrl' - ) - - const raw = await readConcurProxyBody(response) - let parsed: unknown = null - if (raw.length > 0) { - try { - parsed = JSON.parse(raw) - } catch { - parsed = raw - } - } - return { - status: response.status, - body: parsed, - raw, - headers: forwardedSapConcurHeaders(response.headers), - } -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Concur proxy request: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - // boundary-raw-json: internal proxy envelope validated by SapConcurProxyRequestSchema below; not a public boundary - const json = await request.json() - const proxyReq = SapConcurProxyRequestSchema.parse(json) - - const { accessToken, geolocation } = await fetchSapConcurAccessToken(proxyReq, requestId) - const invocation = await callConcur(proxyReq, accessToken, geolocation) - - if (invocation.status >= 200 && invocation.status < 300) { - const data = invocation.status === 204 ? null : invocation.body - return NextResponse.json( - { success: true, output: { status: invocation.status, data } }, - { headers: invocation.headers } - ) - } - - const message = extractSapConcurError(invocation.body, invocation.status) - logger.warn( - `[${requestId}] Concur API error (${invocation.status}) ${proxyReq.path}: ${message}` - ) - return NextResponse.json( - { success: false, error: message, status: invocation.status }, - { status: clampErrorStatus(invocation.status), headers: invocation.headers } - ) - } catch (error) { - if (isZodError(error)) { - logger.warn(`[${requestId}] Validation error:`, error.issues) - return NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Validation failed') }, - { status: 400 } - ) - } - logger.error(`[${requestId}] Unexpected Concur proxy error:`, error) - return NextResponse.json( - { success: false, error: describeSapConcurFetchError(error) }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/sap_concur/shared.test.ts b/apps/sim/app/api/tools/sap_concur/shared.test.ts deleted file mode 100644 index 259f1c6f9ee..00000000000 --- a/apps/sim/app/api/tools/sap_concur/shared.test.ts +++ /dev/null @@ -1,906 +0,0 @@ -/** - * @vitest-environment node - */ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -const { mockSecureFetch, MOCK_MAX_JSON_BYTES } = vi.hoisted(() => ({ - mockSecureFetch: vi.fn(), - MOCK_MAX_JSON_BYTES: 10 * 1024 * 1024, -})) - -vi.mock('@/lib/core/security/input-validation.server', () => ({ - secureFetchWithValidation: mockSecureFetch, - MAX_JSON_API_RESPONSE_BYTES: MOCK_MAX_JSON_BYTES, -})) - -import { - assertSafeExternalUrl, - extractSapConcurError, - fetchSapConcurAccessToken, - forwardedSapConcurHeaders, - SAP_CONCUR_ALLOWED_DATACENTERS, - type SapConcurAuth, - SapConcurDatacenterSchema, - SapConcurProxyPath, - SapConcurProxyRequestSchema, -} from '@/app/api/tools/sap_concur/shared' - -const CLIENT_SECRET = 'super-secret-client-value' -const PASSWORD = 'hunter2-plaintext-password' - -/** - * `TOKEN_CACHE` in shared.ts is module-global and survives every test in this file, so - * each case takes its own `clientId`. That guarantees a cold cache key for the case and - * keeps one test's cached token from silently satisfying the next test's assertions. - */ -let clientIdCounter = 0 -function freshClientId(): string { - clientIdCounter += 1 - return `client-${clientIdCounter}` -} - -function auth(overrides: Partial & { clientId: string }): SapConcurAuth { - return { - datacenter: 'us.api.concursolutions.com', - grantType: 'client_credentials', - clientSecret: CLIENT_SECRET, - ...overrides, - } -} - -function tokenResponse( - body: Record = { access_token: 'token-1', expires_in: 3600 }, - status = 200 -) { - return { - ok: status >= 200 && status < 300, - status, - headers: new Headers(), - json: async () => body, - text: async () => JSON.stringify(body), - } -} - -beforeEach(() => { - vi.clearAllMocks() - // mockReset also drains any `mockResolvedValueOnce` a failing test left queued. - mockSecureFetch.mockReset() - mockSecureFetch.mockResolvedValue(tokenResponse()) -}) - -describe('fetchSapConcurAccessToken token cache key isolation', () => { - /** - * Regression test for the auth-bypass: with the password absent from the cache key, a - * request carrying the wrong password was served a token minted from the correct one. - */ - it('does not share a cache entry across differing passwords', async () => { - const clientId = freshClientId() - const base = auth({ - clientId, - grantType: 'password', - username: 'alice@example.com', - }) - - mockSecureFetch.mockResolvedValueOnce( - tokenResponse({ access_token: 'token-correct', expires_in: 3600 }) - ) - const first = await fetchSapConcurAccessToken({ ...base, password: PASSWORD }, 'req-1') - - mockSecureFetch.mockResolvedValueOnce( - tokenResponse({ access_token: 'token-other', expires_in: 3600 }) - ) - const second = await fetchSapConcurAccessToken({ ...base, password: 'a-different-pw' }, 'req-2') - - expect(mockSecureFetch).toHaveBeenCalledTimes(2) - expect(first.accessToken).toBe('token-correct') - expect(second.accessToken).toBe('token-other') - }) - - it('does not share a cache entry across differing companyUuid', async () => { - const clientId = freshClientId() - const base = auth({ clientId }) - - await fetchSapConcurAccessToken({ ...base, companyUuid: 'company-a' }, 'req-1') - await fetchSapConcurAccessToken({ ...base, companyUuid: 'company-b' }, 'req-2') - - expect(mockSecureFetch).toHaveBeenCalledTimes(2) - }) - - it('does not share a cache entry across differing credtype', async () => { - const clientId = freshClientId() - const base = auth({ - clientId, - grantType: 'password', - username: 'alice@example.com', - password: PASSWORD, - }) - - await fetchSapConcurAccessToken({ ...base, credtype: 'password' }, 'req-1') - await fetchSapConcurAccessToken({ ...base, credtype: 'authtoken' }, 'req-2') - - expect(mockSecureFetch).toHaveBeenCalledTimes(2) - }) - - it('shares the cache for two fully identical requests', async () => { - const clientId = freshClientId() - const base = auth({ - clientId, - grantType: 'password', - username: 'alice@example.com', - password: PASSWORD, - companyUuid: 'company-a', - credtype: 'password', - }) - - mockSecureFetch.mockResolvedValueOnce( - tokenResponse({ access_token: 'token-cached', expires_in: 3600 }) - ) - const first = await fetchSapConcurAccessToken({ ...base }, 'req-1') - const second = await fetchSapConcurAccessToken({ ...base }, 'req-2') - - expect(mockSecureFetch).toHaveBeenCalledTimes(1) - expect(first.accessToken).toBe('token-cached') - expect(second.accessToken).toBe('token-cached') - }) - - it('refetches once a cached token falls inside the 60s safety window', async () => { - vi.useFakeTimers() - try { - vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z')) - const base = auth({ clientId: freshClientId() }) - - mockSecureFetch.mockResolvedValueOnce( - tokenResponse({ access_token: 'token-first', expires_in: 120 }) - ) - const first = await fetchSapConcurAccessToken(base, 'req-1') - expect(first.accessToken).toBe('token-first') - - // 30s in: still outside the 60s safety window, so the cache answers. - vi.setSystemTime(new Date('2026-01-01T00:00:30.000Z')) - const cached = await fetchSapConcurAccessToken(base, 'req-2') - expect(cached.accessToken).toBe('token-first') - expect(mockSecureFetch).toHaveBeenCalledTimes(1) - - // 90s in: expiry (120s) minus the 60s window has passed, so it refetches. - vi.setSystemTime(new Date('2026-01-01T00:01:30.000Z')) - mockSecureFetch.mockResolvedValueOnce( - tokenResponse({ access_token: 'token-second', expires_in: 3600 }) - ) - const refreshed = await fetchSapConcurAccessToken(base, 'req-3') - expect(refreshed.accessToken).toBe('token-second') - expect(mockSecureFetch).toHaveBeenCalledTimes(2) - } finally { - vi.useRealTimers() - } - }) -}) - -/** - * A parallel block fanning out many Concur calls, or a cold container after a deploy, - * misses the token cache on every branch at once. Without coalescing each branch fires - * its own `POST /oauth2/v0/token` into an endpoint Concur rate-limits hard. - */ -describe('fetchSapConcurAccessToken in-flight coalescing', () => { - it('collapses concurrent misses for one key into a single token fetch', async () => { - const base = auth({ clientId: freshClientId() }) - - let release: () => void = () => {} - const gate = new Promise((resolve) => { - release = resolve - }) - mockSecureFetch.mockImplementation(async () => { - await gate - return tokenResponse({ access_token: 'token-shared', expires_in: 3600 }) - }) - - const inFlight = Array.from({ length: 8 }, (_, index) => - fetchSapConcurAccessToken(base, `req-${index}`) - ) - release() - const results = await Promise.all(inFlight) - - expect(mockSecureFetch).toHaveBeenCalledTimes(1) - for (const result of results) { - expect(result.accessToken).toBe('token-shared') - } - }) - - it('does not collapse concurrent misses for different keys', async () => { - const first = auth({ clientId: freshClientId() }) - const second = auth({ clientId: freshClientId() }) - - await Promise.all([ - fetchSapConcurAccessToken(first, 'req-1'), - fetchSapConcurAccessToken(second, 'req-2'), - ]) - - expect(mockSecureFetch).toHaveBeenCalledTimes(2) - }) - - it('does not poison the key when the in-flight request rejects', async () => { - const base = auth({ clientId: freshClientId() }) - - mockSecureFetch.mockRejectedValueOnce(new Error('socket hang up')) - const first = fetchSapConcurAccessToken(base, 'req-1') - const joiner = fetchSapConcurAccessToken(base, 'req-2') - - await expect(first).rejects.toThrow('socket hang up') - await expect(joiner).rejects.toThrow('socket hang up') - - mockSecureFetch.mockResolvedValueOnce( - tokenResponse({ access_token: 'token-after-retry', expires_in: 3600 }) - ) - const retried = await fetchSapConcurAccessToken(base, 'req-3') - - expect(retried.accessToken).toBe('token-after-retry') - expect(mockSecureFetch).toHaveBeenCalledTimes(2) - }) -}) - -describe('fetchSapConcurAccessToken geolocation validation', () => { - const accepted = [ - 'https://us.api.concursolutions.com', - 'https://www-us2.api.concursolutions.com', - 'https://apj1.api.concursolutions.com', - 'https://emea-impl.api.concursolutions.com', - ] - - it.each(accepted)('accepts the Concur geolocation %s', async (geolocation) => { - mockSecureFetch.mockResolvedValueOnce( - tokenResponse({ access_token: 'token-1', expires_in: 3600, geolocation }) - ) - const result = await fetchSapConcurAccessToken(auth({ clientId: freshClientId() }), 'req-1') - expect(result.geolocation).toBe(geolocation) - }) - - const rejected: Array<[string, string]> = [ - ['an unrelated host', 'https://evil.com'], - ['a suffix-confusion host', 'https://concursolutions.com.evil.com'], - ['a subdomain-confusion host', 'https://us.api.concursolutions.com.evil.com'], - ] - - it.each(rejected)('rejects %s', async (_label, geolocation) => { - mockSecureFetch.mockResolvedValueOnce( - tokenResponse({ access_token: 'token-1', expires_in: 3600, geolocation }) - ) - await expect( - fetchSapConcurAccessToken(auth({ clientId: freshClientId() }), 'req-1') - ).rejects.toThrow('not a valid Concur API host') - }) - - it('rejects a plain-http geolocation', async () => { - mockSecureFetch.mockResolvedValueOnce( - tokenResponse({ - access_token: 'token-1', - expires_in: 3600, - geolocation: 'http://us.api.concursolutions.com', - }) - ) - await expect( - fetchSapConcurAccessToken(auth({ clientId: freshClientId() }), 'req-1') - ).rejects.toThrow('geolocation must use https://') - }) - - it('rejects a loopback geolocation', async () => { - mockSecureFetch.mockResolvedValueOnce( - tokenResponse({ access_token: 'token-1', expires_in: 3600, geolocation: 'https://127.0.0.1' }) - ) - await expect( - fetchSapConcurAccessToken(auth({ clientId: freshClientId() }), 'req-1') - ).rejects.toThrow('geolocation host is not allowed') - }) - - it('normalizes a bare hostname to https and still validates it', async () => { - mockSecureFetch.mockResolvedValueOnce( - tokenResponse({ - access_token: 'token-1', - expires_in: 3600, - geolocation: 'us2.api.concursolutions.com', - }) - ) - const result = await fetchSapConcurAccessToken(auth({ clientId: freshClientId() }), 'req-1') - expect(result.geolocation).toBe('https://us2.api.concursolutions.com') - }) - - it('rejects a bare hostname that normalizes to a non-Concur host', async () => { - mockSecureFetch.mockResolvedValueOnce( - tokenResponse({ access_token: 'token-1', expires_in: 3600, geolocation: 'evil.com' }) - ) - await expect( - fetchSapConcurAccessToken(auth({ clientId: freshClientId() }), 'req-1') - ).rejects.toThrow('not a valid Concur API host') - }) - - /** - * DOCUMENTED TRUST ASSUMPTION, asserted as current behavior on purpose: the geolocation - * check validates the *shape* `[label].api.concursolutions.com`, not membership in - * {@link SAP_CONCUR_ALLOWED_DATACENTERS}. That is deliberate — Concur's docs instruct - * clients to store and reuse whatever geolocation the token response returns, and SAP - * adds datacenters (GLZ was one) without clients redeploying, so pinning the response - * to the selectable set would break tenants on a new datacenter. - * - * The consequence is that an attacker-flavored label like `evil-us` is accepted. Such a - * host can only exist if SAP itself creates it under concursolutions.com, which puts it - * inside the same trust boundary as every other Concur host. Narrowing this to the - * allowlist is a deliberate product decision, not a bug fix — do not "harden" it - * without re-reading the geolocation guidance in the authentication docs. - */ - it('accepts any SAP-created label under api.concursolutions.com (trust boundary is the domain)', async () => { - const geolocation = 'https://evil-us.api.concursolutions.com' - mockSecureFetch.mockResolvedValueOnce( - tokenResponse({ access_token: 'token-1', expires_in: 3600, geolocation }) - ) - const result = await fetchSapConcurAccessToken(auth({ clientId: freshClientId() }), 'req-1') - expect(result.geolocation).toBe(geolocation) - }) - - it('rejects a userinfo-form geolocation whose real hostname is attacker-controlled', async () => { - mockSecureFetch.mockResolvedValueOnce( - tokenResponse({ - access_token: 'token-1', - expires_in: 3600, - geolocation: 'https://us.api.concursolutions.com@evil.com', - }) - ) - await expect( - fetchSapConcurAccessToken(auth({ clientId: freshClientId() }), 'req-1') - ).rejects.toThrow('not a valid Concur API host') - }) - - it('accepts a Concur host carrying an explicit port and preserves it', async () => { - const geolocation = 'https://us.api.concursolutions.com:8443' - mockSecureFetch.mockResolvedValueOnce( - tokenResponse({ access_token: 'token-1', expires_in: 3600, geolocation }) - ) - const result = await fetchSapConcurAccessToken(auth({ clientId: freshClientId() }), 'req-1') - expect(result.geolocation).toBe(geolocation) - }) - - it('rejects a non-Concur host even when the port looks Concur-shaped', async () => { - mockSecureFetch.mockResolvedValueOnce( - tokenResponse({ - access_token: 'token-1', - expires_in: 3600, - geolocation: 'https://evil.com:443', - }) - ) - await expect( - fetchSapConcurAccessToken(auth({ clientId: freshClientId() }), 'req-1') - ).rejects.toThrow('not a valid Concur API host') - }) -}) - -/** - * Concur's company-level flow is a password grant that carries the company UUID in - * `username`, the 24-hour App Center request token in `password`, and `credtype=authtoken`. - */ -describe('fetchSapConcurAccessToken company-level auth', () => { - function submittedParams(): URLSearchParams { - const [, init] = mockSecureFetch.mock.calls[0] - return new URLSearchParams(init.body as string) - } - - it('submits the companyUuid as username and defaults credtype to authtoken', async () => { - await fetchSapConcurAccessToken( - auth({ - clientId: freshClientId(), - grantType: 'password', - password: 'company-request-token', - companyUuid: '08BCCA1E-0D4F-4261-9F1B-F778D96617D6', - }), - 'req-1' - ) - - const params = submittedParams() - expect(params.get('grant_type')).toBe('password') - expect(params.get('username')).toBe('08BCCA1E-0D4F-4261-9F1B-F778D96617D6') - expect(params.get('password')).toBe('company-request-token') - expect(params.get('credtype')).toBe('authtoken') - }) - - it('lets an explicit credtype override the company-flow default', async () => { - await fetchSapConcurAccessToken( - auth({ - clientId: freshClientId(), - grantType: 'password', - password: 'company-request-token', - companyUuid: 'company-uuid-1', - credtype: 'password', - }), - 'req-1' - ) - - expect(submittedParams().get('credtype')).toBe('password') - }) - - it('prefers the companyUuid over a supplied username', async () => { - await fetchSapConcurAccessToken( - auth({ - clientId: freshClientId(), - grantType: 'password', - username: 'alice@example.com', - password: 'company-request-token', - companyUuid: 'company-uuid-2', - }), - 'req-1' - ) - - expect(submittedParams().get('username')).toBe('company-uuid-2') - }) - - it('leaves the user-level password grant untouched (no credtype, real username)', async () => { - await fetchSapConcurAccessToken( - auth({ - clientId: freshClientId(), - grantType: 'password', - username: 'alice@example.com', - password: PASSWORD, - }), - 'req-1' - ) - - const params = submittedParams() - expect(params.get('username')).toBe('alice@example.com') - expect(params.has('credtype')).toBe(false) - }) - - it('requires a username or a companyUuid for a password grant', async () => { - await expect( - fetchSapConcurAccessToken( - auth({ clientId: freshClientId(), grantType: 'password', password: PASSWORD }), - 'req-1' - ) - ).rejects.toThrow('username is required for password grant') - }) -}) - -describe('fetchSapConcurAccessToken secret handling', () => { - it('never puts the clientSecret or password into a token-fetch error message', async () => { - mockSecureFetch.mockResolvedValueOnce( - tokenResponse({ error: 'invalid_grant', error_description: 'Bad credentials' }, 401) - ) - - const promise = fetchSapConcurAccessToken( - auth({ - clientId: freshClientId(), - grantType: 'password', - username: 'alice@example.com', - password: PASSWORD, - }), - 'req-1' - ) - - await expect(promise).rejects.toThrow('Concur token request failed: invalid_grant') - const error = await promise.catch((e: Error) => e) - expect(error.message).not.toContain(CLIENT_SECRET) - expect(error.message).not.toContain(PASSWORD) - }) - - /** - * The token request body is form-encoded `client_id=…&client_secret=…&password=…`, so an - * intermediary that rejects the request and echoes it back would otherwise have its page - * surfaced verbatim. The raw fallback is capped on the token path for that reason. - */ - it('truncates an unstructured token-error body instead of echoing it back', async () => { - const echoedRequest = `Request blocked by proxy. Your request was: POST /oauth2/v0/token client_id=abc&client_secret=${CLIENT_SECRET}&grant_type=password&username=alice@example.com&password=${PASSWORD}&credtype=password. Contact your administrator with reference id 0000-1111-2222-3333 for further assistance with this policy decision.` - - mockSecureFetch.mockResolvedValueOnce({ - ok: false, - status: 403, - headers: new Headers(), - json: async () => ({}), - text: async () => echoedRequest, - }) - - const error = await fetchSapConcurAccessToken( - auth({ - clientId: freshClientId(), - grantType: 'password', - username: 'alice@example.com', - password: PASSWORD, - }), - 'req-1' - ).catch((e: Error) => e) - - expect(error.message).not.toContain(CLIENT_SECRET) - expect(error.message).not.toContain(PASSWORD) - expect(error.message.length).toBeLessThan(echoedRequest.length) - expect(error.message).toContain('Request blocked by proxy') - }) - - it('never leaks credentials when the outbound fetch itself throws', async () => { - mockSecureFetch.mockRejectedValueOnce(new Error('socket hang up')) - - const error = await fetchSapConcurAccessToken( - auth({ - clientId: freshClientId(), - grantType: 'password', - username: 'alice@example.com', - password: PASSWORD, - }), - 'req-1' - ).catch((e: Error) => e) - - expect(error.message).not.toContain(CLIENT_SECRET) - expect(error.message).not.toContain(PASSWORD) - }) -}) - -describe('SapConcurProxyPath', () => { - const accepted = [ - '/expensereports/v4/reports/abc123', - 'expensereports/v4/reports/abc123', - '/profile/v1/principals/1234-5678', - ] - - it.each(accepted)('accepts the ordinary path %s', (path) => { - expect(SapConcurProxyPath.safeParse(path).success).toBe(true) - }) - - const rejected = [ - '/expensereports/../../etc/passwd', - '/expensereports/./v4/reports', - '..', - '/expensereports\\..\\v4', - '/expensereports/v4#fragment', - '/expensereports/%2e%2e/v4', - '/expensereports/%2E%2E/v4', - '/expensereports%2fv4', - '/expensereports%5cv4', - '/expensereports%23v4', - ] - - it.each(rejected)('rejects the traversal-shaped path %s', (path) => { - expect(SapConcurProxyPath.safeParse(path).success).toBe(false) - }) - - /** - * KNOWN LIMITATION, asserted as current behavior on purpose: the refine only inspects - * one layer of percent-encoding, so a double-encoded `%252e%252e` passes. That is - * acceptable because the refine is defense-in-depth — the request host is still pinned - * by `assertSafeExternalUrl` against the validated Concur geolocation, so a decoded - * `..` can at worst walk within concursolutions.com and cannot reach another origin. - * Do not "fix" this here without re-checking the host pinning that backs it. - */ - it('does not reject double-encoded traversal (defense-in-depth, host is pinned elsewhere)', () => { - expect(SapConcurProxyPath.safeParse('/expensereports/%252e%252e/v4').success).toBe(true) - }) -}) - -describe('SapConcurProxyRequestSchema accept', () => { - function parse(overrides: Record) { - return SapConcurProxyRequestSchema.parse({ - clientId: 'client-1', - clientSecret: CLIENT_SECRET, - path: '/expensereports/v4/reports', - ...overrides, - }) - } - - it('leaves accept undefined so callConcur applies its application/json default', () => { - expect(parse({}).accept).toBeUndefined() - }) - - /** Itinerary and Travel Profile are XML-only and 406 an Accept they cannot satisfy. */ - it('carries an explicit XML accept through', () => { - expect(parse({ accept: 'application/xml' }).accept).toBe('application/xml') - }) -}) - -/** - * The executor retries 429/5xx for a block with a retry config and paces itself off - * `Retry-After`; dropping the header downgrades a precise wait to blind backoff. - */ -describe('forwardedSapConcurHeaders', () => { - it('forwards Retry-After, Location, and Link', () => { - expect( - forwardedSapConcurHeaders( - new Headers({ - 'Retry-After': '30', - Location: 'https://us.api.concursolutions.com/receipts/v4/receipts/abc', - Link: '; rel="next"', - }) - ) - ).toEqual({ - 'retry-after': '30', - location: 'https://us.api.concursolutions.com/receipts/v4/receipts/abc', - link: '; rel="next"', - }) - }) - - it('omits headers Concur did not send', () => { - expect(forwardedSapConcurHeaders(new Headers({ 'Retry-After': '5' }))).toEqual({ - 'retry-after': '5', - }) - }) - - it('forwards nothing when no interesting header is present', () => { - expect(forwardedSapConcurHeaders(new Headers({ 'Content-Type': 'application/json' }))).toEqual( - {} - ) - }) -}) - -describe('SapConcurDatacenterSchema', () => { - /** Every host in the published Base URIs table, plus the legacy `eu`/`emea` aliases. */ - const accepted = [ - 'us.api.concursolutions.com', - 'www-us.api.concursolutions.com', - 'us2.api.concursolutions.com', - 'www-us2.api.concursolutions.com', - 'eu.api.concursolutions.com', - 'eu2.api.concursolutions.com', - 'www-eu2.api.concursolutions.com', - 'emea.api.concursolutions.com', - 'www-emea.api.concursolutions.com', - 'apj1.api.concursolutions.com', - 'www-apj1.api.concursolutions.com', - 'usg.api.concursolutions.com', - 'www-usg.api.concursolutions.com', - 'glz.api.concursolutions.com', - 'us-impl.api.concursolutions.com', - 'www-us-impl.api.concursolutions.com', - 'emea-impl.api.concursolutions.com', - 'www-emea-impl.api.concursolutions.com', - ] - - it.each(accepted)('accepts the documented datacenter %s', (datacenter) => { - expect(SapConcurDatacenterSchema.safeParse(datacenter).success).toBe(true) - }) - - it('covers exactly the documented set with no extras', () => { - expect([...SAP_CONCUR_ALLOWED_DATACENTERS].sort()).toEqual([...accepted].sort()) - }) - - /** GLZ is the one production row the Base URIs table publishes without a `www-` twin. */ - it('does not offer a www- twin for GLZ', () => { - expect(SapConcurDatacenterSchema.safeParse('www-glz.api.concursolutions.com').success).toBe( - false - ) - }) - - const rejected = [ - 'evil.com', - 'us.api.concursolutions.com.evil.com', - 'evil-us.api.concursolutions.com', - 'https://us.api.concursolutions.com', - ] - - it.each(rejected)('rejects the non-selectable datacenter %s', (datacenter) => { - expect(SapConcurDatacenterSchema.safeParse(datacenter).success).toBe(false) - }) -}) - -describe('assertSafeExternalUrl', () => { - it('accepts a normal Concur https URL', () => { - const url = assertSafeExternalUrl('https://us.api.concursolutions.com/expense/v4', 'apiUrl') - expect(url.hostname).toBe('us.api.concursolutions.com') - }) - - it('rejects a non-URL', () => { - expect(() => assertSafeExternalUrl('not a url', 'apiUrl')).toThrow('must be a valid URL') - }) - - it('rejects a non-https scheme', () => { - expect(() => assertSafeExternalUrl('http://us.api.concursolutions.com', 'apiUrl')).toThrow( - 'must use https://' - ) - }) - - const forbiddenHosts = [ - 'https://localhost/x', - 'https://0.0.0.0/x', - 'https://127.0.0.1/x', - 'https://169.254.169.254/latest/meta-data/', - 'https://metadata.google.internal/x', - 'https://[::1]/x', - ] - - it.each(forbiddenHosts)('rejects the metadata/loopback host %s', (url) => { - expect(() => assertSafeExternalUrl(url, 'apiUrl')).toThrow('is not allowed') - }) - - const privateIps = ['https://10.0.0.5/x', 'https://192.168.1.10/x', 'https://172.16.4.4/x'] - - it.each(privateIps)('rejects the private IP %s', (url) => { - expect(() => assertSafeExternalUrl(url, 'apiUrl')).toThrow('private/loopback range') - }) -}) - -describe('extractSapConcurError', () => { - it('combines the OAuth error and error_description', () => { - expect( - extractSapConcurError({ error: 'invalid_client', error_description: 'Bad client id' }, 401) - ).toBe('invalid_client: Bad client id') - }) - - it('includes the Expense v4 errorMessage with its validation details', () => { - const message = extractSapConcurError( - { - errorMessage: 'Report is not valid', - validationErrors: [{ message: 'purpose is required' }, { message: 'amount must be > 0' }], - }, - 400 - ) - expect(message).toContain('Report is not valid') - expect(message).toContain('purpose is required') - expect(message).toContain('amount must be > 0') - }) - - it('returns a bare Expense v4 errorMessage when there are no validation errors', () => { - expect(extractSapConcurError({ errorMessage: 'Report is not valid' }, 400)).toBe( - 'Report is not valid' - ) - }) - - it('prefixes the SCIM detail with the scimType', () => { - expect( - extractSapConcurError({ scimType: 'invalidValue', detail: 'userName already exists' }, 409) - ).toBe('[invalidValue] userName already exists') - }) - - it('returns a SCIM detail without a scimType', () => { - expect(extractSapConcurError({ detail: 'userName already exists' }, 409)).toBe( - 'userName already exists' - ) - }) - - it('reads the legacy nested Content.Error.Message envelope', () => { - expect( - extractSapConcurError({ Content: { Error: { Message: 'Invalid report key' } } }, 400) - ).toBe('Invalid report key') - }) - - it('reads the legacy top-level Error.Message envelope', () => { - expect(extractSapConcurError({ Error: { Message: 'Invalid itinerary' } }, 400)).toBe( - 'Invalid itinerary' - ) - }) - - it('includes the token-error code alongside the OAuth error', () => { - expect( - extractSapConcurError( - { code: 16, error: 'invalid_request', error_description: 'user lives elsewhere' }, - 400 - ) - ).toBe('[16] invalid_request: user lives elsewhere') - }) - - it('accepts a string-typed token-error code', () => { - expect(extractSapConcurError({ code: '53', error: 'invalid_grant' }, 400)).toBe( - '[53] invalid_grant' - ) - }) - - /** Budget v4 (Budget Category) failure response, verbatim from the API reference. */ - it('joins the Budget v4 errorMessageList with its types and codes', () => { - expect( - extractSapConcurError( - { - status: false, - errorMessageList: [ - { - errorType: 'ERROR', - errorCode: 'BUDGET.BUDGET_CATEGORY_NAME_REQUIRED', - errorMessage: 'Budget category name is required', - }, - { - errorType: 'ERROR', - errorCode: 'BUDGET.BUDGET_CATEGORY_NAME_UNIQUE_ERROR', - errorMessage: 'Budget category must have a unique name', - }, - ], - }, - 400 - ) - ).toBe( - '[ERROR BUDGET.BUDGET_CATEGORY_NAME_REQUIRED] Budget category name is required; ' + - '[ERROR BUDGET.BUDGET_CATEGORY_NAME_UNIQUE_ERROR] Budget category must have a unique name' - ) - }) - - /** Budget Adjustments v4 nests the same object one level down under `message`. */ - it('unwraps an object-valued message to reach a nested errorMessageList', () => { - expect( - extractSapConcurError( - { - message: { - status: false, - errorMessageList: [ - { - errorType: 'ERROR', - errorCode: 'BUDGET.BUDGET_PERIOD_REQUIRED', - errorMessage: 'Record 1) Budget period is missing', - }, - ], - }, - }, - 400 - ) - ).toBe('[ERROR BUDGET.BUDGET_PERIOD_REQUIRED] Record 1) Budget period is missing') - }) - - it('still prefers a string-valued message over the legacy envelope', () => { - expect(extractSapConcurError({ message: 'Report not found' }, 404)).toBe('Report not found') - }) - - it('falls back to the Concur SCIM messages extension when detail is absent', () => { - expect( - extractSapConcurError( - { - schemas: ['urn:ietf:params:scim:api:messages:2.0:Error'], - status: '400', - 'urn:ietf:params:scim:api:messages:concur:2.0:Error': { - messages: [ - { - code: 'ATTRIBUTE_REQUIRED', - message: 'userName is required', - schemaPath: 'userName', - type: 'error', - }, - ], - }, - }, - 400 - ) - ).toBe('[ATTRIBUTE_REQUIRED] userName is required (userName)') - }) - - it('prefers the SCIM detail over the messages extension when both are present', () => { - expect( - extractSapConcurError( - { - detail: 'userName already exists', - 'urn:ietf:params:scim:api:messages:concur:2.0:Error': { - messages: [{ code: 'DUP', message: 'duplicate', type: 'error' }], - }, - }, - 409 - ) - ).toBe('userName already exists') - }) - - it('joins an errors list with its error codes', () => { - expect( - extractSapConcurError( - { - errors: [ - { errorCode: 'E1', errorMessage: 'first problem' }, - { errorCode: 'E2', errorMessage: 'second problem' }, - ], - }, - 400 - ) - ).toBe('[E1] first problem; [E2] second problem') - }) - - it('passes a raw string body through when no cap is set', () => { - expect(extractSapConcurError('Service Unavailable', 503)).toBe('Service Unavailable') - }) - - it('caps a raw string body when maxRawBodyLength is set', () => { - expect(extractSapConcurError('x'.repeat(500), 503, { maxRawBodyLength: 20 })).toBe( - `${'x'.repeat(20)}...` - ) - }) - - it('leaves a structured body uncapped even when maxRawBodyLength is set', () => { - expect(extractSapConcurError({ error: 'invalid_client' }, 401, { maxRawBodyLength: 5 })).toBe( - 'invalid_client' - ) - }) - - it('falls back to the generic HTTP message for an unrecognized shape', () => { - expect(extractSapConcurError({ unexpected: true }, 418)).toBe( - 'Concur request failed with HTTP 418' - ) - }) - - it('falls back to the generic HTTP message for an empty body', () => { - expect(extractSapConcurError('', 500)).toBe('Concur request failed with HTTP 500') - }) -}) - -afterEach(() => { - vi.useRealTimers() -}) diff --git a/apps/sim/app/api/tools/sap_concur/shared.ts b/apps/sim/app/api/tools/sap_concur/shared.ts deleted file mode 100644 index 28800c6256d..00000000000 --- a/apps/sim/app/api/tools/sap_concur/shared.ts +++ /dev/null @@ -1,648 +0,0 @@ -import { createHmac } from 'node:crypto' -import { createLogger } from '@sim/logger' -import { isPrivateIpHost } from '@sim/security/ssrf' -import { getErrorMessage } from '@sim/utils/errors' -import { truncate } from '@sim/utils/string' -import { z } from 'zod' -import { coalesceLocally } from '@/lib/concurrency/singleflight' -import { env } from '@/lib/core/config/env' -import { - MAX_JSON_API_RESPONSE_BYTES, - secureFetchWithValidation, -} from '@/lib/core/security/input-validation.server' -import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' - -const logger = createLogger('SapConcurShared') - -/** - * User-selectable hosts for the token request, from SAP Concur's published Base URIs - * (production, GLZ, APJ, US Gov) and Implementation sandbox hosts. - * - * Every datacenter is published twice: the bare host "optionally requests certs" and is - * the server-side form, while the `www-` twin "does not request certs" and is the form - * the docs point browser/client callers at. Both are legitimate token hosts, so both are - * selectable. GLZ (Global Landing Zone) is the one production row with no `www-` twin. - * - * This set constrains only the datacenter the caller picks; the geolocation Concur - * returns is validated by shape via {@link SAP_CONCUR_GEOLOCATION_HOST_PATTERN}, since - * the docs instruct clients to store and reuse whatever geolocation comes back. - */ -export const SAP_CONCUR_ALLOWED_DATACENTERS = new Set([ - 'us.api.concursolutions.com', - 'www-us.api.concursolutions.com', - 'us2.api.concursolutions.com', - 'www-us2.api.concursolutions.com', - 'eu.api.concursolutions.com', - 'eu2.api.concursolutions.com', - 'www-eu2.api.concursolutions.com', - 'emea.api.concursolutions.com', - 'www-emea.api.concursolutions.com', - 'apj1.api.concursolutions.com', - 'www-apj1.api.concursolutions.com', - 'usg.api.concursolutions.com', - 'www-usg.api.concursolutions.com', - 'glz.api.concursolutions.com', - 'us-impl.api.concursolutions.com', - 'www-us-impl.api.concursolutions.com', - 'emea-impl.api.concursolutions.com', - 'www-emea-impl.api.concursolutions.com', -]) - -/** Documented host form for a Concur geolocation, including `www-` prefixed variants. */ -const SAP_CONCUR_GEOLOCATION_HOST_PATTERN = /^(www-)?[a-z0-9-]+\.api\.concursolutions\.com$/ - -export const SapConcurDatacenterSchema = z - .string() - .min(1) - .refine((d) => SAP_CONCUR_ALLOWED_DATACENTERS.has(d), { - message: `datacenter must be one of: ${Array.from(SAP_CONCUR_ALLOWED_DATACENTERS).join(', ')}`, - }) - -export const SapConcurGrantTypeSchema = z.enum(['client_credentials', 'password']) - -export const SapConcurAuthSchema = z.object({ - datacenter: SapConcurDatacenterSchema.default('us.api.concursolutions.com'), - grantType: SapConcurGrantTypeSchema.default('client_credentials'), - clientId: z.string().min(1, 'clientId is required'), - clientSecret: z.string().min(1, 'clientSecret is required'), - username: z.string().optional(), - password: z.string().optional(), - /** - * Company UUID for the company-level password grant. When set, it is submitted as the - * `username` form field and `credtype` defaults to `authtoken`. See - * {@link fetchSapConcurAccessToken} for the full flow. - */ - companyUuid: z.string().optional(), - /** - * Which credential set is submitted with a password grant. Concur defaults to - * `password` when the form param is absent, so it is only sent when set explicitly or - * implied by {@link SapConcurAuthSchema} `companyUuid`. - */ - credtype: z.enum(['password', 'authtoken']).optional(), -}) - -export type SapConcurAuth = z.infer - -export const SapConcurHttpMethod = z.enum(['GET', 'POST', 'PUT', 'PATCH', 'DELETE']) - -export const SapConcurProxyPath = z - .string() - .min(1, 'path is required') - .refine( - (p) => - !p.split(/[/\\]/).some((seg) => seg === '..' || seg === '.') && - !p.includes('#') && - !/%(?:2[eEfF]|5[cC]|23)/.test(p), - { - message: - 'path must not contain ".." or "." segments, "#", or percent-encoded path/fragment characters', - } - ) - -export const SapConcurProxyRequestSchema = SapConcurAuthSchema.extend({ - path: SapConcurProxyPath, - method: SapConcurHttpMethod.default('GET'), - query: z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])).optional(), - body: z.unknown().optional(), - contentType: z.string().optional(), - /** - * Media type sent as the outbound `Accept` header, defaulting to `application/json`. - * The Itinerary and Travel Profile APIs are XML-only and 406 a JSON-only Accept, so - * those tools set `application/xml` explicitly. - */ - accept: z.string().optional(), -}).superRefine((req, ctx) => { - if (req.grantType === 'password') { - if (!req.username && !req.companyUuid) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ['username'], - message: 'username is required for password grant (or companyUuid for company-level auth)', - }) - } - if (!req.password) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ['password'], - message: 'password is required for password grant', - }) - } - } -}) - -export type SapConcurProxyRequest = z.infer - -export const SapConcurUploadOperation = z.enum([ - 'upload_receipt_image', - 'create_quick_expense_with_image', -]) - -export const SapConcurUploadRequestSchema = SapConcurAuthSchema.extend({ - operation: SapConcurUploadOperation, - userId: z.string().min(1, 'userId is required'), - contextType: z.string().optional(), - receipt: FileInputSchema, - body: z.union([z.record(z.string(), z.unknown()), z.string()]).optional(), -}) - -export type SapConcurUploadRequest = z.infer - -const FORBIDDEN_HOSTS = new Set([ - 'localhost', - '0.0.0.0', - '127.0.0.1', - '169.254.169.254', - 'metadata.google.internal', - 'metadata', - '[::1]', - '[::]', - '[::ffff:127.0.0.1]', - '[fd00:ec2::254]', -]) - -/** Validate a URL is https and not pointing to a private/loopback host. */ -export function assertSafeExternalUrl(rawUrl: string, label: string): URL { - let parsed: URL - try { - parsed = new URL(rawUrl) - } catch { - throw new Error(`${label} must be a valid URL`) - } - if (parsed.protocol !== 'https:') { - throw new Error(`${label} must use https://`) - } - const host = parsed.hostname.toLowerCase() - if (FORBIDDEN_HOSTS.has(host) || FORBIDDEN_HOSTS.has(`[${host}]`)) { - throw new Error(`${label} host is not allowed`) - } - if (isPrivateIpHost(host)) { - throw new Error(`${label} host is not allowed (private/loopback range)`) - } - return parsed -} - -interface CachedToken { - accessToken: string - geolocation: string - expiresAt: number -} - -/** Access token plus the geolocation every subsequent API call for it must be sent to. */ -export interface SapConcurToken { - accessToken: string - geolocation: string -} - -const TOKEN_CACHE = new Map() -const TOKEN_CACHE_MAX_ENTRIES = 500 -const TOKEN_SAFETY_WINDOW_MS = 60_000 -export const SAP_CONCUR_OUTBOUND_FETCH_TIMEOUT_MS = 30_000 - -/** - * Namespace for the process-wide single-flight map so a Concur token cache key cannot - * collide with another subsystem's coalescing key. - */ -const SAP_CONCUR_TOKEN_COALESCE_PREFIX = 'sap-concur:token:' - -/** - * Settle deadline for a coalesced token request, held just above the outbound fetch - * timeout so that timeout is what normally fires. The extra margin covers the response - * read and geolocation validation, and guarantees joiners are released even if the token - * request somehow outlives its own timeout. - */ -const SAP_CONCUR_TOKEN_COALESCE_TIMEOUT_MS = SAP_CONCUR_OUTBOUND_FETCH_TIMEOUT_MS + 5_000 - -/** Cached token for `key`, or `undefined` when absent or inside the expiry safety window. */ -function readCachedToken(key: string): SapConcurToken | undefined { - const cached = TOKEN_CACHE.get(key) - if (!cached || cached.expiresAt - TOKEN_SAFETY_WINDOW_MS <= Date.now()) return undefined - return { accessToken: cached.accessToken, geolocation: cached.geolocation } -} - -/** - * Cache key covering every factor that authenticates the token request. The password - * and company UUID must participate: without them a cache hit skips the token endpoint - * entirely, so a request carrying the wrong password would be served a token minted from - * someone else's correct credentials out of this module-global cache. - * - * The whole tuple is JSON-encoded before hashing rather than concatenated with a - * separator, so a free-form field (clientId, companyUuid) cannot span a field boundary - * and collide with a different tuple. - * - * Keyed with a server-side secret rather than a bare digest. The inputs include a - * user-chosen password, which is low-entropy enough to brute-force from a plain SHA-256 - * if a key ever reached a heap dump or a debug log; an HMAC makes the key useless without - * the secret. A password-hashing KDF would be the wrong tool — this runs on every token - * fetch and the goal is collision-free partitioning, not verification of a stored - * credential. - */ -function tokenCacheKey(req: SapConcurAuth): string { - return createHmac('sha256', env.INTERNAL_API_SECRET) - .update( - JSON.stringify([ - req.datacenter, - req.grantType, - req.clientId, - req.clientSecret, - req.username ?? '', - req.password ?? '', - req.companyUuid ?? '', - req.credtype ?? '', - ]) - ) - .digest('hex') -} - -/** - * Insert a token and evict from the front once the cache is over its cap. - * - * Eviction is FIFO by insertion order, not LRU — a cache *read* does not move an entry - * back. At 500 entries that is deliberate: a token is short-lived and re-minted on the - * next miss, so the extra bookkeeping an LRU needs buys nothing here. - */ -function rememberToken(key: string, token: CachedToken): void { - if (TOKEN_CACHE.has(key)) TOKEN_CACHE.delete(key) - TOKEN_CACHE.set(key, token) - while (TOKEN_CACHE.size > TOKEN_CACHE_MAX_ENTRIES) { - const oldestKey = TOKEN_CACHE.keys().next().value - if (oldestKey === undefined) break - TOKEN_CACHE.delete(oldestKey) - } -} - -function normalizeGeolocation(raw: string | undefined, fallback: string): string { - if (!raw) return `https://${fallback}` - const trimmed = raw.replace(/\/+$/, '') - if (trimmed.startsWith('http://') || trimmed.startsWith('https://')) return trimmed - return `https://${trimmed}` -} - -/** - * Cap for an unstructured token-endpoint error body, applied both to the log line and to - * the surfaced message. The token request body is form-encoded - * `client_id=…&client_secret=…&password=…`, so an intermediary (WAF, proxy, captive - * portal) that echoes the request it rejected would otherwise have its page returned to - * the caller verbatim. Structured Concur error shapes are unaffected — they are matched - * before the raw fallback is reached. - */ -const TOKEN_ERROR_RAW_BODY_MAX_LENGTH = 200 - -/** - * Blank out the credential values this module just submitted, wherever they appear in an - * error body. - * - * Truncation alone bounds the exposure but does not remove it — a secret can sit inside - * the surviving prefix. Because the exact values are known at the callsite, they can be - * substituted out precisely. A genuine Concur error body never contains them, so this is - * a no-op for every documented shape and only bites on an intermediary echoing our - * request back at us. - */ -function redactTokenSecrets(text: string, auth: SapConcurAuth): string { - if (!text) return text - let redacted = text - for (const secret of [auth.clientSecret, auth.password]) { - if (secret && secret.length > 0) redacted = redacted.split(secret).join('[redacted]') - } - return redacted -} - -/** Best-effort JSON parse of an error body, falling back to the raw text. */ -function parseMaybeJson(text: string): unknown { - if (!text) return '' - try { - return JSON.parse(text) - } catch { - return text - } -} - -/** - * Acquire a Concur access token, sharing a cache with the proxy route. - * Validates that the geolocation returned by Concur is a safe external URL. - * - * Misses are coalesced per cache key: a parallel block fanning out many Concur calls, or - * a cold container after a deploy, would otherwise fire one `POST /oauth2/v0/token` per - * branch into an endpoint Concur rate-limits hard. Coalescing also removes an - * interleaving hazard — with concurrent mints, a slow response settling last could cache - * an earlier-expiring token over a fresher one. - * - * Two password-grant shapes are supported: - * - * - User-level: `username` is the user's login and `password` their password. `credtype` - * is omitted, which Concur reads as its `password` default. - * - Company-level: when `companyUuid` is set, Concur's documented company flow is - * `grant_type=password&username=&password=&credtype=authtoken`. - * The company UUID is submitted as `username` and `credtype` defaults to `authtoken`. - * An explicitly supplied `credtype` always wins. - * - * KNOWN LIMITATION of the company flow: the company request token obtained from the App - * Center is valid for 24 hours only, and Concur returns a `refresh_token` alongside the - * access token so the connection can outlive it. Refresh-token exchange is not - * implemented here, so a company connection stops working once the request token expires - * and a fresh one must be issued. - * - * Token-endpoint failures carry `{ code, error, error_description, geolocation? }`. - * Code 16 ("user lives elsewhere") additionally returns the correct geolocation for the - * tenant; retrying the token request against that host is not implemented here. - */ -export async function fetchSapConcurAccessToken( - auth: SapConcurAuth, - requestId: string -): Promise { - if (auth.grantType === 'password') { - if (!auth.username && !auth.companyUuid) { - throw new Error( - 'username is required for password grant (or companyUuid for company-level auth)' - ) - } - if (!auth.password) throw new Error('password is required for password grant') - } - - const cacheKey = tokenCacheKey(auth) - const cached = readCachedToken(cacheKey) - if (cached) return cached - - return coalesceLocally( - `${SAP_CONCUR_TOKEN_COALESCE_PREFIX}${cacheKey}`, - async () => readCachedToken(cacheKey) ?? (await requestAccessToken(auth, requestId, cacheKey)), - SAP_CONCUR_TOKEN_COALESCE_TIMEOUT_MS - ) -} - -/** Mint a fresh token from the Concur token endpoint and cache it under `cacheKey`. */ -async function requestAccessToken( - auth: SapConcurAuth, - requestId: string, - cacheKey: string -): Promise { - const tokenUrl = assertSafeExternalUrl( - `https://${auth.datacenter}/oauth2/v0/token`, - 'tokenUrl' - ).toString() - - const params = new URLSearchParams() - params.set('client_id', auth.clientId) - params.set('client_secret', auth.clientSecret) - params.set('grant_type', auth.grantType) - if (auth.grantType === 'password') { - const companyUuid = auth.companyUuid - params.set('username', companyUuid ?? auth.username ?? '') - params.set('password', auth.password ?? '') - const credtype = auth.credtype ?? (companyUuid ? 'authtoken' : undefined) - if (credtype) params.set('credtype', credtype) - } - - const response = await secureFetchWithValidation( - tokenUrl, - { - method: 'POST', - headers: { - 'Content-Type': 'application/x-www-form-urlencoded', - Accept: 'application/json', - }, - body: params.toString(), - timeout: SAP_CONCUR_OUTBOUND_FETCH_TIMEOUT_MS, - maxRedirects: 0, - stripAuthOnRedirect: true, - maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, - }, - 'tokenUrl' - ) - - if (!response.ok) { - const text = redactTokenSecrets(await response.text().catch(() => ''), auth) - logger.warn( - `[${requestId}] Concur token fetch failed (${response.status}): ${truncate( - text, - TOKEN_ERROR_RAW_BODY_MAX_LENGTH - )}` - ) - throw new Error( - `Concur token request failed: ${extractSapConcurError(parseMaybeJson(text), response.status, { - maxRawBodyLength: TOKEN_ERROR_RAW_BODY_MAX_LENGTH, - })}` - ) - } - - const data = (await response.json()) as { - access_token?: string - expires_in?: number - geolocation?: string - } - - if (!data.access_token) { - throw new Error('Concur token response missing access_token') - } - - const geolocation = normalizeGeolocation(data.geolocation, auth.datacenter) - const geolocationUrl = assertSafeExternalUrl(geolocation, 'geolocation') - if (!SAP_CONCUR_GEOLOCATION_HOST_PATTERN.test(geolocationUrl.hostname.toLowerCase())) { - throw new Error( - `Concur geolocation host is not a valid Concur API host: ${geolocationUrl.hostname}` - ) - } - - const expiresInMs = (data.expires_in ?? 3600) * 1000 - rememberToken(cacheKey, { - accessToken: data.access_token, - geolocation, - expiresAt: Date.now() + expiresInMs, - }) - return { accessToken: data.access_token, geolocation } -} - -/** - * Concur response headers carried through onto the route's own response. - * - * `Retry-After` is the load-bearing one: the executor retries 429/5xx for a block with a - * retry config and paces itself off this header, so dropping it downgrades a precise wait - * into blind exponential backoff against an endpoint that just told us how long to wait. - * `Location` and `Link` identify the resource created by, or the next page of, an - * accepted request. - */ -const FORWARDED_CONCUR_HEADERS = ['retry-after', 'location', 'link'] as const - -/** - * Pick the {@link FORWARDED_CONCUR_HEADERS} present on a Concur response. - * - * Typed structurally rather than as `Headers` so it accepts both a DOM `Headers` and the - * `SecureFetchHeaders` returned by `secureFetchWithValidation`, which exposes only `get`. - */ -export function forwardedSapConcurHeaders(source: { - get(name: string): string | null -}): Record { - const forwarded: Record = {} - for (const name of FORWARDED_CONCUR_HEADERS) { - const value = source.get(name) - if (value) forwarded[name] = value - } - return forwarded -} - -/** - * Turn an outbound-fetch rejection into a message a caller can act on. - * - * `secureFetchWithValidation` runs with `maxRedirects: 0`, so any Concur response that is - * a redirect *with* a `Location` header rejects with `Too many redirects (max: 0)` rather - * than returning a status. That is a deliberate refusal (the bearer token must never be - * replayed to another origin), but the bare message reads like an internal fault, so it - * is restated in terms of what actually happened. - */ -export function describeSapConcurFetchError(error: unknown): string { - const message = getErrorMessage(error, 'Unknown error') - if (message.startsWith('Too many redirects')) { - return 'Concur returned a redirect, which is not followed because the access token must not be replayed to another origin. Check the datacenter/geolocation and the request path.' - } - return message -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value) -} - -function nonEmptyString(value: unknown): string | undefined { - return typeof value === 'string' && value.length > 0 ? value : undefined -} - -/** - * Message from the legacy nested envelope used by Expense v3 and Travel: - * `{ Content: { Error: { Message } } }` or `{ Error: { Message } }`. - */ -function legacyEnvelopeMessage(obj: Record): string | undefined { - const container = isRecord(obj.Content) ? obj.Content : obj - const error = isRecord(container.Error) ? container.Error : undefined - return error ? nonEmptyString(error.Message) : undefined -} - -/** URN of the Concur SCIM error extension carrying per-attribute `messages[]`. */ -const SCIM_CONCUR_ERROR_URN = 'urn:ietf:params:scim:api:messages:concur:2.0:Error' - -/** - * Render one Budget v4 `errorMessageList` entry (`{ errorType, errorCode, errorMessage }`). - * `errorType` is kept because it distinguishes a hard `ERROR` from a `WARNING`. - */ -function formatErrorMessageListEntry(entry: unknown): string { - if (!isRecord(entry)) return String(entry) - const label = [nonEmptyString(entry.errorType), nonEmptyString(entry.errorCode)] - .filter(Boolean) - .join(' ') - const message = nonEmptyString(entry.errorMessage) ?? '' - return label ? `[${label}] ${message}`.trim() : message -} - -/** Render one Concur SCIM extension message (`{ code, message, schemaPath, type }`). */ -function formatScimExtensionMessage(entry: unknown): string { - if (!isRecord(entry)) return String(entry) - const code = nonEmptyString(entry.code) - const message = nonEmptyString(entry.message) ?? '' - const schemaPath = nonEmptyString(entry.schemaPath) - const head = code ? `[${code}] ` : '' - const tail = schemaPath ? ` (${schemaPath})` : '' - return `${head}${message}${tail}`.trim() -} - -function joinNonEmpty(values: unknown[], format: (value: unknown) => string): string | undefined { - const joined = values.map(format).filter(Boolean).join('; ') - return joined.length > 0 ? joined : undefined -} - -/** - * Match a Concur error record against the documented shapes, returning `undefined` when - * none apply so the caller can fall through to its own default. - * - * `depth` bounds the single documented level of nesting: Budget Adjustments v4 wraps the - * same `{ status, errorMessageList }` object under a `message` key, so an object-valued - * `message` is unwrapped once before the string-valued `message` shape is considered. - */ -function extractFromRecord(obj: Record, depth: number): string | undefined { - if (depth === 0 && isRecord(obj.message)) { - const nested = extractFromRecord(obj.message, depth + 1) - if (nested) return nested - } - - const error = nonEmptyString(obj.error) - if (error) { - const description = nonEmptyString(obj.error_description) - const code = obj.code - const codePrefix = typeof code === 'string' || typeof code === 'number' ? `[${code}] ` : '' - return `${codePrefix}${error}${description ? `: ${description}` : ''}` - } - - const errorMessage = nonEmptyString(obj.errorMessage) - if (errorMessage) { - const validationErrors = Array.isArray(obj.validationErrors) - ? obj.validationErrors - .map((v) => (isRecord(v) ? nonEmptyString(v.message) : undefined)) - .filter((m): m is string => Boolean(m)) - : [] - return validationErrors.length > 0 - ? `${errorMessage}: ${validationErrors.join('; ')}` - : errorMessage - } - - if (Array.isArray(obj.errorMessageList) && obj.errorMessageList.length > 0) { - const joined = joinNonEmpty(obj.errorMessageList, formatErrorMessageListEntry) - if (joined) return joined - } - - const detail = nonEmptyString(obj.detail) - if (detail) { - const scimType = nonEmptyString(obj.scimType) - return scimType ? `[${scimType}] ${detail}` : detail - } - - const scimExtension = obj[SCIM_CONCUR_ERROR_URN] - if (isRecord(scimExtension) && Array.isArray(scimExtension.messages)) { - const joined = joinNonEmpty(scimExtension.messages, formatScimExtensionMessage) - if (joined) return joined - } - - const message = nonEmptyString(obj.message) - if (message) return message - - const legacy = legacyEnvelopeMessage(obj) - if (legacy) return legacy - - if (Array.isArray(obj.errors) && obj.errors.length > 0) { - return joinNonEmpty(obj.errors, (e) => { - if (!isRecord(e)) return String(e) - const code = nonEmptyString(e.errorCode) - const msg = nonEmptyString(e.errorMessage) ?? '' - return `${code ? `[${code}] ` : ''}${msg}`.trim() - }) - } - - return undefined -} - -interface ExtractSapConcurErrorOptions { - /** - * Cap applied to an unstructured string body before it is surfaced. Set on the token - * path, where the request body carries credentials an intermediary might echo back. - * Left unset elsewhere so ordinary API errors surface in full. - */ - maxRawBodyLength?: number -} - -/** - * Extract a meaningful error message from a Concur error response body, covering the - * OAuth `{ code, error, error_description }` shape, the Expense v4 `ErrorMessage` schema, - * the Budget v4 `errorMessageList` shape (including the Budget Adjustments v4 variant - * that nests it under `message`), the SCIM (Identity v4.1) `detail` shape and its Concur - * `messages[]` extension, the legacy nested `Content.Error.Message` envelope, and an - * undocumented `{ errors: [...] }` list kept for tolerance. - */ -export function extractSapConcurError( - body: unknown, - status: number, - options: ExtractSapConcurErrorOptions = {} -): string { - if (isRecord(body)) { - const message = extractFromRecord(body, 0) - if (message) return message - } - if (typeof body === 'string' && body.length > 0) { - return options.maxRawBodyLength === undefined ? body : truncate(body, options.maxRawBodyLength) - } - return `Concur request failed with HTTP ${status}` -} diff --git a/apps/sim/app/api/tools/sap_concur/upload/route.ts b/apps/sim/app/api/tools/sap_concur/upload/route.ts deleted file mode 100644 index df51d1db074..00000000000 --- a/apps/sim/app/api/tools/sap_concur/upload/route.ts +++ /dev/null @@ -1,440 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { getValidationErrorMessage, isZodError } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - MAX_JSON_API_RESPONSE_BYTES, - secureFetchWithValidation, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { PayloadSizeLimitError, readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { - assertSafeExternalUrl, - describeSapConcurFetchError, - extractSapConcurError, - fetchSapConcurAccessToken, - forwardedSapConcurHeaders, - SAP_CONCUR_OUTBOUND_FETCH_TIMEOUT_MS, - type SapConcurUploadRequest, - SapConcurUploadRequestSchema, -} from '@/app/api/tools/sap_concur/shared' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('SapConcurUploadAPI') - -type UploadRequest = SapConcurUploadRequest - -const RECEIPT_ALLOWED_MIME_TYPES = new Set([ - 'application/pdf', - 'image/png', - 'image/jpeg', - 'image/jpg', - 'image/gif', - 'image/tiff', - 'image/tif', -]) - -const QUICK_EXPENSE_ALLOWED_MIME_TYPES = new Set([ - 'application/pdf', - 'image/png', - 'image/jpeg', - 'image/jpg', - 'image/tiff', - 'image/tif', -]) - -const ALLOWED_MIME_TYPES = RECEIPT_ALLOWED_MIME_TYPES - -/** - * Concur's documented ceiling for image-only receipts: Receipts "Supported Image Formats" - * states "Image size must not exceed 25MB." - */ -const MAX_RECEIPT_IMAGE_BYTES = 25 * 1024 * 1024 - -/** - * Concur's documented ceiling for quick-expense images: the Quick Expense v4 - * `fileContent` parameter states "Maximum size 50 MB." - * - * A file this large is held in memory more than once on the upload path — the downloaded - * `Buffer`, the `Blob` copy inside the `FormData`, and the serialized multipart `Buffer`. - * What bounds that is the `userFile.size` check performed *before* the download, together - * with the `maxBytes` passed to `downloadServableFileFromStorage`; a file over the cap is - * rejected without ever being materialized. Keep both checks ahead of the download. - */ -const MAX_QUICK_EXPENSE_IMAGE_BYTES = 50 * 1024 * 1024 - -function maxImageBytesForOperation(operation: UploadRequest['operation']): number { - return operation === 'create_quick_expense_with_image' - ? MAX_QUICK_EXPENSE_IMAGE_BYTES - : MAX_RECEIPT_IMAGE_BYTES -} - -function uploadSizeError(bytes: number, maxBytes: number): NextResponse { - const sizeMB = (bytes / (1024 * 1024)).toFixed(2) - const limitMB = Math.round(maxBytes / (1024 * 1024)) - return NextResponse.json( - { - success: false, - error: `File size (${sizeMB}MB) exceeds Concur upload limit of ${limitMB}MB`, - }, - { status: 400 } - ) -} - -/** - * Map a non-2xx Concur status that cannot be re-emitted as an error status onto 502. - * - * With `maxRedirects: 0` a 3xx carrying a `Location` never reaches here — it rejects with - * "Too many redirects" and is handled in the outer catch. What does reach here is a 3xx - * *without* a `Location`, and a 304, which is excluded from the redirect handling - * upstream. Neither is a usable error status to return to the caller. - */ -function clampErrorStatus(status: number): number { - return status >= 400 ? status : 502 -} - -/** Sentinel {@link inferMimeType} returns when neither the declared type nor the extension resolves. */ -const UNKNOWN_MIME_TYPE = 'application/octet-stream' - -function unsupportedMimeTypeError(mimeType: string, allowedLabel: string): NextResponse { - return NextResponse.json( - { - success: false, - error: `Unsupported receipt mime type: ${mimeType}. Allowed: ${allowedLabel}`, - }, - { status: 400 } - ) -} - -/** - * Non-canonical media types Concur callers commonly declare, mapped to the canonical form - * the allowlists and the outbound `Blob` type use. - */ -const MIME_TYPE_ALIASES: Record = { - 'image/jpg': 'image/jpeg', - 'image/tif': 'image/tiff', -} - -function inferMimeType(name: string, declared?: string): string { - if (declared && ALLOWED_MIME_TYPES.has(declared.toLowerCase())) { - const lowerDeclared = declared.toLowerCase() - return MIME_TYPE_ALIASES[lowerDeclared] ?? lowerDeclared - } - const lower = name.toLowerCase() - if (lower.endsWith('.pdf')) return 'application/pdf' - if (lower.endsWith('.png')) return 'image/png' - if (lower.endsWith('.jpg') || lower.endsWith('.jpeg')) return 'image/jpeg' - if (lower.endsWith('.gif')) return 'image/gif' - if (lower.endsWith('.tif') || lower.endsWith('.tiff')) return 'image/tiff' - return UNKNOWN_MIME_TYPE -} - -function stringifyMaybeJson(value: unknown): string { - if (typeof value === 'string') return value - return JSON.stringify(value ?? {}) -} - -interface UploadInvocation { - status: number - body: unknown - /** Concur response headers forwarded onto this route's response. */ - headers: Record -} - -/** - * POST a multipart body to Concur with the bearer token. - * - * `concur-correlationid` is a support/tracing header expected to be a fresh RFC 4122 - * UUID per request; it does not scope a request to a company. Redirects are refused so - * the Authorization header is never forwarded to another origin. - * - * `stripAuthOnRedirect` is unreachable while `maxRedirects` is 0 — no redirect is ever - * followed for it to act on. It is kept as defense-in-depth so raising `maxRedirects` - * later cannot silently start forwarding the bearer token; do not remove it as dead code. - */ -/** - * Read a Concur upload response body under the shared byte cap. - * - * On a success status the body is the result, so a cap breach or a stream - * failure is a real error and must propagate — swallowing it would report an - * incomplete exchange as a successful upload with no data. - * - * On an error status the body only supplies the message, and the upstream - * status is the more important signal: letting a failed read throw here would - * surface Concur's 4xx as a Sim 500 and invite a retry the caller should not - * make. The status is preserved and the message falls back to the generic - * HTTP-status form from {@link extractSapConcurError}. - */ -export async function readConcurUploadBody(response: { - status: number - headers?: { get(name: string): string | null } - body?: ReadableStream | null - arrayBuffer?: () => Promise - text?: () => Promise -}): Promise { - const read = readResponseTextWithLimit(response, { - maxBytes: MAX_JSON_API_RESPONSE_BYTES, - label: 'Concur upload response', - }) - if (response.status >= 200 && response.status < 300) return read - return read.catch(() => '') -} - -async function postMultipart( - url: string, - accessToken: string, - formData: FormData -): Promise { - const headers: Record = { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - 'concur-correlationid': generateId(), - } - - // Serialize FormData (with auto-generated multipart boundary) to a Buffer so we can - // route through secureFetchWithValidation (which doesn't support FormData bodies directly). - const serialized = new Request('http://localhost/internal-multipart-serializer', { - method: 'POST', - body: formData, - }) - const contentType = serialized.headers.get('content-type') - if (contentType) headers['Content-Type'] = contentType - const bodyBuffer = Buffer.from(await serialized.arrayBuffer()) - - const response = await secureFetchWithValidation( - url, - { - method: 'POST', - headers, - body: bodyBuffer, - timeout: SAP_CONCUR_OUTBOUND_FETCH_TIMEOUT_MS, - maxRedirects: 0, - stripAuthOnRedirect: true, - maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, - }, - 'apiUrl' - ) - - const raw = await readConcurUploadBody(response) - let parsed: unknown = null - if (raw.length > 0) { - try { - parsed = JSON.parse(raw) - } catch { - parsed = raw - } - } - // Surface Location/Link headers for receipt endpoints that return 202 with no body. - if ( - parsed === null || - (typeof parsed === 'object' && parsed !== null && Object.keys(parsed).length === 0) - ) { - const location = response.headers.get('Location') - const link = response.headers.get('Link') - if (location || link) { - parsed = { location, link } - } - } - return { - status: response.status, - body: parsed, - headers: forwardedSapConcurHeaders(response.headers), - } -} - -async function handleUploadReceiptImage( - req: UploadRequest, - fileBuffer: Buffer, - fileName: string, - mimeType: string, - accessToken: string, - geolocation: string -): Promise { - const url = assertSafeExternalUrl( - `${geolocation.replace(/\/+$/, '')}/receipts/v4/users/${encodeURIComponent(req.userId)}/image-only-receipts`, - 'apiUrl' - ).toString() - - const formData = new FormData() - formData.append('image', new Blob([new Uint8Array(fileBuffer)], { type: mimeType }), fileName) - - return postMultipart(url, accessToken, formData) -} - -async function handleCreateQuickExpenseWithImage( - req: UploadRequest, - fileBuffer: Buffer, - fileName: string, - mimeType: string, - accessToken: string, - geolocation: string -): Promise { - const contextType = req.contextType?.trim() || 'TRAVELER' - const url = assertSafeExternalUrl( - `${geolocation.replace(/\/+$/, '')}/quickexpense/v4/users/${encodeURIComponent( - req.userId - )}/context/${encodeURIComponent(contextType)}/quickexpenses/image`, - 'apiUrl' - ).toString() - - const quickExpenseRequest = stringifyMaybeJson(req.body ?? {}) - - const formData = new FormData() - formData.append('quickExpenseRequest', quickExpenseRequest) - formData.append( - 'fileContent', - new Blob([new Uint8Array(fileBuffer)], { type: mimeType }), - fileName - ) - - return postMultipart(url, accessToken, formData) -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Concur upload request: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - const userId = authResult.userId - - // boundary-raw-json: internal upload envelope validated by SapConcurUploadRequestSchema below; not a public boundary - const json = await request.json() - const uploadReq = SapConcurUploadRequestSchema.parse(json) - - const userFiles = processFilesToUserFiles( - [uploadReq.receipt as RawFileInput], - requestId, - logger - ) - if (userFiles.length === 0) { - return NextResponse.json( - { success: false, error: 'Invalid receipt file input' }, - { status: 400 } - ) - } - const userFile = userFiles[0] - const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) - if (denied) return denied - - const maxBytes = maxImageBytesForOperation(uploadReq.operation) - const allowedForOperation = - uploadReq.operation === 'create_quick_expense_with_image' - ? QUICK_EXPENSE_ALLOWED_MIME_TYPES - : RECEIPT_ALLOWED_MIME_TYPES - const allowedLabel = - uploadReq.operation === 'create_quick_expense_with_image' - ? 'pdf, png, jpeg, tiff' - : 'pdf, png, jpeg, gif, tiff' - - if (userFile.size > maxBytes) { - return uploadSizeError(userFile.size, maxBytes) - } - - const declaredMimeType = inferMimeType(userFile.name, userFile.type) - if (declaredMimeType !== UNKNOWN_MIME_TYPE && !allowedForOperation.has(declaredMimeType)) { - return unsupportedMimeTypeError(declaredMimeType, allowedLabel) - } - - let fileBuffer: Buffer - let resolvedContentType: string - try { - const resolved = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes, - }) - fileBuffer = resolved.buffer - resolvedContentType = resolved.contentType - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - if (error instanceof PayloadSizeLimitError) { - return uploadSizeError(error.observedBytes ?? userFile.size, maxBytes) - } - logger.error(`[${requestId}] Failed to download Concur receipt file:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error') }, - { status: 500 } - ) - } - - if (fileBuffer.length > maxBytes) { - return uploadSizeError(fileBuffer.length, maxBytes) - } - - const fileName = userFile.name - const mimeType = inferMimeType(fileName, resolvedContentType || userFile.type) - if (!allowedForOperation.has(mimeType)) { - return unsupportedMimeTypeError(mimeType, allowedLabel) - } - - const { accessToken, geolocation } = await fetchSapConcurAccessToken(uploadReq, requestId) - - let invocation: UploadInvocation - if (uploadReq.operation === 'upload_receipt_image') { - invocation = await handleUploadReceiptImage( - uploadReq, - fileBuffer, - fileName, - mimeType, - accessToken, - geolocation - ) - } else { - invocation = await handleCreateQuickExpenseWithImage( - uploadReq, - fileBuffer, - fileName, - mimeType, - accessToken, - geolocation - ) - } - - if (invocation.status >= 200 && invocation.status < 300) { - const data = invocation.status === 204 ? null : invocation.body - logger.info( - `[${requestId}] Concur ${uploadReq.operation} succeeded: HTTP ${invocation.status}` - ) - return NextResponse.json( - { success: true, output: { status: invocation.status, data } }, - { headers: invocation.headers } - ) - } - - const message = extractSapConcurError(invocation.body, invocation.status) - logger.warn( - `[${requestId}] Concur upload error (${invocation.status}) ${uploadReq.operation}: ${message}` - ) - return NextResponse.json( - { success: false, error: message, status: invocation.status }, - { status: clampErrorStatus(invocation.status), headers: invocation.headers } - ) - } catch (error) { - if (isZodError(error)) { - logger.warn(`[${requestId}] Validation error:`, error.issues) - return NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Validation failed') }, - { status: 400 } - ) - } - logger.error(`[${requestId}] Unexpected Concur upload error:`, error) - return NextResponse.json( - { success: false, error: describeSapConcurFetchError(error) }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/sap_s4hana/proxy/route.ts b/apps/sim/app/api/tools/sap_s4hana/proxy/route.ts deleted file mode 100644 index bee4a8b84aa..00000000000 --- a/apps/sim/app/api/tools/sap_s4hana/proxy/route.ts +++ /dev/null @@ -1,380 +0,0 @@ -import { createHash } from 'node:crypto' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { - assertSafeSapExternalUrl, - type SapS4HanaProxyRequest, - sapS4HanaProxyContract, -} from '@/lib/api/contracts/tools/sap' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - type SecureFetchResponse, - secureFetchWithValidation, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('SapS4HanaProxyAPI') - -type ProxyRequest = SapS4HanaProxyRequest - -interface CachedToken { - accessToken: string - expiresAt: number -} - -const TOKEN_CACHE = new Map() -const TOKEN_CACHE_MAX_ENTRIES = 500 -const TOKEN_SAFETY_WINDOW_MS = 60_000 -const OUTBOUND_FETCH_TIMEOUT_MS = 30_000 - -function resolveTokenUrl(req: ProxyRequest): string { - if (req.deploymentType === 'cloud_public') { - return `https://${req.subdomain}.authentication.${req.region}.hana.ondemand.com/oauth/token` - } - if (!req.tokenUrl) { - throw new Error('tokenUrl is required for OAuth on cloud_private/on_premise') - } - return req.tokenUrl -} - -function tokenCacheKey(req: ProxyRequest): string { - const secretHash = req.clientSecret - ? createHash('sha256').update(req.clientSecret).digest('hex').slice(0, 16) - : '' - return `${resolveTokenUrl(req)}::${req.clientId ?? ''}::${secretHash}` -} - -function rememberToken(key: string, token: CachedToken): void { - if (TOKEN_CACHE.has(key)) TOKEN_CACHE.delete(key) - TOKEN_CACHE.set(key, token) - while (TOKEN_CACHE.size > TOKEN_CACHE_MAX_ENTRIES) { - const oldestKey = TOKEN_CACHE.keys().next().value - if (oldestKey === undefined) break - TOKEN_CACHE.delete(oldestKey) - } -} - -async function fetchAccessToken(req: ProxyRequest, requestId: string): Promise { - const cacheKey = tokenCacheKey(req) - const cached = TOKEN_CACHE.get(cacheKey) - if (cached && cached.expiresAt - TOKEN_SAFETY_WINDOW_MS > Date.now()) { - return cached.accessToken - } - - const tokenUrl = assertSafeSapExternalUrl(resolveTokenUrl(req), 'tokenUrl').toString() - const basic = Buffer.from(`${req.clientId}:${req.clientSecret}`).toString('base64') - - const response = await secureFetchWithValidation( - tokenUrl, - { - method: 'POST', - headers: { - Authorization: `Basic ${basic}`, - 'Content-Type': 'application/x-www-form-urlencoded', - Accept: 'application/json', - }, - body: 'grant_type=client_credentials', - timeout: OUTBOUND_FETCH_TIMEOUT_MS, - }, - 'tokenUrl' - ) - - if (!response.ok) { - const text = await response.text().catch(() => '') - logger.warn(`[${requestId}] Token fetch failed (${response.status}): ${text}`) - throw new Error(`SAP token request failed: HTTP ${response.status}`) - } - - const data = (await response.json()) as { - access_token?: string - expires_in?: number - } - - if (!data.access_token) { - throw new Error('SAP token response missing access_token') - } - - const expiresInMs = (data.expires_in ?? 3600) * 1000 - rememberToken(cacheKey, { - accessToken: data.access_token, - expiresAt: Date.now() + expiresInMs, - }) - return data.access_token -} - -interface CsrfBundle { - token: string - cookie: string -} - -function joinSetCookies(response: SecureFetchResponse): string { - return response.headers - .getSetCookie() - .map((c) => c.split(';')[0]?.trim()) - .filter(Boolean) - .join('; ') -} - -function buildAuthHeader(req: ProxyRequest, accessToken: string | null): string { - if (req.authType === 'basic') { - const basic = Buffer.from(`${req.username}:${req.password}`).toString('base64') - return `Basic ${basic}` - } - return `Bearer ${accessToken}` -} - -async function fetchCsrf( - req: ProxyRequest, - accessToken: string | null, - requestId: string -): Promise { - const url = buildOdataUrl(req, '/$metadata') - const response = await secureFetchWithValidation( - url, - { - method: 'GET', - headers: { - Authorization: buildAuthHeader(req, accessToken), - Accept: 'application/xml', - 'X-CSRF-Token': 'Fetch', - }, - timeout: OUTBOUND_FETCH_TIMEOUT_MS, - }, - 'baseUrl' - ) - - if (!response.ok) { - const text = await response.text().catch(() => '') - logger.warn(`[${requestId}] CSRF fetch failed (${response.status}): ${text}`) - return null - } - - const token = response.headers.get('x-csrf-token') - const cookie = joinSetCookies(response) - if (!token) return null - return { token, cookie } -} - -function resolveHost(req: ProxyRequest): string { - if (req.deploymentType === 'cloud_public') { - const constructed = `https://${req.subdomain}-api.s4hana.ondemand.com` - return assertSafeSapExternalUrl(constructed, 'subdomain').toString().replace(/\/+$/, '') - } - if (!req.baseUrl) { - throw new Error('baseUrl is required for cloud_private and on_premise deployments') - } - const trimmed = req.baseUrl.replace(/\/+$/, '') - return assertSafeSapExternalUrl(trimmed, 'baseUrl').toString().replace(/\/+$/, '') -} - -function buildOdataUrl(req: ProxyRequest, pathOverride?: string): string { - const host = resolveHost(req) - const servicePath = `/sap/opu/odata/sap/${req.service}` - const subPath = pathOverride ?? req.path - const normalized = subPath.startsWith('/') ? subPath : `/${subPath}` - const base = `${host}${servicePath}${normalized}` - - if (pathOverride !== undefined) { - return base - } - if (!req.query || Object.keys(req.query).length === 0) { - return base - } - const encode = (s: string) => encodeURIComponent(s).replace(/%24/g, '$') - const parts: string[] = [] - for (const [key, value] of Object.entries(req.query)) { - if (value === undefined || value === null) continue - parts.push(`${encode(key)}=${encode(String(value))}`) - } - const queryString = parts.join('&') - if (!queryString) return base - return base.includes('?') ? `${base}&${queryString}` : `${base}?${queryString}` -} - -const WRITE_METHODS = new Set(['POST', 'PUT', 'PATCH', 'DELETE', 'MERGE']) - -interface OdataInvocation { - status: number - body: unknown - raw: string - csrfHeader: string -} - -async function callOdata( - req: ProxyRequest, - accessToken: string | null, - csrf: CsrfBundle | null -): Promise { - const url = buildOdataUrl(req) - const headers: Record = { - Authorization: buildAuthHeader(req, accessToken), - Accept: 'application/json', - } - - const isWrite = WRITE_METHODS.has(req.method) - const hasBody = req.body !== undefined && req.body !== null - if (hasBody) headers['Content-Type'] = 'application/json' - if (req.ifMatch) headers['If-Match'] = req.ifMatch - - if (isWrite && csrf) { - headers['X-CSRF-Token'] = csrf.token - if (csrf.cookie) headers.Cookie = csrf.cookie - } - - const response = await secureFetchWithValidation( - url, - { - method: req.method, - headers, - body: hasBody ? JSON.stringify(req.body) : undefined, - timeout: OUTBOUND_FETCH_TIMEOUT_MS, - }, - 'baseUrl' - ) - - const raw = await response.text() - let parsed: unknown = null - if (raw.length > 0) { - try { - parsed = JSON.parse(raw) - } catch { - parsed = raw - } - } - - const csrfHeader = response.headers.get('x-csrf-token')?.toLowerCase() ?? '' - return { status: response.status, body: parsed, raw, csrfHeader } -} - -function isCsrfRequired(invocation: OdataInvocation): boolean { - if (invocation.status !== 403) return false - if (invocation.csrfHeader === 'required') return true - if (typeof invocation.body !== 'object' || invocation.body === null) return false - const errorObj = (invocation.body as { error?: { message?: { value?: string } | string } }).error - const messageField = errorObj?.message - const message = typeof messageField === 'string' ? messageField : (messageField?.value ?? '') - return message.toLowerCase().includes('csrf') -} - -function extractOdataError(body: unknown, status: number): string { - if (body && typeof body === 'object') { - const err = ( - body as { - error?: { - message?: { value?: string } | string - code?: string - innererror?: { - errordetails?: Array<{ code?: string; message?: string; severity?: string }> - } - } - } - ).error - if (err) { - const messageField = err.message - const base = - typeof messageField === 'string' ? messageField : (messageField?.value ?? err.code ?? '') - const prefix = err.code ? `[${err.code}] ` : '' - const details = err.innererror?.errordetails - ?.filter((d) => d.message && (!d.severity || d.severity.toLowerCase() !== 'info')) - .map((d) => { - const tag = d.code ? `[${d.code}] ` : '' - return `${tag}${d.message}` - }) - .filter((m): m is string => Boolean(m)) - if (details && details.length > 0) { - const extras = details.filter((d) => !d.endsWith(base)) - return extras.length > 0 ? `${prefix}${base} (${extras.join('; ')})` : `${prefix}${base}` - } - if (base) return `${prefix}${base}` - } - } - if (typeof body === 'string' && body.length > 0) return body - return `SAP request failed with HTTP ${status}` -} - -function unwrapOdata(body: unknown): unknown { - if (!body || typeof body !== 'object') return body - const root = (body as { d?: unknown }).d - if (root === undefined) return body - if (root && typeof root === 'object' && 'results' in (root as Record)) { - const rootObj = root as { results: unknown; __count?: string; __next?: string } - if (rootObj.__count !== undefined || rootObj.__next !== undefined) { - return { - results: rootObj.results, - ...(rootObj.__count !== undefined && { __count: rootObj.__count }), - ...(rootObj.__next !== undefined && { __next: rootObj.__next }), - } - } - return rootObj.results - } - return root -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized SAP proxy request: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest( - sapS4HanaProxyContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Validation failed') }, - { status: 400 } - ), - } - ) - if (!parsed.success) return parsed.response - const proxyReq = parsed.data.body - const isWrite = WRITE_METHODS.has(proxyReq.method) - - const accessToken = - proxyReq.authType === 'oauth_client_credentials' - ? await fetchAccessToken(proxyReq, requestId) - : null - const csrf = isWrite ? await fetchCsrf(proxyReq, accessToken, requestId) : null - - let invocation = await callOdata(proxyReq, accessToken, csrf) - - if (isWrite && isCsrfRequired(invocation)) { - logger.info(`[${requestId}] CSRF token rejected, refetching and retrying`) - const refreshed = await fetchCsrf(proxyReq, accessToken, requestId) - if (refreshed) { - invocation = await callOdata(proxyReq, accessToken, refreshed) - } - } - - if (invocation.status >= 200 && invocation.status < 300) { - const data = invocation.status === 204 ? null : unwrapOdata(invocation.body) - return NextResponse.json({ success: true, output: { status: invocation.status, data } }) - } - - const message = extractOdataError(invocation.body, invocation.status) - logger.warn( - `[${requestId}] SAP API error (${invocation.status}) ${proxyReq.service}${proxyReq.path}: ${message}` - ) - return NextResponse.json( - { success: false, error: message, status: invocation.status }, - { status: invocation.status } - ) - } catch (error) { - logger.error(`[${requestId}] Unexpected SAP proxy error:`, error) - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/search/route.ts b/apps/sim/app/api/tools/search/route.ts deleted file mode 100644 index 41e79bc6c41..00000000000 --- a/apps/sim/app/api/tools/search/route.ts +++ /dev/null @@ -1,132 +0,0 @@ -import { createLogger } from '@sim/logger' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { searchToolContract } from '@/lib/api/contracts/tools/search' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { SEARCH_TOOL_COST } from '@/lib/billing/constants' -import { env } from '@/lib/core/config/env' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { executeTool } from '@/tools' - -const logger = createLogger('search') - -export const maxDuration = 60 -export const dynamic = 'force-dynamic' - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId() - - try { - const { searchParams: urlParams } = new URL(request.url) - const workflowId = urlParams.get('workflowId') || undefined - - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - const errorMessage = workflowId ? 'Workflow not found' : authResult.error || 'Unauthorized' - const statusCode = workflowId ? 404 : 401 - return NextResponse.json({ success: false, error: errorMessage }, { status: statusCode }) - } - - const userId = authResult.userId - - logger.info(`[${requestId}] Authenticated search request via ${authResult.authType}`, { - userId, - }) - - const parsed = await parseRequest(searchToolContract, request, {}) - if (!parsed.success) return parsed.response - const validated = parsed.data.body - - const exaApiKey = env.EXA_API_KEY - - if (!exaApiKey) { - logger.error(`[${requestId}] No Exa API key available`) - return NextResponse.json( - { success: false, error: 'Search service not configured' }, - { status: 503 } - ) - } - - logger.info(`[${requestId}] Executing search`, { - userId, - query: validated.query, - }) - - const result = await executeTool('exa_search', { - query: validated.query, - type: 'auto', - useAutoprompt: true, - highlights: true, - apiKey: exaApiKey, - }) - - if (!result.success) { - logger.error(`[${requestId}] Search failed`, { - userId, - error: result.error, - }) - return NextResponse.json( - { - success: false, - error: result.error || 'Search failed', - }, - { status: 500 } - ) - } - - const results = (result.output.results || []).map((r: any, index: number) => ({ - title: r.title || '', - link: r.url || '', - snippet: Array.isArray(r.highlights) ? r.highlights.join(' ... ') : '', - date: r.publishedDate || undefined, - position: index + 1, - })) - - const cost = { - input: 0, - output: 0, - total: SEARCH_TOOL_COST, - tokens: { - input: 0, - output: 0, - total: 0, - }, - model: 'search-exa', - pricing: { - input: 0, - cachedInput: 0, - output: 0, - updatedAt: new Date().toISOString(), - }, - } - - logger.info(`[${requestId}] Search completed`, { - userId, - resultCount: results.length, - cost: cost.total, - }) - - return NextResponse.json({ - results, - query: validated.query, - totalResults: results.length, - source: 'exa', - cost, - }) - } catch (error: any) { - logger.error(`[${requestId}] Search failed`, { - error: error.message, - stack: error.stack, - }) - - return NextResponse.json( - { - success: false, - error: error.message || 'Search failed', - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/secrets_manager/create-secret/route.ts b/apps/sim/app/api/tools/secrets_manager/create-secret/route.ts deleted file mode 100644 index cf38ee596b0..00000000000 --- a/apps/sim/app/api/tools/secrets_manager/create-secret/route.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSecretsManagerCreateSecretContract } from '@/lib/api/contracts/tools/aws/secrets-manager-create-secret' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSecret, createSecretsManagerClient } from '../utils' - -const logger = createLogger('SecretsManagerCreateSecretAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSecretsManagerCreateSecretContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Creating secret ${params.name}`) - - const client = createSecretsManagerClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await createSecret(client, params.name, params.secretValue, params.description) - - logger.info(`[${requestId}] Secret created: ${result.name}`) - - return NextResponse.json({ - message: `Secret "${result.name}" created successfully`, - ...result, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to create secret:`, error) - - return NextResponse.json({ error: `Failed to create secret: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/secrets_manager/delete-secret/route.ts b/apps/sim/app/api/tools/secrets_manager/delete-secret/route.ts deleted file mode 100644 index 62cdb9ffd58..00000000000 --- a/apps/sim/app/api/tools/secrets_manager/delete-secret/route.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSecretsManagerDeleteSecretContract } from '@/lib/api/contracts/tools/aws/secrets-manager-delete-secret' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSecretsManagerClient, deleteSecret } from '../utils' - -const logger = createLogger('SecretsManagerDeleteSecretAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSecretsManagerDeleteSecretContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Deleting secret ${params.secretId}`) - - const client = createSecretsManagerClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await deleteSecret( - client, - params.secretId, - params.recoveryWindowInDays, - params.forceDelete - ) - - const action = params.forceDelete ? 'permanently deleted' : 'scheduled for deletion' - logger.info(`[${requestId}] Secret ${action}: ${result.name}`) - - return NextResponse.json({ - message: `Secret "${result.name}" ${action}`, - ...result, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to delete secret:`, error) - - return NextResponse.json({ error: `Failed to delete secret: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/secrets_manager/describe-secret/route.ts b/apps/sim/app/api/tools/secrets_manager/describe-secret/route.ts deleted file mode 100644 index 793f8ce802b..00000000000 --- a/apps/sim/app/api/tools/secrets_manager/describe-secret/route.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSecretsManagerDescribeSecretContract } from '@/lib/api/contracts/tools/aws/secrets-manager-describe-secret' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSecretsManagerClient, describeSecret } from '../utils' - -const logger = createLogger('SecretsManagerDescribeSecretAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSecretsManagerDescribeSecretContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Describing secret ${params.secretId}`) - - const client = createSecretsManagerClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await describeSecret(client, params.secretId) - - logger.info(`[${requestId}] Described secret: ${result.name}`) - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to describe secret:`, error) - - return NextResponse.json( - { error: `Failed to describe secret: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/secrets_manager/get-secret/route.ts b/apps/sim/app/api/tools/secrets_manager/get-secret/route.ts deleted file mode 100644 index 31b0ba266c7..00000000000 --- a/apps/sim/app/api/tools/secrets_manager/get-secret/route.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSecretsManagerGetSecretContract } from '@/lib/api/contracts/tools/aws/secrets-manager-get-secret' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSecretsManagerClient, getSecretValue } from '../utils' - -const logger = createLogger('SecretsManagerGetSecretAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSecretsManagerGetSecretContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Retrieving secret ${params.secretId}`) - - const client = createSecretsManagerClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await getSecretValue( - client, - params.secretId, - params.versionId, - params.versionStage - ) - - logger.info(`[${requestId}] Secret retrieved successfully`) - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to retrieve secret:`, error) - - return NextResponse.json( - { error: `Failed to retrieve secret: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/secrets_manager/list-secrets/route.ts b/apps/sim/app/api/tools/secrets_manager/list-secrets/route.ts deleted file mode 100644 index 6345b07b12f..00000000000 --- a/apps/sim/app/api/tools/secrets_manager/list-secrets/route.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSecretsManagerListSecretsContract } from '@/lib/api/contracts/tools/aws/secrets-manager-list-secrets' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSecretsManagerClient, listSecrets } from '../utils' - -const logger = createLogger('SecretsManagerListSecretsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSecretsManagerListSecretsContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Listing secrets`) - - const client = createSecretsManagerClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await listSecrets(client, params.maxResults, params.nextToken) - - logger.info(`[${requestId}] Listed ${result.count} secrets`) - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to list secrets:`, error) - - return NextResponse.json({ error: `Failed to list secrets: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/secrets_manager/restore-secret/route.ts b/apps/sim/app/api/tools/secrets_manager/restore-secret/route.ts deleted file mode 100644 index e73da9582a9..00000000000 --- a/apps/sim/app/api/tools/secrets_manager/restore-secret/route.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSecretsManagerRestoreSecretContract } from '@/lib/api/contracts/tools/aws/secrets-manager-restore-secret' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSecretsManagerClient, restoreSecret } from '../utils' - -const logger = createLogger('SecretsManagerRestoreSecretAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSecretsManagerRestoreSecretContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Restoring secret ${params.secretId}`) - - const client = createSecretsManagerClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await restoreSecret(client, params.secretId) - - logger.info(`[${requestId}] Restored secret: ${result.name}`) - - return NextResponse.json({ - message: `Secret "${result.name}" restored successfully`, - ...result, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to restore secret:`, error) - - return NextResponse.json( - { error: `Failed to restore secret: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/secrets_manager/rotate-secret/route.ts b/apps/sim/app/api/tools/secrets_manager/rotate-secret/route.ts deleted file mode 100644 index 3a0718c4b90..00000000000 --- a/apps/sim/app/api/tools/secrets_manager/rotate-secret/route.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSecretsManagerRotateSecretContract } from '@/lib/api/contracts/tools/aws/secrets-manager-rotate-secret' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSecretsManagerClient, rotateSecret } from '../utils' - -const logger = createLogger('SecretsManagerRotateSecretAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSecretsManagerRotateSecretContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Rotating secret ${params.secretId}`) - - const client = createSecretsManagerClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await rotateSecret( - client, - params.secretId, - params.clientRequestToken, - params.rotationLambdaARN, - { - automaticallyAfterDays: params.automaticallyAfterDays, - duration: params.duration, - scheduleExpression: params.scheduleExpression, - }, - params.rotateImmediately - ) - - logger.info(`[${requestId}] Rotation started for secret: ${result.name}`) - - return NextResponse.json({ - message: `Rotation started for secret "${result.name}"`, - ...result, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to rotate secret:`, error) - - return NextResponse.json({ error: `Failed to rotate secret: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/secrets_manager/tag-resource/route.ts b/apps/sim/app/api/tools/secrets_manager/tag-resource/route.ts deleted file mode 100644 index 6c4bec10989..00000000000 --- a/apps/sim/app/api/tools/secrets_manager/tag-resource/route.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSecretsManagerTagResourceContract } from '@/lib/api/contracts/tools/aws/secrets-manager-tag-resource' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSecretsManagerClient, tagResource } from '../utils' - -const logger = createLogger('SecretsManagerTagResourceAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSecretsManagerTagResourceContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Tagging secret ${params.secretId}`) - - const client = createSecretsManagerClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await tagResource( - client, - params.secretId, - params.tags.map((t) => ({ Key: t.key, Value: t.value })) - ) - - logger.info(`[${requestId}] Tagged secret: ${result.name}`) - - return NextResponse.json({ - message: `Secret "${result.name}" tagged successfully`, - ...result, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to tag secret:`, error) - - return NextResponse.json({ error: `Failed to tag secret: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/secrets_manager/untag-resource/route.ts b/apps/sim/app/api/tools/secrets_manager/untag-resource/route.ts deleted file mode 100644 index 9300e81008f..00000000000 --- a/apps/sim/app/api/tools/secrets_manager/untag-resource/route.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSecretsManagerUntagResourceContract } from '@/lib/api/contracts/tools/aws/secrets-manager-untag-resource' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSecretsManagerClient, untagResource } from '../utils' - -const logger = createLogger('SecretsManagerUntagResourceAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSecretsManagerUntagResourceContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Untagging secret ${params.secretId}`) - - const client = createSecretsManagerClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await untagResource(client, params.secretId, params.tagKeys) - - logger.info(`[${requestId}] Untagged secret: ${result.name}`) - - return NextResponse.json({ - message: `Secret "${result.name}" untagged successfully`, - ...result, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to untag secret:`, error) - - return NextResponse.json({ error: `Failed to untag secret: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/secrets_manager/update-secret/route.ts b/apps/sim/app/api/tools/secrets_manager/update-secret/route.ts deleted file mode 100644 index 1bb386bf564..00000000000 --- a/apps/sim/app/api/tools/secrets_manager/update-secret/route.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSecretsManagerUpdateSecretContract } from '@/lib/api/contracts/tools/aws/secrets-manager-update-secret' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSecretsManagerClient, updateSecretValue } from '../utils' - -const logger = createLogger('SecretsManagerUpdateSecretAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSecretsManagerUpdateSecretContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Updating secret ${params.secretId}`) - - const client = createSecretsManagerClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await updateSecretValue( - client, - params.secretId, - params.secretValue, - params.description - ) - - logger.info(`[${requestId}] Secret updated: ${result.name}`) - - return NextResponse.json({ - message: `Secret "${result.name}" updated successfully`, - ...result, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] Failed to update secret:`, error) - - return NextResponse.json({ error: `Failed to update secret: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/secrets_manager/utils.ts b/apps/sim/app/api/tools/secrets_manager/utils.ts deleted file mode 100644 index fc18d95b2fd..00000000000 --- a/apps/sim/app/api/tools/secrets_manager/utils.ts +++ /dev/null @@ -1,266 +0,0 @@ -import type { RotationRulesType, SecretListEntry, Tag } from '@aws-sdk/client-secrets-manager' -import { - CreateSecretCommand, - DeleteSecretCommand, - DescribeSecretCommand, - GetSecretValueCommand, - ListSecretsCommand, - RestoreSecretCommand, - RotateSecretCommand, - SecretsManagerClient, - TagResourceCommand, - UntagResourceCommand, - UpdateSecretCommand, -} from '@aws-sdk/client-secrets-manager' -import type { SecretsManagerConnectionConfig } from '@/tools/secrets_manager/types' - -function mapRotationRules(rules: RotationRulesType | undefined) { - if (!rules) return null - return { - automaticallyAfterDays: rules.AutomaticallyAfterDays ?? null, - duration: rules.Duration ?? null, - scheduleExpression: rules.ScheduleExpression ?? null, - } -} - -export function createSecretsManagerClient( - config: SecretsManagerConnectionConfig -): SecretsManagerClient { - return new SecretsManagerClient({ - region: config.region, - credentials: { - accessKeyId: config.accessKeyId, - secretAccessKey: config.secretAccessKey, - }, - }) -} - -export async function getSecretValue( - client: SecretsManagerClient, - secretId: string, - versionId?: string | null, - versionStage?: string | null -) { - const command = new GetSecretValueCommand({ - SecretId: secretId, - ...(versionId ? { VersionId: versionId } : {}), - ...(versionStage ? { VersionStage: versionStage } : {}), - }) - - const response = await client.send(command) - - if (!response.SecretString && response.SecretBinary) { - throw new Error( - 'Secret is stored as binary (SecretBinary). This integration only supports string secrets.' - ) - } - - return { - name: response.Name ?? '', - secretValue: response.SecretString ?? '', - arn: response.ARN ?? '', - versionId: response.VersionId ?? '', - versionStages: response.VersionStages ?? [], - createdDate: response.CreatedDate?.toISOString() ?? null, - } -} - -export async function listSecrets( - client: SecretsManagerClient, - maxResults?: number | null, - nextToken?: string | null -) { - const command = new ListSecretsCommand({ - ...(maxResults ? { MaxResults: maxResults } : {}), - ...(nextToken ? { NextToken: nextToken } : {}), - }) - - const response = await client.send(command) - const secrets = (response.SecretList ?? []).map((secret: SecretListEntry) => ({ - name: secret.Name ?? '', - arn: secret.ARN ?? '', - description: secret.Description ?? null, - createdDate: secret.CreatedDate?.toISOString() ?? null, - lastChangedDate: secret.LastChangedDate?.toISOString() ?? null, - lastAccessedDate: secret.LastAccessedDate?.toISOString() ?? null, - rotationEnabled: secret.RotationEnabled ?? false, - tags: secret.Tags?.map((t: Tag) => ({ key: t.Key ?? '', value: t.Value ?? '' })) ?? [], - rotationRules: mapRotationRules(secret.RotationRules), - lastRotatedDate: secret.LastRotatedDate?.toISOString() ?? null, - nextRotationDate: secret.NextRotationDate?.toISOString() ?? null, - deletedDate: secret.DeletedDate?.toISOString() ?? null, - secretVersionsToStages: secret.SecretVersionsToStages ?? null, - })) - - return { - secrets, - nextToken: response.NextToken ?? null, - count: secrets.length, - } -} - -export async function createSecret( - client: SecretsManagerClient, - name: string, - secretValue: string, - description?: string | null -) { - const command = new CreateSecretCommand({ - Name: name, - SecretString: secretValue, - ...(description ? { Description: description } : {}), - }) - - const response = await client.send(command) - return { - name: response.Name ?? '', - arn: response.ARN ?? '', - versionId: response.VersionId ?? '', - } -} - -export async function updateSecretValue( - client: SecretsManagerClient, - secretId: string, - secretValue: string, - description?: string | null -) { - const command = new UpdateSecretCommand({ - SecretId: secretId, - SecretString: secretValue, - ...(description ? { Description: description } : {}), - }) - - const response = await client.send(command) - return { - name: response.Name ?? '', - arn: response.ARN ?? '', - versionId: response.VersionId ?? '', - } -} - -export async function deleteSecret( - client: SecretsManagerClient, - secretId: string, - recoveryWindowInDays?: number | null, - forceDelete?: boolean | null -) { - const command = new DeleteSecretCommand({ - SecretId: secretId, - ...(forceDelete ? { ForceDeleteWithoutRecovery: true } : {}), - ...(!forceDelete && recoveryWindowInDays ? { RecoveryWindowInDays: recoveryWindowInDays } : {}), - }) - - const response = await client.send(command) - return { - name: response.Name ?? '', - arn: response.ARN ?? '', - deletionDate: response.DeletionDate?.toISOString() ?? null, - } -} - -export async function describeSecret(client: SecretsManagerClient, secretId: string) { - const command = new DescribeSecretCommand({ SecretId: secretId }) - const response = await client.send(command) - - return { - name: response.Name ?? '', - arn: response.ARN ?? '', - description: response.Description ?? null, - kmsKeyId: response.KmsKeyId ?? null, - rotationEnabled: response.RotationEnabled ?? false, - rotationLambdaARN: response.RotationLambdaARN ?? null, - rotationRules: mapRotationRules(response.RotationRules), - lastRotatedDate: response.LastRotatedDate?.toISOString() ?? null, - lastChangedDate: response.LastChangedDate?.toISOString() ?? null, - lastAccessedDate: response.LastAccessedDate?.toISOString() ?? null, - deletedDate: response.DeletedDate?.toISOString() ?? null, - nextRotationDate: response.NextRotationDate?.toISOString() ?? null, - tags: response.Tags?.map((t: Tag) => ({ key: t.Key ?? '', value: t.Value ?? '' })) ?? [], - versionIdsToStages: response.VersionIdsToStages ?? null, - owningService: response.OwningService ?? null, - createdDate: response.CreatedDate?.toISOString() ?? null, - primaryRegion: response.PrimaryRegion ?? null, - replicationStatus: - response.ReplicationStatus?.map((r) => ({ - region: r.Region ?? '', - kmsKeyId: r.KmsKeyId ?? null, - status: r.Status ?? null, - statusMessage: r.StatusMessage ?? null, - lastAccessedDate: r.LastAccessedDate?.toISOString() ?? null, - })) ?? [], - } -} - -export async function tagResource(client: SecretsManagerClient, secretId: string, tags: Tag[]) { - const command = new TagResourceCommand({ SecretId: secretId, Tags: tags }) - await client.send(command) - return { name: secretId } -} - -export async function untagResource( - client: SecretsManagerClient, - secretId: string, - tagKeys: string[] -) { - const command = new UntagResourceCommand({ SecretId: secretId, TagKeys: tagKeys }) - await client.send(command) - return { name: secretId } -} - -export async function restoreSecret(client: SecretsManagerClient, secretId: string) { - const command = new RestoreSecretCommand({ SecretId: secretId }) - const response = await client.send(command) - return { - name: response.Name ?? '', - arn: response.ARN ?? '', - } -} - -export async function rotateSecret( - client: SecretsManagerClient, - secretId: string, - clientRequestToken?: string | null, - rotationLambdaARN?: string | null, - rotationRules?: { - automaticallyAfterDays?: number | null - duration?: string | null - scheduleExpression?: string | null - } | null, - rotateImmediately?: boolean | null -) { - const hasRotationRules = Boolean( - rotationRules?.automaticallyAfterDays || - rotationRules?.duration || - rotationRules?.scheduleExpression - ) - - const command = new RotateSecretCommand({ - SecretId: secretId, - ...(clientRequestToken ? { ClientRequestToken: clientRequestToken } : {}), - ...(rotationLambdaARN ? { RotationLambdaARN: rotationLambdaARN } : {}), - ...(hasRotationRules - ? { - RotationRules: { - ...(rotationRules?.automaticallyAfterDays - ? { AutomaticallyAfterDays: rotationRules.automaticallyAfterDays } - : {}), - ...(rotationRules?.duration ? { Duration: rotationRules.duration } : {}), - ...(rotationRules?.scheduleExpression - ? { ScheduleExpression: rotationRules.scheduleExpression } - : {}), - }, - } - : {}), - ...(rotateImmediately === undefined || rotateImmediately === null - ? {} - : { RotateImmediately: rotateImmediately }), - }) - - const response = await client.send(command) - return { - name: response.Name ?? '', - arn: response.ARN ?? '', - versionId: response.VersionId ?? '', - } -} diff --git a/apps/sim/app/api/tools/sendgrid/send-mail/route.ts b/apps/sim/app/api/tools/sendgrid/send-mail/route.ts deleted file mode 100644 index f0bc1b3e660..00000000000 --- a/apps/sim/app/api/tools/sendgrid/send-mail/route.ts +++ /dev/null @@ -1,194 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { sendGridSendMailContract } from '@/lib/api/contracts/tools/communication/email' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('SendGridSendMailAPI') - -/** SendGrid rejects a message whose total attachment payload exceeds 30MB. */ -const MAX_ATTACHMENT_TOTAL_BYTES = 30 * 1024 * 1024 - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized SendGrid send attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const userId = authResult.userId - logger.info(`[${requestId}] Authenticated SendGrid send request via ${authResult.authType}`) - - const parsed = await parseRequest(sendGridSendMailContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Sending SendGrid email`, { - to: validatedData.to, - subject: validatedData.subject || '(template)', - hasAttachments: !!(validatedData.attachments && validatedData.attachments.length > 0), - attachmentCount: validatedData.attachments?.length || 0, - }) - - // Build personalizations - const personalizations: Record = { - to: [ - { email: validatedData.to, ...(validatedData.toName && { name: validatedData.toName }) }, - ], - } - - if (validatedData.cc) { - personalizations.cc = [{ email: validatedData.cc }] - } - - if (validatedData.bcc) { - personalizations.bcc = [{ email: validatedData.bcc }] - } - - if (validatedData.templateId && validatedData.dynamicTemplateData) { - personalizations.dynamic_template_data = - typeof validatedData.dynamicTemplateData === 'string' - ? JSON.parse(validatedData.dynamicTemplateData) - : validatedData.dynamicTemplateData - } - - // Build mail body - const mailBody: Record = { - personalizations: [personalizations], - from: { - email: validatedData.from, - ...(validatedData.fromName && { name: validatedData.fromName }), - }, - subject: validatedData.subject, - } - - if (validatedData.templateId) { - mailBody.template_id = validatedData.templateId - } else { - mailBody.content = [ - { - type: validatedData.contentType || 'text/plain', - value: validatedData.content, - }, - ] - } - - if (validatedData.replyTo) { - mailBody.reply_to = { - email: validatedData.replyTo, - ...(validatedData.replyToName && { name: validatedData.replyToName }), - } - } - - // Process attachments from UserFile objects - if (validatedData.attachments && validatedData.attachments.length > 0) { - const rawAttachments = validatedData.attachments - logger.info(`[${requestId}] Processing ${rawAttachments.length} attachment(s)`) - - const userFiles = processFilesToUserFiles(rawAttachments, requestId, logger) - - if (userFiles.length > 0) { - const accessResults = await Promise.all( - userFiles.map((file) => assertToolFileAccess(file.key, userId, requestId, logger)) - ) - const denied = accessResults.find((r) => r !== null) - if (denied) return denied - - let resolved: Array<{ buffer: Buffer; contentType: string }> - try { - resolved = await downloadServableFilesWithinBudget(userFiles, requestId, logger, { - totalMaxBytes: MAX_ATTACHMENT_TOTAL_BYTES, - label: 'Total attachment size', - }) - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - if (isPayloadSizeLimitError(error)) { - const sizeMB = ( - (error.observedBytes ?? MAX_ATTACHMENT_TOTAL_BYTES) / - (1024 * 1024) - ).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds SendGrid's limit of 30MB`, - }, - { status: 400 } - ) - } - logger.error(`[${requestId}] Failed to download an attachment:`, error) - return NextResponse.json( - { - success: false, - error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`, - }, - { status: 500 } - ) - } - - const sendGridAttachments = userFiles.map((file, i) => ({ - content: resolved[i].buffer.toString('base64'), - filename: file.name, - type: resolved[i].contentType || file.type || 'application/octet-stream', - disposition: 'attachment', - })) - - mailBody.attachments = sendGridAttachments - } - } - - // Send to SendGrid - const response = await fetch('https://api.sendgrid.com/v3/mail/send', { - method: 'POST', - headers: { - Authorization: `Bearer ${validatedData.apiKey}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(mailBody), - }) - - if (!response.ok) { - const errorData = await response.json().catch(() => ({})) - const errorMessage = - errorData.errors?.[0]?.message || errorData.message || 'Failed to send email' - logger.error(`[${requestId}] SendGrid API error:`, { status: response.status, errorData }) - return NextResponse.json({ success: false, error: errorMessage }, { status: response.status }) - } - - const messageId = response.headers.get('X-Message-Id') - logger.info(`[${requestId}] Email sent successfully`, { messageId }) - - return NextResponse.json({ - success: true, - output: { - success: true, - messageId: messageId || undefined, - to: validatedData.to, - subject: validatedData.subject || '', - }, - }) - } catch (error) { - logger.error(`[${requestId}] Unexpected error:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/servicenow/upload-attachment/route.ts b/apps/sim/app/api/tools/servicenow/upload-attachment/route.ts deleted file mode 100644 index 532b3712572..00000000000 --- a/apps/sim/app/api/tools/servicenow/upload-attachment/route.ts +++ /dev/null @@ -1,145 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { servicenowUploadAttachmentContract } from '@/lib/api/contracts/tools/servicenow' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import type { ServiceNowAttachment } from '@/tools/servicenow/types' -import { createBasicAuthHeader } from '@/tools/servicenow/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('ServiceNowUploadAttachmentAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized ServiceNow upload attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(servicenowUploadAttachmentContract, request, {}) - if (!parsed.success) return parsed.response - const body = parsed.data.body - - if (!body.file) { - return NextResponse.json({ success: false, error: 'A file is required' }, { status: 400 }) - } - - let userFile - try { - userFile = processSingleFileToUserFile(body.file, requestId, logger) - } catch (error) { - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Failed to process file') }, - { status: 400 } - ) - } - - const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) - if (denied) return denied - - let fileBuffer: Buffer - let resolvedContentType: string - try { - const servable = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - }) - fileBuffer = servable.buffer - resolvedContentType = servable.contentType - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - logger.error(`[${requestId}] Failed to download file from storage:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Failed to download file') }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } - - const contentType = resolvedContentType || userFile.type || 'application/octet-stream' - - const baseUrl = body.instanceUrl.trim().replace(/\/$/, '') - const uploadParams = new URLSearchParams({ - table_name: body.tableName.trim(), - table_sys_id: body.recordSysId.trim(), - file_name: body.fileName, - }) - const uploadUrl = `${baseUrl}/api/now/attachment/file?${uploadParams.toString()}` - - const response = await secureFetchWithValidation( - uploadUrl, - { - method: 'POST', - headers: { - Authorization: createBasicAuthHeader(body.username, body.password), - 'Content-Type': contentType, - Accept: 'application/json', - }, - body: fileBuffer, - }, - 'instanceUrl' - ) - - if (!response.ok) { - const errorData = (await response.json().catch(() => ({}))) as { - error?: { message?: string } - } - const errorMessage = - errorData?.error?.message ?? - `ServiceNow API error: ${response.status} ${response.statusText}` - logger.error(`[${requestId}] ServiceNow upload attachment failed`, { - status: response.status, - }) - return NextResponse.json({ success: false, error: errorMessage }, { status: response.status }) - } - - const data = (await response.json()) as { result?: ServiceNowAttachment } - const result = data.result - - logger.info(`[${requestId}] File attached to ServiceNow record successfully`, { - tableName: body.tableName, - recordSysId: body.recordSysId, - }) - - return NextResponse.json({ - success: true, - output: { - attachment: result - ? { - sys_id: result.sys_id ?? null, - file_name: result.file_name ?? null, - content_type: result.content_type ?? null, - size_bytes: result.size_bytes ?? null, - table_name: result.table_name ?? null, - table_sys_id: result.table_sys_id ?? null, - download_link: result.download_link ?? null, - } - : null, - metadata: { recordCount: 1 }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error uploading attachment to ServiceNow:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Internal server error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ses/create-configuration-set/route.ts b/apps/sim/app/api/tools/ses/create-configuration-set/route.ts deleted file mode 100644 index f905aff437f..00000000000 --- a/apps/sim/app/api/tools/ses/create-configuration-set/route.ts +++ /dev/null @@ -1,69 +0,0 @@ -import type { SuppressionListReason } from '@aws-sdk/client-sesv2' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSesCreateConfigurationSetContract } from '@/lib/api/contracts/tools/aws/ses-create-configuration-set' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createConfigurationSet, createSESClient } from '../utils' - -const logger = createLogger('SESCreateConfigurationSetAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSesCreateConfigurationSetContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info('Creating SES configuration set') - - const client = createSESClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const suppressedReasons = params.suppressedReasons - ? (params.suppressedReasons - .split(',') - .map((r) => r.trim()) - .filter(Boolean) as SuppressionListReason[]) - : null - - const result = await createConfigurationSet(client, { - configurationSetName: params.configurationSetName, - customRedirectDomain: params.customRedirectDomain, - httpsPolicy: params.httpsPolicy, - tlsPolicy: params.tlsPolicy, - sendingPoolName: params.sendingPoolName, - reputationMetricsEnabled: params.reputationMetricsEnabled, - sendingEnabled: params.sendingEnabled, - suppressedReasons, - tags: params.tags, - }) - - logger.info(`Created configuration set '${params.configurationSetName}'`) - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to create configuration set:', error) - - return NextResponse.json( - { error: `Failed to create configuration set: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ses/create-email-identity/route.ts b/apps/sim/app/api/tools/ses/create-email-identity/route.ts deleted file mode 100644 index 9f58228b173..00000000000 --- a/apps/sim/app/api/tools/ses/create-email-identity/route.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSesCreateEmailIdentityContract } from '@/lib/api/contracts/tools/aws/ses-create-email-identity' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createEmailIdentity, createSESClient } from '../utils' - -const logger = createLogger('SESCreateEmailIdentityAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSesCreateEmailIdentityContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info('Creating SES email identity') - - const client = createSESClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await createEmailIdentity(client, { - emailIdentity: params.emailIdentity, - dkimSigningAttributes: params.dkimSigningAttributes, - tags: params.tags, - configurationSetName: params.configurationSetName, - }) - - logger.info(`Created email identity '${params.emailIdentity}'`) - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to create email identity:', error) - - return NextResponse.json( - { error: `Failed to create email identity: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ses/create-template/route.ts b/apps/sim/app/api/tools/ses/create-template/route.ts deleted file mode 100644 index d8741f0a624..00000000000 --- a/apps/sim/app/api/tools/ses/create-template/route.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSesCreateTemplateContract } from '@/lib/api/contracts/tools/aws/ses-create-template' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSESClient, createTemplate } from '../utils' - -const logger = createLogger('SESCreateTemplateAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSesCreateTemplateContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Creating SES template '${params.templateName}'`) - - const client = createSESClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await createTemplate(client, { - templateName: params.templateName, - subjectPart: params.subjectPart, - textPart: params.textPart, - htmlPart: params.htmlPart, - }) - - logger.info(`Template '${params.templateName}' created successfully`) - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to create template:', error) - - return NextResponse.json( - { error: `Failed to create template: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ses/delete-email-identity/route.ts b/apps/sim/app/api/tools/ses/delete-email-identity/route.ts deleted file mode 100644 index a185fc769ca..00000000000 --- a/apps/sim/app/api/tools/ses/delete-email-identity/route.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSesDeleteEmailIdentityContract } from '@/lib/api/contracts/tools/aws/ses-delete-email-identity' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSESClient, deleteEmailIdentity } from '../utils' - -const logger = createLogger('SESDeleteEmailIdentityAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSesDeleteEmailIdentityContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info('Deleting SES email identity') - - const client = createSESClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await deleteEmailIdentity(client, params.emailIdentity) - - logger.info(`Deleted email identity '${params.emailIdentity}'`) - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to delete email identity:', error) - - return NextResponse.json( - { error: `Failed to delete email identity: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ses/delete-suppressed-destination/route.ts b/apps/sim/app/api/tools/ses/delete-suppressed-destination/route.ts deleted file mode 100644 index c0a047279e6..00000000000 --- a/apps/sim/app/api/tools/ses/delete-suppressed-destination/route.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSesDeleteSuppressedDestinationContract } from '@/lib/api/contracts/tools/aws/ses-delete-suppressed-destination' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSESClient, deleteSuppressedDestination } from '../utils' - -const logger = createLogger('SESDeleteSuppressedDestinationAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSesDeleteSuppressedDestinationContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info('Removing email address from SES suppression list') - - const client = createSESClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await deleteSuppressedDestination(client, params.emailAddress) - - logger.info('Removed email address from suppression list') - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to remove suppressed destination:', error) - - return NextResponse.json( - { error: `Failed to remove suppressed destination: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ses/delete-template/route.ts b/apps/sim/app/api/tools/ses/delete-template/route.ts deleted file mode 100644 index 6a6b9e343a0..00000000000 --- a/apps/sim/app/api/tools/ses/delete-template/route.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSesDeleteTemplateContract } from '@/lib/api/contracts/tools/aws/ses-delete-template' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSESClient, deleteTemplate } from '../utils' - -const logger = createLogger('SESDeleteTemplateAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSesDeleteTemplateContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Deleting SES template '${params.templateName}'`) - - const client = createSESClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await deleteTemplate(client, params.templateName) - - logger.info(`Template '${params.templateName}' deleted successfully`) - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to delete template:', error) - - return NextResponse.json( - { error: `Failed to delete template: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ses/get-account/route.ts b/apps/sim/app/api/tools/ses/get-account/route.ts deleted file mode 100644 index 4316901af58..00000000000 --- a/apps/sim/app/api/tools/ses/get-account/route.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSesGetAccountContract } from '@/lib/api/contracts/tools/aws/ses-get-account' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSESClient, getAccount } from '../utils' - -const logger = createLogger('SESGetAccountAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSesGetAccountContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info('Getting SES account information') - - const client = createSESClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await getAccount(client) - - logger.info('SES account info retrieved successfully') - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to get account information:', error) - - return NextResponse.json( - { error: `Failed to get account information: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ses/get-email-identity/route.ts b/apps/sim/app/api/tools/ses/get-email-identity/route.ts deleted file mode 100644 index c13a11c6fe9..00000000000 --- a/apps/sim/app/api/tools/ses/get-email-identity/route.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSesGetEmailIdentityContract } from '@/lib/api/contracts/tools/aws/ses-get-email-identity' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSESClient, getEmailIdentity } from '../utils' - -const logger = createLogger('SESGetEmailIdentityAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSesGetEmailIdentityContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info('Fetching SES email identity') - - const client = createSESClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await getEmailIdentity(client, params.emailIdentity) - - logger.info(`Fetched email identity '${params.emailIdentity}'`) - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to get email identity:', error) - - return NextResponse.json( - { error: `Failed to get email identity: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ses/get-suppressed-destination/route.ts b/apps/sim/app/api/tools/ses/get-suppressed-destination/route.ts deleted file mode 100644 index 71022496733..00000000000 --- a/apps/sim/app/api/tools/ses/get-suppressed-destination/route.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSesGetSuppressedDestinationContract } from '@/lib/api/contracts/tools/aws/ses-get-suppressed-destination' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSESClient, getSuppressedDestination } from '../utils' - -const logger = createLogger('SESGetSuppressedDestinationAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSesGetSuppressedDestinationContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info('Fetching SES suppressed destination') - - const client = createSESClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await getSuppressedDestination(client, params.emailAddress) - - logger.info('Fetched suppressed destination') - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to get suppressed destination:', error) - - return NextResponse.json( - { error: `Failed to get suppressed destination: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ses/get-template/route.ts b/apps/sim/app/api/tools/ses/get-template/route.ts deleted file mode 100644 index 04cec4dc688..00000000000 --- a/apps/sim/app/api/tools/ses/get-template/route.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSesGetTemplateContract } from '@/lib/api/contracts/tools/aws/ses-get-template' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSESClient, getTemplate } from '../utils' - -const logger = createLogger('SESGetTemplateAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSesGetTemplateContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Getting SES template '${params.templateName}'`) - - const client = createSESClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await getTemplate(client, params.templateName) - - logger.info(`Template '${params.templateName}' retrieved successfully`) - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to get template:', error) - - return NextResponse.json( - { error: `Failed to get template: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ses/list-identities/route.ts b/apps/sim/app/api/tools/ses/list-identities/route.ts deleted file mode 100644 index ec761ac1d15..00000000000 --- a/apps/sim/app/api/tools/ses/list-identities/route.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSesListIdentitiesContract } from '@/lib/api/contracts/tools/aws/ses-list-identities' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSESClient, listIdentities } from '../utils' - -const logger = createLogger('SESListIdentitiesAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSesListIdentitiesContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info('Listing SES email identities') - - const client = createSESClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await listIdentities(client, { - pageSize: params.pageSize, - nextToken: params.nextToken, - }) - - logger.info(`Listed ${result.count} identities`) - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to list identities:', error) - - return NextResponse.json( - { error: `Failed to list identities: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ses/list-suppressed-destinations/route.ts b/apps/sim/app/api/tools/ses/list-suppressed-destinations/route.ts deleted file mode 100644 index 639704da110..00000000000 --- a/apps/sim/app/api/tools/ses/list-suppressed-destinations/route.ts +++ /dev/null @@ -1,80 +0,0 @@ -import type { SuppressionListReason } from '@aws-sdk/client-sesv2' -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSesListSuppressedDestinationsContract } from '@/lib/api/contracts/tools/aws/ses-list-suppressed-destinations' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSESClient, listSuppressedDestinations } from '../utils' - -const logger = createLogger('SESListSuppressedDestinationsAPI') - -const VALID_SUPPRESSION_REASONS: SuppressionListReason[] = ['BOUNCE', 'COMPLAINT'] - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSesListSuppressedDestinationsContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - let reasons: SuppressionListReason[] | null = null - if (params.reasons) { - const candidates = params.reasons - .split(',') - .map((r) => r.trim()) - .filter(Boolean) - const invalid = candidates.filter( - (r) => !VALID_SUPPRESSION_REASONS.includes(r as SuppressionListReason) - ) - if (invalid.length > 0) { - return NextResponse.json( - { - error: `Invalid suppression reason(s): ${invalid.join(', ')}. Must be one of: ${VALID_SUPPRESSION_REASONS.join(', ')}`, - }, - { status: 400 } - ) - } - reasons = candidates as SuppressionListReason[] - } - - logger.info('Listing SES suppressed destinations') - - const client = createSESClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await listSuppressedDestinations(client, { - reasons, - startDate: params.startDate ? new Date(params.startDate) : null, - endDate: params.endDate ? new Date(params.endDate) : null, - pageSize: params.pageSize, - nextToken: params.nextToken, - }) - - logger.info(`Listed ${result.count} suppressed destinations`) - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to list suppressed destinations:', error) - - return NextResponse.json( - { error: `Failed to list suppressed destinations: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ses/list-templates/route.ts b/apps/sim/app/api/tools/ses/list-templates/route.ts deleted file mode 100644 index f0c570cdf44..00000000000 --- a/apps/sim/app/api/tools/ses/list-templates/route.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSesListTemplatesContract } from '@/lib/api/contracts/tools/aws/ses-list-templates' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSESClient, listTemplates } from '../utils' - -const logger = createLogger('SESListTemplatesAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSesListTemplatesContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info('Listing SES email templates') - - const client = createSESClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await listTemplates(client, { - pageSize: params.pageSize, - nextToken: params.nextToken, - }) - - logger.info(`Listed ${result.count} templates`) - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to list templates:', error) - - return NextResponse.json( - { error: `Failed to list templates: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ses/put-suppressed-destination/route.ts b/apps/sim/app/api/tools/ses/put-suppressed-destination/route.ts deleted file mode 100644 index 453ba387893..00000000000 --- a/apps/sim/app/api/tools/ses/put-suppressed-destination/route.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSesPutSuppressedDestinationContract } from '@/lib/api/contracts/tools/aws/ses-put-suppressed-destination' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSESClient, putSuppressedDestination } from '../utils' - -const logger = createLogger('SESPutSuppressedDestinationAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSesPutSuppressedDestinationContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info('Adding email address to SES suppression list') - - const client = createSESClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await putSuppressedDestination(client, { - emailAddress: params.emailAddress, - reason: params.reason, - }) - - logger.info('Added email address to suppression list') - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to add suppressed destination:', error) - - return NextResponse.json( - { error: `Failed to add suppressed destination: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ses/send-bulk-email/route.ts b/apps/sim/app/api/tools/ses/send-bulk-email/route.ts deleted file mode 100644 index 8799b20e303..00000000000 --- a/apps/sim/app/api/tools/ses/send-bulk-email/route.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSesSendBulkEmailContract } from '@/lib/api/contracts/tools/aws/ses-send-bulk-email' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSESClient, parseBulkEmailDestinations, sendBulkEmail } from '../utils' - -const logger = createLogger('SESSendBulkEmailAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSesSendBulkEmailContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - let destinations: ReturnType - try { - destinations = parseBulkEmailDestinations(params.destinations) - } catch { - return NextResponse.json( - { error: 'destinations must be a valid JSON array of destination objects' }, - { status: 400 } - ) - } - - logger.info( - `Sending bulk email from ${params.fromAddress} to ${destinations.length} destination(s) using template '${params.templateName}'` - ) - - const client = createSESClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await sendBulkEmail(client, { - fromAddress: params.fromAddress, - templateName: params.templateName, - destinations, - defaultTemplateData: params.defaultTemplateData, - configurationSetName: params.configurationSetName, - }) - - logger.info( - `Bulk email sent: ${result.successCount} succeeded, ${result.failureCount} failed` - ) - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to send bulk email:', error) - - return NextResponse.json( - { error: `Failed to send bulk email: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ses/send-custom-verification-email/route.ts b/apps/sim/app/api/tools/ses/send-custom-verification-email/route.ts deleted file mode 100644 index dd4bf5387f9..00000000000 --- a/apps/sim/app/api/tools/ses/send-custom-verification-email/route.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSesSendCustomVerificationEmailContract } from '@/lib/api/contracts/tools/aws/ses-send-custom-verification-email' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSESClient, sendCustomVerificationEmail } from '../utils' - -const logger = createLogger('SESSendCustomVerificationEmailAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSesSendCustomVerificationEmailContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info('Sending SES custom verification email') - - const client = createSESClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await sendCustomVerificationEmail(client, { - emailAddress: params.emailAddress, - templateName: params.templateName, - configurationSetName: params.configurationSetName, - }) - - logger.info(`Sent custom verification email to '${params.emailAddress}'`) - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to send custom verification email:', error) - - return NextResponse.json( - { error: `Failed to send custom verification email: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ses/send-email/route.ts b/apps/sim/app/api/tools/ses/send-email/route.ts deleted file mode 100644 index 35f3cc7840b..00000000000 --- a/apps/sim/app/api/tools/ses/send-email/route.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSesSendEmailContract } from '@/lib/api/contracts/tools/aws/ses-send-email' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSESClient, sendEmail } from '../utils' - -const logger = createLogger('SESSendEmailAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSesSendEmailContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const toList = params.toAddresses - .split(',') - .map((s) => s.trim()) - .filter(Boolean) - - logger.info(`Sending email from ${params.fromAddress} to ${toList.length} recipient(s)`) - - const client = createSESClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await sendEmail(client, { - fromAddress: params.fromAddress, - toAddresses: toList, - subject: params.subject, - bodyText: params.bodyText, - bodyHtml: params.bodyHtml, - ccAddresses: params.ccAddresses - ? params.ccAddresses - .split(',') - .map((s) => s.trim()) - .filter(Boolean) - : null, - bccAddresses: params.bccAddresses - ? params.bccAddresses - .split(',') - .map((s) => s.trim()) - .filter(Boolean) - : null, - replyToAddresses: params.replyToAddresses - ? params.replyToAddresses - .split(',') - .map((s) => s.trim()) - .filter(Boolean) - : null, - configurationSetName: params.configurationSetName, - }) - - logger.info(`Email sent successfully, messageId: ${result.messageId}`) - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to send email:', error) - - return NextResponse.json( - { error: `Failed to send email: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ses/send-templated-email/route.ts b/apps/sim/app/api/tools/ses/send-templated-email/route.ts deleted file mode 100644 index 7a9e9f39f70..00000000000 --- a/apps/sim/app/api/tools/ses/send-templated-email/route.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSesSendTemplatedEmailContract } from '@/lib/api/contracts/tools/aws/ses-send-templated-email' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSESClient, sendTemplatedEmail } from '../utils' - -const logger = createLogger('SESSendTemplatedEmailAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSesSendTemplatedEmailContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const toList = params.toAddresses - .split(',') - .map((s) => s.trim()) - .filter(Boolean) - - logger.info( - `Sending templated email from ${params.fromAddress} using template '${params.templateName}'` - ) - - const client = createSESClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await sendTemplatedEmail(client, { - fromAddress: params.fromAddress, - toAddresses: toList, - templateName: params.templateName, - templateData: params.templateData, - ccAddresses: params.ccAddresses - ? params.ccAddresses - .split(',') - .map((s) => s.trim()) - .filter(Boolean) - : null, - bccAddresses: params.bccAddresses - ? params.bccAddresses - .split(',') - .map((s) => s.trim()) - .filter(Boolean) - : null, - configurationSetName: params.configurationSetName, - }) - - logger.info(`Templated email sent successfully, messageId: ${result.messageId}`) - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to send templated email:', error) - - return NextResponse.json( - { error: `Failed to send templated email: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ses/update-template/route.ts b/apps/sim/app/api/tools/ses/update-template/route.ts deleted file mode 100644 index 4bebcff2100..00000000000 --- a/apps/sim/app/api/tools/ses/update-template/route.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSesUpdateTemplateContract } from '@/lib/api/contracts/tools/aws/ses-update-template' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSESClient, updateTemplate } from '../utils' - -const logger = createLogger('SESUpdateTemplateAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSesUpdateTemplateContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info('Updating SES email template') - - const client = createSESClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await updateTemplate(client, { - templateName: params.templateName, - subjectPart: params.subjectPart, - textPart: params.textPart, - htmlPart: params.htmlPart, - }) - - logger.info(`Updated template '${params.templateName}'`) - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to update template:', error) - - return NextResponse.json( - { error: `Failed to update template: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ses/utils.ts b/apps/sim/app/api/tools/ses/utils.ts deleted file mode 100644 index 62fec2b74a0..00000000000 --- a/apps/sim/app/api/tools/ses/utils.ts +++ /dev/null @@ -1,563 +0,0 @@ -import { - CreateConfigurationSetCommand, - CreateEmailIdentityCommand, - CreateEmailTemplateCommand, - DeleteEmailIdentityCommand, - DeleteEmailTemplateCommand, - DeleteSuppressedDestinationCommand, - GetAccountCommand, - GetEmailIdentityCommand, - GetEmailTemplateCommand, - GetSuppressedDestinationCommand, - ListEmailIdentitiesCommand, - ListEmailTemplatesCommand, - ListSuppressedDestinationsCommand, - PutSuppressedDestinationCommand, - SESv2Client, - SendBulkEmailCommand, - SendCustomVerificationEmailCommand, - SendEmailCommand, - type SuppressionListReason, - type TlsPolicy, - UpdateEmailTemplateCommand, -} from '@aws-sdk/client-sesv2' -import { z } from 'zod' -import type { SESConnectionConfig } from '@/tools/ses/types' - -const SesBulkEmailDestinationSchema = z.object({ - toAddresses: z.array(z.string().email()), - templateData: z.string().optional(), -}) - -type SesBulkEmailDestination = z.infer - -export function createSESClient(config: SESConnectionConfig): SESv2Client { - return new SESv2Client({ - region: config.region, - credentials: { - accessKeyId: config.accessKeyId, - secretAccessKey: config.secretAccessKey, - }, - }) -} - -export async function sendEmail( - client: SESv2Client, - params: { - fromAddress: string - toAddresses: string[] - subject: string - bodyText?: string | null - bodyHtml?: string | null - ccAddresses?: string[] | null - bccAddresses?: string[] | null - replyToAddresses?: string[] | null - configurationSetName?: string | null - } -) { - const command = new SendEmailCommand({ - FromEmailAddress: params.fromAddress, - Destination: { - ToAddresses: params.toAddresses, - ...(params.ccAddresses?.length ? { CcAddresses: params.ccAddresses } : {}), - ...(params.bccAddresses?.length ? { BccAddresses: params.bccAddresses } : {}), - }, - Content: { - Simple: { - Subject: { Data: params.subject }, - Body: { - ...(params.bodyText ? { Text: { Data: params.bodyText } } : {}), - ...(params.bodyHtml ? { Html: { Data: params.bodyHtml } } : {}), - }, - }, - }, - ...(params.replyToAddresses?.length ? { ReplyToAddresses: params.replyToAddresses } : {}), - ...(params.configurationSetName ? { ConfigurationSetName: params.configurationSetName } : {}), - }) - - const response = await client.send(command) - - return { - messageId: response.MessageId ?? '', - } -} - -export async function sendTemplatedEmail( - client: SESv2Client, - params: { - fromAddress: string - toAddresses: string[] - templateName: string - templateData: string - ccAddresses?: string[] | null - bccAddresses?: string[] | null - configurationSetName?: string | null - } -) { - const command = new SendEmailCommand({ - FromEmailAddress: params.fromAddress, - Destination: { - ToAddresses: params.toAddresses, - ...(params.ccAddresses?.length ? { CcAddresses: params.ccAddresses } : {}), - ...(params.bccAddresses?.length ? { BccAddresses: params.bccAddresses } : {}), - }, - Content: { - Template: { - TemplateName: params.templateName, - TemplateData: params.templateData, - }, - }, - ...(params.configurationSetName ? { ConfigurationSetName: params.configurationSetName } : {}), - }) - - const response = await client.send(command) - - return { - messageId: response.MessageId ?? '', - } -} - -export function parseBulkEmailDestinations(destinationsJson: string): SesBulkEmailDestination[] { - const destinations = JSON.parse(destinationsJson) - return z.array(SesBulkEmailDestinationSchema).parse(destinations) -} - -export async function sendBulkEmail( - client: SESv2Client, - params: { - fromAddress: string - templateName: string - destinations: SesBulkEmailDestination[] - defaultTemplateData?: string | null - configurationSetName?: string | null - } -) { - const command = new SendBulkEmailCommand({ - FromEmailAddress: params.fromAddress, - DefaultContent: { - Template: { - TemplateName: params.templateName, - ...(params.defaultTemplateData ? { TemplateData: params.defaultTemplateData } : {}), - }, - }, - BulkEmailEntries: params.destinations.map((dest) => ({ - Destination: { ToAddresses: dest.toAddresses }, - ...(dest.templateData - ? { - ReplacementEmailContent: { - ReplacementTemplate: { - ReplacementTemplateData: dest.templateData, - }, - }, - } - : {}), - })), - ...(params.configurationSetName ? { ConfigurationSetName: params.configurationSetName } : {}), - }) - - const response = await client.send(command) - - const results = (response.BulkEmailEntryResults ?? []).map((r) => ({ - messageId: r.MessageId ?? null, - status: r.Status ?? 'UNKNOWN', - error: r.Error ?? null, - })) - - const successCount = results.filter((r) => r.status === 'SUCCESS').length - const failureCount = results.length - successCount - - return { results, successCount, failureCount } -} - -export async function listIdentities( - client: SESv2Client, - params: { - pageSize?: number | null - nextToken?: string | null - } -) { - const command = new ListEmailIdentitiesCommand({ - ...(params.pageSize != null ? { PageSize: params.pageSize } : {}), - ...(params.nextToken ? { NextToken: params.nextToken } : {}), - }) - - const response = await client.send(command) - - const identities = (response.EmailIdentities ?? []).map((identity) => ({ - identityName: identity.IdentityName ?? '', - identityType: identity.IdentityType ?? '', - sendingEnabled: identity.SendingEnabled ?? false, - verificationStatus: identity.VerificationStatus ?? '', - })) - - return { - identities, - nextToken: response.NextToken ?? null, - count: identities.length, - } -} - -export async function getAccount(client: SESv2Client) { - const command = new GetAccountCommand({}) - const response = await client.send(command) - - return { - sendingEnabled: response.SendingEnabled ?? false, - max24HourSend: response.SendQuota?.Max24HourSend ?? 0, - maxSendRate: response.SendQuota?.MaxSendRate ?? 0, - sentLast24Hours: response.SendQuota?.SentLast24Hours ?? 0, - } -} - -export async function createTemplate( - client: SESv2Client, - params: { - templateName: string - subjectPart: string - textPart?: string | null - htmlPart?: string | null - } -) { - const command = new CreateEmailTemplateCommand({ - TemplateName: params.templateName, - TemplateContent: { - Subject: params.subjectPart, - ...(params.textPart ? { Text: params.textPart } : {}), - ...(params.htmlPart ? { Html: params.htmlPart } : {}), - }, - }) - - await client.send(command) - - return { - message: `Template '${params.templateName}' created successfully`, - } -} - -export async function getTemplate(client: SESv2Client, templateName: string) { - const command = new GetEmailTemplateCommand({ TemplateName: templateName }) - const response = await client.send(command) - - return { - templateName: response.TemplateName ?? '', - subjectPart: response.TemplateContent?.Subject ?? '', - textPart: response.TemplateContent?.Text ?? null, - htmlPart: response.TemplateContent?.Html ?? null, - } -} - -export async function listTemplates( - client: SESv2Client, - params: { - pageSize?: number | null - nextToken?: string | null - } -) { - const command = new ListEmailTemplatesCommand({ - ...(params.pageSize != null ? { PageSize: params.pageSize } : {}), - ...(params.nextToken ? { NextToken: params.nextToken } : {}), - }) - - const response = await client.send(command) - - const templates = (response.TemplatesMetadata ?? []).map((t) => ({ - templateName: t.TemplateName ?? '', - createdTimestamp: t.CreatedTimestamp?.toISOString() ?? null, - })) - - return { - templates, - nextToken: response.NextToken ?? null, - count: templates.length, - } -} - -export async function deleteTemplate(client: SESv2Client, templateName: string) { - const command = new DeleteEmailTemplateCommand({ TemplateName: templateName }) - await client.send(command) - - return { - message: `Template '${templateName}' deleted successfully`, - } -} - -export async function updateTemplate( - client: SESv2Client, - params: { - templateName: string - subjectPart: string - textPart?: string | null - htmlPart?: string | null - } -) { - const command = new UpdateEmailTemplateCommand({ - TemplateName: params.templateName, - TemplateContent: { - Subject: params.subjectPart, - ...(params.textPart ? { Text: params.textPart } : {}), - ...(params.htmlPart ? { Html: params.htmlPart } : {}), - }, - }) - - await client.send(command) - - return { - message: `Template '${params.templateName}' updated successfully`, - } -} - -export async function putSuppressedDestination( - client: SESv2Client, - params: { emailAddress: string; reason: SuppressionListReason } -) { - const command = new PutSuppressedDestinationCommand({ - EmailAddress: params.emailAddress, - Reason: params.reason, - }) - - await client.send(command) - - return { - message: `Email address '${params.emailAddress}' added to the suppression list`, - } -} - -export async function deleteSuppressedDestination(client: SESv2Client, emailAddress: string) { - const command = new DeleteSuppressedDestinationCommand({ EmailAddress: emailAddress }) - await client.send(command) - - return { - message: `Email address '${emailAddress}' removed from the suppression list`, - } -} - -export async function getSuppressedDestination(client: SESv2Client, emailAddress: string) { - const command = new GetSuppressedDestinationCommand({ EmailAddress: emailAddress }) - const response = await client.send(command) - const destination = response.SuppressedDestination - - return { - emailAddress: destination?.EmailAddress ?? emailAddress, - reason: destination?.Reason ?? '', - lastUpdateTime: destination?.LastUpdateTime?.toISOString() ?? null, - messageId: destination?.Attributes?.MessageId ?? null, - feedbackId: destination?.Attributes?.FeedbackId ?? null, - } -} - -export async function listSuppressedDestinations( - client: SESv2Client, - params: { - reasons?: SuppressionListReason[] | null - startDate?: Date | null - endDate?: Date | null - pageSize?: number | null - nextToken?: string | null - } -) { - const command = new ListSuppressedDestinationsCommand({ - ...(params.reasons?.length ? { Reasons: params.reasons } : {}), - ...(params.startDate ? { StartDate: params.startDate } : {}), - ...(params.endDate ? { EndDate: params.endDate } : {}), - ...(params.pageSize != null ? { PageSize: params.pageSize } : {}), - ...(params.nextToken ? { NextToken: params.nextToken } : {}), - }) - - const response = await client.send(command) - - const destinations = (response.SuppressedDestinationSummaries ?? []).map((d) => ({ - emailAddress: d.EmailAddress ?? '', - reason: d.Reason ?? '', - lastUpdateTime: d.LastUpdateTime?.toISOString() ?? null, - })) - - return { - destinations, - nextToken: response.NextToken ?? null, - count: destinations.length, - } -} - -export async function createEmailIdentity( - client: SESv2Client, - params: { - emailIdentity: string - dkimSigningAttributes?: { - domainSigningSelector?: string - domainSigningPrivateKey?: string - nextSigningKeyLength?: 'RSA_1024_BIT' | 'RSA_2048_BIT' - } | null - tags?: Array<{ key: string; value: string }> | null - configurationSetName?: string | null - } -) { - const command = new CreateEmailIdentityCommand({ - EmailIdentity: params.emailIdentity, - ...(params.dkimSigningAttributes - ? { - DkimSigningAttributes: { - ...(params.dkimSigningAttributes.domainSigningSelector - ? { DomainSigningSelector: params.dkimSigningAttributes.domainSigningSelector } - : {}), - ...(params.dkimSigningAttributes.domainSigningPrivateKey - ? { DomainSigningPrivateKey: params.dkimSigningAttributes.domainSigningPrivateKey } - : {}), - ...(params.dkimSigningAttributes.nextSigningKeyLength - ? { NextSigningKeyLength: params.dkimSigningAttributes.nextSigningKeyLength } - : {}), - }, - } - : {}), - ...(params.tags?.length - ? { Tags: params.tags.map((t) => ({ Key: t.key, Value: t.value })) } - : {}), - ...(params.configurationSetName ? { ConfigurationSetName: params.configurationSetName } : {}), - }) - - const response = await client.send(command) - - return { - identityType: response.IdentityType ?? '', - verifiedForSendingStatus: response.VerifiedForSendingStatus ?? false, - dkimAttributes: response.DkimAttributes - ? { - signingEnabled: response.DkimAttributes.SigningEnabled ?? null, - status: response.DkimAttributes.Status ?? null, - tokens: response.DkimAttributes.Tokens ?? [], - signingAttributesOrigin: response.DkimAttributes.SigningAttributesOrigin ?? null, - nextSigningKeyLength: response.DkimAttributes.NextSigningKeyLength ?? null, - currentSigningKeyLength: response.DkimAttributes.CurrentSigningKeyLength ?? null, - lastKeyGenerationTimestamp: - response.DkimAttributes.LastKeyGenerationTimestamp?.toISOString() ?? null, - signingHostedZone: response.DkimAttributes.SigningHostedZone ?? null, - } - : null, - } -} - -export async function deleteEmailIdentity(client: SESv2Client, emailIdentity: string) { - const command = new DeleteEmailIdentityCommand({ EmailIdentity: emailIdentity }) - await client.send(command) - - return { - message: `Email identity '${emailIdentity}' deleted successfully`, - } -} - -export async function getEmailIdentity(client: SESv2Client, emailIdentity: string) { - const command = new GetEmailIdentityCommand({ EmailIdentity: emailIdentity }) - const response = await client.send(command) - - return { - identityType: response.IdentityType ?? '', - verifiedForSendingStatus: response.VerifiedForSendingStatus ?? false, - verificationStatus: response.VerificationStatus ?? null, - feedbackForwardingStatus: response.FeedbackForwardingStatus ?? null, - configurationSetName: response.ConfigurationSetName ?? null, - dkimAttributes: response.DkimAttributes - ? { - signingEnabled: response.DkimAttributes.SigningEnabled ?? null, - status: response.DkimAttributes.Status ?? null, - tokens: response.DkimAttributes.Tokens ?? [], - signingAttributesOrigin: response.DkimAttributes.SigningAttributesOrigin ?? null, - nextSigningKeyLength: response.DkimAttributes.NextSigningKeyLength ?? null, - currentSigningKeyLength: response.DkimAttributes.CurrentSigningKeyLength ?? null, - lastKeyGenerationTimestamp: - response.DkimAttributes.LastKeyGenerationTimestamp?.toISOString() ?? null, - signingHostedZone: response.DkimAttributes.SigningHostedZone ?? null, - } - : null, - mailFromAttributes: response.MailFromAttributes - ? { - mailFromDomain: response.MailFromAttributes.MailFromDomain ?? null, - mailFromDomainStatus: response.MailFromAttributes.MailFromDomainStatus ?? null, - behaviorOnMxFailure: response.MailFromAttributes.BehaviorOnMxFailure ?? null, - } - : null, - policies: response.Policies ?? null, - tags: (response.Tags ?? []).map((t) => ({ key: t.Key ?? '', value: t.Value ?? '' })), - verificationInfo: response.VerificationInfo - ? { - errorType: response.VerificationInfo.ErrorType ?? null, - lastCheckedTimestamp: - response.VerificationInfo.LastCheckedTimestamp?.toISOString() ?? null, - lastSuccessTimestamp: - response.VerificationInfo.LastSuccessTimestamp?.toISOString() ?? null, - } - : null, - } -} - -export async function createConfigurationSet( - client: SESv2Client, - params: { - configurationSetName: string - customRedirectDomain?: string | null - httpsPolicy?: 'REQUIRE' | 'REQUIRE_OPEN_ONLY' | 'OPTIONAL' | null - tlsPolicy?: TlsPolicy | null - sendingPoolName?: string | null - reputationMetricsEnabled?: boolean | null - sendingEnabled?: boolean | null - suppressedReasons?: SuppressionListReason[] | null - tags?: Array<{ key: string; value: string }> | null - } -) { - const command = new CreateConfigurationSetCommand({ - ConfigurationSetName: params.configurationSetName, - ...(params.customRedirectDomain - ? { - TrackingOptions: { - CustomRedirectDomain: params.customRedirectDomain, - ...(params.httpsPolicy ? { HttpsPolicy: params.httpsPolicy } : {}), - }, - } - : {}), - ...(params.tlsPolicy || params.sendingPoolName - ? { - DeliveryOptions: { - ...(params.tlsPolicy ? { TlsPolicy: params.tlsPolicy } : {}), - ...(params.sendingPoolName ? { SendingPoolName: params.sendingPoolName } : {}), - }, - } - : {}), - ...(params.reputationMetricsEnabled != null - ? { ReputationOptions: { ReputationMetricsEnabled: params.reputationMetricsEnabled } } - : {}), - ...(params.sendingEnabled != null - ? { SendingOptions: { SendingEnabled: params.sendingEnabled } } - : {}), - ...(params.suppressedReasons?.length - ? { SuppressionOptions: { SuppressedReasons: params.suppressedReasons } } - : {}), - ...(params.tags?.length - ? { Tags: params.tags.map((t) => ({ Key: t.key, Value: t.value })) } - : {}), - }) - - await client.send(command) - - return { - message: `Configuration set '${params.configurationSetName}' created successfully`, - } -} - -export async function sendCustomVerificationEmail( - client: SESv2Client, - params: { - emailAddress: string - templateName: string - configurationSetName?: string | null - } -) { - const command = new SendCustomVerificationEmailCommand({ - EmailAddress: params.emailAddress, - TemplateName: params.templateName, - ...(params.configurationSetName ? { ConfigurationSetName: params.configurationSetName } : {}), - }) - - const response = await client.send(command) - - return { - messageId: response.MessageId ?? '', - } -} diff --git a/apps/sim/app/api/tools/sftp/delete/route.ts b/apps/sim/app/api/tools/sftp/delete/route.ts deleted file mode 100644 index 42e260dfe2e..00000000000 --- a/apps/sim/app/api/tools/sftp/delete/route.ts +++ /dev/null @@ -1,166 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import type { SFTPWrapper } from 'ssh2' -import { sftpDeleteContract } from '@/lib/api/contracts/storage-transfer' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createSftpConnection, - getFileType, - getSftp, - isPathSafe, - sanitizePath, - sftpIsDirectory, -} from '@/app/api/tools/sftp/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('SftpDeleteAPI') - -/** - * Recursively deletes a directory and all its contents - */ -async function deleteRecursive(sftp: SFTPWrapper, dirPath: string): Promise { - const entries = await new Promise>((resolve, reject) => { - sftp.readdir(dirPath, (err, list) => { - if (err) { - reject(err) - } else { - resolve(list) - } - }) - }) - - for (const entry of entries) { - if (entry.filename === '.' || entry.filename === '..') continue - - const entryPath = `${dirPath}/${entry.filename}` - const entryType = getFileType(entry.attrs) - - if (entryType === 'directory') { - await deleteRecursive(sftp, entryPath) - } else { - await new Promise((resolve, reject) => { - sftp.unlink(entryPath, (err) => { - if (err) reject(err) - else resolve() - }) - }) - } - } - - await new Promise((resolve, reject) => { - sftp.rmdir(dirPath, (err) => { - if (err) reject(err) - else resolve() - }) - }) -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized SFTP delete attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated SFTP delete request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const parsed = await parseRequest(sftpDeleteContract, request, {}) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - if (!isPathSafe(params.remotePath)) { - logger.warn(`[${requestId}] Path traversal attempt detected in remotePath`) - return NextResponse.json( - { error: 'Invalid remote path: path traversal sequences are not allowed' }, - { status: 400 } - ) - } - - logger.info(`[${requestId}] Connecting to SFTP server ${params.host}:${params.port}`) - - const client = await createSftpConnection({ - host: params.host, - port: params.port, - username: params.username, - password: params.password, - privateKey: params.privateKey, - passphrase: params.passphrase, - }) - - try { - const sftp = await getSftp(client) - const remotePath = sanitizePath(params.remotePath) - - logger.info(`[${requestId}] Deleting ${remotePath} (recursive: ${params.recursive})`) - - const isDir = await sftpIsDirectory(sftp, remotePath) - - if (isDir) { - if (params.recursive) { - await deleteRecursive(sftp, remotePath) - } else { - await new Promise((resolve, reject) => { - sftp.rmdir(remotePath, (err) => { - if (err) { - if (err.message.includes('not empty')) { - reject( - new Error( - 'Directory is not empty. Use recursive: true to delete non-empty directories.' - ) - ) - } else { - reject(err) - } - } else { - resolve() - } - }) - }) - } - } else { - await new Promise((resolve, reject) => { - sftp.unlink(remotePath, (err) => { - if (err) { - if (err.message.includes('No such file')) { - reject(new Error(`File not found: ${remotePath}`)) - } else { - reject(err) - } - } else { - resolve() - } - }) - }) - } - - logger.info(`[${requestId}] Successfully deleted ${remotePath}`) - - return NextResponse.json({ - success: true, - deletedPath: remotePath, - message: `Successfully deleted ${remotePath}`, - }) - } finally { - client.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] SFTP delete failed:`, error) - - return NextResponse.json({ error: `SFTP delete failed: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/sftp/download/route.ts b/apps/sim/app/api/tools/sftp/download/route.ts deleted file mode 100644 index 8b1bd70680f..00000000000 --- a/apps/sim/app/api/tools/sftp/download/route.ts +++ /dev/null @@ -1,142 +0,0 @@ -import path from 'path' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { sftpDownloadContract } from '@/lib/api/contracts/storage-transfer' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getFileExtension, getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils' -import { - createSftpConnection, - getSftp, - isPathSafe, - MAX_SFTP_READ_BYTES, - readSftpFileCapped, - sanitizePath, -} from '@/app/api/tools/sftp/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('SftpDownloadAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized SFTP download attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated SFTP download request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const parsed = await parseRequest(sftpDownloadContract, request, {}) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - if (!isPathSafe(params.remotePath)) { - logger.warn(`[${requestId}] Path traversal attempt detected in remotePath`) - return NextResponse.json( - { error: 'Invalid remote path: path traversal sequences are not allowed' }, - { status: 400 } - ) - } - - logger.info(`[${requestId}] Connecting to SFTP server ${params.host}:${params.port}`) - - const client = await createSftpConnection({ - host: params.host, - port: params.port, - username: params.username, - password: params.password, - privateKey: params.privateKey, - passphrase: params.passphrase, - }) - - try { - const sftp = await getSftp(client) - const remotePath = sanitizePath(params.remotePath) - - const stats = await new Promise<{ size: number }>((resolve, reject) => { - sftp.stat(remotePath, (err, stats) => { - if (err) { - if (err.message.includes('No such file')) { - reject(new Error(`File not found: ${remotePath}`)) - } else { - reject(err) - } - } else { - resolve(stats) - } - }) - }) - - if (stats.size > MAX_SFTP_READ_BYTES) { - const sizeMB = (stats.size / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { success: false, error: `File size (${sizeMB}MB) exceeds download limit of 50MB` }, - { status: 413 } - ) - } - - logger.info(`[${requestId}] Downloading file ${remotePath} (${stats.size} bytes)`) - - const buffer = await readSftpFileCapped( - sftp, - remotePath, - MAX_SFTP_READ_BYTES, - 'SFTP download' - ) - const fileName = path.basename(remotePath) - const extension = getFileExtension(fileName) - const mimeType = getMimeTypeFromExtension(extension) - - let content: string - if (params.encoding === 'base64') { - content = buffer.toString('base64') - } else { - content = buffer.toString('utf-8') - } - - logger.info(`[${requestId}] Downloaded ${fileName} (${buffer.length} bytes)`) - - return NextResponse.json({ - success: true, - fileName, - file: { - name: fileName, - mimeType, - data: buffer.toString('base64'), - size: buffer.length, - }, - content, - size: buffer.length, - encoding: params.encoding, - message: `Successfully downloaded ${fileName}`, - }) - } finally { - client.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - - if (isPayloadSizeLimitError(error)) { - logger.warn(`[${requestId}] SFTP download aborted: ${errorMessage}`) - return NextResponse.json({ success: false, error: errorMessage }, { status: 413 }) - } - - logger.error(`[${requestId}] SFTP download failed:`, error) - - return NextResponse.json({ error: `SFTP download failed: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/sftp/mkdir/route.ts b/apps/sim/app/api/tools/sftp/mkdir/route.ts deleted file mode 100644 index 122568e7918..00000000000 --- a/apps/sim/app/api/tools/sftp/mkdir/route.ts +++ /dev/null @@ -1,146 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import type { SFTPWrapper } from 'ssh2' -import { sftpMkdirContract } from '@/lib/api/contracts/storage-transfer' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createSftpConnection, - getSftp, - isPathSafe, - sanitizePath, - sftpExists, -} from '@/app/api/tools/sftp/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('SftpMkdirAPI') - -/** - * Creates directory recursively (like mkdir -p) - */ -async function mkdirRecursive(sftp: SFTPWrapper, dirPath: string): Promise { - const parts = dirPath.split('/').filter(Boolean) - let currentPath = dirPath.startsWith('/') ? '' : '' - - for (const part of parts) { - currentPath = currentPath - ? `${currentPath}/${part}` - : dirPath.startsWith('/') - ? `/${part}` - : part - - const exists = await sftpExists(sftp, currentPath) - if (!exists) { - await new Promise((resolve, reject) => { - sftp.mkdir(currentPath, (err) => { - if (err && !err.message.includes('already exists')) { - reject(err) - } else { - resolve() - } - }) - }) - } - } -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized SFTP mkdir attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated SFTP mkdir request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const parsed = await parseRequest(sftpMkdirContract, request, {}) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - if (!isPathSafe(params.remotePath)) { - logger.warn(`[${requestId}] Path traversal attempt detected in remotePath`) - return NextResponse.json( - { error: 'Invalid remote path: path traversal sequences are not allowed' }, - { status: 400 } - ) - } - - logger.info(`[${requestId}] Connecting to SFTP server ${params.host}:${params.port}`) - - const client = await createSftpConnection({ - host: params.host, - port: params.port, - username: params.username, - password: params.password, - privateKey: params.privateKey, - passphrase: params.passphrase, - }) - - try { - const sftp = await getSftp(client) - const remotePath = sanitizePath(params.remotePath) - - logger.info( - `[${requestId}] Creating directory ${remotePath} (recursive: ${params.recursive})` - ) - - if (params.recursive) { - await mkdirRecursive(sftp, remotePath) - } else { - const exists = await sftpExists(sftp, remotePath) - if (exists) { - return NextResponse.json( - { error: `Directory already exists: ${remotePath}` }, - { status: 409 } - ) - } - - await new Promise((resolve, reject) => { - sftp.mkdir(remotePath, (err) => { - if (err) { - if (err.message.includes('No such file')) { - reject( - new Error( - 'Parent directory does not exist. Use recursive: true to create parent directories.' - ) - ) - } else { - reject(err) - } - } else { - resolve() - } - }) - }) - } - - logger.info(`[${requestId}] Successfully created directory ${remotePath}`) - - return NextResponse.json({ - success: true, - createdPath: remotePath, - message: `Successfully created directory ${remotePath}`, - }) - } finally { - client.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] SFTP mkdir failed:`, error) - - return NextResponse.json({ error: `SFTP mkdir failed: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/sftp/upload/route.ts b/apps/sim/app/api/tools/sftp/upload/route.ts deleted file mode 100644 index bca922e7d20..00000000000 --- a/apps/sim/app/api/tools/sftp/upload/route.ts +++ /dev/null @@ -1,234 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { sftpUploadContract } from '@/lib/api/contracts/storage-transfer' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { - createSftpConnection, - getSftp, - isPathSafe, - sanitizeFileName, - sanitizePath, - sftpExists, -} from '@/app/api/tools/sftp/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('SftpUploadAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized SFTP upload attempt: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated SFTP upload request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const parsed = await parseRequest(sftpUploadContract, request, {}) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const hasFiles = params.files && params.files.length > 0 - const hasDirectContent = params.fileContent && params.fileName - - if (!hasFiles && !hasDirectContent) { - return NextResponse.json( - { error: 'Either files or fileContent with fileName must be provided' }, - { status: 400 } - ) - } - - if (!isPathSafe(params.remotePath)) { - logger.warn(`[${requestId}] Path traversal attempt detected in remotePath`) - return NextResponse.json( - { error: 'Invalid remote path: path traversal sequences are not allowed' }, - { status: 400 } - ) - } - - logger.info(`[${requestId}] Connecting to SFTP server ${params.host}:${params.port}`) - - const client = await createSftpConnection({ - host: params.host, - port: params.port, - username: params.username, - password: params.password, - privateKey: params.privateKey, - passphrase: params.passphrase, - }) - - try { - const sftp = await getSftp(client) - const remotePath = sanitizePath(params.remotePath) - const uploadedFiles: Array<{ name: string; remotePath: string; size: number }> = [] - - if (hasFiles) { - const rawFiles = params.files! - logger.info(`[${requestId}] Processing ${rawFiles.length} file(s) for upload`) - - const userFiles = processFilesToUserFiles(rawFiles, requestId, logger) - - const totalSize = userFiles.reduce((sum, file) => sum + file.size, 0) - const maxSize = 100 * 1024 * 1024 - - if (totalSize > maxSize) { - const sizeMB = (totalSize / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { success: false, error: `Total file size (${sizeMB}MB) exceeds limit of 100MB` }, - { status: 400 } - ) - } - - let resolvedTotal = 0 - for (const file of userFiles) { - try { - const denied = await assertToolFileAccess( - file.key, - authResult.userId, - requestId, - logger - ) - if (denied) return denied - logger.info( - `[${requestId}] Downloading file for upload: ${file.name} (${file.size} bytes)` - ) - const { buffer } = await downloadServableFileFromStorage(file, requestId, logger, { - maxBytes: maxSize - resolvedTotal, - }) - - resolvedTotal += buffer.length - - const safeFileName = sanitizeFileName(file.name) - const fullRemotePath = remotePath.endsWith('/') - ? `${remotePath}${safeFileName}` - : `${remotePath}/${safeFileName}` - - const sanitizedRemotePath = sanitizePath(fullRemotePath) - - if (!params.overwrite) { - const exists = await sftpExists(sftp, sanitizedRemotePath) - if (exists) { - logger.warn(`[${requestId}] File ${sanitizedRemotePath} already exists, skipping`) - continue - } - } - - await new Promise((resolve, reject) => { - const writeStream = sftp.createWriteStream(sanitizedRemotePath, { - mode: params.permissions ? Number.parseInt(params.permissions, 8) : 0o644, - }) - - writeStream.on('error', reject) - writeStream.on('close', () => resolve()) - writeStream.end(buffer) - }) - - uploadedFiles.push({ - name: safeFileName, - remotePath: sanitizedRemotePath, - size: buffer.length, - }) - - logger.info(`[${requestId}] Uploaded ${safeFileName} to ${sanitizedRemotePath}`) - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - if (isPayloadSizeLimitError(error)) { - const observed = resolvedTotal + (error.observedBytes ?? file.size) - const sizeMB = (observed / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { success: false, error: `Total file size (${sizeMB}MB) exceeds limit of 100MB` }, - { status: 400 } - ) - } - logger.error(`[${requestId}] Failed to upload file ${file.name}:`, error) - throw new Error( - `Failed to upload file "${file.name}": ${getErrorMessage(error, 'Unknown error')}` - ) - } - } - } - - if (hasDirectContent) { - const safeFileName = sanitizeFileName(params.fileName!) - const fullRemotePath = remotePath.endsWith('/') - ? `${remotePath}${safeFileName}` - : `${remotePath}/${safeFileName}` - - const sanitizedRemotePath = sanitizePath(fullRemotePath) - - if (!params.overwrite) { - const exists = await sftpExists(sftp, sanitizedRemotePath) - if (exists) { - return NextResponse.json( - { error: 'File already exists and overwrite is disabled' }, - { status: 409 } - ) - } - } - - let content: Buffer - try { - content = Buffer.from(params.fileContent!, 'base64') - const reEncoded = content.toString('base64') - if (reEncoded !== params.fileContent) { - content = Buffer.from(params.fileContent!, 'utf-8') - } - } catch { - content = Buffer.from(params.fileContent!, 'utf-8') - } - - await new Promise((resolve, reject) => { - const writeStream = sftp.createWriteStream(sanitizedRemotePath, { - mode: params.permissions ? Number.parseInt(params.permissions, 8) : 0o644, - }) - - writeStream.on('error', reject) - writeStream.on('close', () => resolve()) - writeStream.end(content) - }) - - uploadedFiles.push({ - name: safeFileName, - remotePath: sanitizedRemotePath, - size: content.length, - }) - - logger.info(`[${requestId}] Uploaded direct content to ${sanitizedRemotePath}`) - } - - logger.info(`[${requestId}] SFTP upload completed: ${uploadedFiles.length} file(s)`) - - return NextResponse.json({ - success: true, - uploadedFiles, - message: `Successfully uploaded ${uploadedFiles.length} file(s)`, - }) - } finally { - client.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] SFTP upload failed:`, error) - - return NextResponse.json({ error: `SFTP upload failed: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/sftp/utils.test.ts b/apps/sim/app/api/tools/sftp/utils.test.ts deleted file mode 100644 index a9e3121717e..00000000000 --- a/apps/sim/app/api/tools/sftp/utils.test.ts +++ /dev/null @@ -1,72 +0,0 @@ -/** - * @vitest-environment node - */ -import { Readable } from 'stream' -import type { SFTPWrapper } from 'ssh2' -import { describe, expect, it, vi } from 'vitest' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { MAX_SFTP_READ_BYTES, readSftpFileCapped } from '@/app/api/tools/sftp/utils' - -/** - * Builds a fake SFTP wrapper whose read stream emits `chunkCount` chunks of - * `chunkSize` bytes — the shape of a malicious server that understates the - * file size in its stat reply and then streams unbounded data. - */ -function fakeSftp(chunkSize: number, chunkCount: number) { - let emitted = 0 - const stream = new Readable({ - read() { - if (emitted >= chunkCount) { - this.push(null) - return - } - emitted++ - this.push(Buffer.alloc(chunkSize, 0x41)) - }, - }) - const createReadStream = vi.fn(() => stream) - return { sftp: { createReadStream } as unknown as SFTPWrapper, stream, createReadStream } -} - -describe('readSftpFileCapped', () => { - it('resolves with the full contents when under the cap', async () => { - const { sftp, createReadStream } = fakeSftp(4, 3) - - const buffer = await readSftpFileCapped(sftp, '/file', 1024, 'file') - - expect(buffer.toString()).toBe('A'.repeat(12)) - expect(createReadStream).toHaveBeenCalledWith('/file') - }) - - it('rejects and destroys the stream once received bytes exceed the cap', async () => { - const { sftp, stream } = fakeSftp(8, 1_000_000) - - await expect(readSftpFileCapped(sftp, '/bomb', 16, 'file')).rejects.toSatisfy( - isPayloadSizeLimitError - ) - expect(stream.destroyed).toBe(true) - }) - - it('enforces the cap on actual bytes even when the file was reported as tiny', async () => { - const { sftp, stream } = fakeSftp(1024, 1_000_000) - - await expect(readSftpFileCapped(sftp, '/bomb', 4096, 'file')).rejects.toThrow( - /exceeds maximum size of 4096 bytes/ - ) - expect(stream.destroyed).toBe(true) - }) - - it('survives the late stream error ssh2 emits when the channel closes after an abort', async () => { - const { sftp, stream } = fakeSftp(8, 1_000_000) - - await expect(readSftpFileCapped(sftp, '/bomb', 16, 'file')).rejects.toSatisfy( - isPayloadSizeLimitError - ) - - expect(() => stream.emit('error', new Error('No response from server'))).not.toThrow() - }) - - it('caps remote reads at 50MB', () => { - expect(MAX_SFTP_READ_BYTES).toBe(50 * 1024 * 1024) - }) -}) diff --git a/apps/sim/app/api/tools/sftp/utils.ts b/apps/sim/app/api/tools/sftp/utils.ts deleted file mode 100644 index 17ad9c57623..00000000000 --- a/apps/sim/app/api/tools/sftp/utils.ts +++ /dev/null @@ -1,418 +0,0 @@ -import { createHash } from 'node:crypto' -import { createLogger } from '@sim/logger' -import { safeCompare } from '@sim/security/compare' -import { toError } from '@sim/utils/errors' -import { type Attributes, Client, type ConnectConfig, type SFTPWrapper } from 'ssh2' -import { validateDatabaseHost } from '@/lib/core/security/input-validation.server' -import { readNodeStreamToBufferWithLimit } from '@/lib/core/utils/stream-limits' - -const logger = createLogger('SftpUtils') - -const S_IFMT = 0o170000 -const S_IFDIR = 0o040000 -const S_IFREG = 0o100000 -const S_IFLNK = 0o120000 - -export interface SftpConnectionConfig { - host: string - port: number - username: string - password?: string | null - privateKey?: string | null - passphrase?: string | null - /** - * Idle socket timeout in ms, forwarded to ssh2's `sock.setTimeout`. Left - * unset the socket has no idle timeout at all (ssh2 defaults it to `0`). - */ - timeout?: number - keepaliveInterval?: number - readyTimeout?: number - /** - * Expected SHA-256 host key fingerprint in the format `ssh-keyscan` and - * OpenSSH print (`SHA256:`). The `SHA256:` prefix and any base64 - * padding are optional. When set, a server presenting a different host key is - * rejected before authentication runs. When omitted, the host is not - * verified — ssh2's default behavior. - */ - hostFingerprint?: string | null -} - -/** - * Normalizes a user-supplied SHA-256 fingerprint for comparison: trims, drops - * an optional `SHA256:` prefix, and strips base64 `=` padding, which OpenSSH - * omits but copy/paste sources sometimes include. - */ -function normalizeSha256Fingerprint(value: string): string { - return value - .trim() - .replace(/^sha256:/i, '') - .replace(/=+$/, '') - .trim() -} - -/** - * Computes the OpenSSH SHA-256 fingerprint of a host key. ssh2 hands the - * verifier the raw SSH wire-format public key blob — the same bytes OpenSSH - * base64-encodes into `known_hosts` — so hashing it directly reproduces the - * unpadded base64 digest that `ssh-keyscan | ssh-keygen -lf -` prints. - */ -function computeHostKeyFingerprint(hostKey: Buffer): string { - return createHash('sha256').update(hostKey).digest('base64').replace(/=+$/, '') -} - -/** - * Formats SSH/SFTP errors with helpful troubleshooting context - */ -function formatSftpError(err: Error, config: { host: string; port: number }): Error { - const errorMessage = err.message.toLowerCase() - const { host, port } = config - - if (errorMessage.includes('econnrefused') || errorMessage.includes('connection refused')) { - return new Error( - `Connection refused to ${host}:${port}. ` + - `Please verify: (1) SSH/SFTP server is running, ` + - `(2) Port ${port} is correct, ` + - `(3) Firewall allows connections.` - ) - } - - if (errorMessage.includes('econnreset') || errorMessage.includes('connection reset')) { - return new Error( - `Connection reset by ${host}:${port}. ` + - `This usually means: (1) Wrong port number, ` + - `(2) Server rejected the connection, ` + - `(3) Network/firewall interrupted the connection.` - ) - } - - if (errorMessage.includes('etimedout') || errorMessage.includes('timeout')) { - return new Error( - `Connection timed out to ${host}:${port}. ` + - `Please verify: (1) Host is reachable, ` + - `(2) No firewall is blocking the connection, ` + - `(3) The SFTP server is responding.` - ) - } - - if (errorMessage.includes('enotfound') || errorMessage.includes('getaddrinfo')) { - return new Error( - `Could not resolve hostname "${host}". Please verify the hostname or IP address is correct.` - ) - } - - if (errorMessage.includes('authentication') || errorMessage.includes('auth')) { - return new Error( - `Authentication failed on ${host}:${port}. ` + - `Please verify: (1) Username is correct, ` + - `(2) Password or private key is valid, ` + - `(3) User has SFTP access on the server.` - ) - } - - if ( - errorMessage.includes('key') && - (errorMessage.includes('parse') || errorMessage.includes('invalid')) - ) { - return new Error( - `Invalid private key format. ` + - `Please ensure you're using a valid OpenSSH private key ` + - `(starts with "-----BEGIN" and ends with "-----END").` - ) - } - - if (errorMessage.includes('host key') || errorMessage.includes('hostkey')) { - return new Error( - `Host key verification issue for ${host}. ` + - `This may be the first connection or the server's key has changed.` - ) - } - - return new Error(`SFTP connection to ${host}:${port} failed: ${err.message}`) -} - -/** - * Creates an SSH connection for SFTP using the provided configuration. - * Uses ssh2 library defaults which align with OpenSSH standards. - * - * When `hostFingerprint` is supplied the server's host key is pinned to it and - * a mismatch aborts the handshake before any credential is sent. Without it - * ssh2 accepts whatever host key answers, which is the pre-existing behavior - * kept for backward compatibility. - */ -export async function createSftpConnection(config: SftpConnectionConfig): Promise { - const host = config.host - - if (!host || host.trim() === '') { - throw new Error('Host is required. Please provide a valid hostname or IP address.') - } - - const hostValidation = await validateDatabaseHost(host, 'host') - if (!hostValidation.isValid) { - throw new Error(hostValidation.error) - } - - const resolvedHost = hostValidation.resolvedIP ?? host.trim() - - return new Promise((resolve, reject) => { - const client = new Client() - const port = config.port || 22 - - const hasPassword = config.password && config.password.trim() !== '' - const hasPrivateKey = config.privateKey && config.privateKey.trim() !== '' - - if (!hasPassword && !hasPrivateKey) { - reject(new Error('Authentication required. Please provide either a password or private key.')) - return - } - - const connectConfig: ConnectConfig = { - host: resolvedHost, - port, - username: config.username, - } - - if (config.readyTimeout !== undefined) { - connectConfig.readyTimeout = config.readyTimeout - } - if (config.keepaliveInterval !== undefined) { - connectConfig.keepaliveInterval = config.keepaliveInterval - } - if (config.timeout !== undefined) { - connectConfig.timeout = config.timeout - } - - const suppliedFingerprint = config.hostFingerprint?.trim() - const expectedFingerprint = suppliedFingerprint - ? normalizeSha256Fingerprint(suppliedFingerprint) - : undefined - - /** - * Fail closed rather than silently skipping verification. A value that is - * non-blank but normalizes away (`SHA256:`, `=`) would otherwise leave no - * `hostVerifier` installed, trusting whatever host answers — the opposite - * of what supplying a fingerprint asks for. - */ - if (suppliedFingerprint && !expectedFingerprint) { - throw new Error( - 'Host key fingerprint is not a valid SHA-256 fingerprint. Expected the base64 form printed by `ssh-keyscan | ssh-keygen -lf -`.' - ) - } - - /** - * Set when the pinned fingerprint does not match. ssh2 reports the - * rejection through a generic `'error'` event, so the precise cause is - * carried out of the verifier rather than re-derived from that message. - */ - let hostKeyRejection: Error | undefined - - if (expectedFingerprint) { - connectConfig.hostVerifier = (hostKey: Buffer): boolean => { - const actualFingerprint = computeHostKeyFingerprint(hostKey) - if (safeCompare(actualFingerprint, expectedFingerprint)) { - return true - } - hostKeyRejection = new Error( - `Host key verification failed for ${host}:${port}. ` + - `Expected SHA256:${expectedFingerprint} but the server presented SHA256:${actualFingerprint}. ` + - `Either the server's host key changed, or the connection was intercepted. ` + - `Re-run "ssh-keyscan -t rsa,ecdsa,ed25519 ${host}" to confirm the current key before updating the fingerprint.` - ) - logger.warn('SFTP host key fingerprint mismatch', { host, port }) - return false - } - } - - if (hasPrivateKey) { - connectConfig.privateKey = config.privateKey! - if (config.passphrase && config.passphrase.trim() !== '') { - connectConfig.passphrase = config.passphrase - } - } else if (hasPassword) { - connectConfig.password = config.password! - } - - client.on('ready', () => { - resolve(client) - }) - - client.on('error', (err) => { - reject(hostKeyRejection ?? formatSftpError(err, { host, port })) - }) - - /** - * ssh2 only re-emits the socket's `'timeout'` event; it never destroys the - * socket, so without this the connection would sit open forever after the - * idle timeout elapsed. - */ - client.on('timeout', () => { - client.destroy() - reject( - new Error( - `Connection to ${host}:${port} timed out after ${config.timeout}ms of inactivity.` - ) - ) - }) - - try { - client.connect(connectConfig) - } catch (err) { - reject(formatSftpError(toError(err), { host, port })) - } - }) -} - -/** - * Gets SFTP subsystem from SSH client - */ -export function getSftp(client: Client): Promise { - return new Promise((resolve, reject) => { - client.sftp((err, sftp) => { - if (err) { - reject(new Error(`Failed to start SFTP session: ${err.message}`)) - } else { - resolve(sftp) - } - }) - }) -} - -/** Maximum bytes a route will buffer from a remote SFTP file. */ -export const MAX_SFTP_READ_BYTES = 50 * 1024 * 1024 - -/** - * Reads a remote file into memory, enforcing the cap on the bytes actually - * received rather than on the `stat()` size the remote server reports. - * A caller-supplied SSH server can understate the size in its `SSH_FXP_STAT` - * reply and then stream unbounded data, so the stream is destroyed as soon as - * the running total exceeds `maxBytes`. Rejects with a `PayloadSizeLimitError`. - */ -export function readSftpFileCapped( - sftp: SFTPWrapper, - remotePath: string, - maxBytes: number, - label: string -): Promise { - const stream = sftp.createReadStream(remotePath) - - /** - * Closing the SSH client rejects every still-pending SFTP request with - * "No response from server", which lands as a late `error` on a stream the - * limiter has already detached from once it destroyed it. An `error` event - * with no listener is an uncaught exception, so keep one attached for the - * stream's whole life; the limiter's own handler still settles the promise. - */ - stream.on('error', () => {}) - - return readNodeStreamToBufferWithLimit(stream, { maxBytes, label }) -} - -/** - * Sanitizes a remote path to prevent path traversal attacks. - * Removes null bytes, normalizes path separators, and collapses traversal sequences. - * Based on OWASP Path Traversal prevention guidelines. - */ -export function sanitizePath(path: string): string { - let sanitized = path - sanitized = sanitized.replace(/\0/g, '') - sanitized = decodeURIComponent(sanitized) - sanitized = sanitized.replace(/\\/g, '/') - sanitized = sanitized.replace(/\/+/g, '/') - sanitized = sanitized.trim() - return sanitized -} - -/** - * Sanitizes a filename to prevent path traversal and injection attacks. - * Removes directory traversal sequences, path separators, null bytes, and dangerous patterns. - * Based on OWASP Input Validation Cheat Sheet recommendations. - */ -export function sanitizeFileName(fileName: string): string { - let sanitized = fileName - sanitized = sanitized.replace(/\0/g, '') - - try { - sanitized = decodeURIComponent(sanitized) - } catch { - // Keep original if decode fails (malformed encoding) - } - - sanitized = sanitized.replace(/\.\.[/\\]?/g, '') - sanitized = sanitized.replace(/[/\\]/g, '_') - sanitized = sanitized.replace(/^\.+/, '') - sanitized = sanitized.replace(/[\x00-\x1f\x7f]/g, '') - sanitized = sanitized.trim() - - return sanitized || 'unnamed_file' -} - -/** - * Validates that a path doesn't contain traversal sequences. - * Returns true if the path is safe, false if it contains potential traversal attacks. - */ -export function isPathSafe(path: string): boolean { - const normalizedPath = path.replace(/\\/g, '/') - - if (normalizedPath.includes('../') || normalizedPath.includes('..\\')) { - return false - } - - try { - const decoded = decodeURIComponent(normalizedPath) - if (decoded.includes('../') || decoded.includes('..\\')) { - return false - } - } catch { - return false - } - - if (normalizedPath.includes('\0')) { - return false - } - - return true -} - -/** - * Parses file permissions from mode bits to octal string representation. - */ -export function parsePermissions(mode: number): string { - return `0${(mode & 0o777).toString(8)}` -} - -/** - * Determines file type from SFTP attributes mode bits. - */ -export function getFileType(attrs: Attributes): 'file' | 'directory' | 'symlink' | 'other' { - const fileType = attrs.mode & S_IFMT - - if (fileType === S_IFDIR) return 'directory' - if (fileType === S_IFREG) return 'file' - if (fileType === S_IFLNK) return 'symlink' - return 'other' -} - -/** - * Checks if a path exists on the SFTP server. - */ -export function sftpExists(sftp: SFTPWrapper, path: string): Promise { - return new Promise((resolve) => { - sftp.stat(path, (err) => { - resolve(!err) - }) - }) -} - -/** - * Checks if a path is a directory on the SFTP server. - */ -export function sftpIsDirectory(sftp: SFTPWrapper, path: string): Promise { - return new Promise((resolve) => { - sftp.stat(path, (err, stats) => { - if (err) { - resolve(false) - } else { - resolve(getFileType(stats) === 'directory') - } - }) - }) -} diff --git a/apps/sim/app/api/tools/sharepoint/download-file/route.ts b/apps/sim/app/api/tools/sharepoint/download-file/route.ts deleted file mode 100644 index 1d9adcd5552..00000000000 --- a/apps/sim/app/api/tools/sharepoint/download-file/route.ts +++ /dev/null @@ -1,178 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { sharepointDownloadFileContract } from '@/lib/api/contracts/tools/microsoft' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' - -export const dynamic = 'force-dynamic' - -/** Microsoft Graph API error response structure */ -interface GraphApiError { - error?: { - code?: string - message?: string - } -} - -/** Microsoft Graph API drive item metadata response */ -interface DriveItemMetadata { - id?: string - name?: string - folder?: Record - file?: { - mimeType?: string - } -} - -const logger = createLogger('SharepointDownloadFileAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized SharePoint download attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - const parsed = await parseRequest(sharepointDownloadFileContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, driveId, itemId, fileName } = parsed.data.body - const authHeader = `Bearer ${accessToken}` - - logger.info(`[${requestId}] Getting file metadata from SharePoint`, { driveId, itemId }) - - const metadataUrl = `https://graph.microsoft.com/v1.0/drives/${encodeURIComponent(driveId)}/items/${encodeURIComponent(itemId)}` - const metadataUrlValidation = await validateUrlWithDNS(metadataUrl, 'metadataUrl') - if (!metadataUrlValidation.isValid) { - return NextResponse.json( - { success: false, error: metadataUrlValidation.error }, - { status: 400 } - ) - } - - const metadataResponse = await secureFetchWithPinnedIP( - metadataUrl, - metadataUrlValidation.resolvedIP!, - { - headers: { Authorization: authHeader }, - } - ) - - if (!metadataResponse.ok) { - const errorDetails = (await metadataResponse.json().catch(() => ({}))) as GraphApiError - logger.error(`[${requestId}] Failed to get file metadata`, { - status: metadataResponse.status, - error: errorDetails, - }) - return NextResponse.json( - { success: false, error: errorDetails.error?.message || 'Failed to get file metadata' }, - { status: 400 } - ) - } - - const metadata = (await metadataResponse.json()) as DriveItemMetadata - - if (metadata.folder && !metadata.file) { - logger.error(`[${requestId}] Attempted to download a folder`, { - itemId: metadata.id, - itemName: metadata.name, - }) - return NextResponse.json( - { - success: false, - error: `Cannot download folder "${metadata.name}". Please select a file instead.`, - }, - { status: 400 } - ) - } - - const mimeType = metadata.file?.mimeType || 'application/octet-stream' - - logger.info(`[${requestId}] Downloading file from SharePoint`, { driveId, itemId, mimeType }) - - const downloadUrl = `https://graph.microsoft.com/v1.0/drives/${encodeURIComponent(driveId)}/items/${encodeURIComponent(itemId)}/content` - const downloadUrlValidation = await validateUrlWithDNS(downloadUrl, 'downloadUrl') - if (!downloadUrlValidation.isValid) { - return NextResponse.json( - { success: false, error: downloadUrlValidation.error }, - { status: 400 } - ) - } - - const downloadResponse = await secureFetchWithPinnedIP( - downloadUrl, - downloadUrlValidation.resolvedIP!, - { - headers: { Authorization: authHeader }, - // The content endpoint 302s to a preauthenticated URL on a different origin that needs no auth. - stripAuthOnRedirect: true, - maxResponseBytes: MAX_FILE_SIZE, - } - ) - - if (!downloadResponse.ok) { - const downloadError = (await downloadResponse.json().catch(() => ({}))) as GraphApiError - logger.error(`[${requestId}] Failed to download file`, { - status: downloadResponse.status, - error: downloadError, - }) - return NextResponse.json( - { success: false, error: downloadError.error?.message || 'Failed to download file' }, - { status: 400 } - ) - } - - const arrayBuffer = await downloadResponse.arrayBuffer() - const fileBuffer = Buffer.from(arrayBuffer) - - const resolvedName = fileName || metadata.name || 'download' - - logger.info(`[${requestId}] File downloaded successfully`, { - driveId, - itemId, - name: resolvedName, - size: fileBuffer.length, - mimeType, - }) - - const base64Data = fileBuffer.toString('base64') - - return NextResponse.json({ - success: true, - output: { - file: { - name: resolvedName, - mimeType, - data: base64Data, - size: fileBuffer.length, - }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error downloading SharePoint file:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/sharepoint/site/route.ts b/apps/sim/app/api/tools/sharepoint/site/route.ts deleted file mode 100644 index 4dc2b508917..00000000000 --- a/apps/sim/app/api/tools/sharepoint/site/route.ts +++ /dev/null @@ -1,96 +0,0 @@ -import { createLogger } from '@sim/logger' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { sharepointSiteQuerySchema } from '@/lib/api/contracts/selectors/sharepoint' -import { getValidationErrorMessage } from '@/lib/api/server' -import { authorizeCredentialUse } from '@/lib/auth/credential-access' -import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('SharePointSiteAPI') - -export const GET = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const { searchParams } = new URL(request.url) - const validation = sharepointSiteQuerySchema.safeParse({ - credentialId: searchParams.get('credentialId') ?? '', - siteId: searchParams.get('siteId') ?? '', - }) - if (!validation.success) { - return NextResponse.json( - { error: getValidationErrorMessage(validation.error, 'Invalid request') }, - { status: 400 } - ) - } - const { credentialId, siteId } = validation.data - - const siteIdValidation = validateMicrosoftGraphId(siteId, 'siteId') - if (!siteIdValidation.isValid) { - return NextResponse.json({ error: siteIdValidation.error }, { status: 400 }) - } - - const authz = await authorizeCredentialUse(request, { credentialId }) - if (!authz.ok || !authz.credentialOwnerUserId || !authz.resolvedCredentialId) { - return NextResponse.json({ error: authz.error || 'Unauthorized' }, { status: 403 }) - } - - const accessToken = await refreshAccessTokenIfNeeded( - authz.resolvedCredentialId, - authz.credentialOwnerUserId, - requestId - ) - if (!accessToken) { - return NextResponse.json({ error: 'Failed to obtain valid access token' }, { status: 401 }) - } - - let endpoint: string - if (siteId === 'root') { - endpoint = 'sites/root' - } else if (siteId.includes(':')) { - endpoint = `sites/${siteId}` - } else if (siteId.includes('groups/')) { - endpoint = siteId - } else { - endpoint = `sites/${siteId}` - } - - const response = await fetch( - `https://graph.microsoft.com/v1.0/${endpoint}?$select=id,name,displayName,webUrl,createdDateTime,lastModifiedDateTime`, - { - headers: { - Authorization: `Bearer ${accessToken}`, - }, - } - ) - - if (!response.ok) { - const errorData = await response.json().catch(() => ({ error: { message: 'Unknown error' } })) - return NextResponse.json( - { error: errorData.error?.message || 'Failed to fetch site from SharePoint' }, - { status: response.status } - ) - } - - const site = await response.json() - - const transformedSite = { - id: site.id, - name: site.displayName || site.name, - mimeType: 'application/vnd.microsoft.graph.site', - webViewLink: site.webUrl, - createdTime: site.createdDateTime, - modifiedTime: site.lastModifiedDateTime, - } - - logger.info(`[${requestId}] Successfully fetched SharePoint site: ${transformedSite.name}`) - return NextResponse.json({ site: transformedSite }, { status: 200 }) - } catch (error) { - logger.error(`[${requestId}] Error fetching site from SharePoint`, error) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/sharepoint/upload/route.ts b/apps/sim/app/api/tools/sharepoint/upload/route.ts deleted file mode 100644 index b29a51fd387..00000000000 --- a/apps/sim/app/api/tools/sharepoint/upload/route.ts +++ /dev/null @@ -1,281 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { sharepointUploadContract } from '@/lib/api/contracts/tools/microsoft' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import type { MicrosoftGraphDriveItem } from '@/tools/onedrive/types' -import type { SharepointSkippedFile, SharepointUploadError } from '@/tools/sharepoint/types' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('SharepointUploadAPI') -const MAX_SHAREPOINT_UPLOAD_BYTES = 250 * 1024 * 1024 - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized SharePoint upload attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info( - `[${requestId}] Authenticated SharePoint upload request via ${authResult.authType}`, - { - userId: authResult.userId, - } - ) - - const parsed = await parseRequest(sharepointUploadContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Uploading files to SharePoint`, { - siteId: validatedData.siteId, - driveId: validatedData.driveId, - folderPath: validatedData.folderPath, - hasFiles: !!(validatedData.files && validatedData.files.length > 0), - fileCount: validatedData.files?.length || 0, - }) - - if (!validatedData.files || validatedData.files.length === 0) { - return NextResponse.json( - { - success: false, - error: 'At least one file is required for upload', - }, - { status: 400 } - ) - } - - const userFiles = processFilesToUserFiles(validatedData.files, requestId, logger) - - if (userFiles.length === 0) { - return NextResponse.json( - { - success: false, - error: 'No valid files to upload', - }, - { status: 400 } - ) - } - - const siteId = validatedData.siteId.trim() || 'root' - const driveId = validatedData.driveId?.trim() || null - const uploadedFiles: MicrosoftGraphDriveItem[] = [] - const skippedFiles: SharepointSkippedFile[] = [] - const errors: SharepointUploadError[] = [] - - for (const userFile of userFiles) { - const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) - if (denied) return denied - logger.info(`[${requestId}] Uploading file: ${userFile.name}`) - - const fileName = validatedData.fileName || userFile.name - const folderPath = validatedData.folderPath?.trim() || '' - - const skipOversized = (size: number) => { - logger.warn( - `[${requestId}] File ${fileName} is ${(size / (1024 * 1024)).toFixed(2)}MB, exceeds 250MB limit` - ) - skippedFiles.push({ - name: fileName, - size, - limit: MAX_SHAREPOINT_UPLOAD_BYTES, - reason: 'File exceeds the 250 MB Microsoft Graph small upload limit', - }) - } - - let buffer: Buffer - let downloadedContentType = '' - try { - const result = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_SHAREPOINT_UPLOAD_BYTES, - }) - buffer = result.buffer - downloadedContentType = result.contentType - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - // An oversized file is skipped rather than failing the whole batch, exactly as - // it was when the size was only discovered after the download. - if (isPayloadSizeLimitError(error)) { - skipOversized(error.observedBytes ?? userFile.size) - continue - } - throw error - } - - let uploadPath = '' - if (folderPath) { - const normalizedPath = folderPath.startsWith('/') ? folderPath : `/${folderPath}` - const cleanPath = normalizedPath.endsWith('/') - ? normalizedPath.slice(0, -1) - : normalizedPath - uploadPath = `${cleanPath}/${fileName}` - } else { - uploadPath = `/${fileName}` - } - - const encodedPath = uploadPath - .split('/') - .map((segment) => (segment ? encodeURIComponent(segment) : '')) - .join('/') - - const uploadUrl = driveId - ? `https://graph.microsoft.com/v1.0/drives/${encodeURIComponent(driveId)}/root:${encodedPath}:/content` - : `https://graph.microsoft.com/v1.0/sites/${encodeURIComponent(siteId)}/drive/root:${encodedPath}:/content` - - logger.info(`[${requestId}] Uploading to: ${uploadUrl}`) - - const uploadResponse = await secureFetchWithValidation( - uploadUrl, - { - method: 'PUT', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': downloadedContentType || userFile.type || 'application/octet-stream', - }, - body: buffer, - }, - 'uploadUrl' - ) - - if (!uploadResponse.ok) { - const errorData = await uploadResponse.json().catch(() => ({})) - logger.error(`[${requestId}] Failed to upload file ${fileName}:`, errorData) - - if (uploadResponse.status === 409) { - // File exists - retry with conflict behavior set to replace - logger.warn(`[${requestId}] File ${fileName} already exists, retrying with replace`) - const replaceUrl = `${uploadUrl}?@microsoft.graph.conflictBehavior=replace` - const replaceResponse = await secureFetchWithValidation( - replaceUrl, - { - method: 'PUT', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Content-Type': - downloadedContentType || userFile.type || 'application/octet-stream', - }, - body: buffer, - }, - 'replaceUrl' - ) - - if (!replaceResponse.ok) { - const replaceErrorData = (await replaceResponse.json().catch(() => ({}))) as { - error?: { message?: string } - } - logger.error(`[${requestId}] Failed to replace file ${fileName}:`, replaceErrorData) - errors.push({ - name: fileName, - status: replaceResponse.status, - error: replaceErrorData.error?.message || `Failed to replace file: ${fileName}`, - }) - continue - } - - const replaceData = (await replaceResponse.json()) as { - id: string - name: string - webUrl: string - size: number - createdDateTime: string - lastModifiedDateTime: string - } - logger.info(`[${requestId}] File replaced successfully: ${fileName}`) - - uploadedFiles.push({ - id: replaceData.id, - name: replaceData.name, - webUrl: replaceData.webUrl, - size: replaceData.size, - createdDateTime: replaceData.createdDateTime, - lastModifiedDateTime: replaceData.lastModifiedDateTime, - }) - continue - } - - errors.push({ - name: fileName, - status: uploadResponse.status, - error: - (errorData as { error?: { message?: string } }).error?.message || - `Failed to upload file: ${fileName}`, - }) - continue - } - - const uploadData = (await uploadResponse.json()) as MicrosoftGraphDriveItem - logger.info(`[${requestId}] File uploaded successfully: ${fileName}`) - - uploadedFiles.push({ - id: uploadData.id, - name: uploadData.name, - webUrl: uploadData.webUrl, - size: uploadData.size, - createdDateTime: uploadData.createdDateTime, - lastModifiedDateTime: uploadData.lastModifiedDateTime, - }) - } - - if (uploadedFiles.length === 0) { - return NextResponse.json({ - success: false, - error: 'No files were uploaded successfully', - output: { - uploadedFiles, - fileCount: 0, - skippedFiles, - skippedCount: skippedFiles.length, - errors, - }, - }) - } - - logger.info(`[${requestId}] Completed SharePoint upload`, { - uploadedCount: uploadedFiles.length, - skippedCount: skippedFiles.length, - errorCount: errors.length, - }) - - return NextResponse.json({ - success: true, - output: { - uploadedFiles, - fileCount: uploadedFiles.length, - skippedFiles, - skippedCount: skippedFiles.length, - errors, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error uploading files to SharePoint:`, error) - return NextResponse.json( - { - success: false, - error: toError(error).message, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/slack/add-reaction/route.ts b/apps/sim/app/api/tools/slack/add-reaction/route.ts deleted file mode 100644 index a266f890b79..00000000000 --- a/apps/sim/app/api/tools/slack/add-reaction/route.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { slackAddReactionContract } from '@/lib/api/contracts/tools/communication/slack' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - const parsed = await parseRequest(slackAddReactionContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const slackResponse = await fetch('https://slack.com/api/reactions.add', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${validatedData.accessToken}`, - }, - body: JSON.stringify({ - channel: validatedData.channel, - timestamp: validatedData.timestamp, - name: validatedData.name, - }), - }) - - const data = await slackResponse.json() - - if (!data.ok) { - return NextResponse.json( - { - success: false, - error: data.error || 'Failed to add reaction', - }, - { status: slackResponse.status } - ) - } - - return NextResponse.json({ - success: true, - output: { - content: `Successfully added :${validatedData.name}: reaction`, - metadata: { - channel: validatedData.channel, - timestamp: validatedData.timestamp, - reaction: validatedData.name, - }, - }, - }) - } catch (error) { - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/slack/delete-message/route.ts b/apps/sim/app/api/tools/slack/delete-message/route.ts deleted file mode 100644 index 4634a3da073..00000000000 --- a/apps/sim/app/api/tools/slack/delete-message/route.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { slackDeleteMessageContract } from '@/lib/api/contracts/tools/communication/slack' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - const parsed = await parseRequest(slackDeleteMessageContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const slackResponse = await fetch('https://slack.com/api/chat.delete', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${validatedData.accessToken}`, - }, - body: JSON.stringify({ - channel: validatedData.channel, - ts: validatedData.timestamp, - }), - }) - - const data = await slackResponse.json() - - if (!data.ok) { - return NextResponse.json( - { - success: false, - error: data.error || 'Failed to delete message', - }, - { status: slackResponse.status } - ) - } - - return NextResponse.json({ - success: true, - output: { - content: 'Message deleted successfully', - metadata: { - channel: data.channel, - timestamp: data.ts, - }, - }, - }) - } catch (error) { - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/slack/download/route.test.ts b/apps/sim/app/api/tools/slack/download/route.test.ts deleted file mode 100644 index 8e31d5fcc53..00000000000 --- a/apps/sim/app/api/tools/slack/download/route.test.ts +++ /dev/null @@ -1,99 +0,0 @@ -/** - * @vitest-environment node - */ -import { - createMockRequest, - hybridAuthMockFns, - inputValidationMock, - inputValidationMockFns, -} from '@sim/testing' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' - -vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) - -import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { POST } from '@/app/api/tools/slack/download/route' - -const { mockValidateUrlWithDNS, mockSecureFetchWithPinnedIP } = inputValidationMockFns - -const PINNED_IP = '93.184.216.34' - -const baseBody = { - accessToken: 'token-123', - fileId: 'file-abc', -} - -function fileResponse(bytes: number) { - return { - ok: true, - status: 200, - statusText: '', - headers: new Headers(), - body: null, - text: async () => '', - json: async () => ({}), - arrayBuffer: async () => new ArrayBuffer(bytes), - } -} - -const originalFetch = global.fetch - -beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - mockValidateUrlWithDNS.mockResolvedValue({ - isValid: true, - resolvedIP: PINNED_IP, - originalHostname: 'files.slack.com', - }) - global.fetch = vi.fn().mockResolvedValue({ - ok: true, - json: async () => ({ - ok: true, - file: { - name: 'report.pdf', - mimetype: 'application/pdf', - url_private: 'https://files.slack.com/files-pri/T000-F000/report.pdf', - }, - }), - }) as unknown as typeof fetch -}) - -afterEach(() => { - global.fetch = originalFetch -}) - -describe('POST /api/tools/slack/download', () => { - it('downloads a normal file under the size cap', async () => { - mockSecureFetchWithPinnedIP.mockResolvedValueOnce(fileResponse(1024)) - - const response = await POST(createMockRequest('POST', baseBody)) - expect(response.status).toBe(200) - const data = (await response.json()) as { success: boolean; output: { file: { size: number } } } - expect(data.success).toBe(true) - expect(data.output.file.size).toBe(1024) - - const downloadCall = mockSecureFetchWithPinnedIP.mock.calls[0] - expect(downloadCall[2]).toMatchObject({ maxResponseBytes: MAX_FILE_SIZE }) - }) - - it('surfaces a clean 413 when the streamed content exceeds the cap', async () => { - mockSecureFetchWithPinnedIP.mockRejectedValueOnce( - new PayloadSizeLimitError({ - label: 'response body', - maxBytes: MAX_FILE_SIZE, - observedBytes: MAX_FILE_SIZE + 1, - }) - ) - - const response = await POST(createMockRequest('POST', baseBody)) - expect(response.status).toBe(413) - const data = (await response.json()) as { success: boolean } - expect(data.success).toBe(false) - }) -}) diff --git a/apps/sim/app/api/tools/slack/download/route.ts b/apps/sim/app/api/tools/slack/download/route.ts deleted file mode 100644 index 68eef0e7048..00000000000 --- a/apps/sim/app/api/tools/slack/download/route.ts +++ /dev/null @@ -1,169 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { slackDownloadContract } from '@/lib/api/contracts/tools/communication/slack' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('SlackDownloadAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Slack download attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated Slack download request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const parsed = await parseRequest(slackDownloadContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, fileId, fileName } = parsed.data.body - - logger.info(`[${requestId}] Getting file info from Slack`, { fileId }) - - const infoResponse = await fetch(`https://slack.com/api/files.info?file=${fileId}`, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!infoResponse.ok) { - const errorDetails = await infoResponse.json().catch(() => ({})) - logger.error(`[${requestId}] Failed to get file info from Slack`, { - status: infoResponse.status, - statusText: infoResponse.statusText, - error: errorDetails, - }) - return NextResponse.json( - { - success: false, - error: errorDetails.error || 'Failed to get file info', - }, - { status: 400 } - ) - } - - const data = await infoResponse.json() - - if (!data.ok) { - logger.error(`[${requestId}] Slack API returned error`, { error: data.error }) - return NextResponse.json( - { - success: false, - error: data.error || 'Slack API error', - }, - { status: 400 } - ) - } - - const file = data.file - const resolvedFileName = fileName || file.name || 'download' - const mimeType = file.mimetype || 'application/octet-stream' - const urlPrivate = file.url_private - - if (!urlPrivate) { - return NextResponse.json( - { - success: false, - error: 'File does not have a download URL', - }, - { status: 400 } - ) - } - - const urlValidation = await validateUrlWithDNS(urlPrivate, 'urlPrivate') - if (!urlValidation.isValid) { - return NextResponse.json( - { - success: false, - error: urlValidation.error, - }, - { status: 400 } - ) - } - - logger.info(`[${requestId}] Downloading file from Slack`, { - fileId, - fileName: resolvedFileName, - mimeType, - }) - - const downloadResponse = await secureFetchWithPinnedIP(urlPrivate, urlValidation.resolvedIP!, { - headers: { - Authorization: `Bearer ${accessToken}`, - }, - maxResponseBytes: MAX_FILE_SIZE, - }) - - if (!downloadResponse.ok) { - logger.error(`[${requestId}] Failed to download file content`, { - status: downloadResponse.status, - statusText: downloadResponse.statusText, - }) - return NextResponse.json( - { - success: false, - error: 'Failed to download file content', - }, - { status: 400 } - ) - } - - const arrayBuffer = await downloadResponse.arrayBuffer() - const fileBuffer = Buffer.from(arrayBuffer) - - logger.info(`[${requestId}] File downloaded successfully`, { - fileId, - name: resolvedFileName, - size: fileBuffer.length, - mimeType, - }) - - const base64Data = fileBuffer.toString('base64') - - return NextResponse.json({ - success: true, - output: { - file: { - name: resolvedFileName, - mimeType, - data: base64Data, - size: fileBuffer.length, - }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error downloading Slack file:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/slack/read-messages/route.ts b/apps/sim/app/api/tools/slack/read-messages/route.ts deleted file mode 100644 index 712ccbc7f0f..00000000000 --- a/apps/sim/app/api/tools/slack/read-messages/route.ts +++ /dev/null @@ -1,186 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { slackReadMessagesContract } from '@/lib/api/contracts/tools/communication/slack' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { openDMChannel } from '../utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('SlackReadMessagesAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Slack read messages attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info( - `[${requestId}] Authenticated Slack read messages request via ${authResult.authType}`, - { - userId: authResult.userId, - } - ) - - const parsed = await parseRequest(slackReadMessagesContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - let channel = validatedData.channel - if (!channel && validatedData.userId) { - logger.info(`[${requestId}] Opening DM channel for user: ${validatedData.userId}`) - channel = await openDMChannel( - validatedData.accessToken, - validatedData.userId, - requestId, - logger - ) - } - - const url = new URL('https://slack.com/api/conversations.history') - url.searchParams.append('channel', channel!) - const limit = validatedData.limit ?? 10 - url.searchParams.append('limit', String(limit)) - - if (validatedData.oldest) { - url.searchParams.append('oldest', validatedData.oldest) - } - if (validatedData.latest) { - url.searchParams.append('latest', validatedData.latest) - } - - logger.info(`[${requestId}] Reading Slack messages`, { - channel, - limit, - }) - - const slackResponse = await fetch(url.toString(), { - method: 'GET', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${validatedData.accessToken}`, - }, - }) - - const data = await slackResponse.json() - - if (!data.ok) { - logger.error(`[${requestId}] Slack API error:`, data) - - if (data.error === 'not_in_channel') { - return NextResponse.json( - { - success: false, - error: - 'Bot is not in the channel. Please invite the Sim bot to your Slack channel by typing: /invite @Sim Studio', - }, - { status: 400 } - ) - } - if (data.error === 'channel_not_found') { - return NextResponse.json( - { - success: false, - error: 'Channel not found. Please check the channel ID and try again.', - }, - { status: 400 } - ) - } - if (data.error === 'missing_scope') { - return NextResponse.json( - { - success: false, - error: - 'Missing required permissions. Reconnect your Slack account to grant channel history access (channels:history, groups:history). Reading direct message history is not supported with the Sim bot.', - }, - { status: 400 } - ) - } - - return NextResponse.json( - { - success: false, - error: data.error || 'Failed to fetch messages', - }, - { status: 400 } - ) - } - - const messages = (data.messages || []).map((message: any) => ({ - type: message.type || 'message', - ts: message.ts, - text: message.text || '', - user: message.user, - bot_id: message.bot_id, - username: message.username, - channel: message.channel, - team: message.team, - thread_ts: message.thread_ts, - parent_user_id: message.parent_user_id, - reply_count: message.reply_count, - reply_users_count: message.reply_users_count, - latest_reply: message.latest_reply, - subscribed: message.subscribed, - last_read: message.last_read, - unread_count: message.unread_count, - subtype: message.subtype, - reactions: message.reactions?.map((reaction: any) => ({ - name: reaction.name, - count: reaction.count, - users: reaction.users || [], - })), - is_starred: message.is_starred, - pinned_to: message.pinned_to, - files: message.files?.map((file: any) => ({ - id: file.id, - name: file.name, - mimetype: file.mimetype, - size: file.size, - url_private: file.url_private, - permalink: file.permalink, - mode: file.mode, - })), - attachments: message.attachments, - blocks: message.blocks, - edited: message.edited - ? { - user: message.edited.user, - ts: message.edited.ts, - } - : undefined, - permalink: message.permalink, - })) - - logger.info(`[${requestId}] Successfully read ${messages.length} messages`) - - return NextResponse.json({ - success: true, - output: { - messages, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error reading Slack messages:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/slack/remove-reaction/route.ts b/apps/sim/app/api/tools/slack/remove-reaction/route.ts deleted file mode 100644 index 108d08b52e6..00000000000 --- a/apps/sim/app/api/tools/slack/remove-reaction/route.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { slackRemoveReactionContract } from '@/lib/api/contracts/tools/communication/slack' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - const parsed = await parseRequest(slackRemoveReactionContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const slackResponse = await fetch('https://slack.com/api/reactions.remove', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${validatedData.accessToken}`, - }, - body: JSON.stringify({ - channel: validatedData.channel, - timestamp: validatedData.timestamp, - name: validatedData.name, - }), - }) - - const data = await slackResponse.json() - - if (!data.ok) { - return NextResponse.json( - { - success: false, - error: data.error || 'Failed to remove reaction', - }, - { status: slackResponse.status } - ) - } - - return NextResponse.json({ - success: true, - output: { - content: `Successfully removed :${validatedData.name}: reaction`, - metadata: { - channel: validatedData.channel, - timestamp: validatedData.timestamp, - reaction: validatedData.name, - }, - }, - }) - } catch (error) { - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/slack/send-ephemeral/route.ts b/apps/sim/app/api/tools/slack/send-ephemeral/route.ts deleted file mode 100644 index 058ce45d0d3..00000000000 --- a/apps/sim/app/api/tools/slack/send-ephemeral/route.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { slackSendEphemeralContract } from '@/lib/api/contracts/tools/communication/slack' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('SlackSendEphemeralAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Slack ephemeral send attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info( - `[${requestId}] Authenticated Slack ephemeral send request via ${authResult.authType}`, - { userId: authResult.userId } - ) - - const parsed = await parseRequest(slackSendEphemeralContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Sending ephemeral message`, { - channel: validatedData.channel, - user: validatedData.user, - threadTs: validatedData.thread_ts ?? undefined, - }) - - const response = await fetch('https://slack.com/api/chat.postEphemeral', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${validatedData.accessToken}`, - }, - body: JSON.stringify({ - channel: validatedData.channel, - user: validatedData.user, - text: validatedData.text, - ...(validatedData.thread_ts && { thread_ts: validatedData.thread_ts }), - ...(validatedData.blocks && - validatedData.blocks.length > 0 && { blocks: validatedData.blocks }), - }), - }) - - const data = await response.json() - - if (!data.ok) { - logger.error(`[${requestId}] Slack API error:`, data.error) - return NextResponse.json( - { success: false, error: data.error || 'Failed to send ephemeral message' }, - { status: 400 } - ) - } - - logger.info(`[${requestId}] Ephemeral message sent successfully`) - - return NextResponse.json({ - success: true, - output: { - messageTs: data.message_ts, - channel: validatedData.channel, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error sending ephemeral message:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/slack/send-message/route.ts b/apps/sim/app/api/tools/slack/send-message/route.ts deleted file mode 100644 index 8a972fd4c85..00000000000 --- a/apps/sim/app/api/tools/slack/send-message/route.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { slackSendMessageContract } from '@/lib/api/contracts/tools/communication/slack' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { FileAccessDeniedError } from '@/app/api/files/authorization' -import { sendSlackMessage } from '@/app/api/tools/slack/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('SlackSendMessageAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Slack send attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - const userId = authResult.userId - logger.info(`[${requestId}] Authenticated Slack send request via ${authResult.authType}`, { - userId, - }) - - const parsed = await parseRequest(slackSendMessageContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const isDM = !!validatedData.userId - logger.info(`[${requestId}] Sending Slack message`, { - channel: validatedData.channel, - userId: validatedData.userId, - isDM, - hasFiles: !!(validatedData.files && validatedData.files.length > 0), - fileCount: validatedData.files?.length || 0, - }) - - const result = await sendSlackMessage( - { - accessToken: validatedData.accessToken, - channel: validatedData.channel ?? undefined, - userId: validatedData.userId ?? undefined, - ownerUserId: userId, - text: validatedData.text, - threadTs: validatedData.thread_ts ?? undefined, - blocks: validatedData.blocks ?? undefined, - files: validatedData.files ?? undefined, - }, - requestId, - logger - ) - - if (!result.success) { - return NextResponse.json({ success: false, error: result.error }, { status: 400 }) - } - - return NextResponse.json({ success: true, output: result.output }) - } catch (error) { - if (error instanceof FileAccessDeniedError) { - return NextResponse.json({ success: false, error: 'File not found' }, { status: 404 }) - } - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - logger.error(`[${requestId}] Error sending Slack message:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/slack/update-message/route.ts b/apps/sim/app/api/tools/slack/update-message/route.ts deleted file mode 100644 index fdfd675cae7..00000000000 --- a/apps/sim/app/api/tools/slack/update-message/route.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { slackUpdateMessageContract } from '@/lib/api/contracts/tools/communication/slack' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('SlackUpdateMessageAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Slack update message attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info( - `[${requestId}] Authenticated Slack update message request via ${authResult.authType}`, - { - userId: authResult.userId, - } - ) - - const parsed = await parseRequest(slackUpdateMessageContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Updating Slack message`, { - channel: validatedData.channel, - timestamp: validatedData.timestamp, - }) - - const slackResponse = await fetch('https://slack.com/api/chat.update', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${validatedData.accessToken}`, - }, - body: JSON.stringify({ - channel: validatedData.channel, - ts: validatedData.timestamp, - text: validatedData.text, - ...(validatedData.blocks && - validatedData.blocks.length > 0 && { blocks: validatedData.blocks }), - }), - }) - - const data = await slackResponse.json() - - if (!data.ok) { - logger.error(`[${requestId}] Slack API error:`, data) - return NextResponse.json( - { - success: false, - error: data.error || 'Failed to update message', - }, - { status: slackResponse.status } - ) - } - - logger.info(`[${requestId}] Message updated successfully`, { - channel: data.channel, - timestamp: data.ts, - }) - - const messageObj = data.message || { - type: 'message', - ts: data.ts, - text: data.text || validatedData.text, - channel: data.channel, - } - - return NextResponse.json({ - success: true, - output: { - message: messageObj, - content: 'Message updated successfully', - metadata: { - channel: data.channel, - timestamp: data.ts, - text: data.text || validatedData.text, - }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error updating Slack message:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/slack/utils.ts b/apps/sim/app/api/tools/slack/utils.ts deleted file mode 100644 index d40b3a81a66..00000000000 --- a/apps/sim/app/api/tools/slack/utils.ts +++ /dev/null @@ -1,349 +0,0 @@ -import type { Logger } from '@sim/logger' -import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { FileAccessDeniedError, verifyFileAccess } from '@/app/api/files/authorization' -import type { ToolFileData } from '@/tools/types' - -/** - * Sends a message to a Slack channel using chat.postMessage - */ -async function postSlackMessage( - accessToken: string, - channel: string, - text: string, - threadTs?: string | null, - blocks?: unknown[] | null -): Promise<{ ok: boolean; ts?: string; channel?: string; message?: any; error?: string }> { - const response = await fetch('https://slack.com/api/chat.postMessage', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - body: JSON.stringify({ - channel, - text, - ...(threadTs && { thread_ts: threadTs }), - ...(blocks && blocks.length > 0 && { blocks }), - }), - }) - - return response.json() -} - -/** - * Creates a default message object when the API doesn't return one - */ -export function createDefaultMessageObject( - ts: string, - text: string, - channel: string -): Record { - return { - type: 'message', - ts, - text, - channel, - } -} - -/** - * Formats the success response for a sent message - */ -export function formatMessageSuccessResponse( - data: any, - text: string -): { - message: any - ts: string - channel: string -} { - const messageObj = data.message || createDefaultMessageObject(data.ts, text, data.channel) - return { - message: messageObj, - ts: data.ts, - channel: data.channel, - } -} - -/** - * Uploads files to Slack and returns the uploaded file IDs - */ -async function uploadFilesToSlack( - files: any[], - accessToken: string, - requestId: string, - logger: Logger, - ownerUserId: string -): Promise<{ fileIds: string[]; files: ToolFileData[] }> { - const userFiles = processFilesToUserFiles(files, requestId, logger) - const uploadedFileIds: string[] = [] - const uploadedFiles: ToolFileData[] = [] - // One share can carry several files, so the ceiling spans the set: each file may - // only use what its predecessors left. - let remainingBytes = MAX_BUFFERED_TRANSFER_BYTES - - for (const userFile of userFiles) { - logger.info(`[${requestId}] Uploading file: ${userFile.name}`) - - const hasAccess = await verifyFileAccess(userFile.key, ownerUserId) - if (!hasAccess) { - throw new FileAccessDeniedError() - } - - const { buffer, contentType } = await downloadServableFileFromStorage( - userFile, - requestId, - logger, - { maxBytes: remainingBytes } - ) - remainingBytes -= buffer.length - - const getUrlResponse = await fetch('https://slack.com/api/files.getUploadURLExternal', { - method: 'POST', - headers: { - 'Content-Type': 'application/x-www-form-urlencoded', - Authorization: `Bearer ${accessToken}`, - }, - body: new URLSearchParams({ - filename: userFile.name, - length: buffer.length.toString(), - }), - }) - - const urlData = await getUrlResponse.json() - - if (!urlData.ok) { - logger.error(`[${requestId}] Failed to get upload URL:`, urlData.error) - continue - } - - logger.info(`[${requestId}] Got upload URL for ${userFile.name}, file_id: ${urlData.file_id}`) - - const uploadResponse = await secureFetchWithValidation( - urlData.upload_url, - { - method: 'POST', - body: buffer, - }, - 'uploadUrl' - ) - - if (!uploadResponse.ok) { - logger.error(`[${requestId}] Failed to upload file data: ${uploadResponse.status}`) - continue - } - - logger.info(`[${requestId}] File data uploaded successfully`) - uploadedFileIds.push(urlData.file_id) - // Only add to uploadedFiles after successful upload to keep arrays in sync - uploadedFiles.push({ - name: userFile.name, - mimeType: contentType || userFile.type || 'application/octet-stream', - data: buffer.toString('base64'), - size: buffer.length, - }) - } - - return { fileIds: uploadedFileIds, files: uploadedFiles } -} - -/** - * Completes the file upload process by associating files with a channel - */ -async function completeSlackFileUpload( - uploadedFileIds: string[], - channel: string, - text: string, - accessToken: string, - threadTs?: string | null, - blocks?: unknown[] | null -): Promise<{ ok: boolean; files?: any[]; error?: string }> { - const response = await fetch('https://slack.com/api/files.completeUploadExternal', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - body: JSON.stringify({ - files: uploadedFileIds.map((id) => ({ id })), - channel_id: channel, - // Per Slack docs for files.completeUploadExternal: if `initial_comment` - // is provided, `blocks` is silently ignored. So when blocks are present - // we omit initial_comment and let blocks render instead. - ...(blocks && blocks.length > 0 ? { blocks } : { initial_comment: text }), - ...(threadTs && { thread_ts: threadTs }), - }), - }) - - return response.json() -} - -/** - * Creates a message object for file uploads - */ -export function createFileMessageObject( - text: string, - channel: string, - files: any[] -): Record { - const fileTs = files?.[0]?.created?.toString() || (Date.now() / 1000).toString() - return { - type: 'message', - ts: fileTs, - text, - channel, - files: files?.map((file: any) => ({ - id: file?.id, - name: file?.name, - mimetype: file?.mimetype, - size: file?.size, - url_private: file?.url_private, - permalink: file?.permalink, - })), - } -} - -/** - * Opens a DM channel with a user and returns the channel ID - */ -export async function openDMChannel( - accessToken: string, - userId: string, - requestId: string, - logger: Logger -): Promise { - const response = await fetch('https://slack.com/api/conversations.open', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - body: JSON.stringify({ - users: userId, - }), - }) - - const data = await response.json() - - if (!data.ok) { - logger.error(`[${requestId}] Failed to open DM channel:`, data.error) - throw new Error(data.error || 'Failed to open DM channel with user') - } - - logger.info(`[${requestId}] Opened DM channel: ${data.channel.id}`) - return data.channel.id -} - -export interface SlackMessageParams { - accessToken: string - channel?: string - userId?: string - ownerUserId: string - text: string - threadTs?: string | null - blocks?: unknown[] | null - files?: any[] | null -} - -/** - * Sends a Slack message with optional file attachments - * Supports both channel messages and direct messages via userId - */ -export async function sendSlackMessage( - params: SlackMessageParams, - requestId: string, - logger: Logger -): Promise<{ - success: boolean - output?: { - message: any - ts: string - channel: string - fileCount?: number - files?: ToolFileData[] - } - error?: string -}> { - const { accessToken, text, threadTs, blocks, files, ownerUserId } = params - let { channel } = params - - if (!channel && params.userId) { - logger.info(`[${requestId}] Opening DM channel for user: ${params.userId}`) - channel = await openDMChannel(accessToken, params.userId, requestId, logger) - } - - if (!channel) { - return { success: false, error: 'Either channel or userId is required' } - } - - // No files - simple message - if (!files || files.length === 0) { - logger.info(`[${requestId}] No files, using chat.postMessage`) - - const data = await postSlackMessage(accessToken, channel, text, threadTs, blocks) - - if (!data.ok) { - logger.error(`[${requestId}] Slack API error:`, data.error) - return { success: false, error: data.error || 'Failed to send message' } - } - - logger.info(`[${requestId}] Message sent successfully`) - return { success: true, output: formatMessageSuccessResponse(data, text) } - } - - // Process files - logger.info(`[${requestId}] Processing ${files.length} file(s)`) - const { fileIds, files: uploadedFiles } = await uploadFilesToSlack( - files, - accessToken, - requestId, - logger, - ownerUserId - ) - - // No valid files uploaded - send text-only - if (fileIds.length === 0) { - logger.warn(`[${requestId}] No valid files to upload, sending text-only message`) - - const data = await postSlackMessage(accessToken, channel, text, threadTs, blocks) - - if (!data.ok) { - return { success: false, error: data.error || 'Failed to send message' } - } - - return { success: true, output: formatMessageSuccessResponse(data, text) } - } - - // Complete file upload with thread support - const completeData = await completeSlackFileUpload( - fileIds, - channel, - text, - accessToken, - threadTs, - blocks - ) - - if (!completeData.ok) { - logger.error(`[${requestId}] Failed to complete upload:`, completeData.error) - return { success: false, error: completeData.error || 'Failed to complete file upload' } - } - - logger.info(`[${requestId}] Files uploaded and shared successfully`) - - const fileMessage = createFileMessageObject(text, channel, completeData.files || []) - - return { - success: true, - output: { - message: fileMessage, - ts: fileMessage.ts, - channel, - fileCount: fileIds.length, - files: uploadedFiles, - }, - } -} diff --git a/apps/sim/app/api/tools/sms/send/route.ts b/apps/sim/app/api/tools/sms/send/route.ts deleted file mode 100644 index e19840c2a04..00000000000 --- a/apps/sim/app/api/tools/sms/send/route.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { smsSendContract } from '@/lib/api/contracts/tools/communication/messaging' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { env } from '@/lib/core/config/env' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { type SMSOptions, sendSMS } from '@/lib/messaging/sms/service' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('SMSSendAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized SMS send attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - message: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated SMS request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const parsed = await parseRequest(smsSendContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const fromNumber = env.TWILIO_PHONE_NUMBER - - if (!fromNumber) { - logger.error(`[${requestId}] SMS sending failed: No phone number configured`) - return NextResponse.json( - { - success: false, - message: 'SMS sending failed: No phone number configured.', - }, - { status: 500 } - ) - } - - logger.info(`[${requestId}] Sending SMS via internal SMS API`, { - to: validatedData.to, - bodyLength: validatedData.body.length, - from: fromNumber, - }) - - const smsOptions: SMSOptions = { - to: validatedData.to, - body: validatedData.body, - from: fromNumber, - } - - const result = await sendSMS(smsOptions) - - logger.info(`[${requestId}] SMS send result`, { - success: result.success, - message: result.message, - }) - - return NextResponse.json(result) - } catch (error) { - logger.error(`[${requestId}] Error sending SMS via API:`, error) - - return NextResponse.json( - { - success: false, - message: 'Internal server error while sending SMS', - data: {}, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/smtp/send/route.ts b/apps/sim/app/api/tools/smtp/send/route.ts deleted file mode 100644 index 83a48f3ce2b..00000000000 --- a/apps/sim/app/api/tools/smtp/send/route.ts +++ /dev/null @@ -1,234 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import nodemailer from 'nodemailer' -import { smtpSendContract } from '@/lib/api/contracts/tools/communication/email' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateDatabaseHost } from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getSmtpEhloName } from '@/lib/messaging/email/ehlo' -import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('SmtpSendAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized SMTP send attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - const userId = authResult.userId - logger.info(`[${requestId}] Authenticated SMTP request via ${authResult.authType}`, { - userId, - }) - - const parsed = await parseRequest(smtpSendContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const hostValidation = await validateDatabaseHost(validatedData.smtpHost, 'smtpHost') - if (!hostValidation.isValid) { - logger.warn(`[${requestId}] SMTP host validation failed`, { - host: validatedData.smtpHost, - error: hostValidation.error, - }) - return NextResponse.json({ success: false, error: hostValidation.error }, { status: 400 }) - } - - logger.info(`[${requestId}] Sending email via SMTP`, { - host: validatedData.smtpHost, - port: validatedData.smtpPort, - to: validatedData.to, - subject: validatedData.subject, - secure: validatedData.smtpSecure, - }) - - // Pin the pre-resolved IP to prevent DNS rebinding (TOCTOU) attacks. - // Pass resolvedIP as the host so nodemailer connects to the validated address, - // and set servername for correct TLS SNI/certificate validation. - const pinnedHost = hostValidation.resolvedIP ?? validatedData.smtpHost - - const transporter = nodemailer.createTransport({ - host: pinnedHost, - port: validatedData.smtpPort, - secure: validatedData.smtpSecure === 'SSL', - auth: { - user: validatedData.smtpUsername, - pass: validatedData.smtpPassword, - }, - name: getSmtpEhloName(), - tls: - validatedData.smtpSecure === 'None' - ? { rejectUnauthorized: false, servername: validatedData.smtpHost } - : { rejectUnauthorized: true, servername: validatedData.smtpHost }, - }) - - const contentType = validatedData.contentType || 'text' - const fromAddress = validatedData.fromName - ? `"${validatedData.fromName}" <${validatedData.from}>` - : validatedData.from - - const mailOptions: nodemailer.SendMailOptions = { - from: fromAddress, - to: validatedData.to, - subject: validatedData.subject, - [contentType === 'html' ? 'html' : 'text']: validatedData.body, - } - - if (validatedData.cc) { - mailOptions.cc = validatedData.cc - } - if (validatedData.bcc) { - mailOptions.bcc = validatedData.bcc - } - if (validatedData.replyTo) { - mailOptions.replyTo = validatedData.replyTo - } - - if (validatedData.attachments && validatedData.attachments.length > 0) { - const rawAttachments = validatedData.attachments - logger.info(`[${requestId}] Processing ${rawAttachments.length} attachment(s)`) - - const attachments = processFilesToUserFiles(rawAttachments, requestId, logger) - - if (attachments.length > 0) { - const totalSize = attachments.reduce((sum, file) => sum + file.size, 0) - const maxSize = 25 * 1024 * 1024 - - if (totalSize > maxSize) { - const sizeMB = (totalSize / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds SMTP limit of 25MB`, - }, - { status: 400 } - ) - } - - const accessResults = await Promise.all( - attachments.map((file) => assertToolFileAccess(file.key, userId, requestId, logger)) - ) - const denied = accessResults.find((r) => r !== null) - if (denied) return denied - - let resolved: Array<{ buffer: Buffer; contentType: string }> - try { - resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, { - totalMaxBytes: maxSize, - label: 'Total attachment size', - }) - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - if (isPayloadSizeLimitError(error)) { - const sizeMB = ((error.observedBytes ?? totalSize) / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `Total attachment size (${sizeMB}MB) exceeds SMTP limit of 25MB`, - }, - { status: 400 } - ) - } - logger.error(`[${requestId}] Failed to download an attachment:`, error) - return NextResponse.json( - { - success: false, - error: `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`, - }, - { status: 500 } - ) - } - - const attachmentBuffers = attachments.map((file, i) => ({ - filename: file.name, - content: resolved[i].buffer, - contentType: resolved[i].contentType || file.type || 'application/octet-stream', - })) - - logger.info(`[${requestId}] Processed ${attachmentBuffers.length} attachment(s)`) - mailOptions.attachments = attachmentBuffers - } - } - - const result = await transporter.sendMail(mailOptions) - - logger.info(`[${requestId}] Email sent successfully via SMTP`, { - messageId: result.messageId, - to: validatedData.to, - }) - - return NextResponse.json({ - success: true, - messageId: result.messageId, - to: validatedData.to, - subject: validatedData.subject, - }) - } catch (error: unknown) { - // Type guard for error objects with code property - const isNodeError = (err: unknown): err is NodeJS.ErrnoException => { - return err instanceof Error && 'code' in err - } - - let errorMessage = 'Failed to send email via SMTP' - - if (isNodeError(error)) { - if (error.code === 'EAUTH') { - errorMessage = 'SMTP authentication failed - check username and password' - } else if ( - error.code === 'ECONNECTION' || - error.code === 'ECONNREFUSED' || - error.code === 'ECONNRESET' || - error.code === 'ETIMEDOUT' - ) { - errorMessage = 'Could not connect to SMTP server - check host and port' - } - } - - const hasResponseCode = (err: unknown): err is { responseCode: number } => { - return typeof err === 'object' && err !== null && 'responseCode' in err - } - - if (hasResponseCode(error)) { - if (error.responseCode >= 500) { - errorMessage = 'SMTP server error - please try again later' - } else if (error.responseCode >= 400) { - errorMessage = 'Email rejected by SMTP server - check recipient addresses' - } - } - - logger.error(`[${requestId}] Error sending email via SMTP:`, { - error: toError(error).message, - code: isNodeError(error) ? error.code : undefined, - responseCode: hasResponseCode(error) ? error.responseCode : undefined, - }) - - return NextResponse.json( - { - success: false, - error: errorMessage, - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/sqs/send/route.ts b/apps/sim/app/api/tools/sqs/send/route.ts deleted file mode 100644 index 497cac1d99a..00000000000 --- a/apps/sim/app/api/tools/sqs/send/route.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { awsSqsSendContract } from '@/lib/api/contracts/tools/aws/sqs-send' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSqsClient, sendMessage } from '../utils' - -const logger = createLogger('SQSSendMessageAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsSqsSendContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Sending message to SQS queue ${params.queueUrl}`) - - const client = createSqsClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await sendMessage( - client, - params.queueUrl, - params.data, - params.messageGroupId, - params.messageDeduplicationId - ) - - logger.info(`[${requestId}] Message sent to SQS queue ${params.queueUrl}`) - - return NextResponse.json({ - message: `Message sent to SQS queue ${params.queueUrl}`, - id: result?.id, - }) - } finally { - client.destroy() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] SQS send message failed:`, error) - - return NextResponse.json({ error: `SQS send message failed: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/sqs/utils.ts b/apps/sim/app/api/tools/sqs/utils.ts deleted file mode 100644 index e3853af1306..00000000000 --- a/apps/sim/app/api/tools/sqs/utils.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { SendMessageCommand, type SendMessageCommandOutput, SQSClient } from '@aws-sdk/client-sqs' -import type { SqsConnectionConfig } from '@/tools/sqs/types' - -export function createSqsClient(config: SqsConnectionConfig): SQSClient { - return new SQSClient({ - region: config.region, - credentials: { - accessKeyId: config.accessKeyId, - secretAccessKey: config.secretAccessKey, - }, - }) -} - -export async function sendMessage( - client: SQSClient, - queueUrl: string, - data: Record, - messageGroupId?: string | null, - messageDeduplicationId?: string | null -): Promise | null> { - const command = new SendMessageCommand({ - QueueUrl: queueUrl, - MessageBody: JSON.stringify(data), - MessageGroupId: messageGroupId ?? undefined, - ...(messageDeduplicationId ? { MessageDeduplicationId: messageDeduplicationId } : {}), - }) - - const response = await client.send(command) - return parseSendMessageResponse(response) -} - -function parseSendMessageResponse( - response: SendMessageCommandOutput -): Record | null { - if (!response) { - return null - } - - return { id: response.MessageId } -} diff --git a/apps/sim/app/api/tools/square/catalog-image/route.ts b/apps/sim/app/api/tools/square/catalog-image/route.ts deleted file mode 100644 index 229e70deff3..00000000000 --- a/apps/sim/app/api/tools/square/catalog-image/route.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { squareCatalogImageContract } from '@/lib/api/contracts/tools/square' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { SQUARE_API_VERSION, SQUARE_BASE_URL } from '@/tools/square/types' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('SquareCatalogImageAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Square catalog image upload: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(squareCatalogImageContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - if (!validatedData.file) { - return NextResponse.json({ success: false, error: 'File is required' }, { status: 400 }) - } - - const userFiles = processFilesToUserFiles( - [validatedData.file as RawFileInput], - requestId, - logger - ) - - if (userFiles.length === 0) { - return NextResponse.json({ success: false, error: 'Invalid file input' }, { status: 400 }) - } - - const userFile = userFiles[0] - const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) - if (denied) return denied - - const fileBuffer = await downloadFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - }) - const fileName = validatedData.fileName || userFile.name - const mimeType = userFile.type || 'application/octet-stream' - - const imageRequest: Record = { - idempotency_key: validatedData.idempotencyKey || generateId(), - image: { - type: 'IMAGE', - id: '#square_catalog_image', - image_data: validatedData.caption ? { caption: validatedData.caption } : {}, - }, - } - if (validatedData.objectId) imageRequest.object_id = validatedData.objectId - - const formData = new FormData() - formData.append('request', JSON.stringify(imageRequest)) - formData.append('file', new Blob([new Uint8Array(fileBuffer)], { type: mimeType }), fileName) - - const response = await fetch(`${SQUARE_BASE_URL}/v2/catalog/images`, { - method: 'POST', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - 'Square-Version': SQUARE_API_VERSION, - }, - body: formData, - }) - - if (!response.ok) { - const errorText = await response.text() - let detail: string | undefined - try { - detail = JSON.parse(errorText)?.errors?.[0]?.detail - } catch { - detail = undefined - } - logger.error(`[${requestId}] Square API error:`, { status: response.status, body: errorText }) - return NextResponse.json( - { - success: false, - error: detail || `Failed to upload catalog image (HTTP ${response.status})`, - }, - { status: response.status } - ) - } - - const data = await response.json() - const object = data.image ?? {} - - return NextResponse.json({ - success: true, - output: { - object, - metadata: { - id: object.id ?? '', - type: object.type ?? null, - version: object.version ?? null, - }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Unexpected error:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ssh/check-command-exists/route.ts b/apps/sim/app/api/tools/ssh/check-command-exists/route.ts deleted file mode 100644 index ea0238b01f3..00000000000 --- a/apps/sim/app/api/tools/ssh/check-command-exists/route.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { sshCheckCommandExistsContract } from '@/lib/api/contracts/storage-transfer' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSSHConnection, escapeShellArg, executeSSHCommand } from '@/app/api/tools/ssh/utils' - -const logger = createLogger('SSHCheckCommandExistsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized SSH check command exists attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(sshCheckCommandExistsContract, request, {}) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Checking if command '${params.commandName}' exists on ${params.host}:${params.port}` - ) - - const client = await createSSHConnection({ - host: params.host, - port: params.port, - username: params.username, - password: params.password, - privateKey: params.privateKey, - passphrase: params.passphrase, - }) - - try { - const escapedCommand = escapeShellArg(params.commandName) - - const result = await executeSSHCommand( - client, - `command -v '${escapedCommand}' 2>/dev/null || which '${escapedCommand}' 2>/dev/null` - ) - - const exists = result.exitCode === 0 && result.stdout.trim().length > 0 - const path = exists ? result.stdout.trim() : undefined - - let version: string | undefined - if (exists) { - try { - const versionResult = await executeSSHCommand( - client, - `'${escapedCommand}' --version 2>&1 | head -1 || '${escapedCommand}' -v 2>&1 | head -1` - ) - if (versionResult.exitCode === 0 && versionResult.stdout.trim()) { - version = versionResult.stdout.trim() - } - } catch { - // Version check failed, that's okay - } - } - - logger.info( - `[${requestId}] Command '${params.commandName}' ${exists ? 'exists' : 'does not exist'}` - ) - - return NextResponse.json({ - exists, - path, - version, - message: exists - ? `Command '${params.commandName}' found at ${path}` - : `Command '${params.commandName}' not found`, - }) - } finally { - client.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] SSH check command exists failed:`, error) - - return NextResponse.json( - { error: `SSH check command exists failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ssh/check-file-exists/route.ts b/apps/sim/app/api/tools/ssh/check-file-exists/route.ts deleted file mode 100644 index b5aaf72a62b..00000000000 --- a/apps/sim/app/api/tools/ssh/check-file-exists/route.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import type { Client, SFTPWrapper, Stats } from 'ssh2' -import { sshCheckFileExistsContract } from '@/lib/api/contracts/storage-transfer' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createSSHConnection, - getFileType, - parsePermissions, - sanitizePath, -} from '@/app/api/tools/ssh/utils' - -const logger = createLogger('SSHCheckFileExistsAPI') - -function getSFTP(client: Client): Promise { - return new Promise((resolve, reject) => { - client.sftp((err, sftp) => { - if (err) { - reject(err) - } else { - resolve(sftp) - } - }) - }) -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized SSH check file exists attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(sshCheckFileExistsContract, request, {}) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Checking if path exists: ${params.path} on ${params.host}:${params.port}` - ) - - const client = await createSSHConnection({ - host: params.host, - port: params.port, - username: params.username, - password: params.password, - privateKey: params.privateKey, - passphrase: params.passphrase, - }) - - try { - const sftp = await getSFTP(client) - const filePath = sanitizePath(params.path) - - const stats = await new Promise((resolve) => { - sftp.stat(filePath, (err, stats) => { - if (err) { - resolve(null) - } else { - resolve(stats) - } - }) - }) - - if (!stats) { - logger.info(`[${requestId}] Path does not exist: ${filePath}`) - return NextResponse.json({ - exists: false, - type: 'not_found', - message: `Path does not exist: ${filePath}`, - }) - } - - const fileType = getFileType(stats) - - // Check if the type matches the expected type - if (params.type !== 'any' && fileType !== params.type) { - logger.info(`[${requestId}] Path exists but is not a ${params.type}: ${filePath}`) - return NextResponse.json({ - exists: false, - type: fileType, - size: stats.size, - permissions: parsePermissions(stats.mode), - modified: new Date((stats.mtime || 0) * 1000).toISOString(), - message: `Path exists but is a ${fileType}, not a ${params.type}`, - }) - } - - logger.info(`[${requestId}] Path exists: ${filePath} (${fileType})`) - - return NextResponse.json({ - exists: true, - type: fileType, - size: stats.size, - permissions: parsePermissions(stats.mode), - modified: new Date((stats.mtime || 0) * 1000).toISOString(), - message: `Path exists: ${filePath} (${fileType})`, - }) - } finally { - client.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] SSH check file exists failed:`, error) - - return NextResponse.json( - { error: `SSH check file exists failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ssh/create-directory/route.ts b/apps/sim/app/api/tools/ssh/create-directory/route.ts deleted file mode 100644 index abd3214d7f1..00000000000 --- a/apps/sim/app/api/tools/ssh/create-directory/route.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { sshCreateDirectoryContract } from '@/lib/api/contracts/storage-transfer' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createSSHConnection, - escapeShellArg, - executeSSHCommand, - sanitizePath, -} from '@/app/api/tools/ssh/utils' - -const logger = createLogger('SSHCreateDirectoryAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized SSH create directory attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(sshCreateDirectoryContract, request, {}) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Creating directory ${params.path} on ${params.host}:${params.port}`) - - const client = await createSSHConnection({ - host: params.host, - port: params.port, - username: params.username, - password: params.password, - privateKey: params.privateKey, - passphrase: params.passphrase, - }) - - try { - const dirPath = sanitizePath(params.path) - const escapedPath = escapeShellArg(dirPath) - - const checkResult = await executeSSHCommand( - client, - `test -d '${escapedPath}' && echo "exists"` - ) - const alreadyExists = checkResult.stdout.trim() === 'exists' - - if (alreadyExists) { - logger.info(`[${requestId}] Directory already exists: ${dirPath}`) - return NextResponse.json({ - created: false, - path: dirPath, - alreadyExists: true, - message: `Directory already exists: ${dirPath}`, - }) - } - - const mkdirFlag = params.recursive ? '-p' : '' - const command = `mkdir ${mkdirFlag} -m ${params.permissions} '${escapedPath}'` - const result = await executeSSHCommand(client, command) - - if (result.exitCode !== 0) { - throw new Error(result.stderr || 'Failed to create directory') - } - - logger.info(`[${requestId}] Directory created successfully: ${dirPath}`) - - return NextResponse.json({ - created: true, - path: dirPath, - alreadyExists: false, - message: `Directory created successfully: ${dirPath}`, - }) - } finally { - client.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] SSH create directory failed:`, error) - - return NextResponse.json( - { error: `SSH create directory failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ssh/delete-file/route.ts b/apps/sim/app/api/tools/ssh/delete-file/route.ts deleted file mode 100644 index 52b1c714c82..00000000000 --- a/apps/sim/app/api/tools/ssh/delete-file/route.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { sshDeleteFileContract } from '@/lib/api/contracts/storage-transfer' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createSSHConnection, - escapeShellArg, - executeSSHCommand, - sanitizePath, -} from '@/app/api/tools/ssh/utils' - -const logger = createLogger('SSHDeleteFileAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized SSH delete file attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(sshDeleteFileContract, request, {}) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Deleting ${params.path} on ${params.host}:${params.port}`) - - const client = await createSSHConnection({ - host: params.host, - port: params.port, - username: params.username, - password: params.password, - privateKey: params.privateKey, - passphrase: params.passphrase, - }) - - try { - const filePath = sanitizePath(params.path) - const escapedPath = escapeShellArg(filePath) - - const checkResult = await executeSSHCommand( - client, - `test -e '${escapedPath}' && echo "exists"` - ) - if (checkResult.stdout.trim() !== 'exists') { - return NextResponse.json({ error: `Path does not exist: ${filePath}` }, { status: 404 }) - } - - let command: string - if (params.recursive) { - command = params.force ? `rm -rf '${escapedPath}'` : `rm -r '${escapedPath}'` - } else { - command = params.force ? `rm -f '${escapedPath}'` : `rm '${escapedPath}'` - } - - const result = await executeSSHCommand(client, command) - - if (result.exitCode !== 0) { - throw new Error(result.stderr || 'Failed to delete path') - } - - logger.info(`[${requestId}] Path deleted successfully: ${filePath}`) - - return NextResponse.json({ - deleted: true, - path: filePath, - message: `Successfully deleted: ${filePath}`, - }) - } finally { - client.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] SSH delete file failed:`, error) - - return NextResponse.json({ error: `SSH delete file failed: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/ssh/download-file/route.ts b/apps/sim/app/api/tools/ssh/download-file/route.ts deleted file mode 100644 index 9c0af06f10e..00000000000 --- a/apps/sim/app/api/tools/ssh/download-file/route.ts +++ /dev/null @@ -1,128 +0,0 @@ -import path from 'path' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import type { Client, SFTPWrapper } from 'ssh2' -import { sshDownloadFileContract } from '@/lib/api/contracts/storage-transfer' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getFileExtension, getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils' -import { MAX_SFTP_READ_BYTES, readSftpFileCapped } from '@/app/api/tools/sftp/utils' -import { createSSHConnection, sanitizePath } from '@/app/api/tools/ssh/utils' - -const logger = createLogger('SSHDownloadFileAPI') - -function getSFTP(client: Client): Promise { - return new Promise((resolve, reject) => { - client.sftp((err, sftp) => { - if (err) { - reject(err) - } else { - resolve(sftp) - } - }) - }) -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized SSH download file attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(sshDownloadFileContract, request, {}) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Downloading file from ${params.host}:${params.port}${params.remotePath}` - ) - - const client = await createSSHConnection({ - host: params.host, - port: params.port, - username: params.username, - password: params.password, - privateKey: params.privateKey, - passphrase: params.passphrase, - }) - - try { - const sftp = await getSFTP(client) - const remotePath = sanitizePath(params.remotePath) - - // Check if file exists - const stats = await new Promise<{ size: number }>((resolve, reject) => { - sftp.stat(remotePath, (err, stats) => { - if (err) { - reject(new Error(`File not found: ${remotePath}`)) - } else { - resolve(stats) - } - }) - }) - - if (stats.size > MAX_SFTP_READ_BYTES) { - const sizeMB = (stats.size / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { error: `File size (${sizeMB}MB) exceeds download limit of 50MB` }, - { status: 413 } - ) - } - - const content = await readSftpFileCapped( - sftp, - remotePath, - MAX_SFTP_READ_BYTES, - 'SSH file download' - ) - - const fileName = path.basename(remotePath) - const extension = getFileExtension(fileName) - const mimeType = getMimeTypeFromExtension(extension) - - // Encode content as base64 for binary safety - const base64Content = content.toString('base64') - - logger.info(`[${requestId}] File downloaded successfully from ${remotePath}`) - - return NextResponse.json({ - downloaded: true, - file: { - name: fileName, - mimeType, - data: base64Content, - size: content.length, - }, - content: base64Content, - fileName: fileName, - remotePath: remotePath, - size: content.length, - message: `File downloaded successfully from ${remotePath}`, - }) - } finally { - client.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - - if (isPayloadSizeLimitError(error)) { - logger.warn(`[${requestId}] SSH file download aborted: ${errorMessage}`) - return NextResponse.json({ error: errorMessage }, { status: 413 }) - } - - logger.error(`[${requestId}] SSH file download failed:`, error) - - return NextResponse.json( - { error: `SSH file download failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ssh/execute-command/route.ts b/apps/sim/app/api/tools/ssh/execute-command/route.ts deleted file mode 100644 index 3b192957d97..00000000000 --- a/apps/sim/app/api/tools/ssh/execute-command/route.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { sshExecuteCommandContract } from '@/lib/api/contracts/storage-transfer' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createSSHConnection, - escapeShellArg, - executeSSHCommand, - sanitizeCommand, -} from '@/app/api/tools/ssh/utils' - -const logger = createLogger('SSHExecuteCommandAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized SSH execute command attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(sshExecuteCommandContract, request, {}) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Executing SSH command on ${params.host}:${params.port}`) - - const client = await createSSHConnection({ - host: params.host, - port: params.port, - username: params.username, - password: params.password, - privateKey: params.privateKey, - passphrase: params.passphrase, - }) - - try { - let command = sanitizeCommand(params.command) - if (params.workingDirectory) { - const escapedWorkDir = escapeShellArg(params.workingDirectory) - command = `cd '${escapedWorkDir}' && ${command}` - } - - const result = await executeSSHCommand(client, command) - - logger.info(`[${requestId}] Command executed successfully with exit code ${result.exitCode}`) - - return NextResponse.json({ - stdout: result.stdout, - stderr: result.stderr, - exitCode: result.exitCode, - success: result.exitCode === 0, - message: `Command executed with exit code ${result.exitCode}`, - }) - } finally { - client.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] SSH command execution failed:`, error) - - return NextResponse.json( - { error: `SSH command execution failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ssh/execute-script/route.ts b/apps/sim/app/api/tools/ssh/execute-script/route.ts deleted file mode 100644 index ce6cf309c71..00000000000 --- a/apps/sim/app/api/tools/ssh/execute-script/route.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { sshExecuteScriptContract } from '@/lib/api/contracts/storage-transfer' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSSHConnection, escapeShellArg, executeSSHCommand } from '@/app/api/tools/ssh/utils' - -const logger = createLogger('SSHExecuteScriptAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized SSH execute script attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(sshExecuteScriptContract, request, {}) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Executing SSH script on ${params.host}:${params.port}`) - - const client = await createSSHConnection({ - host: params.host, - port: params.port, - username: params.username, - password: params.password, - privateKey: params.privateKey, - passphrase: params.passphrase, - }) - - try { - const scriptPath = `/tmp/sim_script_${requestId}.sh` - const escapedScriptPath = escapeShellArg(scriptPath) - const escapedInterpreter = escapeShellArg(params.interpreter) - - const heredocDelimiter = `SIMEOF_${generateId().replace(/-/g, '')}` - let command = `cat > '${escapedScriptPath}' << '${heredocDelimiter}' -${params.script} -${heredocDelimiter} -chmod +x '${escapedScriptPath}'` - - if (params.workingDirectory) { - const escapedWorkDir = escapeShellArg(params.workingDirectory) - command += ` -cd '${escapedWorkDir}'` - } - - command += ` -'${escapedInterpreter}' '${escapedScriptPath}' -exit_code=$? -rm -f '${escapedScriptPath}' -exit $exit_code` - - const result = await executeSSHCommand(client, command) - - logger.info(`[${requestId}] Script executed successfully with exit code ${result.exitCode}`) - - return NextResponse.json({ - stdout: result.stdout, - stderr: result.stderr, - exitCode: result.exitCode, - success: result.exitCode === 0, - scriptPath: scriptPath, - message: `Script executed with exit code ${result.exitCode}`, - }) - } finally { - client.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] SSH script execution failed:`, error) - - return NextResponse.json( - { error: `SSH script execution failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ssh/get-system-info/route.ts b/apps/sim/app/api/tools/ssh/get-system-info/route.ts deleted file mode 100644 index 9dc315afeba..00000000000 --- a/apps/sim/app/api/tools/ssh/get-system-info/route.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { sshGetSystemInfoContract } from '@/lib/api/contracts/storage-transfer' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSSHConnection, executeSSHCommand } from '@/app/api/tools/ssh/utils' - -const logger = createLogger('SSHGetSystemInfoAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized SSH get system info attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(sshGetSystemInfoContract, request, {}) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Getting system info from ${params.host}:${params.port}`) - - const client = await createSSHConnection({ - host: params.host, - port: params.port, - username: params.username, - password: params.password, - privateKey: params.privateKey, - passphrase: params.passphrase, - }) - - try { - // Get hostname - const hostnameResult = await executeSSHCommand(client, 'hostname') - const hostname = hostnameResult.stdout.trim() - - // Get OS info - const osResult = await executeSSHCommand(client, 'uname -s') - const os = osResult.stdout.trim() - - // Get architecture - const archResult = await executeSSHCommand(client, 'uname -m') - const architecture = archResult.stdout.trim() - - // Get uptime in seconds - const uptimeResult = await executeSSHCommand( - client, - "cat /proc/uptime 2>/dev/null | awk '{print int($1)}' || sysctl -n kern.boottime 2>/dev/null | awk '{print int(($(date +%s)) - $4)}'" - ) - const uptime = Number.parseInt(uptimeResult.stdout.trim()) || 0 - - // Get memory info - const memoryResult = await executeSSHCommand( - client, - "free -b 2>/dev/null | awk '/Mem:/ {print $2, $7, $3}' || vm_stat 2>/dev/null | awk '/Pages free|Pages active|Pages speculative|Pages wired|page size/ {gsub(/[^0-9]/, \"\"); print}'" - ) - const memParts = memoryResult.stdout.trim().split(/\s+/) - let memory = { total: 0, free: 0, used: 0 } - if (memParts.length >= 3) { - memory = { - total: Number.parseInt(memParts[0]) || 0, - free: Number.parseInt(memParts[1]) || 0, - used: Number.parseInt(memParts[2]) || 0, - } - } - - // Get disk space - const diskResult = await executeSSHCommand( - client, - "df -B1 / 2>/dev/null | awk 'NR==2 {print $2, $4, $3}' || df -k / 2>/dev/null | awk 'NR==2 {print $2*1024, $4*1024, $3*1024}'" - ) - const diskParts = diskResult.stdout.trim().split(/\s+/) - let diskSpace = { total: 0, free: 0, used: 0 } - if (diskParts.length >= 3) { - diskSpace = { - total: Number.parseInt(diskParts[0]) || 0, - free: Number.parseInt(diskParts[1]) || 0, - used: Number.parseInt(diskParts[2]) || 0, - } - } - - logger.info(`[${requestId}] System info retrieved successfully`) - - return NextResponse.json({ - hostname, - os, - architecture, - uptime, - memory, - diskSpace, - message: `System info retrieved for ${hostname}`, - }) - } finally { - client.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] SSH get system info failed:`, error) - - return NextResponse.json( - { error: `SSH get system info failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ssh/list-directory/route.ts b/apps/sim/app/api/tools/ssh/list-directory/route.ts deleted file mode 100644 index b8d4b795619..00000000000 --- a/apps/sim/app/api/tools/ssh/list-directory/route.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import type { Client, FileEntry, SFTPWrapper } from 'ssh2' -import { sshListDirectoryContract } from '@/lib/api/contracts/storage-transfer' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createSSHConnection, - getFileType, - parsePermissions, - sanitizePath, -} from '@/app/api/tools/ssh/utils' - -const logger = createLogger('SSHListDirectoryAPI') - -function getSFTP(client: Client): Promise { - return new Promise((resolve, reject) => { - client.sftp((err, sftp) => { - if (err) { - reject(err) - } else { - resolve(sftp) - } - }) - }) -} - -interface FileInfo { - name: string - type: 'file' | 'directory' | 'symlink' | 'other' - size: number - permissions: string - modified: string -} - -async function listDir(sftp: SFTPWrapper, dirPath: string): Promise { - return new Promise((resolve, reject) => { - sftp.readdir(dirPath, (err, list) => { - if (err) { - reject(err) - } else { - resolve(list) - } - }) - }) -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized SSH list directory attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(sshListDirectoryContract, request, {}) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`[${requestId}] Listing directory ${params.path} on ${params.host}:${params.port}`) - - const client = await createSSHConnection({ - host: params.host, - port: params.port, - username: params.username, - password: params.password, - privateKey: params.privateKey, - passphrase: params.passphrase, - }) - - try { - const sftp = await getSFTP(client) - const dirPath = sanitizePath(params.path) - - const list = await listDir(sftp, dirPath) - - const entries: FileInfo[] = list.map((entry) => ({ - name: entry.filename, - type: getFileType(entry.attrs), - size: entry.attrs.size, - permissions: parsePermissions(entry.attrs.mode), - modified: new Date((entry.attrs.mtime || 0) * 1000).toISOString(), - })) - - const totalFiles = entries.filter((e) => e.type === 'file').length - const totalDirectories = entries.filter((e) => e.type === 'directory').length - - logger.info( - `[${requestId}] Directory listed successfully: ${totalFiles} files, ${totalDirectories} directories` - ) - - return NextResponse.json({ - entries, - totalFiles, - totalDirectories, - message: `Found ${totalFiles} files and ${totalDirectories} directories`, - }) - } finally { - client.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] SSH list directory failed:`, error) - - return NextResponse.json( - { error: `SSH list directory failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ssh/move-rename/route.ts b/apps/sim/app/api/tools/ssh/move-rename/route.ts deleted file mode 100644 index 2433bdd08f4..00000000000 --- a/apps/sim/app/api/tools/ssh/move-rename/route.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { sshMoveRenameContract } from '@/lib/api/contracts/storage-transfer' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createSSHConnection, - escapeShellArg, - executeSSHCommand, - sanitizePath, -} from '@/app/api/tools/ssh/utils' - -const logger = createLogger('SSHMoveRenameAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized SSH move/rename attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(sshMoveRenameContract, request, {}) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Moving ${params.sourcePath} to ${params.destinationPath} on ${params.host}:${params.port}` - ) - - const client = await createSSHConnection({ - host: params.host, - port: params.port, - username: params.username, - password: params.password, - privateKey: params.privateKey, - passphrase: params.passphrase, - }) - - try { - const sourcePath = sanitizePath(params.sourcePath) - const destPath = sanitizePath(params.destinationPath) - const escapedSource = escapeShellArg(sourcePath) - const escapedDest = escapeShellArg(destPath) - - const sourceCheck = await executeSSHCommand( - client, - `test -e '${escapedSource}' && echo "exists"` - ) - if (sourceCheck.stdout.trim() !== 'exists') { - return NextResponse.json( - { error: `Source path does not exist: ${sourcePath}` }, - { status: 404 } - ) - } - - if (!params.overwrite) { - const destCheck = await executeSSHCommand( - client, - `test -e '${escapedDest}' && echo "exists"` - ) - if (destCheck.stdout.trim() === 'exists') { - return NextResponse.json( - { error: `Destination already exists and overwrite is disabled: ${destPath}` }, - { status: 409 } - ) - } - } - - const command = params.overwrite - ? `mv -f '${escapedSource}' '${escapedDest}'` - : `mv '${escapedSource}' '${escapedDest}'` - const result = await executeSSHCommand(client, command) - - if (result.exitCode !== 0) { - throw new Error(result.stderr || 'Failed to move/rename') - } - - logger.info(`[${requestId}] Successfully moved ${sourcePath} to ${destPath}`) - - return NextResponse.json({ - success: true, - sourcePath, - destinationPath: destPath, - message: `Successfully moved ${sourcePath} to ${destPath}`, - }) - } finally { - client.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] SSH move/rename failed:`, error) - - return NextResponse.json({ error: `SSH move/rename failed: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/ssh/read-file-content/route.ts b/apps/sim/app/api/tools/ssh/read-file-content/route.ts deleted file mode 100644 index 7594803fe53..00000000000 --- a/apps/sim/app/api/tools/ssh/read-file-content/route.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import type { Client, SFTPWrapper } from 'ssh2' -import { sshReadFileContentContract } from '@/lib/api/contracts/storage-transfer' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { readSftpFileCapped } from '@/app/api/tools/sftp/utils' -import { createSSHConnection, sanitizePath } from '@/app/api/tools/ssh/utils' - -const logger = createLogger('SSHReadFileContentAPI') - -function getSFTP(client: Client): Promise { - return new Promise((resolve, reject) => { - client.sftp((err, sftp) => { - if (err) { - reject(err) - } else { - resolve(sftp) - } - }) - }) -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized SSH read file content attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(sshReadFileContentContract, request, {}) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Reading file content from ${params.path} on ${params.host}:${params.port}` - ) - - const client = await createSSHConnection({ - host: params.host, - port: params.port, - username: params.username, - password: params.password, - privateKey: params.privateKey, - passphrase: params.passphrase, - }) - - try { - const sftp = await getSFTP(client) - const filePath = sanitizePath(params.path) - const maxBytes = params.maxSize * 1024 * 1024 // Convert MB to bytes - - const stats = await new Promise<{ size: number }>((resolve, reject) => { - sftp.stat(filePath, (err, stats) => { - if (err) { - reject(new Error(`File not found: ${filePath}`)) - } else { - resolve(stats) - } - }) - }) - - if (stats.size > maxBytes) { - return NextResponse.json( - { error: `File size (${stats.size} bytes) exceeds maximum allowed (${maxBytes} bytes)` }, - { status: 413 } - ) - } - - const buffer = await readSftpFileCapped(sftp, filePath, maxBytes, `File '${filePath}'`) - const content = buffer.toString(params.encoding as BufferEncoding) - - const lines = content.split('\n').length - - logger.info( - `[${requestId}] File content read successfully: ${buffer.length} bytes, ${lines} lines` - ) - - return NextResponse.json({ - content, - size: buffer.length, - lines, - path: filePath, - message: `File read successfully: ${buffer.length} bytes, ${lines} lines`, - }) - } finally { - client.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - - if (isPayloadSizeLimitError(error)) { - logger.warn(`[${requestId}] SSH read file content aborted: ${errorMessage}`) - return NextResponse.json({ error: errorMessage }, { status: 413 }) - } - - logger.error(`[${requestId}] SSH read file content failed:`, error) - - return NextResponse.json( - { error: `SSH read file content failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/ssh/upload-file/route.ts b/apps/sim/app/api/tools/ssh/upload-file/route.ts deleted file mode 100644 index 106aeff7c79..00000000000 --- a/apps/sim/app/api/tools/ssh/upload-file/route.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import type { Client, SFTPWrapper } from 'ssh2' -import { sshUploadFileContract } from '@/lib/api/contracts/storage-transfer' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSSHConnection, sanitizePath } from '@/app/api/tools/ssh/utils' - -const logger = createLogger('SSHUploadFileAPI') - -function getSFTP(client: Client): Promise { - return new Promise((resolve, reject) => { - client.sftp((err, sftp) => { - if (err) { - reject(err) - } else { - resolve(sftp) - } - }) - }) -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized SSH upload file attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(sshUploadFileContract, request, {}) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Uploading file to ${params.host}:${params.port}${params.remotePath}` - ) - - const client = await createSSHConnection({ - host: params.host, - port: params.port, - username: params.username, - password: params.password, - privateKey: params.privateKey, - passphrase: params.passphrase, - }) - - try { - const sftp = await getSFTP(client) - const remotePath = sanitizePath(params.remotePath) - - if (!params.overwrite) { - const exists = await new Promise((resolve) => { - sftp.stat(remotePath, (err) => { - resolve(!err) - }) - }) - - if (exists) { - return NextResponse.json( - { error: 'File already exists and overwrite is disabled' }, - { status: 409 } - ) - } - } - - let content: Buffer - try { - content = Buffer.from(params.fileContent, 'base64') - const reEncoded = content.toString('base64') - if (reEncoded !== params.fileContent) { - content = Buffer.from(params.fileContent, 'utf-8') - } - } catch { - content = Buffer.from(params.fileContent, 'utf-8') - } - - await new Promise((resolve, reject) => { - const writeStream = sftp.createWriteStream(remotePath, { - mode: params.permissions ? Number.parseInt(params.permissions, 8) : 0o644, - }) - - writeStream.on('error', reject) - writeStream.on('close', () => resolve()) - - writeStream.end(content) - }) - - logger.info(`[${requestId}] File uploaded successfully to ${remotePath}`) - - return NextResponse.json({ - uploaded: true, - remotePath: remotePath, - size: content.length, - message: `File uploaded successfully to ${remotePath}`, - }) - } finally { - client.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - logger.error(`[${requestId}] SSH file upload failed:`, error) - - return NextResponse.json({ error: `SSH file upload failed: ${errorMessage}` }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/ssh/utils.ts b/apps/sim/app/api/tools/ssh/utils.ts deleted file mode 100644 index 9f375ca2796..00000000000 --- a/apps/sim/app/api/tools/ssh/utils.ts +++ /dev/null @@ -1,386 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type Attributes, Client, type ConnectConfig } from 'ssh2' -import { validateDatabaseHost } from '@/lib/core/security/input-validation.server' - -const logger = createLogger('SSHUtils') - -// File type constants from POSIX -const S_IFMT = 0o170000 // bit mask for the file type bit field -const S_IFDIR = 0o040000 // directory -const S_IFREG = 0o100000 // regular file -const S_IFLNK = 0o120000 // symbolic link - -export interface SSHConnectionConfig { - host: string - port: number - username: string - password?: string | null - privateKey?: string | null - passphrase?: string | null - timeout?: number - keepaliveInterval?: number - readyTimeout?: number -} - -export interface SSHCommandResult { - stdout: string - stderr: string - exitCode: number -} - -/** - * Format SSH error with helpful troubleshooting context - */ -function formatSSHError(err: Error, config: { host: string; port: number }): Error { - const errorMessage = err.message.toLowerCase() - const host = config.host - const port = config.port - - if (errorMessage.includes('econnrefused') || errorMessage.includes('connection refused')) { - return new Error( - `Connection refused to ${host}:${port}. ` + - `Please verify: (1) SSH server is running on the target machine, ` + - `(2) Port ${port} is correct (default SSH port is 22), ` + - `(3) Firewall allows connections to port ${port}.` - ) - } - - if (errorMessage.includes('econnreset') || errorMessage.includes('connection reset')) { - return new Error( - `Connection reset by ${host}:${port}. ` + - `This usually means: (1) Wrong port number (SSH default is 22), ` + - `(2) Server rejected the connection, ` + - `(3) Network/firewall interrupted the connection. ` + - `Verify your SSH server configuration and port number.` - ) - } - - if (errorMessage.includes('etimedout') || errorMessage.includes('timeout')) { - return new Error( - `Connection timed out to ${host}:${port}. ` + - `Please verify: (1) Host "${host}" is reachable, ` + - `(2) No firewall is blocking the connection, ` + - `(3) The SSH server is responding.` - ) - } - - if (errorMessage.includes('enotfound') || errorMessage.includes('getaddrinfo')) { - return new Error( - `Could not resolve hostname "${host}". ` + - `Please verify the hostname or IP address is correct.` - ) - } - - if (errorMessage.includes('authentication') || errorMessage.includes('auth')) { - return new Error( - `Authentication failed for user on ${host}:${port}. ` + - `Please verify: (1) Username is correct, ` + - `(2) Password or private key is valid, ` + - `(3) User has SSH access on the server.` - ) - } - - if ( - errorMessage.includes('key') && - (errorMessage.includes('parse') || errorMessage.includes('invalid')) - ) { - return new Error( - `Invalid private key format. ` + - `Please ensure you're using a valid OpenSSH private key. ` + - `The key should start with "-----BEGIN" and end with "-----END".` - ) - } - - if (errorMessage.includes('host key') || errorMessage.includes('hostkey')) { - return new Error( - `Host key verification issue for ${host}. ` + - `This may be the first connection to this server or the server's key has changed.` - ) - } - - return new Error(`SSH connection to ${host}:${port} failed: ${err.message}`) -} - -/** - * Create an SSH connection using the provided configuration - * - * Uses ssh2 library defaults which align with OpenSSH standards: - * - readyTimeout: 20000ms (20 seconds) - * - keepaliveInterval: 0 (disabled, same as OpenSSH ServerAliveInterval) - * - keepaliveCountMax: 3 (same as OpenSSH ServerAliveCountMax) - */ -export async function createSSHConnection(config: SSHConnectionConfig): Promise { - const host = config.host - - if (!host || host.trim() === '') { - throw new Error('Host is required. Please provide a valid hostname or IP address.') - } - - const hostValidation = await validateDatabaseHost(host, 'host') - if (!hostValidation.isValid) { - throw new Error(hostValidation.error) - } - - const resolvedHost = hostValidation.resolvedIP ?? host.trim() - - return new Promise((resolve, reject) => { - const client = new Client() - const port = config.port || 22 - - const hasPassword = config.password && config.password.trim() !== '' - const hasPrivateKey = config.privateKey && config.privateKey.trim() !== '' - - if (!hasPassword && !hasPrivateKey) { - reject(new Error('Authentication required. Please provide either a password or private key.')) - return - } - - const connectConfig: ConnectConfig = { - host: resolvedHost, - port, - username: config.username, - } - - if (config.readyTimeout !== undefined) { - connectConfig.readyTimeout = config.readyTimeout - } - if (config.keepaliveInterval !== undefined) { - connectConfig.keepaliveInterval = config.keepaliveInterval - } - - if (hasPrivateKey) { - connectConfig.privateKey = config.privateKey! - if (config.passphrase && config.passphrase.trim() !== '') { - connectConfig.passphrase = config.passphrase - } - } else if (hasPassword) { - connectConfig.password = config.password! - } - - client.on('ready', () => { - resolve(client) - }) - - client.on('error', (err) => { - reject(formatSSHError(err, { host, port })) - }) - - try { - client.connect(connectConfig) - } catch (err) { - reject(formatSSHError(toError(err), { host, port })) - } - }) -} - -const MAX_OUTPUT_BYTES = 16 * 1024 * 1024 - -/** - * Execute a command on the SSH connection - */ -export function executeSSHCommand(client: Client, command: string): Promise { - return new Promise((resolve, reject) => { - client.exec(command, (err, stream) => { - if (err) { - reject(err) - return - } - - let stdout = '' - let stderr = '' - let stdoutBytes = 0 - let stderrBytes = 0 - let stdoutTruncated = false - let stderrTruncated = false - - stream.on('close', (code: number) => { - resolve({ - stdout: stdoutTruncated - ? `${stdout.trim()}\n[output truncated: exceeded 16MB limit]` - : stdout.trim(), - stderr: stderrTruncated - ? `${stderr.trim()}\n[stderr truncated: exceeded 16MB limit]` - : stderr.trim(), - exitCode: code ?? -1, - }) - }) - - stream.on('data', (data: Buffer) => { - const remaining = MAX_OUTPUT_BYTES - stdoutBytes - if (remaining <= 0) { - stdoutTruncated = true - return - } - const chunk = data.subarray(0, remaining) - stdout += chunk.toString() - stdoutBytes += chunk.length - if (data.length > remaining) stdoutTruncated = true - }) - - stream.stderr.on('data', (data: Buffer) => { - const remaining = MAX_OUTPUT_BYTES - stderrBytes - if (remaining <= 0) { - stderrTruncated = true - return - } - const chunk = data.subarray(0, remaining) - stderr += chunk.toString() - stderrBytes += chunk.length - if (data.length > remaining) stderrTruncated = true - }) - }) - }) -} - -/** - * Sanitize command input to prevent command injection - * - * Removes null bytes and other dangerous control characters while preserving - * legitimate shell syntax. Logs warnings for potentially dangerous patterns. - * - * Note: This function does not block complex shell commands (pipes, redirects, etc.) - * as users legitimately need these features for remote command execution. - * - * @param command - The command to sanitize - * @returns The sanitized command string - * - * @example - * ```typescript - * const safeCommand = sanitizeCommand(userInput) - * // Use safeCommand for SSH execution - * ``` - */ -export function sanitizeCommand(command: string): string { - let sanitized = command.replace(/\0/g, '') - - sanitized = sanitized.replace(/[\x0B\x0C]/g, '') - - sanitized = sanitized.trim() - - const dangerousPatterns = [ - { pattern: /\$\(.*\)/, name: 'command substitution $()' }, - { pattern: /`.*`/, name: 'backtick command substitution' }, - { pattern: /;\s*rm\s+-rf/i, name: 'destructive rm -rf command' }, - { pattern: /;\s*dd\s+/i, name: 'dd command (disk operations)' }, - { pattern: /mkfs/i, name: 'filesystem formatting command' }, - { pattern: />\s*\/dev\/sd[a-z]/i, name: 'direct disk write' }, - ] - - for (const { pattern, name } of dangerousPatterns) { - if (pattern.test(sanitized)) { - logger.warn(`Command contains ${name}`, { - command: sanitized.substring(0, 100) + (sanitized.length > 100 ? '...' : ''), - }) - } - } - - return sanitized -} - -/** - * Sanitize and validate file path to prevent path traversal attacks - * - * This function validates that a file path does not contain: - * - Null bytes - * - Path traversal sequences (.. or ../) - * - URL-encoded path traversal attempts - * - * @param path - The file path to sanitize and validate - * @returns The sanitized path if valid - * @throws Error if path traversal is detected - * - * @example - * ```typescript - * try { - * const safePath = sanitizePath(userInput) - * // Use safePath safely - * } catch (error) { - * // Handle invalid path - * } - * ``` - */ -export function sanitizePath(path: string): string { - let sanitized = path.replace(/\0/g, '') - sanitized = sanitized.trim() - - if (sanitized.includes('%00')) { - logger.warn('Path contains URL-encoded null bytes', { - path: path.substring(0, 100), - }) - throw new Error('Path contains invalid characters') - } - - const pathTraversalPatterns = [ - '../', // Standard Unix path traversal - '..\\', // Windows path traversal - '/../', // Mid-path traversal - '\\..\\', // Windows mid-path traversal - '%2e%2e%2f', // Fully encoded ../ - '%2e%2e/', // Partially encoded ../ - '%2e%2e%5c', // Fully encoded ..\ - '%2e%2e\\', // Partially encoded ..\ - '..%2f', // .. with encoded / - '..%5c', // .. with encoded \ - '%252e%252e', // Double URL encoded .. - '..%252f', // .. with double encoded / - '..%255c', // .. with double encoded \ - ] - - const lowerPath = sanitized.toLowerCase() - for (const pattern of pathTraversalPatterns) { - if (lowerPath.includes(pattern.toLowerCase())) { - logger.warn('Path traversal attempt detected', { - pattern, - path: path.substring(0, 100), - }) - throw new Error('Path contains invalid path traversal sequences') - } - } - - const segments = sanitized.split(/[/\\]/) - for (const segment of segments) { - if (segment === '..') { - logger.warn('Path traversal attempt detected (.. as path segment)', { - path: path.substring(0, 100), - }) - throw new Error('Path contains invalid path traversal sequences') - } - } - - return sanitized -} - -/** - * Escape a string for safe use in single-quoted shell arguments - * This is standard practice for shell command construction. - * e.g., "/tmp/test'file" becomes "/tmp/test'\''file" - * - * The pattern 'foo'\''bar' works because: - * - First ' ends the current single-quoted string - * - \' inserts a literal single quote (escaped outside quotes) - * - Next ' starts a new single-quoted string - */ -export function escapeShellArg(arg: string): string { - return arg.replace(/'/g, "'\\''") -} - -/** - * Parse file permissions from octal string - */ -export function parsePermissions(mode: number): string { - return `0${(mode & 0o777).toString(8)}` -} - -/** - * Get file type from attributes mode bits - */ -export function getFileType(attrs: Attributes): 'file' | 'directory' | 'symlink' | 'other' { - const mode = attrs.mode - const fileType = mode & S_IFMT - - if (fileType === S_IFDIR) return 'directory' - if (fileType === S_IFREG) return 'file' - if (fileType === S_IFLNK) return 'symlink' - return 'other' -} diff --git a/apps/sim/app/api/tools/ssh/write-file-content/route.ts b/apps/sim/app/api/tools/ssh/write-file-content/route.ts deleted file mode 100644 index ce742d0e62a..00000000000 --- a/apps/sim/app/api/tools/ssh/write-file-content/route.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import type { Client, SFTPWrapper } from 'ssh2' -import { sshWriteFileContentContract } from '@/lib/api/contracts/storage-transfer' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { MAX_SFTP_READ_BYTES, readSftpFileCapped } from '@/app/api/tools/sftp/utils' -import { createSSHConnection, sanitizePath } from '@/app/api/tools/ssh/utils' - -const logger = createLogger('SSHWriteFileContentAPI') - -function getSFTP(client: Client): Promise { - return new Promise((resolve, reject) => { - client.sftp((err, sftp) => { - if (err) { - reject(err) - } else { - resolve(sftp) - } - }) - }) -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId().slice(0, 8) - - try { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - logger.warn(`[${requestId}] Unauthorized SSH write file content attempt`) - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(sshWriteFileContentContract, request, {}) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info( - `[${requestId}] Writing file content to ${params.path} on ${params.host}:${params.port} (mode: ${params.mode})` - ) - - const client = await createSSHConnection({ - host: params.host, - port: params.port, - username: params.username, - password: params.password, - privateKey: params.privateKey, - passphrase: params.passphrase, - }) - - try { - const sftp = await getSFTP(client) - const filePath = sanitizePath(params.path) - - // Check if file exists for 'create' mode - if (params.mode === 'create') { - const exists = await new Promise((resolve) => { - sftp.stat(filePath, (err) => { - resolve(!err) - }) - }) - - if (exists) { - return NextResponse.json( - { error: `File already exists and mode is 'create': ${filePath}` }, - { status: 409 } - ) - } - } - - // Handle append mode by reading existing content first - let finalContent = params.content - if (params.mode === 'append') { - let existingContent = '' - try { - const existing = await readSftpFileCapped( - sftp, - filePath, - MAX_SFTP_READ_BYTES, - `Existing file '${filePath}'` - ) - existingContent = existing.toString('utf-8') - } catch (error) { - if (isPayloadSizeLimitError(error)) throw error - } - finalContent = existingContent + params.content - } - - // Write file - const fileMode = params.permissions ? Number.parseInt(params.permissions, 8) : 0o644 - await new Promise((resolve, reject) => { - const writeStream = sftp.createWriteStream(filePath, { mode: fileMode }) - - writeStream.on('error', reject) - writeStream.on('close', () => resolve()) - - writeStream.end(Buffer.from(finalContent, 'utf-8')) - }) - - // Get final file size - const stats = await new Promise<{ size: number }>((resolve, reject) => { - sftp.stat(filePath, (err, stats) => { - if (err) reject(err) - else resolve(stats) - }) - }) - - logger.info(`[${requestId}] File written successfully: ${stats.size} bytes`) - - return NextResponse.json({ - written: true, - path: filePath, - size: stats.size, - message: `File written successfully: ${stats.size} bytes`, - }) - } finally { - client.end() - } - } catch (error) { - const errorMessage = getErrorMessage(error, 'Unknown error occurred') - - if (isPayloadSizeLimitError(error)) { - logger.warn(`[${requestId}] SSH write file content aborted: ${errorMessage}`) - return NextResponse.json({ error: errorMessage }, { status: 413 }) - } - - logger.error(`[${requestId}] SSH write file content failed:`, error) - - return NextResponse.json( - { error: `SSH write file content failed: ${errorMessage}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/stagehand/agent/route.ts b/apps/sim/app/api/tools/stagehand/agent/route.ts deleted file mode 100644 index a1fa2ce7706..00000000000 --- a/apps/sim/app/api/tools/stagehand/agent/route.ts +++ /dev/null @@ -1,393 +0,0 @@ -import type { Stagehand as StagehandType } from '@browserbasehq/stagehand' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { stagehandAgentContract } from '@/lib/api/contracts/tools/stagehand' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { env } from '@/lib/core/config/env' -import { validateUrlWithDNS } from '@/lib/core/security/input-validation.server' -import { isSensitiveKey, REDACTED_MARKER } from '@/lib/core/security/redaction' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { ensureZodObject, normalizeUrl } from '@/app/api/tools/stagehand/utils' - -const logger = createLogger('StagehandAgentAPI') - -const BROWSERBASE_API_KEY = env.BROWSERBASE_API_KEY -const BROWSERBASE_PROJECT_ID = env.BROWSERBASE_PROJECT_ID - -/** - * Extracts the inner schema object from a potentially nested schema structure - */ -function getSchemaObject(outputSchema: Record): Record { - if (outputSchema.schema && typeof outputSchema.schema === 'object') { - return outputSchema.schema - } - return outputSchema -} - -/** - * Formats a schema object as a string for inclusion in agent instructions - */ -function formatSchemaForInstructions(schema: Record): string { - try { - return JSON.stringify(schema, null, 2) - } catch (error) { - logger.error('Error formatting schema for instructions', { error }) - return JSON.stringify(schema) - } -} - -/** - * Processes variables from various input formats into a standardized key-value object - */ -function processVariables(variables: any): Record | undefined { - if (!variables) return undefined - - let variablesObject: Record = {} - - if (Array.isArray(variables)) { - variables.forEach((item: any) => { - if (item?.cells?.Key && typeof item.cells.Key === 'string') { - variablesObject[item.cells.Key] = item.cells.Value || '' - } - }) - } else if (typeof variables === 'object' && variables !== null) { - variablesObject = { ...variables } - } else if (typeof variables === 'string') { - try { - variablesObject = JSON.parse(variables) - } catch (_e) { - logger.warn('Failed to parse variables string as JSON', { variables }) - return undefined - } - } - - if (Object.keys(variablesObject).length === 0) { - return undefined - } - - return variablesObject -} - -/** - * Substitutes variable placeholders in text with their actual values - * Variables are referenced using %key% syntax - */ -function substituteVariables(text: string, variables: Record | undefined): string { - if (!variables) return text - - let result = text - for (const [key, value] of Object.entries(variables)) { - const placeholder = `%${key}%` - result = result.split(placeholder).join(value) - } - return result -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - let stagehand: StagehandType | null = null - - try { - const parsed = await parseRequest( - stagehandAgentContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.error('Invalid request body', { errors: error.issues }) - return NextResponse.json( - { - error: getValidationErrorMessage(error, 'Invalid request parameters'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info('Received Stagehand agent request', { - startUrl: params.startUrl, - hasTask: !!params.task, - hasVariables: !!params.variables, - hasSchema: !!params.outputSchema, - }) - - const { task, startUrl: rawStartUrl, outputSchema, provider, apiKey, mode, maxSteps } = params - const variablesObject = processVariables(params.variables) - - const startUrl = normalizeUrl(rawStartUrl) - const urlValidation = await validateUrlWithDNS(startUrl, 'startUrl') - if (!urlValidation.isValid) { - return NextResponse.json({ error: urlValidation.error }, { status: 400 }) - } - - logger.info('Starting Stagehand agent process', { - rawStartUrl, - startUrl, - hasTask: !!task, - hasVariables: !!variablesObject, - provider, - }) - - if (!BROWSERBASE_API_KEY || !BROWSERBASE_PROJECT_ID) { - logger.error('Missing required environment variables', { - hasBrowserbaseApiKey: !!BROWSERBASE_API_KEY, - hasBrowserbaseProjectId: !!BROWSERBASE_PROJECT_ID, - }) - - return NextResponse.json( - { error: 'Server configuration error: Missing required environment variables' }, - { status: 500 } - ) - } - - if (!apiKey || typeof apiKey !== 'string') { - logger.error('API key is required') - return NextResponse.json({ error: 'API key is required' }, { status: 400 }) - } - - if (provider === 'openai' && !apiKey.startsWith('sk-')) { - logger.error('Invalid OpenAI API key format') - return NextResponse.json({ error: 'Invalid OpenAI API key format' }, { status: 400 }) - } - - if (provider === 'anthropic' && !apiKey.startsWith('sk-ant-')) { - logger.error('Invalid Anthropic API key format') - return NextResponse.json({ error: 'Invalid Anthropic API key format' }, { status: 400 }) - } - - const modelName = provider === 'anthropic' ? 'anthropic/claude-sonnet-4-6' : 'openai/gpt-5' - - let sessionId: string | null = null - let liveViewUrl: string | null = null - - try { - logger.info('Initializing Stagehand with Browserbase (v3)', { provider, modelName }) - - const { Stagehand } = await import('@browserbasehq/stagehand') - - stagehand = new Stagehand({ - env: 'BROWSERBASE', - apiKey: BROWSERBASE_API_KEY, - projectId: BROWSERBASE_PROJECT_ID, - verbose: 1, - disableAPI: true, // Use local agent handler instead of Browserbase API - logger: (msg) => logger.info(typeof msg === 'string' ? msg : JSON.stringify(msg)), - model: { - modelName, - apiKey: apiKey, - }, - }) - - logger.info('Starting stagehand.init()') - await stagehand.init() - logger.info('Stagehand initialized successfully') - - sessionId = stagehand.browserbaseSessionID ?? null - if (sessionId) { - try { - const debugResponse = await fetch( - `https://api.browserbase.com/v1/sessions/${sessionId}/debug`, - { - method: 'GET', - headers: { - 'X-BB-API-Key': BROWSERBASE_API_KEY, - }, - } - ) - if (debugResponse.ok) { - const debugData = (await debugResponse.json()) as { - debuggerFullscreenUrl?: string - debuggerUrl?: string - } - liveViewUrl = debugData.debuggerFullscreenUrl ?? debugData.debuggerUrl ?? null - if (liveViewUrl) { - logger.info(`Browserbase live view URL: ${liveViewUrl}`) - } - } else { - logger.warn(`Failed to fetch Browserbase debug URL: ${debugResponse.statusText}`) - } - } catch (debugError) { - logger.warn('Error fetching Browserbase debug URL', { error: debugError }) - } - } - - const page = stagehand.context.pages()[0] - logger.info(`Navigating to ${startUrl}`) - await page.goto(startUrl, { waitUntil: 'networkidle' }) - logger.info('Navigation complete') - - const taskWithVariables = substituteVariables(task, variablesObject) - - let agentInstructions = `You are a helpful web browsing assistant. Complete the following task: ${taskWithVariables}` - - if (variablesObject && Object.keys(variablesObject).length > 0) { - const safeVarKeys = Object.keys(variablesObject).map((key) => { - return isSensitiveKey(key) ? `${key}: ${REDACTED_MARKER}` : key - }) - logger.info('Variables available for task', { variables: safeVarKeys }) - } - - if (outputSchema && typeof outputSchema === 'object' && outputSchema !== null) { - const schemaObj = getSchemaObject(outputSchema) - agentInstructions += `\n\nIMPORTANT: You MUST return your final result in the following JSON format exactly:\n${formatSchemaForInstructions(schemaObj)}\n\nYour response should consist of valid JSON only, with no additional text.` - } - - logger.info('Creating Stagehand agent') - - const agent = stagehand.agent({ - model: { - modelName, - apiKey: apiKey, - }, - executionModel: { - modelName, - apiKey: apiKey, - }, - systemPrompt: agentInstructions, - mode, - }) - - logger.info('Executing agent task', { task: taskWithVariables, mode, maxSteps }) - - const agentExecutionResult = await agent.execute({ - instruction: taskWithVariables, - maxSteps, - }) - - const agentResult = { - success: agentExecutionResult.success, - completed: agentExecutionResult.completed, - message: agentExecutionResult.message, - actions: agentExecutionResult.actions, - } - - logger.info('Agent execution complete', { - success: agentResult.success, - completed: agentResult.completed, - actionCount: agentResult.actions?.length || 0, - }) - - let structuredOutput = null - const hasOutputSchema = - outputSchema && typeof outputSchema === 'object' && outputSchema !== null - - if (agentResult.message) { - try { - let jsonContent = agentResult.message - - const jsonBlockMatch = jsonContent.match(/```(?:json)?\s*([\s\S]*?)\s*```/) - if (jsonBlockMatch?.[1]) { - jsonContent = jsonBlockMatch[1] - } - - structuredOutput = JSON.parse(jsonContent) - logger.info('Successfully parsed structured output from agent response') - } catch (parseError) { - if (hasOutputSchema) { - logger.warn('Failed to parse JSON from agent message, attempting fallback extraction', { - error: parseError, - }) - - if (stagehand) { - try { - logger.info('Attempting to extract structured data using Stagehand extract') - const schemaObj = getSchemaObject(outputSchema) - const zodSchema = ensureZodObject(logger, schemaObj) - - structuredOutput = await stagehand.extract( - 'Extract the requested information from this page according to the schema', - zodSchema - ) - - logger.info('Successfully extracted structured data as fallback', { - keys: structuredOutput ? Object.keys(structuredOutput) : [], - }) - } catch (extractError) { - logger.error('Fallback extraction also failed', { error: extractError }) - } - } - } else { - logger.info('Agent returned plain text response (no schema provided)') - } - } - } - - return NextResponse.json({ - agentResult, - structuredOutput, - liveViewUrl, - sessionId, - }) - } catch (error) { - logger.error('Stagehand agent execution error', { - error, - message: getErrorMessage(error, 'Unknown error'), - stack: error instanceof Error ? error.stack : undefined, - }) - - let errorMessage = 'Unknown error during agent execution' - let errorDetails: Record = {} - - if (error instanceof Error) { - errorMessage = error.message - errorDetails = { - name: error.name, - stack: error.stack, - } - - const errorObj = error as any - if (typeof errorObj.code !== 'undefined') { - errorDetails.code = errorObj.code - } - if (typeof errorObj.statusCode !== 'undefined') { - errorDetails.statusCode = errorObj.statusCode - } - if (typeof errorObj.response !== 'undefined') { - errorDetails.response = errorObj.response - } - } - - return NextResponse.json( - { - error: errorMessage, - details: errorDetails, - liveViewUrl, - sessionId, - }, - { status: 500 } - ) - } - } catch (error) { - logger.error('Unexpected error in agent API route', { - error, - message: getErrorMessage(error, 'Unknown error'), - stack: error instanceof Error ? error.stack : undefined, - }) - return NextResponse.json( - { - error: 'Internal server error', - details: getErrorMessage(error, 'Unknown error'), - }, - { status: 500 } - ) - } finally { - if (stagehand) { - try { - logger.info('Closing Stagehand instance') - await stagehand.close() - } catch (closeError) { - logger.error('Error closing Stagehand instance', { error: closeError }) - } - } - } -}) diff --git a/apps/sim/app/api/tools/stagehand/extract/route.ts b/apps/sim/app/api/tools/stagehand/extract/route.ts deleted file mode 100644 index c72f96c4e7c..00000000000 --- a/apps/sim/app/api/tools/stagehand/extract/route.ts +++ /dev/null @@ -1,249 +0,0 @@ -import type { Stagehand as StagehandType } from '@browserbasehq/stagehand' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { stagehandExtractContract } from '@/lib/api/contracts/tools/stagehand' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { env } from '@/lib/core/config/env' -import { validateUrlWithDNS } from '@/lib/core/security/input-validation.server' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { ensureZodObject, normalizeUrl } from '@/app/api/tools/stagehand/utils' - -const logger = createLogger('StagehandExtractAPI') - -const BROWSERBASE_API_KEY = env.BROWSERBASE_API_KEY -const BROWSERBASE_PROJECT_ID = env.BROWSERBASE_PROJECT_ID - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - let stagehand: StagehandType | null = null - - try { - const parsed = await parseRequest( - stagehandExtractContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.error('Invalid request body', { errors: error.issues }) - return NextResponse.json( - { - error: getValidationErrorMessage(error, 'Invalid request parameters'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info('Received extraction request', { - url: params.url, - hasInstruction: !!params.instruction, - schema: params.schema ? typeof params.schema : 'none', - }) - - const { url: rawUrl, instruction, provider, apiKey, schema } = params - const url = normalizeUrl(rawUrl) - const urlValidation = await validateUrlWithDNS(url, 'url') - if (!urlValidation.isValid) { - return NextResponse.json({ error: urlValidation.error }, { status: 400 }) - } - - logger.info('Starting Stagehand extraction process', { - rawUrl, - url, - hasInstruction: !!instruction, - schemaType: typeof schema, - }) - - if (!schema || typeof schema !== 'object') { - logger.error('Invalid schema format', { schema }) - return NextResponse.json( - { error: 'Invalid schema format. Schema must be a valid JSON object.' }, - { status: 400 } - ) - } - - if (!BROWSERBASE_API_KEY || !BROWSERBASE_PROJECT_ID) { - logger.error('Missing required environment variables', { - hasBrowserbaseApiKey: !!BROWSERBASE_API_KEY, - hasBrowserbaseProjectId: !!BROWSERBASE_PROJECT_ID, - }) - - return NextResponse.json( - { error: 'Server configuration error: Missing required environment variables' }, - { status: 500 } - ) - } - - if (!apiKey || typeof apiKey !== 'string') { - logger.error('API key is required') - return NextResponse.json({ error: 'API key is required' }, { status: 400 }) - } - - if (provider === 'openai' && !apiKey.startsWith('sk-')) { - logger.error('Invalid OpenAI API key format') - return NextResponse.json({ error: 'Invalid OpenAI API key format' }, { status: 400 }) - } - - if (provider === 'anthropic' && !apiKey.startsWith('sk-ant-')) { - logger.error('Invalid Anthropic API key format') - return NextResponse.json({ error: 'Invalid Anthropic API key format' }, { status: 400 }) - } - - try { - const modelName = provider === 'anthropic' ? 'anthropic/claude-sonnet-4-6' : 'openai/gpt-5' - - logger.info('Initializing Stagehand with Browserbase (v3)', { provider, modelName }) - - const { Stagehand } = await import('@browserbasehq/stagehand') - - stagehand = new Stagehand({ - env: 'BROWSERBASE', - apiKey: BROWSERBASE_API_KEY, - projectId: BROWSERBASE_PROJECT_ID, - verbose: 1, - logger: (msg) => logger.info(typeof msg === 'string' ? msg : JSON.stringify(msg)), - model: { - modelName, - apiKey: apiKey, - }, - }) - - logger.info('Starting stagehand.init()') - await stagehand.init() - logger.info('Stagehand initialized successfully') - - const page = stagehand.context.pages()[0] - - logger.info(`Navigating to ${url}`) - await page.goto(url, { waitUntil: 'networkidle' }) - logger.info('Navigation complete') - - logger.info('Preparing extraction schema', { - schema: `${JSON.stringify(schema).substring(0, 100)}...`, - }) - - logger.info('Extracting data with Stagehand') - - try { - const schemaToConvert = schema.schema || schema - - let zodSchema - try { - logger.info('Creating Zod schema from JSON schema', { - schemaType: typeof schemaToConvert, - hasNestedSchema: !!schema.schema, - }) - - zodSchema = ensureZodObject(logger, schemaToConvert) - - logger.info('Successfully created Zod schema') - } catch (schemaError) { - logger.error('Failed to convert JSON schema to Zod schema', { - error: schemaError, - message: getErrorMessage(schemaError, 'Unknown schema error'), - }) - - logger.info('Falling back to simple extraction without schema') - zodSchema = undefined - } - - logger.info('Calling stagehand.extract with options', { - hasInstruction: !!instruction, - hasSchema: !!zodSchema, - }) - - let extractedData - if (zodSchema) { - extractedData = await stagehand.extract(instruction, zodSchema) - } else { - extractedData = await stagehand.extract(instruction) - } - - logger.info('Extraction successful', { - hasData: !!extractedData, - dataType: typeof extractedData, - dataKeys: extractedData ? Object.keys(extractedData) : [], - }) - - return NextResponse.json({ - data: extractedData, - schema, - }) - } catch (extractError) { - logger.error('Error during extraction operation', { - error: extractError, - message: getErrorMessage(extractError, 'Unknown extraction error'), - }) - throw extractError - } - } catch (error) { - logger.error('Stagehand extraction error', { - error, - message: getErrorMessage(error, 'Unknown error'), - stack: error instanceof Error ? error.stack : undefined, - }) - - let errorMessage = 'Unknown error during extraction' - let errorDetails: Record = {} - - if (error instanceof Error) { - errorMessage = error.message - errorDetails = { - name: error.name, - stack: error.stack, - } - - const errorObj = error as any - if (typeof errorObj.code !== 'undefined') { - errorDetails.code = errorObj.code - } - if (typeof errorObj.statusCode !== 'undefined') { - errorDetails.statusCode = errorObj.statusCode - } - if (typeof errorObj.response !== 'undefined') { - errorDetails.response = errorObj.response - } - } - - return NextResponse.json( - { - error: errorMessage, - details: errorDetails, - }, - { status: 500 } - ) - } - } catch (error) { - logger.error('Unexpected error in extraction API route', { - error, - message: getErrorMessage(error, 'Unknown error'), - stack: error instanceof Error ? error.stack : undefined, - }) - return NextResponse.json( - { - error: 'Internal server error', - details: getErrorMessage(error, 'Unknown error'), - }, - { status: 500 } - ) - } finally { - if (stagehand) { - try { - logger.info('Closing Stagehand instance') - await stagehand.close() - } catch (closeError) { - logger.error('Error closing Stagehand instance', { error: closeError }) - } - } - } -}) diff --git a/apps/sim/app/api/tools/stagehand/utils.ts b/apps/sim/app/api/tools/stagehand/utils.ts deleted file mode 100644 index 1e61f2971f7..00000000000 --- a/apps/sim/app/api/tools/stagehand/utils.ts +++ /dev/null @@ -1,133 +0,0 @@ -import type { Logger } from '@sim/logger' -import { z } from 'zod' - -function jsonSchemaToZod(logger: Logger, jsonSchema: Record): z.ZodTypeAny { - if (!jsonSchema) { - logger.error('Invalid schema: Schema is null or undefined') - throw new Error('Invalid schema: Schema is required') - } - - // Handle non-object schemas (strings, numbers, etc.) - if (typeof jsonSchema !== 'object' || jsonSchema === null) { - logger.warn('Schema is not an object, defaulting to any', { type: typeof jsonSchema }) - return z.any() - } - - // Handle different schema types - if (jsonSchema.type === 'object' && jsonSchema.properties) { - const shape: Record = {} - - // Create a zod object for each property - for (const [key, propSchema] of Object.entries(jsonSchema.properties)) { - shape[key] = jsonSchemaToZod(logger, propSchema as Record) - - // Add description if available - if ((propSchema as Record).description) { - shape[key] = shape[key].describe((propSchema as Record).description) - } - } - - // Create the base object - let zodObject = z.object(shape) - - // Handle required fields if specified - if (jsonSchema.required && Array.isArray(jsonSchema.required)) { - // For each property that's not in required, make it optional - for (const key of Object.keys(jsonSchema.properties)) { - if (!jsonSchema.required.includes(key)) { - shape[key] = shape[key].optional() - } - } - - // Recreate the object with the updated shape - zodObject = z.object(shape) - } - - return zodObject - } - if (jsonSchema.type === 'array' && jsonSchema.items) { - const itemSchema = jsonSchemaToZod(logger, jsonSchema.items as Record) - let arraySchema = z.array(itemSchema) - - // Add description if available - if (jsonSchema.description) { - arraySchema = arraySchema.describe(jsonSchema.description) - } - - return arraySchema - } - if (jsonSchema.type === 'string') { - let stringSchema = z.string() - - // Add description if available - if (jsonSchema.description) { - stringSchema = stringSchema.describe(jsonSchema.description) - } - - return stringSchema - } - if (jsonSchema.type === 'number') { - let numberSchema = z.number() - - // Add description if available - if (jsonSchema.description) { - numberSchema = numberSchema.describe(jsonSchema.description) - } - - return numberSchema - } - if (jsonSchema.type === 'boolean') { - let boolSchema = z.boolean() - - // Add description if available - if (jsonSchema.description) { - boolSchema = boolSchema.describe(jsonSchema.description) - } - - return boolSchema - } - if (jsonSchema.type === 'null') { - return z.null() - } - if (jsonSchema.type === 'integer') { - let intSchema = z.number().int() - - // Add description if available - if (jsonSchema.description) { - intSchema = intSchema.describe(jsonSchema.description) - } - - return intSchema - } - // For unknown types, return any - logger.warn('Unknown schema type, defaulting to any', { type: jsonSchema.type }) - return z.any() -} - -// Helper function to ensure we have a ZodObject -export function ensureZodObject(logger: Logger, schema: Record): z.ZodObject { - const zodSchema = jsonSchemaToZod(logger, schema) - - // If not already an object type, wrap it in an object - if (schema.type !== 'object') { - logger.warn('Schema is not an object type, wrapping in an object', { - type: schema.type, - }) - return z.object({ value: zodSchema }) - } - - // Safe cast since we know it's a ZodObject if type is 'object' - return zodSchema as z.ZodObject -} - -export function normalizeUrl(url: string): string { - // Normalize the URL - only add https:// if needed - let normalizedUrl = url - - // Add https:// if no protocol is specified - if (!normalizedUrl.startsWith('http://') && !normalizedUrl.startsWith('https://')) { - normalizedUrl = `https://${normalizedUrl}` - } - - return normalizedUrl -} diff --git a/apps/sim/app/api/tools/sts/assume-role-with-saml/route.ts b/apps/sim/app/api/tools/sts/assume-role-with-saml/route.ts deleted file mode 100644 index d8762ee4e52..00000000000 --- a/apps/sim/app/api/tools/sts/assume-role-with-saml/route.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsStsAssumeRoleWithSAMLContract } from '@/lib/api/contracts/tools/aws/sts-assume-role-with-saml' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { assumeRoleWithSAML, createUnauthenticatedSTSClient } from '../utils' - -const logger = createLogger('STSAssumeRoleWithSAMLAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsStsAssumeRoleWithSAMLContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Assuming role ${params.roleArn} with SAML`) - - const client = createUnauthenticatedSTSClient(params.region) - - try { - const result = await assumeRoleWithSAML( - client, - params.roleArn, - params.principalArn, - params.samlAssertion, - params.policyArns, - params.policy, - params.durationSeconds - ) - - logger.info('Role assumed successfully with SAML') - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to assume role with SAML', { error: toError(error).message }) - - return NextResponse.json( - { error: `Failed to assume role with SAML: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/sts/assume-role-with-web-identity/route.ts b/apps/sim/app/api/tools/sts/assume-role-with-web-identity/route.ts deleted file mode 100644 index bd951266bad..00000000000 --- a/apps/sim/app/api/tools/sts/assume-role-with-web-identity/route.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsStsAssumeRoleWithWebIdentityContract } from '@/lib/api/contracts/tools/aws/sts-assume-role-with-web-identity' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { assumeRoleWithWebIdentity, createUnauthenticatedSTSClient } from '../utils' - -const logger = createLogger('STSAssumeRoleWithWebIdentityAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsStsAssumeRoleWithWebIdentityContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Assuming role ${params.roleArn} with web identity`) - - const client = createUnauthenticatedSTSClient(params.region) - - try { - const result = await assumeRoleWithWebIdentity( - client, - params.roleArn, - params.roleSessionName, - params.webIdentityToken, - params.providerId, - params.policyArns, - params.policy, - params.durationSeconds - ) - - logger.info('Role assumed successfully with web identity') - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to assume role with web identity', { error: toError(error).message }) - - return NextResponse.json( - { error: `Failed to assume role with web identity: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/sts/assume-role/route.ts b/apps/sim/app/api/tools/sts/assume-role/route.ts deleted file mode 100644 index a66513a96b9..00000000000 --- a/apps/sim/app/api/tools/sts/assume-role/route.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsStsAssumeRoleContract } from '@/lib/api/contracts/tools/aws/sts-assume-role' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { assumeRole, createSTSClient } from '../utils' - -const logger = createLogger('STSAssumeRoleAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsStsAssumeRoleContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Assuming role ${params.roleArn}`) - - const client = createSTSClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await assumeRole( - client, - params.roleArn, - params.roleSessionName, - params.durationSeconds, - params.policy, - params.externalId, - params.serialNumber, - params.tokenCode, - params.policyArns, - params.tags, - params.transitiveTagKeys - ) - - logger.info('Role assumed successfully') - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to assume role', { error: toError(error).message }) - - return NextResponse.json( - { error: `Failed to assume role: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/sts/get-access-key-info/route.ts b/apps/sim/app/api/tools/sts/get-access-key-info/route.ts deleted file mode 100644 index 381f33b2057..00000000000 --- a/apps/sim/app/api/tools/sts/get-access-key-info/route.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsStsGetAccessKeyInfoContract } from '@/lib/api/contracts/tools/aws/sts-get-access-key-info' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSTSClient, getAccessKeyInfo } from '../utils' - -const logger = createLogger('STSGetAccessKeyInfoAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsStsGetAccessKeyInfoContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info(`Getting access key info for ${params.targetAccessKeyId}`) - - const client = createSTSClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await getAccessKeyInfo(client, params.targetAccessKeyId) - - logger.info('Access key info retrieved successfully') - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to get access key info', { error: toError(error).message }) - - return NextResponse.json( - { error: `Failed to get access key info: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/sts/get-caller-identity/route.ts b/apps/sim/app/api/tools/sts/get-caller-identity/route.ts deleted file mode 100644 index 4b24f2fb9ff..00000000000 --- a/apps/sim/app/api/tools/sts/get-caller-identity/route.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsStsGetCallerIdentityContract } from '@/lib/api/contracts/tools/aws/sts-get-caller-identity' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSTSClient, getCallerIdentity } from '../utils' - -const logger = createLogger('STSGetCallerIdentityAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsStsGetCallerIdentityContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info('Getting caller identity') - - const client = createSTSClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await getCallerIdentity(client) - - logger.info('Caller identity retrieved successfully') - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to get caller identity', { error: toError(error).message }) - - return NextResponse.json( - { error: `Failed to get caller identity: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/sts/get-session-token/route.ts b/apps/sim/app/api/tools/sts/get-session-token/route.ts deleted file mode 100644 index 2ebcbf2c627..00000000000 --- a/apps/sim/app/api/tools/sts/get-session-token/route.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { awsStsGetSessionTokenContract } from '@/lib/api/contracts/tools/aws/sts-get-session-token' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createSTSClient, getSessionToken } from '../utils' - -const logger = createLogger('STSGetSessionTokenAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const auth = await checkInternalAuth(request) - if (!auth.success || !auth.userId) { - return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) - } - - try { - const parsed = await parseToolRequest(awsStsGetSessionTokenContract, request, { - errorFormat: 'details', - logger, - }) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - logger.info('Getting session token') - - const client = createSTSClient({ - region: params.region, - accessKeyId: params.accessKeyId, - secretAccessKey: params.secretAccessKey, - }) - - try { - const result = await getSessionToken( - client, - params.durationSeconds, - params.serialNumber, - params.tokenCode - ) - - logger.info('Session token retrieved successfully') - - return NextResponse.json(result) - } finally { - client.destroy() - } - } catch (error) { - logger.error('Failed to get session token', { error: toError(error).message }) - - return NextResponse.json( - { error: `Failed to get session token: ${toError(error).message}` }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/sts/utils.ts b/apps/sim/app/api/tools/sts/utils.ts deleted file mode 100644 index a0caec02260..00000000000 --- a/apps/sim/app/api/tools/sts/utils.ts +++ /dev/null @@ -1,242 +0,0 @@ -import { - AssumeRoleCommand, - AssumeRoleWithSAMLCommand, - AssumeRoleWithWebIdentityCommand, - GetAccessKeyInfoCommand, - GetCallerIdentityCommand, - GetSessionTokenCommand, - type PolicyDescriptorType, - STSClient, - type Tag, -} from '@aws-sdk/client-sts' -import type { STSConnectionConfig } from '@/tools/sts/types' - -export function createSTSClient(config: STSConnectionConfig): STSClient { - return new STSClient({ - region: config.region, - credentials: { - accessKeyId: config.accessKeyId, - secretAccessKey: config.secretAccessKey, - }, - }) -} - -/** - * Creates an STS client for AssumeRoleWithWebIdentity / AssumeRoleWithSAML, - * which authenticate the caller via the supplied token/assertion rather than - * an IAM access key — AWS does not check the request signature for these two - * operations. The SDK's signing middleware still requires a `credentials` - * value to be resolvable, though, so static placeholder credentials are - * supplied explicitly to skip the default credential provider chain (env - * vars, shared config, container/IMDS role). Without this, the client would - * throw a CredentialsProviderError before the request is even sent in - * environments with no ambient AWS identity, even though a real IAM identity - * was never required. - */ -export function createUnauthenticatedSTSClient(region: string): STSClient { - return new STSClient({ - region, - credentials: { accessKeyId: 'anonymous', secretAccessKey: 'anonymous' }, - }) -} - -function parsePolicyArns(policyArns?: string | null): PolicyDescriptorType[] | undefined { - if (!policyArns) return undefined - const arns = policyArns - .split(',') - .map((arn) => arn.trim()) - .filter((arn) => arn.length > 0) - return arns.length > 0 ? arns.map((arn) => ({ arn })) : undefined -} - -function parseTags(tags?: string | null): Tag[] | undefined { - if (!tags) return undefined - const parsed = JSON.parse(tags) as Record - const entries = Object.entries(parsed) - return entries.length > 0 - ? entries.map(([Key, Value]) => ({ Key, Value: String(Value) })) - : undefined -} - -function parseTransitiveTagKeys(transitiveTagKeys?: string | null): string[] | undefined { - if (!transitiveTagKeys) return undefined - const keys = transitiveTagKeys - .split(',') - .map((key) => key.trim()) - .filter((key) => key.length > 0) - return keys.length > 0 ? keys : undefined -} - -export async function assumeRole( - client: STSClient, - roleArn: string, - roleSessionName: string, - durationSeconds?: number | null, - policy?: string | null, - externalId?: string | null, - serialNumber?: string | null, - tokenCode?: string | null, - policyArns?: string | null, - tags?: string | null, - transitiveTagKeys?: string | null -) { - const command = new AssumeRoleCommand({ - RoleArn: roleArn, - RoleSessionName: roleSessionName, - ...(durationSeconds ? { DurationSeconds: durationSeconds } : {}), - ...(policy ? { Policy: policy } : {}), - ...(externalId ? { ExternalId: externalId } : {}), - ...(serialNumber ? { SerialNumber: serialNumber } : {}), - ...(tokenCode ? { TokenCode: tokenCode } : {}), - ...(() => { - const arns = parsePolicyArns(policyArns) - return arns ? { PolicyArns: arns } : {} - })(), - ...(() => { - const sessionTags = parseTags(tags) - return sessionTags ? { Tags: sessionTags } : {} - })(), - ...(() => { - const keys = parseTransitiveTagKeys(transitiveTagKeys) - return keys ? { TransitiveTagKeys: keys } : {} - })(), - }) - - const response = await client.send(command) - - return { - accessKeyId: response.Credentials?.AccessKeyId ?? '', - secretAccessKey: response.Credentials?.SecretAccessKey ?? '', - sessionToken: response.Credentials?.SessionToken ?? '', - expiration: response.Credentials?.Expiration?.toISOString() ?? null, - assumedRoleArn: response.AssumedRoleUser?.Arn ?? '', - assumedRoleId: response.AssumedRoleUser?.AssumedRoleId ?? '', - packedPolicySize: response.PackedPolicySize ?? null, - sourceIdentity: response.SourceIdentity ?? null, - } -} - -export async function assumeRoleWithWebIdentity( - client: STSClient, - roleArn: string, - roleSessionName: string, - webIdentityToken: string, - providerId?: string | null, - policyArns?: string | null, - policy?: string | null, - durationSeconds?: number | null -) { - const command = new AssumeRoleWithWebIdentityCommand({ - RoleArn: roleArn, - RoleSessionName: roleSessionName, - WebIdentityToken: webIdentityToken, - ...(providerId ? { ProviderId: providerId } : {}), - ...(policy ? { Policy: policy } : {}), - ...(durationSeconds ? { DurationSeconds: durationSeconds } : {}), - ...(() => { - const arns = parsePolicyArns(policyArns) - return arns ? { PolicyArns: arns } : {} - })(), - }) - - const response = await client.send(command) - - return { - accessKeyId: response.Credentials?.AccessKeyId ?? '', - secretAccessKey: response.Credentials?.SecretAccessKey ?? '', - sessionToken: response.Credentials?.SessionToken ?? '', - expiration: response.Credentials?.Expiration?.toISOString() ?? null, - assumedRoleArn: response.AssumedRoleUser?.Arn ?? '', - assumedRoleId: response.AssumedRoleUser?.AssumedRoleId ?? '', - subjectFromWebIdentityToken: response.SubjectFromWebIdentityToken ?? '', - audience: response.Audience ?? null, - provider: response.Provider ?? null, - packedPolicySize: response.PackedPolicySize ?? null, - sourceIdentity: response.SourceIdentity ?? null, - } -} - -export async function assumeRoleWithSAML( - client: STSClient, - roleArn: string, - principalArn: string, - samlAssertion: string, - policyArns?: string | null, - policy?: string | null, - durationSeconds?: number | null -) { - const command = new AssumeRoleWithSAMLCommand({ - RoleArn: roleArn, - PrincipalArn: principalArn, - SAMLAssertion: samlAssertion, - ...(policy ? { Policy: policy } : {}), - ...(durationSeconds ? { DurationSeconds: durationSeconds } : {}), - ...(() => { - const arns = parsePolicyArns(policyArns) - return arns ? { PolicyArns: arns } : {} - })(), - }) - - const response = await client.send(command) - - return { - accessKeyId: response.Credentials?.AccessKeyId ?? '', - secretAccessKey: response.Credentials?.SecretAccessKey ?? '', - sessionToken: response.Credentials?.SessionToken ?? '', - expiration: response.Credentials?.Expiration?.toISOString() ?? null, - assumedRoleArn: response.AssumedRoleUser?.Arn ?? '', - assumedRoleId: response.AssumedRoleUser?.AssumedRoleId ?? '', - subject: response.Subject ?? null, - subjectType: response.SubjectType ?? null, - issuer: response.Issuer ?? null, - audience: response.Audience ?? null, - nameQualifier: response.NameQualifier ?? null, - packedPolicySize: response.PackedPolicySize ?? null, - sourceIdentity: response.SourceIdentity ?? null, - } -} - -export async function getCallerIdentity(client: STSClient) { - const command = new GetCallerIdentityCommand({}) - const response = await client.send(command) - - return { - account: response.Account ?? '', - arn: response.Arn ?? '', - userId: response.UserId ?? '', - } -} - -export async function getSessionToken( - client: STSClient, - durationSeconds?: number | null, - serialNumber?: string | null, - tokenCode?: string | null -) { - const command = new GetSessionTokenCommand({ - ...(durationSeconds ? { DurationSeconds: durationSeconds } : {}), - ...(serialNumber ? { SerialNumber: serialNumber } : {}), - ...(tokenCode ? { TokenCode: tokenCode } : {}), - }) - - const response = await client.send(command) - - return { - accessKeyId: response.Credentials?.AccessKeyId ?? '', - secretAccessKey: response.Credentials?.SecretAccessKey ?? '', - sessionToken: response.Credentials?.SessionToken ?? '', - expiration: response.Credentials?.Expiration?.toISOString() ?? null, - } -} - -export async function getAccessKeyInfo(client: STSClient, accessKeyId: string) { - const command = new GetAccessKeyInfoCommand({ - AccessKeyId: accessKeyId, - }) - - const response = await client.send(command) - - return { - account: response.Account ?? '', - } -} diff --git a/apps/sim/app/api/tools/stt/route.test.ts b/apps/sim/app/api/tools/stt/route.test.ts deleted file mode 100644 index 137ae94ad90..00000000000 --- a/apps/sim/app/api/tools/stt/route.test.ts +++ /dev/null @@ -1,227 +0,0 @@ -/** - * @vitest-environment node - */ -import { - createMockRequest, - hybridAuthMockFns, - inputValidationMock, - inputValidationMockFns, -} from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' -import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { PRIVATE_MODEL_INPUT_PROVENANCE_HEADER } from '@/lib/execution/model-input-provenance' -import { - RESOLVED_SECRET_PROVENANCE_FIELD, - RESOLVED_SECRET_PROVENANCE_METADATA_V1, -} from '@/lib/execution/private-tool-metadata' - -const { - mockIsInternalFileUrl, - mockDownloadFileFromStorage, - mockIsModelSafeWorkspaceFileKey, - mockResolveInternalFileUrl, -} = vi.hoisted(() => ({ - mockIsInternalFileUrl: vi.fn(), - mockDownloadFileFromStorage: vi.fn(), - mockIsModelSafeWorkspaceFileKey: vi.fn(), - mockResolveInternalFileUrl: vi.fn(), -})) - -vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) -vi.mock('@/lib/uploads/utils/file-utils', () => ({ - extractStorageKey: vi.fn(() => 'storage-key'), - isInternalFileUrl: mockIsInternalFileUrl, - getMimeTypeFromExtension: vi.fn(() => 'application/octet-stream'), -})) -vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ - downloadFileFromStorage: mockDownloadFileFromStorage, - resolveInternalFileUrl: mockResolveInternalFileUrl, -})) -vi.mock('@/app/api/files/authorization', () => ({ - assertToolFileAccess: vi.fn().mockResolvedValue(null), -})) -vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({ - isModelSafeWorkspaceFileKey: mockIsModelSafeWorkspaceFileKey, - MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE: - 'File cannot be sent to a model because its secret provenance is unavailable', -})) -vi.mock('@/lib/audio/extractor', () => ({ - isVideoFile: vi.fn(() => false), - extractAudioFromVideo: vi.fn(), -})) - -import { POST } from '@/app/api/tools/stt/route' - -const PINNED_IP = '93.184.216.34' - -const baseBody = { - provider: 'whisper', - apiKey: 'test-api-key', - audioUrl: 'https://example.com/audio.mp3', -} - -function createVerifiedSttRequest(body: Record) { - return createMockRequest( - 'POST', - { - ...body, - [RESOLVED_SECRET_PROVENANCE_FIELD]: { - version: 1, - complete: true, - entries: [], - }, - }, - { [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1 } - ) -} - -function mockSecureFetchResponse(body: { ok?: boolean; contentType?: string }) { - return { - ok: body.ok ?? true, - status: 200, - statusText: '', - headers: new Headers({ 'content-type': body.contentType ?? 'audio/mpeg' }), - body: null, - text: async () => '', - json: async () => ({}), - arrayBuffer: async () => new ArrayBuffer(8), - } -} - -describe('POST /api/tools/stt', () => { - beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ - isValid: true, - resolvedIP: PINNED_IP, - originalHostname: 'example.com', - }) - mockIsInternalFileUrl.mockReturnValue(false) - mockIsModelSafeWorkspaceFileKey.mockResolvedValue(true) - mockDownloadFileFromStorage.mockResolvedValue(Buffer.from('audio')) - - vi.stubGlobal( - 'fetch', - vi.fn().mockResolvedValue({ - ok: true, - json: async () => ({ text: 'hello world', language: 'en', duration: 1.2 }), - }) - ) - }) - - it('bounds the audioUrl download and rejects oversized responses cleanly', async () => { - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockRejectedValueOnce( - new PayloadSizeLimitError({ - label: 'response body', - maxBytes: 100 * 1024 * 1024, - observedBytes: 200 * 1024 * 1024, - }) - ) - - const response = await POST(createVerifiedSttRequest(baseBody)) - - expect(response.status).toBe(413) - const data = (await response.json()) as { error: string } - expect(data.error).toMatch(/exceeds the maximum supported size/i) - - const call = inputValidationMockFns.mockSecureFetchWithPinnedIP.mock.calls[0] - expect(call[1]).toBe(PINNED_IP) - expect(call[2]).toMatchObject({ maxResponseBytes: 100 * 1024 * 1024 }) - }) - - it('transcribes a normal, well-under-cap audio download successfully', async () => { - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( - mockSecureFetchResponse({}) - ) - - const response = await POST(createVerifiedSttRequest(baseBody)) - - expect(response.status).toBe(200) - const data = (await response.json()) as { transcript: string } - expect(data.transcript).toBe('hello world') - }) - - it('rejects an authenticated but incomplete private provenance envelope before downloading', async () => { - const response = await POST( - createMockRequest( - 'POST', - { - ...baseBody, - [RESOLVED_SECRET_PROVENANCE_FIELD]: { - version: 1, - complete: false, - entries: [], - }, - }, - { [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1 } - ) - ) - - expect(response.status).toBe(400) - expect(await response.json()).toEqual({ error: 'Model input provenance is unavailable' }) - expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() - expect(fetch).not.toHaveBeenCalled() - }) - - it('accepts a verified empty private provenance envelope', async () => { - inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( - mockSecureFetchResponse({}) - ) - - const response = await POST(createVerifiedSttRequest(baseBody)) - - expect(response.status).toBe(200) - expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenCalledOnce() - }) - - it('rejects a tracked unsafe workspace audio file before reading its bytes', async () => { - mockIsModelSafeWorkspaceFileKey.mockResolvedValueOnce(false) - - const response = await POST( - createVerifiedSttRequest({ - provider: 'whisper', - apiKey: 'test-api-key', - audioFile: { - id: 'file-1', - name: 'audio.mp3', - size: 5, - type: 'audio/mpeg', - key: 'workspace/workspace-1/audio.mp3', - }, - }) - ) - - expect(response.status).toBe(400) - expect(await response.json()).toEqual({ - error: 'File cannot be sent to a model because its secret provenance is unavailable', - }) - expect(mockDownloadFileFromStorage).not.toHaveBeenCalled() - expect(fetch).not.toHaveBeenCalled() - }) - - it('rejects a tracked unsafe internal audio URL after access resolution', async () => { - mockIsInternalFileUrl.mockReturnValue(true) - mockResolveInternalFileUrl.mockResolvedValueOnce({ - fileUrl: 'https://storage.example.com/signed-audio.mp3', - }) - mockIsModelSafeWorkspaceFileKey.mockResolvedValueOnce(false) - - const response = await POST( - createVerifiedSttRequest({ - ...baseBody, - audioUrl: '/api/files/serve/workspace/workspace-1/audio.mp3', - }) - ) - - expect(response.status).toBe(400) - expect(mockResolveInternalFileUrl).toHaveBeenCalledOnce() - expect(mockIsModelSafeWorkspaceFileKey).toHaveBeenCalledWith('storage-key') - expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() - expect(fetch).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/app/api/tools/stt/route.ts b/apps/sim/app/api/tools/stt/route.ts deleted file mode 100644 index 3ff3cb3eaaa..00000000000 --- a/apps/sim/app/api/tools/stt/route.ts +++ /dev/null @@ -1,837 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { sleep } from '@sim/utils/helpers' -import { generateId } from '@sim/utils/id' -import { type NextRequest, NextResponse } from 'next/server' -import { sttToolContract } from '@/lib/api/contracts/tools/media/stt' -import { getValidationErrorMessage, parseRequest, validationErrorResponse } from '@/lib/api/server' -import { extractAudioFromVideo, isVideoFile } from '@/lib/audio/extractor' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' -import { - isModelSafeWorkspaceFileKey, - MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, -} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' -import { - extractStorageKey, - getMimeTypeFromExtension, - isInternalFileUrl, -} from '@/lib/uploads/utils/file-utils' -import { - downloadFileFromStorage, - resolveInternalFileUrl, -} from '@/lib/uploads/utils/file-utils.server' -import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import type { TranscriptSegment } from '@/tools/stt/types' - -const logger = createLogger('SttProxyAPI') -const ELEVENLABS_STT_MODEL = 'scribe_v2' - -export const dynamic = 'force-dynamic' -/** - * Mirrors the hosted workflow execution ceiling (7 days) used by - * `getMaxExecutionTimeout()` for the transcript polling loop below. Next.js requires a - * static literal for `maxDuration`, so this value must be kept in sync with that source. - */ -export const maxDuration = 604800 - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId() - logger.info(`[${requestId}] STT transcription request started`) - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const userId = authResult.userId - - const parsed = await parseRequest( - sttToolContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid STT request:`, error.issues) - return validationErrorResponse( - error, - getValidationErrorMessage(error, 'Invalid request data') - ) - }, - } - ) - if (!parsed.success) return parsed.response - - const body = parsed.data.body - const modelInputProvenance = validateOpaqueModelInputProvenance({ - headers: request.headers, - payload: body, - isInternalRequest: true, - }) - if (!modelInputProvenance.success) { - return NextResponse.json( - { error: modelInputProvenance.error }, - { status: modelInputProvenance.status } - ) - } - - const { - provider, - apiKey, - model, - language, - timestamps, - diarization, - translateToEnglish, - sentiment, - entityDetection, - piiRedaction, - summarization, - } = body - - let audioBuffer: Buffer - let audioFileName: string - let audioMimeType: string - - if (body.audioFile) { - if (Array.isArray(body.audioFile) && body.audioFile.length !== 1) { - return NextResponse.json({ error: 'audioFile must be a single file' }, { status: 400 }) - } - const file = Array.isArray(body.audioFile) ? body.audioFile[0] : body.audioFile - logger.info(`[${requestId}] Processing uploaded audio`) - - const deniedAudio = await assertToolFileAccess(file.key, userId, requestId, logger) - if (deniedAudio) return deniedAudio - if (!(await isModelSafeWorkspaceFileKey(file.key))) { - return NextResponse.json( - { error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE }, - { status: 400 } - ) - } - audioBuffer = await downloadFileFromStorage(file, requestId, logger, { - maxBytes: MAX_FILE_SIZE, - }) - audioFileName = file.name - // file.type may be missing if the file came from a block that doesn't preserve it - // Infer from filename extension as fallback - const ext = file.name.split('.').pop()?.toLowerCase() || '' - audioMimeType = file.type || getMimeTypeFromExtension(ext) - } else if (body.audioFileReference) { - if (Array.isArray(body.audioFileReference) && body.audioFileReference.length !== 1) { - return NextResponse.json( - { error: 'audioFileReference must be a single file' }, - { status: 400 } - ) - } - const file = Array.isArray(body.audioFileReference) - ? body.audioFileReference[0] - : body.audioFileReference - logger.info(`[${requestId}] Processing referenced audio`) - - const deniedRef = await assertToolFileAccess(file.key, userId, requestId, logger) - if (deniedRef) return deniedRef - if (!(await isModelSafeWorkspaceFileKey(file.key))) { - return NextResponse.json( - { error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE }, - { status: 400 } - ) - } - audioBuffer = await downloadFileFromStorage(file, requestId, logger, { - maxBytes: MAX_FILE_SIZE, - }) - audioFileName = file.name - - const ext = file.name.split('.').pop()?.toLowerCase() || '' - audioMimeType = file.type || getMimeTypeFromExtension(ext) - } else if (body.audioUrl) { - let audioUrl = body.audioUrl.trim() - const internalAudioUrl = isInternalFileUrl(audioUrl) - logger.info(`[${requestId}] Downloading audio source`, { internal: internalAudioUrl }) - if (audioUrl.startsWith('/') && !isInternalFileUrl(audioUrl)) { - return NextResponse.json( - { - error: 'Invalid file path. Only uploaded files are supported for internal paths.', - }, - { status: 400 } - ) - } - - if (internalAudioUrl) { - if (!userId) { - return NextResponse.json( - { error: 'Authentication required for internal file access' }, - { status: 401 } - ) - } - const resolution = await resolveInternalFileUrl(audioUrl, userId, requestId, logger) - if (resolution.error) { - return NextResponse.json( - { error: resolution.error.message }, - { status: resolution.error.status } - ) - } - audioUrl = resolution.fileUrl || audioUrl - if (!(await isModelSafeWorkspaceFileKey(extractStorageKey(body.audioUrl)))) { - return NextResponse.json( - { error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE }, - { status: 400 } - ) - } - } - - const urlValidation = await validateUrlWithDNS(audioUrl, 'audioUrl') - if (!urlValidation.isValid) { - return NextResponse.json({ error: urlValidation.error }, { status: 400 }) - } - - const response = await secureFetchWithPinnedIP(audioUrl, urlValidation.resolvedIP!, { - method: 'GET', - maxResponseBytes: MAX_FILE_SIZE, - }) - if (!response.ok) { - await response.text().catch(() => {}) - throw new Error(`Failed to download audio from URL: ${response.statusText}`) - } - - const arrayBuffer = await response.arrayBuffer() - audioBuffer = Buffer.from(arrayBuffer) - audioFileName = audioUrl.split('/').pop() || 'audio_file' - audioMimeType = response.headers.get('content-type') || 'audio/mpeg' - } else { - return NextResponse.json( - { error: 'No audio source provided. Provide audioFile, audioFileReference, or audioUrl' }, - { status: 400 } - ) - } - - if (isVideoFile(audioMimeType)) { - logger.info(`[${requestId}] Extracting audio from video file`) - try { - const extracted = await extractAudioFromVideo(audioBuffer, audioMimeType, { - outputFormat: 'mp3', - sampleRate: 16000, - channels: 1, - }) - audioBuffer = extracted.buffer - audioMimeType = 'audio/mpeg' - audioFileName = audioFileName.replace(/\.[^.]+$/, '.mp3') - } catch (error) { - logger.error(`[${requestId}] Video extraction failed:`, error) - return NextResponse.json( - { - error: `Failed to extract audio from video: ${getErrorMessage(error, 'Unknown error')}`, - }, - { status: 500 } - ) - } - } - - logger.info(`[${requestId}] Transcribing audio`, { provider }) - - let transcript: string - let segments: TranscriptSegment[] | undefined - let detectedLanguage: string | undefined - let duration: number | undefined - let confidence: number | undefined - let sentimentResults: any[] | undefined - let entities: any[] | undefined - let summary: string | undefined - - try { - if (provider === 'whisper') { - const result = await transcribeWithWhisper( - audioBuffer, - apiKey, - language, - timestamps, - translateToEnglish, - model, - body.prompt, - body.temperature, - audioMimeType, - audioFileName - ) - transcript = result.transcript - segments = result.segments - detectedLanguage = result.language - duration = result.duration - } else if (provider === 'deepgram') { - const result = await transcribeWithDeepgram( - audioBuffer, - apiKey, - language, - timestamps, - diarization, - model, - audioMimeType - ) - transcript = result.transcript - segments = result.segments - detectedLanguage = result.language - duration = result.duration - confidence = result.confidence - } else if (provider === 'elevenlabs') { - const result = await transcribeWithElevenLabs(audioBuffer, apiKey, language, timestamps) - transcript = result.transcript - segments = result.segments - detectedLanguage = result.language - duration = result.duration - } else if (provider === 'assemblyai') { - const result = await transcribeWithAssemblyAI( - audioBuffer, - apiKey, - language, - timestamps, - diarization, - sentiment, - entityDetection, - piiRedaction, - summarization, - model - ) - transcript = result.transcript - segments = result.segments - detectedLanguage = result.language - duration = result.duration - confidence = result.confidence - sentimentResults = result.sentiment - entities = result.entities - summary = result.summary - } else if (provider === 'gemini') { - const result = await transcribeWithGemini( - audioBuffer, - apiKey, - audioMimeType, - language, - timestamps, - model - ) - transcript = result.transcript - segments = result.segments - detectedLanguage = result.language - duration = result.duration - confidence = result.confidence - } else { - return NextResponse.json({ error: `Unknown provider: ${provider}` }, { status: 400 }) - } - } catch (error) { - logger.error(`[${requestId}] Transcription failed:`, error) - const errorMessage = getErrorMessage(error, 'Transcription failed') - return NextResponse.json({ error: errorMessage }, { status: 500 }) - } - - logger.info(`[${requestId}] Transcription completed successfully`) - - const response: Record = { transcript } - if (segments !== undefined) response.segments = segments - if (detectedLanguage !== undefined) response.language = detectedLanguage - if (duration !== undefined) response.duration = duration - if (confidence !== undefined) response.confidence = confidence - if (sentimentResults !== undefined) response.sentiment = sentimentResults - if (entities !== undefined) response.entities = entities - if (summary !== undefined) response.summary = summary - - return NextResponse.json(response) - } catch (error) { - logger.error(`[${requestId}] STT proxy error:`, error) - const isSizeLimit = isPayloadSizeLimitError(error) - const errorMessage = isSizeLimit - ? 'Audio file exceeds the maximum supported size' - : getErrorMessage(error, 'Unknown error') - return NextResponse.json({ error: errorMessage }, { status: isSizeLimit ? 413 : 500 }) - } -}) - -async function transcribeWithWhisper( - audioBuffer: Buffer, - apiKey: string, - language?: string, - timestamps?: 'none' | 'sentence' | 'word', - translate?: boolean, - model?: string, - prompt?: string, - temperature?: number, - mimeType?: string, - fileName?: string -): Promise<{ - transcript: string - segments?: TranscriptSegment[] - language?: string - duration?: number -}> { - const formData = new FormData() - - // Use actual MIME type and filename if provided - const actualMimeType = mimeType || 'audio/mpeg' - const actualFileName = fileName || 'audio.mp3' - const blob = new Blob([new Uint8Array(audioBuffer)], { type: actualMimeType }) - formData.append('file', blob, actualFileName) - formData.append('model', model || 'whisper-1') - - if (language && language !== 'auto') { - formData.append('language', language) - } - - if (prompt) { - formData.append('prompt', prompt) - } - - if (temperature !== undefined) { - formData.append('temperature', temperature.toString()) - } - - formData.append('response_format', 'verbose_json') - - // OpenAI API uses array notation for timestamp_granularities - if (timestamps === 'word') { - formData.append('timestamp_granularities[]', 'word') - } else if (timestamps === 'sentence') { - formData.append('timestamp_granularities[]', 'segment') - } - - const endpoint = translate ? 'translations' : 'transcriptions' - const response = await fetch(`https://api.openai.com/v1/audio/${endpoint}`, { - method: 'POST', - headers: { - Authorization: `Bearer ${apiKey}`, - }, - body: formData, - }) - - if (!response.ok) { - const error = await response.json() - const errorMessage = error.error?.message || error.message || JSON.stringify(error) - throw new Error(`Whisper API error: ${errorMessage}`) - } - - const data = await response.json() - - let segments: TranscriptSegment[] | undefined - if (timestamps !== 'none') { - segments = (data.segments || data.words || []).map((seg: any) => ({ - text: seg.text, - start: seg.start, - end: seg.end, - })) - } - - return { - transcript: data.text, - segments, - language: data.language, - duration: data.duration, - } -} - -async function transcribeWithDeepgram( - audioBuffer: Buffer, - apiKey: string, - language?: string, - timestamps?: 'none' | 'sentence' | 'word', - diarization?: boolean, - model?: string, - mimeType?: string -): Promise<{ - transcript: string - segments?: TranscriptSegment[] - language?: string - duration?: number - confidence?: number -}> { - const params = new URLSearchParams({ - model: model || 'nova-3', - smart_format: 'true', - punctuate: 'true', - }) - - if (language && language !== 'auto') { - params.append('language', language) - } else if (language === 'auto') { - params.append('detect_language', 'true') - } - - if (timestamps === 'sentence') { - params.append('utterances', 'true') - } - - if (diarization) { - params.append('diarize', 'true') - } - - const response = await fetch(`https://api.deepgram.com/v1/listen?${params.toString()}`, { - method: 'POST', - headers: { - Authorization: `Token ${apiKey}`, - 'Content-Type': mimeType || 'audio/mpeg', - }, - body: new Uint8Array(audioBuffer), - }) - - if (!response.ok) { - const error = await response.json() - const errorMessage = error.err_msg || error.message || JSON.stringify(error) - throw new Error(`Deepgram API error: ${errorMessage}`) - } - - const data = await response.json() - const result = data.results?.channels?.[0]?.alternatives?.[0] - - if (!result) { - throw new Error('No transcription result from Deepgram') - } - - const transcript = result.transcript - const detectedLanguage = data.results?.channels?.[0]?.detected_language - const confidence = result.confidence - - let segments: TranscriptSegment[] | undefined - if (result.words && timestamps === 'word') { - segments = result.words.map((word: any) => ({ - text: word.word, - start: word.start, - end: word.end, - speaker: word.speaker !== undefined ? `Speaker ${word.speaker}` : undefined, - confidence: word.confidence, - })) - } else if (data.results?.utterances && timestamps === 'sentence') { - segments = data.results.utterances.map((utterance: any) => ({ - text: utterance.transcript, - start: utterance.start, - end: utterance.end, - speaker: utterance.speaker !== undefined ? `Speaker ${utterance.speaker}` : undefined, - confidence: utterance.confidence, - })) - } - - return { - transcript, - segments, - language: detectedLanguage, - duration: data.metadata?.duration, - confidence, - } -} - -async function transcribeWithElevenLabs( - audioBuffer: Buffer, - apiKey: string, - language?: string, - timestamps?: 'none' | 'sentence' | 'word' -): Promise<{ - transcript: string - segments?: TranscriptSegment[] - language?: string - duration?: number -}> { - const formData = new FormData() - const blob = new Blob([new Uint8Array(audioBuffer)], { type: 'audio/mpeg' }) - formData.append('file', blob, 'audio.mp3') - formData.append('model_id', ELEVENLABS_STT_MODEL) - - if (language && language !== 'auto') { - formData.append('language_code', language) - } - - if (timestamps && timestamps !== 'none') { - const granularity = timestamps === 'word' ? 'word' : 'word' - formData.append('timestamps_granularity', granularity) - } else { - formData.append('timestamps_granularity', 'word') - } - - const response = await fetch('https://api.elevenlabs.io/v1/speech-to-text', { - method: 'POST', - headers: { - 'xi-api-key': apiKey, - }, - body: formData, - }) - - if (!response.ok) { - const error = await response.json() - const errorMessage = - typeof error.detail === 'string' - ? error.detail - : error.detail?.message || error.message || JSON.stringify(error) - throw new Error(`ElevenLabs API error: ${errorMessage}`) - } - - const data = await response.json() - - const words = data.words || [] - const segments: TranscriptSegment[] = words - .filter((w: any) => w.type === 'word') - .map((w: any) => ({ - text: w.text, - start: w.start, - end: w.end, - speaker: w.speaker_id, - })) - - return { - transcript: data.text || '', - segments: segments.length > 0 ? segments : undefined, - language: data.language_code, - duration: undefined, // ElevenLabs doesn't return duration in response - } -} - -async function transcribeWithAssemblyAI( - audioBuffer: Buffer, - apiKey: string, - language?: string, - timestamps?: 'none' | 'sentence' | 'word', - diarization?: boolean, - sentiment?: boolean, - entityDetection?: boolean, - piiRedaction?: boolean, - summarization?: boolean, - model?: string -): Promise<{ - transcript: string - segments?: TranscriptSegment[] - language?: string - duration?: number - confidence?: number - sentiment?: any[] - entities?: any[] - summary?: string -}> { - const uploadResponse = await fetch('https://api.assemblyai.com/v2/upload', { - method: 'POST', - headers: { - authorization: apiKey, - 'content-type': 'application/octet-stream', - }, - body: new Uint8Array(audioBuffer), - }) - - if (!uploadResponse.ok) { - const error = await uploadResponse.json() - throw new Error(`AssemblyAI upload error: ${error.error || JSON.stringify(error)}`) - } - - const { upload_url } = await uploadResponse.json() - - const transcriptRequest: any = { - audio_url: upload_url, - } - - // AssemblyAI supports 'best', 'slam-1', or 'universal' for speech_model - if (model === 'best' || model === 'slam-1' || model === 'universal') { - transcriptRequest.speech_model = model - } - - if (language && language !== 'auto') { - transcriptRequest.language_code = language - } else if (language === 'auto') { - transcriptRequest.language_detection = true - } - - if (diarization) { - transcriptRequest.speaker_labels = true - } - - if (sentiment) { - transcriptRequest.sentiment_analysis = true - } - - if (entityDetection) { - transcriptRequest.entity_detection = true - } - - if (piiRedaction) { - transcriptRequest.redact_pii = true - transcriptRequest.redact_pii_policies = [ - 'us_social_security_number', - 'email_address', - 'phone_number', - ] - } - - if (summarization) { - transcriptRequest.summarization = true - transcriptRequest.summary_model = 'informative' - transcriptRequest.summary_type = 'bullets' - } - - const transcriptResponse = await fetch('https://api.assemblyai.com/v2/transcript', { - method: 'POST', - headers: { - authorization: apiKey, - 'content-type': 'application/json', - }, - body: JSON.stringify(transcriptRequest), - }) - - if (!transcriptResponse.ok) { - const error = await transcriptResponse.json() - throw new Error(`AssemblyAI transcript error: ${error.error || JSON.stringify(error)}`) - } - - const { id } = await transcriptResponse.json() - - let transcript: any - let attempts = 0 - const pollIntervalMs = 5000 - const maxAttempts = Math.ceil(getMaxExecutionTimeout() / pollIntervalMs) - - while (attempts < maxAttempts) { - const statusResponse = await fetch(`https://api.assemblyai.com/v2/transcript/${id}`, { - headers: { - authorization: apiKey, - }, - }) - - if (!statusResponse.ok) { - const error = await statusResponse.json() - throw new Error(`AssemblyAI status error: ${error.error || JSON.stringify(error)}`) - } - - transcript = await statusResponse.json() - - if (transcript.status === 'completed') { - break - } - if (transcript.status === 'error') { - throw new Error(`AssemblyAI transcription failed: ${transcript.error}`) - } - - await sleep(5000) - attempts++ - } - - if (transcript.status !== 'completed') { - throw new Error('AssemblyAI transcription timed out') - } - - let segments: TranscriptSegment[] | undefined - if (timestamps !== 'none' && transcript.words) { - segments = transcript.words.map((word: any) => ({ - text: word.text, - start: word.start / 1000, - end: word.end / 1000, - speaker: word.speaker ? `Speaker ${word.speaker}` : undefined, - confidence: word.confidence, - })) - } - - const result: any = { - transcript: transcript.text, - segments, - language: transcript.language_code, - duration: transcript.audio_duration, - confidence: transcript.confidence, - } - - if (sentiment && transcript.sentiment_analysis_results) { - result.sentiment = transcript.sentiment_analysis_results - } - - if (entityDetection && transcript.entities) { - result.entities = transcript.entities - } - - if (summarization && transcript.summary) { - result.summary = transcript.summary - } - - return result -} - -async function transcribeWithGemini( - audioBuffer: Buffer, - apiKey: string, - mimeType: string, - language?: string, - timestamps?: 'none' | 'sentence' | 'word', - model?: string -): Promise<{ - transcript: string - segments?: TranscriptSegment[] - language?: string - duration?: number - confidence?: number -}> { - const modelName = model || 'gemini-2.5-flash' - - const estimatedSize = audioBuffer.length * 1.34 - if (estimatedSize > 20 * 1024 * 1024) { - throw new Error('Audio file exceeds 20MB limit for inline data') - } - - const base64Audio = audioBuffer.toString('base64') - - const languagePrompt = language && language !== 'auto' ? ` The audio is in ${language}.` : '' - - const timestampPrompt = - timestamps === 'sentence' || timestamps === 'word' - ? ' Include timestamps in MM:SS format for each sentence.' - : '' - - const requestBody = { - contents: [ - { - parts: [ - { - inline_data: { - mime_type: mimeType, - data: base64Audio, - }, - }, - { - text: `Please transcribe this audio file.${languagePrompt}${timestampPrompt} Provide the full transcript.`, - }, - ], - }, - ], - } - - const response = await fetch( - `https://generativelanguage.googleapis.com/v1beta/models/${modelName}:generateContent?key=${apiKey}`, - { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(requestBody), - } - ) - - if (!response.ok) { - const error = await response.json() - if (response.status === 404) { - throw new Error( - `Model not found: ${modelName}. Use gemini-3.1-pro-preview, gemini-3-pro-preview, gemini-2.5-pro, gemini-2.5-flash, gemini-2.5-flash-lite, or gemini-2.0-flash-exp` - ) - } - const errorMessage = error.error?.message || JSON.stringify(error) - throw new Error(`Gemini API error: ${errorMessage}`) - } - - const data = await response.json() - - if (!data.candidates?.[0]?.content?.parts?.[0]?.text) { - const candidate = data.candidates?.[0] - if (candidate?.finishReason === 'SAFETY') { - throw new Error('Content was blocked by safety filters') - } - throw new Error('Invalid response structure from Gemini API') - } - - const transcript = data.candidates[0].content.parts[0].text - - return { - transcript, - language: language !== 'auto' ? language : undefined, - } -} diff --git a/apps/sim/app/api/tools/supabase/storage-upload/route.ts b/apps/sim/app/api/tools/supabase/storage-upload/route.ts deleted file mode 100644 index ac9d6bda605..00000000000 --- a/apps/sim/app/api/tools/supabase/storage-upload/route.ts +++ /dev/null @@ -1,283 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { supabaseStorageUploadContract } from '@/lib/api/contracts/tools/databases/supabase' -import { parseToolRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateSupabaseProjectId } from '@/lib/core/security/input-validation' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { encodeStoragePath, encodeStorageSegment } from '@/tools/supabase/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('SupabaseStorageUploadAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn( - `[${requestId}] Unauthorized Supabase storage upload attempt: ${authResult.error}` - ) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info( - `[${requestId}] Authenticated Supabase storage upload request via ${authResult.authType}`, - { - userId: authResult.userId, - } - ) - - const parsed = await parseToolRequest(supabaseStorageUploadContract, request, { - errorFormat: 'toolDetails', - logger, - }) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - const fileData = validatedData.fileData - const isStringInput = typeof fileData === 'string' - - logger.info(`[${requestId}] Uploading to Supabase Storage`, { - bucket: validatedData.bucket, - fileName: validatedData.fileName, - path: validatedData.path, - fileDataType: isStringInput ? 'string' : 'object', - }) - - if (!fileData) { - return NextResponse.json( - { - success: false, - error: 'fileData is required', - }, - { status: 400 } - ) - } - - let uploadBody: Buffer - let uploadContentType: string | undefined - - if (isStringInput) { - let content = fileData as string - - const dataUrlMatch = content.match(/^data:([^;]+);base64,(.+)$/s) - if (dataUrlMatch) { - const [, mimeType, base64Data] = dataUrlMatch - content = base64Data - if (!validatedData.contentType) { - uploadContentType = mimeType - } - logger.info(`[${requestId}] Extracted base64 from data URL (MIME: ${mimeType})`) - } - - const cleanedContent = content.replace(/[\s\r\n]/g, '') - const isLikelyBase64 = /^[A-Za-z0-9+/]*={0,2}$/.test(cleanedContent) - - if (isLikelyBase64 && cleanedContent.length >= 4) { - try { - uploadBody = Buffer.from(cleanedContent, 'base64') - - const expectedMinSize = Math.floor(cleanedContent.length * 0.7) - const expectedMaxSize = Math.ceil(cleanedContent.length * 0.8) - - if ( - uploadBody.length >= expectedMinSize && - uploadBody.length <= expectedMaxSize && - uploadBody.length > 0 - ) { - logger.info( - `[${requestId}] Decoded base64 content: ${cleanedContent.length} chars -> ${uploadBody.length} bytes` - ) - } else { - const reEncoded = uploadBody.toString('base64') - if (reEncoded !== cleanedContent) { - logger.info( - `[${requestId}] Content looked like base64 but re-encoding didn't match, using as plain text` - ) - uploadBody = Buffer.from(content, 'utf-8') - } else { - logger.info( - `[${requestId}] Decoded base64 content (verified): ${uploadBody.length} bytes` - ) - } - } - } catch (decodeError) { - logger.info( - `[${requestId}] Failed to decode as base64, using as plain text: ${decodeError}` - ) - uploadBody = Buffer.from(content, 'utf-8') - } - } else { - uploadBody = Buffer.from(content, 'utf-8') - logger.info(`[${requestId}] Using content as plain text (${uploadBody.length} bytes)`) - } - - uploadContentType = - uploadContentType || validatedData.contentType || 'application/octet-stream' - } else { - const rawFile = fileData - logger.info(`[${requestId}] Processing file object: ${rawFile.name || 'unknown'}`) - - let userFile - try { - userFile = processSingleFileToUserFile(rawFile, requestId, logger) - } catch (error) { - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Failed to process file'), - }, - { status: 400 } - ) - } - - const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) - if (denied) return denied - let buffer: Buffer - let resolvedContentType: string - try { - const resolved = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - }) - buffer = resolved.buffer - resolvedContentType = resolved.contentType - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - logger.error(`[${requestId}] Failed to download file for Supabase upload:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Internal server error') }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } - - uploadBody = buffer - uploadContentType = - validatedData.contentType || - resolvedContentType || - userFile.type || - 'application/octet-stream' - } - - let fullPath = validatedData.fileName - if (validatedData.path) { - const folderPath = validatedData.path.endsWith('/') - ? validatedData.path - : `${validatedData.path}/` - fullPath = `${folderPath}${validatedData.fileName}` - } - - const projectValidation = validateSupabaseProjectId(validatedData.projectId) - if (!projectValidation.isValid) { - return NextResponse.json({ success: false, error: projectValidation.error }, { status: 400 }) - } - - const encodedBucket = encodeStorageSegment(validatedData.bucket) - const encodedPath = encodeStoragePath(fullPath) - const supabaseUrl = `https://${projectValidation.sanitized}.supabase.co/storage/v1/object/${encodedBucket}/${encodedPath}` - - const headers: Record = { - apikey: validatedData.apiKey, - Authorization: `Bearer ${validatedData.apiKey}`, - 'Content-Type': uploadContentType, - } - - if (validatedData.cacheControl) { - const cacheControl = validatedData.cacheControl.trim() - headers['cache-control'] = /^\d+$/.test(cacheControl) - ? `max-age=${cacheControl}` - : cacheControl - } - - if (validatedData.upsert) { - headers['x-upsert'] = 'true' - } - - logger.info(`[${requestId}] Sending to Supabase: ${supabaseUrl}`, { - contentType: uploadContentType, - bodySize: uploadBody.length, - upsert: validatedData.upsert, - }) - - const response = await fetch(supabaseUrl, { - method: 'POST', - headers, - body: new Uint8Array(uploadBody), - }) - - if (!response.ok) { - const errorText = await response.text() - let errorData - try { - errorData = JSON.parse(errorText) - } catch { - errorData = { message: errorText } - } - - logger.error(`[${requestId}] Supabase Storage upload failed:`, { - status: response.status, - statusText: response.statusText, - error: errorData, - }) - - return NextResponse.json( - { - success: false, - error: errorData.message || errorData.error || `Upload failed: ${response.statusText}`, - details: errorData, - }, - { status: response.status } - ) - } - - const result = await response.json() - - logger.info(`[${requestId}] File uploaded successfully to Supabase Storage`, { - bucket: validatedData.bucket, - path: fullPath, - }) - - const publicUrl = `https://${projectValidation.sanitized}.supabase.co/storage/v1/object/public/${encodedBucket}/${encodedPath}` - - return NextResponse.json({ - success: true, - output: { - message: 'Successfully uploaded file to storage', - results: { - ...result, - path: fullPath, - bucket: validatedData.bucket, - publicUrl, - }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error uploading to Supabase Storage:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/telegram/send-document/route.ts b/apps/sim/app/api/tools/telegram/send-document/route.ts deleted file mode 100644 index 6d4cf533d64..00000000000 --- a/apps/sim/app/api/tools/telegram/send-document/route.ts +++ /dev/null @@ -1,204 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { telegramSendDocumentContract } from '@/lib/api/contracts/tools/communication/messaging' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { convertMarkdownToHTML } from '@/tools/telegram/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('TelegramSendDocumentAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { - requireWorkflowId: false, - }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Telegram send attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated Telegram send request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const parsed = await parseRequest(telegramSendDocumentContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Sending Telegram document`, { - chatId: validatedData.chatId, - hasFiles: !!(validatedData.files && validatedData.files.length > 0), - fileCount: validatedData.files?.length || 0, - }) - - if (!validatedData.files || validatedData.files.length === 0) { - return NextResponse.json( - { - success: false, - error: 'At least one document file is required for sendDocument operation', - }, - { status: 400 } - ) - } - - const userFiles = processFilesToUserFiles(validatedData.files, requestId, logger) - - if (userFiles.length === 0) { - logger.warn(`[${requestId}] No valid files to upload`) - return NextResponse.json( - { - success: false, - error: 'No valid files provided for upload', - }, - { status: 400 } - ) - } - - const maxSize = 50 * 1024 * 1024 // 50MB - const tooLargeFiles = userFiles.filter((file) => file.size > maxSize) - - if (tooLargeFiles.length > 0) { - const filesInfo = tooLargeFiles - .map((f) => `${f.name} (${(f.size / (1024 * 1024)).toFixed(2)}MB)`) - .join(', ') - return NextResponse.json( - { - success: false, - error: `The following files exceed Telegram's 50MB limit: ${filesInfo}`, - }, - { status: 400 } - ) - } - - const userFile = userFiles[0] - logger.info(`[${requestId}] Uploading document: ${userFile.name}`) - - const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) - if (denied) return denied - - let buffer: Buffer - let contentType: string - try { - const downloaded = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: maxSize, - }) - buffer = downloaded.buffer - contentType = downloaded.contentType - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - if (isPayloadSizeLimitError(error)) { - const sizeMB = ((error.observedBytes ?? userFile.size) / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `The following files exceed Telegram's 50MB limit: ${userFile.name} (${sizeMB}MB)`, - }, - { status: 400 } - ) - } - logger.error(`[${requestId}] Failed to download document ${userFile.name}:`, error) - return NextResponse.json( - { - success: false, - error: `Failed to download document: ${getErrorMessage(error, 'Unknown error')}`, - }, - { status: 500 } - ) - } - - if (buffer.length > maxSize) { - const sizeMB = (buffer.length / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { - success: false, - error: `The following files exceed Telegram's 50MB limit: ${userFile.name} (${sizeMB}MB)`, - }, - { status: 400 } - ) - } - - const resolvedMimeType = contentType || userFile.type || 'application/octet-stream' - const filesOutput = [ - { - name: userFile.name, - mimeType: resolvedMimeType, - data: buffer.toString('base64'), - size: buffer.length, - }, - ] - - logger.info(`[${requestId}] Downloaded file: ${buffer.length} bytes`) - - const formData = new FormData() - formData.append('chat_id', validatedData.chatId) - - const blob = new Blob([new Uint8Array(buffer)], { type: resolvedMimeType }) - formData.append('document', blob, userFile.name) - - if (validatedData.caption) { - formData.append('caption', convertMarkdownToHTML(validatedData.caption)) - formData.append('parse_mode', 'HTML') - } - - const telegramApiUrl = `https://api.telegram.org/bot${validatedData.botToken}/sendDocument` - logger.info(`[${requestId}] Sending request to Telegram API`) - - const response = await fetch(telegramApiUrl, { - method: 'POST', - body: formData, - }) - - const data = await response.json() - - if (!data.ok) { - logger.error(`[${requestId}] Telegram API error:`, data) - return NextResponse.json( - { - success: false, - error: data.description || 'Failed to send document to Telegram', - }, - { status: response.status } - ) - } - - logger.info(`[${requestId}] Document sent successfully`) - - return NextResponse.json({ - success: true, - output: { - message: 'Document sent successfully', - data: data.result, - files: filesOutput, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error sending Telegram document:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/textract/analyze-expense/route.test.ts b/apps/sim/app/api/tools/textract/analyze-expense/route.test.ts deleted file mode 100644 index 76cb459128f..00000000000 --- a/apps/sim/app/api/tools/textract/analyze-expense/route.test.ts +++ /dev/null @@ -1,81 +0,0 @@ -/** - * @vitest-environment node - */ -import { describe, expect, it } from 'vitest' -import { normalizeExpenseDocuments } from '@/app/api/tools/textract/analyze-expense/route' - -describe('normalizeExpenseDocuments', () => { - it('maps a documented AWS AnalyzeExpense response shape', () => { - const result = normalizeExpenseDocuments([ - { - ExpenseIndex: 1, - SummaryFields: [ - { - Type: { Text: 'VENDOR_NAME', Confidence: 98.1 }, - ValueDetection: { Text: 'Acme Corp', Confidence: 97.5 }, - LabelDetection: { Text: 'Vendor', Confidence: 90 }, - PageNumber: 1, - Currency: { Code: 'USD', Confidence: 95 }, - GroupProperties: [{ Id: 'g1', Types: ['VENDOR'] }], - }, - ], - LineItemGroups: [ - { - LineItemGroupIndex: 1, - LineItems: [ - { - LineItemExpenseFields: [ - { - Type: { Text: 'ITEM', Confidence: 91 }, - ValueDetection: { Text: 'Widget', Confidence: 93 }, - }, - ], - }, - ], - }, - ], - }, - ]) - - expect(result).toEqual([ - { - expenseIndex: 1, - summaryFields: [ - { - type: { text: 'VENDOR_NAME', confidence: 98.1 }, - valueDetection: { text: 'Acme Corp', confidence: 97.5 }, - labelDetection: { text: 'Vendor', confidence: 90 }, - pageNumber: 1, - currency: { code: 'USD', confidence: 95 }, - groupProperties: [{ id: 'g1', types: ['VENDOR'] }], - }, - ], - lineItemGroups: [ - { - lineItemGroupIndex: 1, - lineItems: [ - { - lineItemExpenseFields: [ - { - type: { text: 'ITEM', confidence: 91 }, - valueDetection: { text: 'Widget', confidence: 93 }, - labelDetection: undefined, - pageNumber: undefined, - currency: undefined, - groupProperties: undefined, - }, - ], - }, - ], - }, - ], - }, - ]) - }) - - it('defaults missing arrays to empty arrays', () => { - expect(normalizeExpenseDocuments([{ ExpenseIndex: 0 }])).toEqual([ - { expenseIndex: 0, summaryFields: [], lineItemGroups: [] }, - ]) - }) -}) diff --git a/apps/sim/app/api/tools/textract/analyze-expense/route.ts b/apps/sim/app/api/tools/textract/analyze-expense/route.ts deleted file mode 100644 index 02ba90a882f..00000000000 --- a/apps/sim/app/api/tools/textract/analyze-expense/route.ts +++ /dev/null @@ -1,230 +0,0 @@ -import { - AnalyzeExpenseCommand, - type ExpenseDocument, - GetExpenseAnalysisCommand, - StartExpenseAnalysisCommand, - TextractClient, -} from '@aws-sdk/client-textract' -import { createLogger } from '@sim/logger' -import type { NextRequest } from 'next/server' -import { NextResponse } from 'next/server' -import { textractAnalyzeExpenseContract } from '@/lib/api/contracts/tools/media/document-parse' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' -import { - mapTextractSdkError, - parseS3Uri, - pollTextractJob, - resolveDocumentInput, - textractErrorResponse, -} from '@/app/api/tools/textract/shared' - -export const dynamic = 'force-dynamic' -/** Mirrors maxDuration in ../parse/route.ts — see that file's TSDoc for details. */ -export const maxDuration = 604800 - -const logger = createLogger('TextractAnalyzeExpenseAPI') - -/** Response shape shared by AnalyzeExpense and its async Get* counterpart. */ -interface TextractExpenseResult { - JobStatus?: string - StatusMessage?: string - NextToken?: string - ExpenseDocuments?: ExpenseDocument[] - DocumentMetadata?: { Pages?: number } - AnalyzeExpenseModelVersion?: string -} - -export function normalizeExpenseField(field: { - Type?: { Text?: string; Confidence?: number } - ValueDetection?: { Text?: string; Confidence?: number } - LabelDetection?: { Text?: string; Confidence?: number } - PageNumber?: number - Currency?: { Code?: string; Confidence?: number } - GroupProperties?: { Id?: string; Types?: string[] }[] -}) { - return { - type: { text: field.Type?.Text, confidence: field.Type?.Confidence }, - valueDetection: { - text: field.ValueDetection?.Text, - confidence: field.ValueDetection?.Confidence, - }, - labelDetection: field.LabelDetection - ? { text: field.LabelDetection.Text, confidence: field.LabelDetection.Confidence } - : undefined, - pageNumber: field.PageNumber, - currency: field.Currency - ? { code: field.Currency.Code, confidence: field.Currency.Confidence } - : undefined, - groupProperties: field.GroupProperties?.map((group) => ({ - id: group.Id ?? '', - types: group.Types ?? [], - })), - } -} - -export function normalizeExpenseDocuments(documents: ExpenseDocument[]) { - return documents.map((doc) => ({ - expenseIndex: doc.ExpenseIndex, - summaryFields: (doc.SummaryFields ?? []).map(normalizeExpenseField), - lineItemGroups: (doc.LineItemGroups ?? []).map((group) => ({ - lineItemGroupIndex: group.LineItemGroupIndex, - lineItems: (group.LineItems ?? []).map((item) => ({ - lineItemExpenseFields: (item.LineItemExpenseFields ?? []).map(normalizeExpenseField), - })), - })), - })) -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Textract analyze-expense attempt`, { - error: authResult.error || 'Missing userId', - }) - return NextResponse.json( - { success: false, error: authResult.error || 'Unauthorized' }, - { status: 401 } - ) - } - const userId = authResult.userId - - const parsed = await parseRequest( - textractAnalyzeExpenseContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - - const validatedData = parsed.data.body - const modelInputProvenance = validateOpaqueModelInputProvenance({ - headers: request.headers, - payload: validatedData, - isInternalRequest: true, - }) - if (!modelInputProvenance.success) { - return NextResponse.json( - { success: false, error: modelInputProvenance.error }, - { status: modelInputProvenance.status } - ) - } - const processingMode = validatedData.processingMode || 'sync' - - logger.info(`[${requestId}] Textract analyze-expense request`, { - processingMode, - hasFile: Boolean(validatedData.file), - hasS3Uri: Boolean(validatedData.s3Uri), - userId, - }) - - const client = new TextractClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - if (processingMode === 'async') { - if (!validatedData.s3Uri) { - return NextResponse.json( - { - success: false, - error: 'S3 URI is required for multi-page processing (s3://bucket/key)', - }, - { status: 400 } - ) - } - - const { bucket, key } = parseS3Uri(validatedData.s3Uri) - logger.info(`[${requestId}] Starting async Textract expense analysis job`, { - s3Bucket: bucket, - s3Key: key, - }) - - const { JobId: jobId } = await client.send( - new StartExpenseAnalysisCommand({ - DocumentLocation: { S3Object: { Bucket: bucket, Name: key } }, - }) - ) - if (!jobId) { - throw new Error('Failed to start Textract expense analysis job: No JobId returned') - } - logger.info(`[${requestId}] Async expense analysis job started`, { jobId }) - - const result = await pollTextractJob( - requestId, - logger, - (nextToken) => - client.send(new GetExpenseAnalysisCommand({ JobId: jobId, NextToken: nextToken })), - (accumulated, page) => ({ - ...accumulated, - ...page, - ExpenseDocuments: [ - ...(accumulated.ExpenseDocuments ?? []), - ...(page.ExpenseDocuments ?? []), - ], - }) - ) - - return NextResponse.json({ - success: true, - output: { - expenseDocuments: normalizeExpenseDocuments(result.ExpenseDocuments ?? []), - documentMetadata: { pages: result.DocumentMetadata?.Pages ?? 0 }, - modelVersion: result.AnalyzeExpenseModelVersion, - }, - }) - } - - const resolved = await resolveDocumentInput( - { file: validatedData.file, filePath: validatedData.filePath }, - userId, - requestId, - logger - ) - if (!resolved.ok) return resolved.response - const { bytes, isPdf } = resolved.document - - let result: TextractExpenseResult - try { - result = await client.send(new AnalyzeExpenseCommand({ Document: { Bytes: bytes } })) - } catch (error) { - throw mapTextractSdkError(error, isPdf) - } - - logger.info(`[${requestId}] Textract analyze-expense successful`, { - pageCount: result.DocumentMetadata?.Pages ?? 0, - expenseDocumentCount: result.ExpenseDocuments?.length ?? 0, - }) - - return NextResponse.json({ - success: true, - output: { - expenseDocuments: normalizeExpenseDocuments(result.ExpenseDocuments ?? []), - documentMetadata: { pages: result.DocumentMetadata?.Pages ?? 0 }, - }, - }) - } catch (error) { - return textractErrorResponse(error, requestId, logger) - } -}) diff --git a/apps/sim/app/api/tools/textract/analyze-id/route.test.ts b/apps/sim/app/api/tools/textract/analyze-id/route.test.ts deleted file mode 100644 index 5798f2441fd..00000000000 --- a/apps/sim/app/api/tools/textract/analyze-id/route.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -/** - * @vitest-environment node - */ -import { describe, expect, it } from 'vitest' -import { normalizeIdentityDocuments } from '@/app/api/tools/textract/analyze-id/route' - -describe('normalizeIdentityDocuments', () => { - it('maps a documented AWS AnalyzeID response shape', () => { - const result = normalizeIdentityDocuments([ - { - DocumentIndex: 1, - IdentityDocumentFields: [ - { - Type: { Text: 'FIRST_NAME', Confidence: 99 }, - ValueDetection: { Text: 'Jane', Confidence: 98 }, - }, - { - Type: { - Text: 'DATE_OF_BIRTH', - Confidence: 97, - NormalizedValue: { Value: '1990-01-01', ValueType: 'Date' }, - }, - ValueDetection: { - Text: '01/01/1990', - Confidence: 96, - NormalizedValue: { Value: '1990-01-01T00:00:00', ValueType: 'Date' }, - }, - }, - ], - }, - ]) - - expect(result).toEqual([ - { - documentIndex: 1, - identityDocumentFields: [ - { - type: { text: 'FIRST_NAME', confidence: 99, normalizedValue: undefined }, - valueDetection: { text: 'Jane', confidence: 98, normalizedValue: undefined }, - }, - { - type: { - text: 'DATE_OF_BIRTH', - confidence: 97, - normalizedValue: { value: '1990-01-01', valueType: 'Date' }, - }, - valueDetection: { - text: '01/01/1990', - confidence: 96, - normalizedValue: { value: '1990-01-01T00:00:00', valueType: 'Date' }, - }, - }, - ], - }, - ]) - }) - - it('defaults missing fields to an empty array', () => { - expect(normalizeIdentityDocuments([{ DocumentIndex: 0 }])).toEqual([ - { documentIndex: 0, identityDocumentFields: [] }, - ]) - }) -}) diff --git a/apps/sim/app/api/tools/textract/analyze-id/route.ts b/apps/sim/app/api/tools/textract/analyze-id/route.ts deleted file mode 100644 index cfe7a0d4263..00000000000 --- a/apps/sim/app/api/tools/textract/analyze-id/route.ts +++ /dev/null @@ -1,162 +0,0 @@ -import { AnalyzeIDCommand, type IdentityDocument, TextractClient } from '@aws-sdk/client-textract' -import { createLogger } from '@sim/logger' -import type { NextRequest } from 'next/server' -import { NextResponse } from 'next/server' -import { textractAnalyzeIdContract } from '@/lib/api/contracts/tools/media/document-parse' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' -import { - mapTextractSdkError, - resolveDocumentInput, - textractErrorResponse, -} from '@/app/api/tools/textract/shared' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('TextractAnalyzeIdAPI') - -export function normalizeIdentityDocuments(documents: IdentityDocument[]) { - return documents.map((doc) => ({ - documentIndex: doc.DocumentIndex, - identityDocumentFields: (doc.IdentityDocumentFields ?? []).map((field) => ({ - type: { - text: field.Type?.Text, - confidence: field.Type?.Confidence, - normalizedValue: field.Type?.NormalizedValue - ? { - value: field.Type.NormalizedValue.Value, - valueType: field.Type.NormalizedValue.ValueType, - } - : undefined, - }, - valueDetection: { - text: field.ValueDetection?.Text, - confidence: field.ValueDetection?.Confidence, - normalizedValue: field.ValueDetection?.NormalizedValue - ? { - value: field.ValueDetection.NormalizedValue.Value, - valueType: field.ValueDetection.NormalizedValue.ValueType, - } - : undefined, - }, - })), - })) -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Textract analyze-id attempt`, { - error: authResult.error || 'Missing userId', - }) - return NextResponse.json( - { success: false, error: authResult.error || 'Unauthorized' }, - { status: 401 } - ) - } - const userId = authResult.userId - - const parsed = await parseRequest( - textractAnalyzeIdContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - - const validatedData = parsed.data.body - const modelInputProvenance = validateOpaqueModelInputProvenance({ - headers: request.headers, - payload: validatedData, - isInternalRequest: true, - }) - if (!modelInputProvenance.success) { - return NextResponse.json( - { success: false, error: modelInputProvenance.error }, - { status: modelInputProvenance.status } - ) - } - - logger.info(`[${requestId}] Textract analyze-id request`, { - hasFile: Boolean(validatedData.file), - hasBackFile: Boolean(validatedData.fileBack || validatedData.filePathBack), - userId, - }) - - const front = await resolveDocumentInput( - { file: validatedData.file, filePath: validatedData.filePath }, - userId, - requestId, - logger - ) - if (!front.ok) return front.response - - const documentPages = [{ Bytes: front.document.bytes }] - let isPdf = front.document.isPdf - - if (validatedData.fileBack || validatedData.filePathBack) { - const back = await resolveDocumentInput( - { file: validatedData.fileBack, filePath: validatedData.filePathBack }, - userId, - requestId, - logger - ) - if (!back.ok) return back.response - documentPages.push({ Bytes: back.document.bytes }) - isPdf = isPdf || back.document.isPdf - } - - const client = new TextractClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - let result: { - AnalyzeIDModelVersion?: string - DocumentMetadata?: { Pages?: number } - IdentityDocuments?: IdentityDocument[] - } - try { - result = await client.send(new AnalyzeIDCommand({ DocumentPages: documentPages })) - } catch (error) { - throw mapTextractSdkError(error, isPdf, { hasAsyncMode: false }) - } - - logger.info(`[${requestId}] Textract analyze-id successful`, { - pageCount: result.DocumentMetadata?.Pages ?? 0, - documentCount: result.IdentityDocuments?.length ?? 0, - }) - - return NextResponse.json({ - success: true, - output: { - identityDocuments: normalizeIdentityDocuments(result.IdentityDocuments ?? []), - documentMetadata: { pages: result.DocumentMetadata?.Pages ?? 0 }, - modelVersion: result.AnalyzeIDModelVersion, - }, - }) - } catch (error) { - return textractErrorResponse(error, requestId, logger) - } -}) diff --git a/apps/sim/app/api/tools/textract/parse/route.ts b/apps/sim/app/api/tools/textract/parse/route.ts deleted file mode 100644 index c4effb087a5..00000000000 --- a/apps/sim/app/api/tools/textract/parse/route.ts +++ /dev/null @@ -1,232 +0,0 @@ -import { - AnalyzeDocumentCommand, - DetectDocumentTextCommand, - type FeatureType, - GetDocumentAnalysisCommand, - GetDocumentTextDetectionCommand, - StartDocumentAnalysisCommand, - StartDocumentTextDetectionCommand, - TextractClient, -} from '@aws-sdk/client-textract' -import { createLogger } from '@sim/logger' -import type { NextRequest } from 'next/server' -import { NextResponse } from 'next/server' -import { textractParseContract } from '@/lib/api/contracts/tools/media/document-parse' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' -import { - mapTextractSdkError, - parseS3Uri, - pollTextractJob, - resolveDocumentInput, - textractErrorResponse, -} from '@/app/api/tools/textract/shared' - -export const dynamic = 'force-dynamic' -/** - * Mirrors the hosted workflow execution ceiling (7 days) used by - * `getMaxExecutionTimeout()` for the job polling loop below. Next.js requires a static - * literal for `maxDuration`, so this value must be kept in sync with that source. - */ -export const maxDuration = 604800 - -const logger = createLogger('TextractParseAPI') - -/** Response shape shared by AnalyzeDocument/DetectDocumentText and their async Get* counterparts. */ -interface TextractDocumentResult { - JobStatus?: string - StatusMessage?: string - NextToken?: string - Blocks?: unknown[] - DocumentMetadata?: { Pages?: number } - AnalyzeDocumentModelVersion?: string - DetectDocumentTextModelVersion?: string -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Textract parse attempt`, { - error: authResult.error || 'Missing userId', - }) - return NextResponse.json( - { success: false, error: authResult.error || 'Unauthorized' }, - { status: 401 } - ) - } - const userId = authResult.userId - - const parsed = await parseRequest( - textractParseContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid request data`, { errors: error.issues }) - return NextResponse.json( - { - success: false, - error: getValidationErrorMessage(error, 'Invalid request data'), - details: error.issues, - }, - { status: 400 } - ) - }, - } - ) - if (!parsed.success) return parsed.response - - const validatedData = parsed.data.body - const modelInputProvenance = validateOpaqueModelInputProvenance({ - headers: request.headers, - payload: validatedData, - isInternalRequest: true, - }) - if (!modelInputProvenance.success) { - return NextResponse.json( - { success: false, error: modelInputProvenance.error }, - { status: modelInputProvenance.status } - ) - } - const processingMode = validatedData.processingMode || 'sync' - const featureTypes = (validatedData.featureTypes ?? []) as FeatureType[] - const useAnalyzeDocument = featureTypes.length > 0 - const queriesConfig = - validatedData.queries && validatedData.queries.length > 0 && featureTypes.includes('QUERIES') - ? { - Queries: validatedData.queries.map((q) => ({ - Text: q.Text, - Alias: q.Alias, - Pages: q.Pages, - })), - } - : undefined - - logger.info(`[${requestId}] Textract parse request`, { - processingMode, - hasFile: Boolean(validatedData.file), - hasS3Uri: Boolean(validatedData.s3Uri), - featureTypes, - userId, - }) - - const client = new TextractClient({ - region: validatedData.region, - credentials: { - accessKeyId: validatedData.accessKeyId, - secretAccessKey: validatedData.secretAccessKey, - }, - }) - - if (processingMode === 'async') { - if (!validatedData.s3Uri) { - return NextResponse.json( - { - success: false, - error: 'S3 URI is required for multi-page processing (s3://bucket/key)', - }, - { status: 400 } - ) - } - - const { bucket, key } = parseS3Uri(validatedData.s3Uri) - logger.info(`[${requestId}] Starting async Textract job`, { s3Bucket: bucket, s3Key: key }) - - const { JobId: jobId } = useAnalyzeDocument - ? await client.send( - new StartDocumentAnalysisCommand({ - DocumentLocation: { S3Object: { Bucket: bucket, Name: key } }, - FeatureTypes: featureTypes, - QueriesConfig: queriesConfig, - }) - ) - : await client.send( - new StartDocumentTextDetectionCommand({ - DocumentLocation: { S3Object: { Bucket: bucket, Name: key } }, - }) - ) - if (!jobId) { - throw new Error('Failed to start Textract job: No JobId returned') - } - logger.info(`[${requestId}] Async job started`, { jobId }) - - const result = await pollTextractJob( - requestId, - logger, - async (nextToken) => - useAnalyzeDocument - ? await client.send( - new GetDocumentAnalysisCommand({ JobId: jobId, NextToken: nextToken }) - ) - : await client.send( - new GetDocumentTextDetectionCommand({ JobId: jobId, NextToken: nextToken }) - ), - (accumulated, page) => ({ - ...accumulated, - ...page, - Blocks: [...(accumulated.Blocks ?? []), ...(page.Blocks ?? [])], - }) - ) - - logger.info(`[${requestId}] Textract async parse successful`, { - pageCount: result.DocumentMetadata?.Pages ?? 0, - blockCount: result.Blocks?.length ?? 0, - }) - - return NextResponse.json({ - success: true, - output: { - blocks: result.Blocks ?? [], - documentMetadata: { pages: result.DocumentMetadata?.Pages ?? 0 }, - modelVersion: result.AnalyzeDocumentModelVersion ?? result.DetectDocumentTextModelVersion, - }, - }) - } - - const resolved = await resolveDocumentInput( - { file: validatedData.file, filePath: validatedData.filePath }, - userId, - requestId, - logger - ) - if (!resolved.ok) return resolved.response - const { bytes, isPdf } = resolved.document - - let result: TextractDocumentResult - try { - result = useAnalyzeDocument - ? await client.send( - new AnalyzeDocumentCommand({ - Document: { Bytes: bytes }, - FeatureTypes: featureTypes, - QueriesConfig: queriesConfig, - }) - ) - : await client.send(new DetectDocumentTextCommand({ Document: { Bytes: bytes } })) - } catch (error) { - throw mapTextractSdkError(error, isPdf) - } - - logger.info(`[${requestId}] Textract parse successful`, { - pageCount: result.DocumentMetadata?.Pages ?? 0, - blockCount: result.Blocks?.length ?? 0, - }) - - return NextResponse.json({ - success: true, - output: { - blocks: result.Blocks ?? [], - documentMetadata: { pages: result.DocumentMetadata?.Pages ?? 0 }, - modelVersion: result.AnalyzeDocumentModelVersion ?? result.DetectDocumentTextModelVersion, - }, - }) - } catch (error) { - return textractErrorResponse(error, requestId, logger) - } -}) diff --git a/apps/sim/app/api/tools/textract/shared.test.ts b/apps/sim/app/api/tools/textract/shared.test.ts deleted file mode 100644 index b95b6753453..00000000000 --- a/apps/sim/app/api/tools/textract/shared.test.ts +++ /dev/null @@ -1,165 +0,0 @@ -/** - * @vitest-environment node - */ -import { createLogger } from '@sim/logger' -import { describe, expect, it } from 'vitest' -import { - mapTextractSdkError, - parseS3Uri, - pollTextractJob, - TextractRouteError, -} from '@/app/api/tools/textract/shared' - -const logger = createLogger('TextractSharedTest') - -describe('parseS3Uri', () => { - it('parses a valid s3 URI', () => { - expect(parseS3Uri('s3://my-bucket/path/to/doc.pdf')).toEqual({ - bucket: 'my-bucket', - key: 'path/to/doc.pdf', - }) - }) - - it('rejects a malformed URI', () => { - expect(() => parseS3Uri('not-an-s3-uri')).toThrow(TextractRouteError) - }) - - it('rejects path traversal in the key', () => { - expect(() => parseS3Uri('s3://my-bucket/../secrets.pdf')).toThrow('path traversal') - }) -}) - -describe('mapTextractSdkError', () => { - it('gives a friendly hint for unsupported PDFs in single-page mode', () => { - const mapped = mapTextractSdkError( - { name: 'UnsupportedDocumentException', message: 'Unsupported document' }, - true - ) - expect(mapped.status).toBe(400) - expect(mapped.message).toContain('Multi-Page (PDF, TIFF via S3)') - }) - - it('omits the multi-page hint for operations without an async mode', () => { - const mapped = mapTextractSdkError( - { name: 'UnsupportedDocumentException', message: 'Unsupported document' }, - true, - { hasAsyncMode: false } - ) - expect(mapped.message).not.toContain('Multi-Page') - expect(mapped.message).toContain('Only JPEG, PNG, and single-page PDF files are supported') - }) - - it('does not rewrite the message for non-PDF unsupported documents', () => { - const mapped = mapTextractSdkError( - { name: 'UnsupportedDocumentException', message: 'Unsupported document' }, - false - ) - expect(mapped.message).toBe('Unsupported document') - }) - - it('uses the SDK http status when under 500', () => { - const mapped = mapTextractSdkError( - { - name: 'InvalidParameterException', - message: 'Bad param', - $metadata: { httpStatusCode: 400 }, - }, - false - ) - expect(mapped.status).toBe(400) - expect(mapped.message).toBe('Bad param') - }) - - it('passes through a 5xx SDK status so tool-execution retry logic still fires', () => { - const mapped = mapTextractSdkError( - { message: 'Internal failure', $metadata: { httpStatusCode: 500 } }, - false - ) - expect(mapped.status).toBe(500) - }) - - it('defaults to 500 when the SDK gives no http status, since that implies a server-side failure', () => { - const mapped = mapTextractSdkError({ message: 'Unknown failure' }, false) - expect(mapped.status).toBe(500) - }) -}) - -describe('pollTextractJob', () => { - it('returns immediately on SUCCEEDED with no NextToken', async () => { - const result = await pollTextractJob( - 'req-1', - logger, - async () => ({ JobStatus: 'SUCCEEDED', Blocks: [{ Id: '1' }] }), - (accumulated, page) => ({ - ...page, - Blocks: [...(accumulated.Blocks ?? []), ...(page.Blocks ?? [])], - }) - ) - - expect(result.JobStatus).toBe('SUCCEEDED') - expect(result.Blocks).toHaveLength(1) - }) - - it('follows NextToken pagination and merges pages', async () => { - let calls = 0 - const result = await pollTextractJob( - 'req-2', - logger, - async (nextToken) => { - calls += 1 - if (!nextToken) return { JobStatus: 'SUCCEEDED', Blocks: [{ Id: '1' }], NextToken: 'next' } - return { JobStatus: 'SUCCEEDED', Blocks: [{ Id: '2' }] } - }, - (accumulated, page) => ({ - ...accumulated, - ...page, - Blocks: [...(accumulated.Blocks ?? []), ...(page.Blocks ?? [])], - }) - ) - - expect(calls).toBe(2) - expect(result.Blocks).toHaveLength(2) - }) - - it('preserves fields the first page has but a later page omits (e.g. DocumentMetadata)', async () => { - const result = await pollTextractJob<{ - JobStatus?: string - NextToken?: string - Blocks?: unknown[] - DocumentMetadata?: { Pages?: number } - }>( - 'req-4', - logger, - async (nextToken) => { - if (!nextToken) { - return { - JobStatus: 'SUCCEEDED', - Blocks: [{ Id: '1' }], - DocumentMetadata: { Pages: 3 }, - NextToken: 'next', - } - } - return { JobStatus: 'SUCCEEDED', Blocks: [{ Id: '2' }] } - }, - (accumulated, page) => ({ - ...accumulated, - ...page, - Blocks: [...(accumulated.Blocks ?? []), ...(page.Blocks ?? [])], - }) - ) - - expect(result.Blocks).toHaveLength(2) - expect(result.DocumentMetadata).toEqual({ Pages: 3 }) - }) - - it('throws a TextractRouteError when the job fails', async () => { - await expect( - pollTextractJob( - 'req-3', - logger, - async () => ({ JobStatus: 'FAILED', StatusMessage: 'boom' }), - (accumulated) => accumulated - ) - ).rejects.toThrow('Textract job failed: boom') - }) -}) diff --git a/apps/sim/app/api/tools/textract/shared.ts b/apps/sim/app/api/tools/textract/shared.ts deleted file mode 100644 index 0150c8f0c99..00000000000 --- a/apps/sim/app/api/tools/textract/shared.ts +++ /dev/null @@ -1,341 +0,0 @@ -import type { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { sleep } from '@sim/utils/helpers' -import { NextResponse } from 'next/server' -import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' -import { validateS3BucketName } from '@/lib/core/security/input-validation' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { - isModelSafeWorkspaceFileKey, - MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, -} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import type { RawFileInput } from '@/lib/uploads/utils/file-utils' -import { - extractStorageKey, - isInternalFileUrl, - processSingleFileToUserFile, -} from '@/lib/uploads/utils/file-utils' -import { - downloadServableFileFromStorage, - resolveInternalFileUrl, -} from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' - -type RouteLogger = ReturnType - -/** Thrown by AWS SDK call sites so route handlers can map failures to the right HTTP status. */ -export class TextractRouteError extends Error { - status: number - - constructor(message: string, status = 500) { - super(message) - this.name = 'TextractRouteError' - this.status = status - } -} - -export function textractErrorResponse( - error: unknown, - requestId: string, - logger: RouteLogger -): NextResponse { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - - logger.error(`[${requestId}] Error in Textract request:`, error) - const status = error instanceof TextractRouteError ? error.status : 500 - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Internal server error') }, - { status } - ) -} - -/** - * Maps an AWS SDK TextractClient rejection to a client-facing error, with a friendly hint for the - * common "PDF used in single-page mode" mistake. The real AWS HTTP status (including 5xx) is - * passed through so the tool-execution layer's retry logic can still treat throttling/internal - * errors as retryable, matching the pre-migration hand-rolled-signing behavior. - */ -export function mapTextractSdkError( - error: unknown, - isPdf: boolean, - options?: { hasAsyncMode?: boolean } -): TextractRouteError { - const err = error as { - name?: string - message?: string - $metadata?: { httpStatusCode?: number } - } - const hasAsyncMode = options?.hasAsyncMode ?? true - - const isUnsupportedFormat = - err.name === 'UnsupportedDocumentException' || - Boolean(err.message?.toLowerCase().includes('unsupported document')) - - if (isUnsupportedFormat && isPdf) { - const hint = hasAsyncMode - ? ' If this is a multi-page PDF, please use "Multi-Page (PDF, TIFF via S3)" mode instead, which requires uploading your document to S3 first. Single Page mode only supports JPEG, PNG, and single-page PDF files.' - : ' Only JPEG, PNG, and single-page PDF files are supported.' - return new TextractRouteError(`This document format is not supported.${hint}`, 400) - } - - const status = err.$metadata?.httpStatusCode ?? 500 - return new TextractRouteError(err.message || 'Textract API error', status) -} - -export interface ResolvedDocument { - bytes: Buffer - contentType: string - isPdf: boolean -} - -export type ResolveDocumentResult = - | { ok: true; document: ResolvedDocument } - | { ok: false; response: NextResponse } - -/** Passes through the document host's real HTTP status on failure, so tool-execution retry logic can still treat a transient 5xx as retryable. */ -async function fetchDocumentBytes(url: string): Promise<{ bytes: Buffer; contentType: string }> { - const urlValidation = await validateUrlWithDNS(url, 'Document URL') - if (!urlValidation.isValid) { - throw new TextractRouteError(urlValidation.error || 'Invalid document URL', 400) - } - - const response = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP!, { - method: 'GET', - }) - if (!response.ok) { - await response.text().catch(() => {}) - throw new TextractRouteError( - `Failed to fetch document: ${response.statusText}`, - response.status - ) - } - - const arrayBuffer = await response.arrayBuffer() - const contentType = response.headers.get('content-type') || 'application/octet-stream' - - return { bytes: Buffer.from(arrayBuffer), contentType } -} - -/** Resolves a document input (uploaded file reference or URL) to raw bytes for the Textract Document.Bytes field. */ -export async function resolveDocumentInput( - input: { file?: RawFileInput; filePath?: string }, - userId: string, - requestId: string, - logger: RouteLogger -): Promise { - if (input.file) { - let userFile: ReturnType - try { - userFile = processSingleFileToUserFile(input.file, requestId, logger) - } catch (error) { - return { - ok: false, - response: NextResponse.json( - { success: false, error: getErrorMessage(error, 'Failed to process file') }, - { status: 400 } - ), - } - } - - const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) - if (denied) return { ok: false, response: denied } - if (!(await isModelSafeWorkspaceFileKey(userFile.key))) { - return { - ok: false, - response: NextResponse.json( - { - success: false, - error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, - }, - { status: 400 } - ), - } - } - - const { buffer, contentType } = await downloadServableFileFromStorage( - userFile, - requestId, - logger, - { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - } - ) - const resolvedContentType = contentType || userFile.type || 'application/octet-stream' - - return { - ok: true, - document: { - bytes: buffer, - contentType: resolvedContentType, - isPdf: - resolvedContentType.includes('pdf') || - Boolean(userFile.name?.toLowerCase().endsWith('.pdf')), - }, - } - } - - if (input.filePath) { - let fileUrl = input.filePath - const isInternalFilePath = isInternalFileUrl(fileUrl) - - if (isInternalFilePath) { - const resolution = await resolveInternalFileUrl(fileUrl, userId, requestId, logger) - if (resolution.error) { - return { - ok: false, - response: NextResponse.json( - { success: false, error: resolution.error.message }, - { status: resolution.error.status } - ), - } - } - fileUrl = resolution.fileUrl || fileUrl - if (!(await isModelSafeWorkspaceFileKey(extractStorageKey(input.filePath)))) { - return { - ok: false, - response: NextResponse.json( - { - success: false, - error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, - }, - { status: 400 } - ), - } - } - } else if (fileUrl.startsWith('/')) { - logger.warn(`[${requestId}] Invalid internal path`, { - userId, - path: fileUrl.substring(0, 50), - }) - return { - ok: false, - response: NextResponse.json( - { - success: false, - error: 'Invalid file path. Only uploaded files are supported for internal paths.', - }, - { status: 400 } - ), - } - } else { - const urlValidation = await validateUrlWithDNS(fileUrl, 'Document URL') - if (!urlValidation.isValid) { - logger.warn(`[${requestId}] SSRF attempt blocked`, { - userId, - url: fileUrl.substring(0, 100), - error: urlValidation.error, - }) - return { - ok: false, - response: NextResponse.json( - { success: false, error: urlValidation.error }, - { status: 400 } - ), - } - } - } - - const fetched = await fetchDocumentBytes(fileUrl) - return { - ok: true, - document: { - bytes: fetched.bytes, - contentType: fetched.contentType, - isPdf: fetched.contentType.includes('pdf') || fileUrl.toLowerCase().endsWith('.pdf'), - }, - } - } - - return { - ok: false, - response: NextResponse.json( - { success: false, error: 'Document input is required' }, - { status: 400 } - ), - } -} - -export function parseS3Uri(s3Uri: string): { bucket: string; key: string } { - const match = s3Uri.match(/^s3:\/\/([^/]+)\/(.+)$/) - if (!match) { - throw new TextractRouteError( - `Invalid S3 URI format: ${s3Uri}. Expected format: s3://bucket-name/path/to/object`, - 400 - ) - } - - const bucket = match[1] - const key = match[2] - - const bucketValidation = validateS3BucketName(bucket, 'S3 bucket name') - if (!bucketValidation.isValid) { - throw new TextractRouteError(bucketValidation.error || 'Invalid S3 bucket name', 400) - } - - if (key.includes('..') || key.startsWith('/')) { - throw new TextractRouteError('S3 key contains invalid path traversal sequences', 400) - } - - return { bucket, key } -} - -interface PollableJobResult { - JobStatus?: string - StatusMessage?: string - NextToken?: string -} - -/** Polls a started async Textract job (StartDocumentAnalysis/StartDocumentTextDetection/StartExpenseAnalysis) until it completes, following NextToken pagination on success. */ -export async function pollTextractJob( - requestId: string, - logger: RouteLogger, - getPage: (nextToken?: string) => Promise, - mergePage: (accumulated: TResult, page: TResult) => TResult -): Promise { - const pollIntervalMs = 5000 - const maxPollTimeMs = getMaxExecutionTimeout() - const maxAttempts = Math.ceil(maxPollTimeMs / pollIntervalMs) - - for (let attempt = 0; attempt < maxAttempts; attempt++) { - const result = await getPage() - const jobStatus = result.JobStatus - - if (jobStatus === 'SUCCEEDED' || jobStatus === 'PARTIAL_SUCCESS') { - if (jobStatus === 'PARTIAL_SUCCESS') { - logger.warn(`[${requestId}] Job completed with partial success: ${result.StatusMessage}`) - } else { - logger.info(`[${requestId}] Async job completed successfully after ${attempt + 1} polls`) - } - - let merged = result - let nextToken = result.NextToken - while (nextToken) { - const page = await getPage(nextToken) - merged = mergePage(merged, page) - nextToken = page.NextToken - } - return merged - } - - if (jobStatus === 'FAILED') { - throw new TextractRouteError( - `Textract job failed: ${result.StatusMessage || 'Unknown error'}`, - 502 - ) - } - - logger.info(`[${requestId}] Job status: ${jobStatus}, attempt ${attempt + 1}/${maxAttempts}`) - await sleep(pollIntervalMs) - } - - throw new TextractRouteError( - `Timeout waiting for Textract job to complete (max ${maxPollTimeMs / 1000} seconds)`, - 504 - ) -} diff --git a/apps/sim/app/api/tools/thinking/route.ts b/apps/sim/app/api/tools/thinking/route.ts deleted file mode 100644 index 75510e49c89..00000000000 --- a/apps/sim/app/api/tools/thinking/route.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { thinkingToolContract } from '@/lib/api/contracts/tools/thinking' -import { parseRequest } from '@/lib/api/server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import type { ThinkingToolResponse } from '@/tools/thinking/types' - -const logger = createLogger('ThinkingToolAPI') - -export const dynamic = 'force-dynamic' - -/** - * POST - Process a thinking tool request - * Simply acknowledges the thought by returning it in the output - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const parsed = await parseRequest(thinkingToolContract, request, {}) - if (!parsed.success) return parsed.response - const { body } = parsed.data - - logger.info(`[${requestId}] Processing thinking tool request`) - - // Simply acknowledge the thought by returning it in the output - const response: ThinkingToolResponse = { - success: true, - output: { - acknowledgedThought: body.thought, - }, - } - - logger.info(`[${requestId}] Thinking tool processed successfully`) - return NextResponse.json(response) - } catch (error) { - logger.error(`[${requestId}] Error processing thinking tool:`, error) - return NextResponse.json( - { - success: false, - error: 'Failed to process thinking tool request', - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/tiktok/upload-video-draft/route.test.ts b/apps/sim/app/api/tools/tiktok/upload-video-draft/route.test.ts deleted file mode 100644 index 4eafdea3693..00000000000 --- a/apps/sim/app/api/tools/tiktok/upload-video-draft/route.test.ts +++ /dev/null @@ -1,128 +0,0 @@ -/** - * @vitest-environment node - */ -import { createMockRequest, hybridAuthMockFns } from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' -import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' - -const { - mockAssertToolFileAccess, - mockComputeTikTokChunkPlan, - mockGetStoredVideoSize, - mockStreamStoredVideoToTikTok, -} = vi.hoisted(() => ({ - mockAssertToolFileAccess: vi.fn(), - mockComputeTikTokChunkPlan: vi.fn(() => ({ - chunkSize: 10_000_000, - totalChunkCount: 2, - })), - mockGetStoredVideoSize: vi.fn(), - mockStreamStoredVideoToTikTok: vi.fn(), -})) - -vi.mock('@/app/api/files/authorization', () => ({ - assertToolFileAccess: mockAssertToolFileAccess, -})) - -vi.mock('@/app/api/tools/tiktok/upload-video-draft/upload', () => ({ - computeTikTokChunkPlan: mockComputeTikTokChunkPlan, - getStoredVideoSize: mockGetStoredVideoSize, - streamStoredVideoToTikTok: mockStreamStoredVideoToTikTok, - TIKTOK_MAX_VIDEO_BYTES: 250 * 1024 * 1024, -})) - -import { POST } from '@/app/api/tools/tiktok/upload-video-draft/route' - -const file = { - key: 'workspace/workspace-1/video.mp4', - name: 'video.mp4', - size: 1, - type: 'video/mp4', -} - -describe('POST /api/tools/tiktok/upload-video-draft', () => { - beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - mockAssertToolFileAccess.mockResolvedValue(null) - mockGetStoredVideoSize.mockResolvedValue(20_000_000) - mockComputeTikTokChunkPlan.mockReturnValue({ - chunkSize: 10_000_000, - totalChunkCount: 2, - }) - mockStreamStoredVideoToTikTok.mockResolvedValue(undefined) - }) - - it('uses authoritative storage size for initialization and streaming', async () => { - const fetchMock = vi.fn().mockResolvedValue( - Response.json({ - data: { publish_id: 'publish-1', upload_url: 'https://upload.example/video' }, - error: { code: 'ok' }, - }) - ) - vi.stubGlobal('fetch', fetchMock) - const request = createMockRequest('POST', { - accessToken: 'access-token', - file, - }) - - const response = await POST(request) - - expect(response.status).toBe(200) - await expect(response.json()).resolves.toEqual({ - success: true, - output: { publishId: 'publish-1' }, - }) - expect(mockGetStoredVideoSize).toHaveBeenCalledWith({ - key: file.key, - context: 'workspace', - signal: request.signal, - }) - const init = JSON.parse(fetchMock.mock.calls[0][1]?.body as string) as { - source_info: Record - } - expect(init.source_info).toEqual({ - source: 'FILE_UPLOAD', - video_size: 20_000_000, - chunk_size: 10_000_000, - total_chunk_count: 2, - }) - expect(fetchMock.mock.calls[0][1]?.signal).toBe(request.signal) - expect(mockStreamStoredVideoToTikTok).toHaveBeenCalledWith({ - key: file.key, - context: 'workspace', - uploadUrl: 'https://upload.example/video', - totalBytes: 20_000_000, - mimeType: 'video/mp4', - requestId: 'mock-request-id', - signal: request.signal, - }) - }) - - it('returns 413 when the storage object exceeds the relay limit', async () => { - mockGetStoredVideoSize.mockRejectedValue( - new PayloadSizeLimitError({ - label: 'TikTok video upload', - maxBytes: 250 * 1024 * 1024, - observedBytes: 251 * 1024 * 1024, - }) - ) - - const response = await POST( - createMockRequest('POST', { - accessToken: 'access-token', - file, - }) - ) - - expect(response.status).toBe(413) - await expect(response.json()).resolves.toEqual({ - success: false, - error: 'Video exceeds the 250MB limit for file uploads.', - }) - }) -}) diff --git a/apps/sim/app/api/tools/tiktok/upload-video-draft/route.ts b/apps/sim/app/api/tools/tiktok/upload-video-draft/route.ts deleted file mode 100644 index e65b7fe1671..00000000000 --- a/apps/sim/app/api/tools/tiktok/upload-video-draft/route.ts +++ /dev/null @@ -1,206 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { tiktokUploadVideoDraftContract } from '@/lib/api/contracts/tiktok-tools' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - getFileExtension, - getMimeTypeFromExtension, - processSingleFileToUserFile, - resolveTrustedFileContext, -} from '@/lib/uploads/utils/file-utils' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { - computeTikTokChunkPlan, - getStoredVideoSize, - streamStoredVideoToTikTok, - TIKTOK_MAX_VIDEO_BYTES, -} from '@/app/api/tools/tiktok/upload-video-draft/upload' -import type { UserFile } from '@/executor/types' -import { tiktokPublishInitApiDataSchema } from '@/tools/tiktok/api-schemas' -import { readTikTokApiResponse } from '@/tools/tiktok/utils' - -export const dynamic = 'force-dynamic' -export const maxDuration = 900 - -const logger = createLogger('TikTokUploadVideoDraftAPI') - -const TIKTOK_VIDEO_MIME_TYPES = new Set(['video/mp4', 'video/quicktime', 'video/webm']) - -function resolveVideoMimeType(fileName: string, fileType: string | undefined): string | null { - if (fileType && TIKTOK_VIDEO_MIME_TYPES.has(fileType)) return fileType - const fromExtension = getMimeTypeFromExtension(getFileExtension(fileName)) - return TIKTOK_VIDEO_MIME_TYPES.has(fromExtension) ? fromExtension : null -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn( - `[${requestId}] Unauthorized TikTok upload-video-draft attempt: ${authResult.error}` - ) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(tiktokUploadVideoDraftContract, request, {}) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - let userFile: UserFile - try { - userFile = processSingleFileToUserFile(data.file, requestId, logger) - } catch (error) { - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Failed to process file') }, - { status: 400 } - ) - } - - const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) - if (denied) return denied - - const mimeType = resolveVideoMimeType(userFile.name, userFile.type) - if (!mimeType) { - return NextResponse.json( - { - success: false, - error: 'Unsupported video type. TikTok accepts MP4, MOV/QuickTime, or WebM files.', - }, - { status: 400 } - ) - } - - const context = resolveTrustedFileContext(userFile.key, userFile.context) - const videoSize = await getStoredVideoSize({ - key: userFile.key, - context, - signal: request.signal, - }) - if (videoSize === 0) { - return NextResponse.json( - { success: false, error: 'The video file is empty.' }, - { status: 400 } - ) - } - - logger.info(`[${requestId}] Resolved video from storage`, { - fileName: userFile.name, - declaredSize: userFile.size, - storageSize: videoSize, - }) - - const { chunkSize, totalChunkCount } = computeTikTokChunkPlan(videoSize) - const initBody = { - source_info: { - source: 'FILE_UPLOAD', - video_size: videoSize, - chunk_size: chunkSize, - total_chunk_count: totalChunkCount, - }, - } - - logger.info(`[${requestId}] Initializing TikTok video draft`, { - videoSize, - chunkSize, - totalChunkCount, - }) - - const initResponse = await fetch( - 'https://open.tiktokapis.com/v2/post/publish/inbox/video/init/', - { - method: 'POST', - headers: { - Authorization: `Bearer ${data.accessToken}`, - 'Content-Type': 'application/json; charset=UTF-8', - }, - body: JSON.stringify(initBody), - signal: request.signal, - } - ) - - const { data: initData, error: initError } = await readTikTokApiResponse( - initResponse, - tiktokPublishInitApiDataSchema, - { signal: request.signal } - ) - - if (initError) { - logger.error(`[${requestId}] TikTok init failed`, { error: initError }) - return NextResponse.json( - { - success: false, - error: initError.message || initError.code || 'Failed to initialize TikTok upload', - }, - { status: initResponse.status >= 400 ? initResponse.status : 502 } - ) - } - - const publishId = initData?.publish_id - const uploadUrl = initData?.upload_url - - if (!publishId || !uploadUrl) { - return NextResponse.json( - { success: false, error: 'TikTok did not return a publish ID and upload URL' }, - { status: 502 } - ) - } - - try { - await streamStoredVideoToTikTok({ - key: userFile.key, - context, - uploadUrl, - totalBytes: videoSize, - mimeType, - requestId, - signal: request.signal, - }) - } catch (error) { - if (request.signal.aborted) throw error - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Failed to upload video to TikTok') }, - { status: 502 } - ) - } - - logger.info(`[${requestId}] TikTok video upload complete`, { publishId }) - - return NextResponse.json({ success: true, output: { publishId } }) - } catch (error) { - if (isPayloadSizeLimitError(error)) { - logger.warn(`[${requestId}] Rejected oversized TikTok video upload`, { - maxBytes: error.maxBytes, - observedBytes: error.observedBytes, - }) - const maxMb = Math.floor(TIKTOK_MAX_VIDEO_BYTES / (1024 * 1024)) - return NextResponse.json( - { - success: false, - error: `Video exceeds the ${maxMb}MB limit for file uploads.`, - }, - { status: 413 } - ) - } - if (request.signal.aborted) { - return NextResponse.json( - { success: false, error: 'TikTok video upload was cancelled.' }, - { status: 499 } - ) - } - logger.error(`[${requestId}] Error uploading TikTok video draft:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Internal server error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/tiktok/upload-video-draft/upload.test.ts b/apps/sim/app/api/tools/tiktok/upload-video-draft/upload.test.ts deleted file mode 100644 index d32cee7852f..00000000000 --- a/apps/sim/app/api/tools/tiktok/upload-video-draft/upload.test.ts +++ /dev/null @@ -1,195 +0,0 @@ -/** - * @vitest-environment node - */ -import { Readable } from 'node:stream' -import { beforeEach, describe, expect, it, vi } from 'vitest' -import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' - -const { mockBackoffWithJitter, mockDownloadFileStream, mockHeadObject, mockParseRetryAfter } = - vi.hoisted(() => ({ - mockBackoffWithJitter: vi.fn(() => 0), - mockDownloadFileStream: vi.fn(), - mockHeadObject: vi.fn(), - mockParseRetryAfter: vi.fn(() => 25), - })) - -vi.mock('@sim/utils/retry', () => ({ - backoffWithJitter: mockBackoffWithJitter, - parseRetryAfter: mockParseRetryAfter, -})) - -vi.mock('@/lib/uploads/core/storage-service', () => ({ - downloadFileStream: mockDownloadFileStream, - headObject: mockHeadObject, -})) - -import { - computeTikTokChunkPlan, - getStoredVideoSize, - streamStoredVideoToTikTok, - TIKTOK_MAX_VIDEO_BYTES, -} from '@/app/api/tools/tiktok/upload-video-draft/upload' - -const baseStreamOptions = { - key: 'workspace/workspace-1/video.mp4', - context: 'workspace' as const, - uploadUrl: 'https://upload.example/video', - mimeType: 'video/mp4', - requestId: 'request-1', -} - -describe('TikTok video upload streaming', () => { - beforeEach(() => { - vi.clearAllMocks() - mockBackoffWithJitter.mockReturnValue(0) - mockParseRetryAfter.mockReturnValue(25) - }) - - it('uses provider metadata as the authoritative bounded size', async () => { - mockHeadObject.mockResolvedValue({ size: 1234, contentType: 'video/mp4' }) - - await expect( - getStoredVideoSize({ - key: baseStreamOptions.key, - context: baseStreamOptions.context, - signal: new AbortController().signal, - }) - ).resolves.toBe(1234) - expect(mockDownloadFileStream).not.toHaveBeenCalled() - }) - - it('counts a stream without accumulating it when provider metadata is unavailable', async () => { - mockHeadObject.mockResolvedValue(null) - mockDownloadFileStream.mockResolvedValue( - Readable.from([Buffer.alloc(3), Buffer.alloc(5), Buffer.alloc(7)]) - ) - - await expect( - getStoredVideoSize({ - key: baseStreamOptions.key, - context: baseStreamOptions.context, - signal: new AbortController().signal, - }) - ).resolves.toBe(15) - }) - - it('rejects an oversized provider object before opening its body', async () => { - mockHeadObject.mockResolvedValue({ size: TIKTOK_MAX_VIDEO_BYTES + 1 }) - - await expect( - getStoredVideoSize({ - key: baseStreamOptions.key, - context: baseStreamOptions.context, - signal: new AbortController().signal, - }) - ).rejects.toBeInstanceOf(PayloadSizeLimitError) - expect(mockDownloadFileStream).not.toHaveBeenCalled() - }) - - it('computes TikTok chunk counts with the final chunk absorbing the remainder', () => { - expect(computeTikTokChunkPlan(4_000_000)).toEqual({ - chunkSize: 4_000_000, - totalChunkCount: 1, - }) - expect(computeTikTokChunkPlan(20_000_001)).toEqual({ - chunkSize: 10_000_000, - totalChunkCount: 2, - }) - }) - - it('streams sequential chunks with exact 206 intermediate and 201 final ranges', async () => { - const totalBytes = 20_000_001 - mockDownloadFileStream.mockResolvedValue( - Readable.from([Buffer.alloc(7_000_000, 1), Buffer.alloc(13_000_001, 2)]) - ) - const fetchMock = vi - .fn() - .mockResolvedValueOnce(new Response(null, { status: 206 })) - .mockResolvedValueOnce(new Response(null, { status: 201 })) - vi.stubGlobal('fetch', fetchMock) - const controller = new AbortController() - - await streamStoredVideoToTikTok({ - ...baseStreamOptions, - totalBytes, - signal: controller.signal, - }) - - expect(fetchMock).toHaveBeenCalledTimes(2) - expect(fetchMock.mock.calls[0][1]).toMatchObject({ - method: 'PUT', - signal: controller.signal, - headers: { - 'Content-Length': '10000000', - 'Content-Range': 'bytes 0-9999999/20000001', - 'Content-Type': 'video/mp4', - }, - }) - expect(fetchMock.mock.calls[1][1]).toMatchObject({ - method: 'PUT', - signal: controller.signal, - headers: { - 'Content-Length': '10000001', - 'Content-Range': 'bytes 10000000-20000000/20000001', - 'Content-Type': 'video/mp4', - }, - }) - }) - - it('retries only 5xx responses with the same bounded chunk', async () => { - mockDownloadFileStream.mockResolvedValue(Readable.from([Buffer.from('video')])) - const fetchMock = vi - .fn() - .mockResolvedValueOnce( - new Response('temporary one', { status: 500, headers: { 'Retry-After': '1' } }) - ) - .mockResolvedValueOnce(new Response('temporary two', { status: 503 })) - .mockResolvedValueOnce(new Response(null, { status: 201 })) - vi.stubGlobal('fetch', fetchMock) - - await streamStoredVideoToTikTok({ - ...baseStreamOptions, - totalBytes: 5, - signal: new AbortController().signal, - }) - - expect(fetchMock).toHaveBeenCalledTimes(3) - expect(mockParseRetryAfter).toHaveBeenCalledTimes(2) - expect(mockBackoffWithJitter).toHaveBeenNthCalledWith(1, 1, 25) - expect(mockBackoffWithJitter).toHaveBeenNthCalledWith(2, 2, 25) - const uploadedBodies = fetchMock.mock.calls.map((call) => - Buffer.from(call[1]?.body as Uint8Array).toString('utf8') - ) - expect(uploadedBodies).toEqual(['video', 'video', 'video']) - }) - - it('rejects a successful but protocol-invalid final status without retrying', async () => { - mockDownloadFileStream.mockResolvedValue(Readable.from([Buffer.from('video')])) - const fetchMock = vi.fn().mockResolvedValue(new Response(null, { status: 200 })) - vi.stubGlobal('fetch', fetchMock) - - await expect( - streamStoredVideoToTikTok({ - ...baseStreamOptions, - totalBytes: 5, - signal: new AbortController().signal, - }) - ).rejects.toThrow('expected HTTP 201, received HTTP 200') - expect(fetchMock).toHaveBeenCalledTimes(1) - }) - - it('detects storage-size drift before sending the final chunk', async () => { - mockDownloadFileStream.mockResolvedValue(Readable.from([Buffer.from('video-extra')])) - const fetchMock = vi.fn() - vi.stubGlobal('fetch', fetchMock) - - await expect( - streamStoredVideoToTikTok({ - ...baseStreamOptions, - totalBytes: 5, - signal: new AbortController().signal, - }) - ).rejects.toThrow('Stored video grew after its size was resolved') - expect(fetchMock).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/app/api/tools/tts/route.ts b/apps/sim/app/api/tools/tts/route.ts deleted file mode 100644 index 366d2ee03ee..00000000000 --- a/apps/sim/app/api/tools/tts/route.ts +++ /dev/null @@ -1,174 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import type { NextRequest } from 'next/server' -import { NextResponse } from 'next/server' -import { ttsToolContract } from '@/lib/api/contracts/tools/media/tts' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' -import { validateAlphanumericId } from '@/lib/core/security/input-validation' -import { - isPayloadSizeLimitError, - readResponseToBufferWithLimit, -} from '@/lib/core/utils/stream-limits' -import { getBaseUrl } from '@/lib/core/utils/urls' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { StorageService } from '@/lib/uploads' - -const logger = createLogger('ProxyTTSAPI') -const MAX_TTS_AUDIO_BYTES = 25 * 1024 * 1024 - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - logger.error('Authentication failed for TTS proxy:', authResult.error) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest( - ttsToolContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { error: getValidationErrorMessage(error, 'Missing required parameters') }, - { status: 400 } - ), - } - ) - if (!parsed.success) return parsed.response - - const { - text, - voiceId, - apiKey, - modelId, - stability, - similarityBoost, - workspaceId, - workflowId, - executionId, - } = parsed.data.body - - const voiceIdValidation = validateAlphanumericId(voiceId, 'voiceId', 255) - if (!voiceIdValidation.isValid) { - logger.error(`Invalid voice ID: ${voiceIdValidation.error}`) - return NextResponse.json({ error: voiceIdValidation.error }, { status: 400 }) - } - - // Check if this is an execution context (from workflow tool execution) - const executionContext = - workspaceId && workflowId && executionId ? { workspaceId, workflowId, executionId } : null - logger.info('Proxying TTS request for voice:', { - voiceId, - hasExecutionContext: Boolean(executionContext), - workspaceId, - workflowId, - executionId, - }) - - const endpoint = `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}` - - const hasVoiceSetting = stability !== undefined || similarityBoost !== undefined - const voiceSettings = hasVoiceSetting - ? { - stability: stability ?? 0.5, - similarity_boost: similarityBoost ?? 0.75, - } - : undefined - - const response = await fetch(endpoint, { - method: 'POST', - headers: { - Accept: 'audio/mpeg', - 'Content-Type': 'application/json', - 'xi-api-key': apiKey, - }, - body: JSON.stringify({ - text, - model_id: modelId, - ...(voiceSettings ? { voice_settings: voiceSettings } : {}), - }), - signal: AbortSignal.timeout(DEFAULT_EXECUTION_TIMEOUT_MS), - }) - - if (!response.ok) { - await response.body?.cancel().catch(() => {}) - logger.error(`Failed to generate TTS: ${response.status} ${response.statusText}`) - return NextResponse.json( - { error: `Failed to generate TTS: ${response.status} ${response.statusText}` }, - { status: response.status } - ) - } - - const audioBuffer = await readResponseToBufferWithLimit(response, { - maxBytes: MAX_TTS_AUDIO_BYTES, - label: 'TTS audio response', - signal: request.signal, - }) - - if (audioBuffer.length === 0) { - logger.error('Empty audio received from ElevenLabs') - return NextResponse.json({ error: 'Empty audio received' }, { status: 422 }) - } - - const timestamp = Date.now() - - // Use execution storage for workflow tool calls, copilot for chat UI - if (executionContext) { - const { uploadExecutionFile } = await import('@/lib/uploads/contexts/execution') - const fileName = `tts-${timestamp}.mp3` - - const userFile = await uploadExecutionFile( - executionContext, - audioBuffer, - fileName, - 'audio/mpeg', - authResult.userId - ) - - logger.info('TTS audio stored in execution context:', { - executionId, - fileName, - size: userFile.size, - }) - - return NextResponse.json({ - audioFile: userFile, - audioUrl: userFile.url, - }) - } - - // Chat UI usage - no execution context, use copilot context - const fileName = `tts-${timestamp}.mp3` - const fileInfo = await StorageService.uploadFile({ - file: audioBuffer, - fileName, - contentType: 'audio/mpeg', - context: 'copilot', - }) - - const audioUrl = `${getBaseUrl()}${fileInfo.path}` - - logger.info('TTS audio stored in copilot context (chat UI):', { - fileName, - size: fileInfo.size, - }) - - return NextResponse.json({ - audioUrl, - size: fileInfo.size, - }) - } catch (error) { - logger.error('Error proxying TTS:', error) - - return NextResponse.json( - { - error: `Internal Server Error: ${getErrorMessage(error, 'Unknown error')}`, - }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/tts/unified/route.ts b/apps/sim/app/api/tools/tts/unified/route.ts deleted file mode 100644 index f86d6332970..00000000000 --- a/apps/sim/app/api/tools/tts/unified/route.ts +++ /dev/null @@ -1,844 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { isRecordLike } from '@sim/utils/object' -import type { NextRequest } from 'next/server' -import { NextResponse } from 'next/server' -import { - playHtOutputFormatSchema, - ttsUnifiedToolContract, -} from '@/lib/api/contracts/tools/media/tts' -import { getValidationErrorMessage, parseRequest, validationErrorResponse } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { validateAlphanumericId } from '@/lib/core/security/input-validation' -import { - assertKnownSizeWithinLimit, - isPayloadSizeLimitError, - readResponseJsonWithLimit, - readResponseTextWithLimit, - readResponseToBufferWithLimit, -} from '@/lib/core/utils/stream-limits' -import { getBaseUrl } from '@/lib/core/utils/urls' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { StorageService } from '@/lib/uploads' -import type { - AzureTtsParams, - CartesiaTtsParams, - DeepgramTtsParams, - ElevenLabsTtsUnifiedParams, - GoogleTtsParams, - OpenAiTtsParams, - PlayHtTtsParams, - TtsResponse, -} from '@/tools/tts/types' -import { getFileExtension, getMimeType } from '@/tools/tts/types' - -const logger = createLogger('TtsUnifiedProxyAPI') -const MAX_TTS_AUDIO_BYTES = 25 * 1024 * 1024 -const MAX_TTS_ERROR_BYTES = 64 * 1024 -const MAX_TTS_JSON_BYTES = Math.ceil((MAX_TTS_AUDIO_BYTES * 4) / 3) + 256 * 1024 - -async function readTtsErrorJson( - response: Response, - label: string -): Promise> { - return readResponseJsonWithLimit>(response, { - maxBytes: MAX_TTS_ERROR_BYTES, - label, - }).catch(() => ({})) -} - -function getTtsErrorMessage(error: Record, fallback: string): string { - const nested = error.error - if (typeof nested === 'object' && nested !== null && 'message' in nested) { - const message = (nested as { message?: unknown }).message - if (typeof message === 'string') return message - } - for (const key of ['message', 'err_msg', 'error_message', 'error', 'detail']) { - const value = error[key] - if (typeof value === 'string') return value - if (typeof value === 'object' && value !== null && 'message' in value) { - const message = (value as { message?: unknown }).message - if (typeof message === 'string') return message - } - } - return fallback -} - -export const dynamic = 'force-dynamic' -export const maxDuration = 60 // 1 minute - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId() - logger.info(`[${requestId}] TTS unified request started`) - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - logger.error('Authentication failed for TTS unified proxy:', authResult.error) - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest( - ttsUnifiedToolContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid TTS unified request:`, error.issues) - return validationErrorResponse( - error, - getValidationErrorMessage(error, 'Invalid request data') - ) - }, - } - ) - if (!parsed.success) return parsed.response - - const body = parsed.data.body - const { provider, text, apiKey, workspaceId, workflowId, executionId } = body - - const executionContext = - workspaceId && workflowId && executionId ? { workspaceId, workflowId, executionId } : null - logger.info(`[${requestId}] Processing TTS with ${provider}`, { - hasExecutionContext: Boolean(executionContext), - textLength: text.length, - }) - - let audioBuffer: Buffer - let format: string - let mimeType: string - let duration: number | undefined - - try { - if (provider === 'openai') { - const result = await synthesizeWithOpenAi({ - text, - apiKey, - model: body.model, - voice: body.voice as OpenAiTtsParams['voice'], - responseFormat: body.responseFormat, - speed: body.speed, - }) - audioBuffer = result.audioBuffer - format = result.format - mimeType = result.mimeType - } else if (provider === 'deepgram') { - const result = await synthesizeWithDeepgram({ - text, - apiKey, - model: body.voice, - encoding: body.encoding, - sampleRate: body.sampleRate, - bitRate: body.bitRate, - container: body.container, - }) - audioBuffer = result.audioBuffer - format = result.format - mimeType = result.mimeType - duration = result.duration - } else if (provider === 'elevenlabs') { - if (!body.voiceId) { - return NextResponse.json( - { error: 'voiceId is required for ElevenLabs provider' }, - { status: 400 } - ) - } - const voiceIdValidation = validateAlphanumericId(body.voiceId, 'voiceId') - if (!voiceIdValidation.isValid) { - return NextResponse.json({ error: voiceIdValidation.error }, { status: 400 }) - } - const result = await synthesizeWithElevenLabs({ - text, - apiKey, - voiceId: body.voiceId, - modelId: body.modelId, - stability: body.stability, - similarityBoost: body.similarityBoost, - style: body.style as number | undefined, - useSpeakerBoost: body.useSpeakerBoost, - }) - audioBuffer = result.audioBuffer - format = result.format - mimeType = result.mimeType - } else if (provider === 'cartesia') { - const result = await synthesizeWithCartesia({ - text, - apiKey, - modelId: body.modelId, - voice: body.voice, - language: body.language, - outputFormat: isRecordLike(body.outputFormat) - ? (body.outputFormat as CartesiaTtsParams['outputFormat']) - : undefined, - speed: body.speed, - emotion: body.emotion, - }) - audioBuffer = result.audioBuffer - format = result.format - mimeType = result.mimeType - } else if (provider === 'google') { - const result = await synthesizeWithGoogle({ - text, - apiKey, - voiceId: body.voiceId, - languageCode: body.languageCode, - gender: body.gender, - audioEncoding: body.audioEncoding, - speakingRate: body.speakingRate, - pitch: typeof body.pitch === 'number' ? body.pitch : undefined, - volumeGainDb: body.volumeGainDb, - sampleRateHertz: body.sampleRateHertz, - effectsProfileId: body.effectsProfileId, - }) - audioBuffer = result.audioBuffer - format = result.format - mimeType = result.mimeType - } else if (provider === 'azure') { - const result = await synthesizeWithAzure({ - text, - apiKey, - voiceId: body.voiceId, - region: body.region, - outputFormat: - typeof body.outputFormat === 'string' - ? (body.outputFormat as AzureTtsParams['outputFormat']) - : undefined, - rate: body.rate, - pitch: body.pitch as string | undefined, - style: body.style as string | undefined, - styleDegree: body.styleDegree, - role: body.role, - }) - audioBuffer = result.audioBuffer - format = result.format - mimeType = result.mimeType - } else if (provider === 'playht') { - if (!body.userId) { - return NextResponse.json( - { error: 'userId is required for PlayHT provider' }, - { status: 400 } - ) - } - const playHtOutputFormat = playHtOutputFormatSchema.safeParse(body.outputFormat) - const result = await synthesizeWithPlayHT({ - text, - apiKey, - userId: body.userId, - voice: body.voice, - quality: body.quality, - outputFormat: playHtOutputFormat.success ? playHtOutputFormat.data : undefined, - speed: body.speed, - temperature: body.temperature, - voiceGuidance: body.voiceGuidance, - textGuidance: body.textGuidance, - sampleRate: body.sampleRate, - }) - audioBuffer = result.audioBuffer - format = result.format - mimeType = result.mimeType - } else { - return NextResponse.json({ error: `Unknown provider: ${provider}` }, { status: 400 }) - } - } catch (error) { - logger.error(`[${requestId}] TTS synthesis failed:`, error) - const errorMessage = getErrorMessage(error, 'TTS synthesis failed') - return NextResponse.json( - { error: errorMessage }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } - - const timestamp = Date.now() - const fileExtension = getFileExtension(format) - const fileName = `tts-${provider}-${timestamp}.${fileExtension}` - - if (executionContext) { - const { uploadExecutionFile } = await import('@/lib/uploads/contexts/execution') - - const userFile = await uploadExecutionFile( - executionContext, - audioBuffer, - fileName, - mimeType, - authResult.userId - ) - - logger.info(`[${requestId}] TTS audio stored in execution context:`, { - executionId, - fileName, - size: userFile.size, - }) - - const response: TtsResponse = { - audioUrl: userFile.url, - audioFile: userFile, - characterCount: text.length, - format, - provider, - } - - if (duration) { - response.duration = duration - } - - return NextResponse.json(response) - } - - // Chat UI / copilot usage - no execution context - const fileInfo = await StorageService.uploadFile({ - file: audioBuffer, - fileName, - contentType: mimeType, - context: 'copilot', - }) - - const audioUrl = `${getBaseUrl()}${fileInfo.path}` - - logger.info(`[${requestId}] TTS audio stored in copilot context:`, { - fileName, - size: fileInfo.size, - }) - - const response: TtsResponse = { - audioUrl, - characterCount: text.length, - format, - provider, - } - - if (duration) { - response.duration = duration - } - - return NextResponse.json(response) - } catch (error) { - logger.error(`[${requestId}] TTS unified proxy error:`, error) - const errorMessage = getErrorMessage(error, 'Unknown error') - return NextResponse.json( - { error: errorMessage }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } -}) - -async function synthesizeWithOpenAi( - params: OpenAiTtsParams -): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> { - const { text, apiKey, model = 'tts-1', responseFormat = 'mp3', speed = 1.0 } = params - const voice = (params.voice || 'alloy') as OpenAiTtsParams['voice'] - - const response = await fetch('https://api.openai.com/v1/audio/speech', { - method: 'POST', - headers: { - Authorization: `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - model, - voice, - input: text, - response_format: responseFormat, - speed: Math.max(0.25, Math.min(4.0, speed)), - }), - }) - - if (!response.ok) { - const error = await readTtsErrorJson(response, 'OpenAI TTS error response') - const errorMessage = getTtsErrorMessage(error, response.statusText) - throw new Error(`OpenAI TTS API error: ${errorMessage}`) - } - - const audioBuffer = await readResponseToBufferWithLimit(response, { - maxBytes: MAX_TTS_AUDIO_BYTES, - label: 'OpenAI TTS audio response', - }) - const mimeType = getMimeType(responseFormat) - - return { - audioBuffer, - format: responseFormat, - mimeType, - } -} - -async function synthesizeWithDeepgram( - params: DeepgramTtsParams -): Promise<{ audioBuffer: Buffer; format: string; mimeType: string; duration?: number }> { - const { - text, - apiKey, - model = 'aura-asteria-en', - encoding = 'mp3', - sampleRate, - bitRate, - container, - } = params - - const queryParams = new URLSearchParams({ - model: model, - encoding: encoding, - }) - - if (sampleRate && encoding === 'linear16') { - queryParams.append('sample_rate', sampleRate.toString()) - } - - if (bitRate) { - queryParams.append('bit_rate', bitRate.toString()) - } - - if (container && container !== 'none') { - queryParams.append('container', container) - } - - const response = await fetch(`https://api.deepgram.com/v1/speak?${queryParams.toString()}`, { - method: 'POST', - headers: { - Authorization: `Token ${apiKey}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ text }), - }) - - if (!response.ok) { - const error = await readTtsErrorJson(response, 'Deepgram TTS error response') - const errorMessage = getTtsErrorMessage(error, response.statusText) - throw new Error(`Deepgram TTS API error: ${errorMessage}`) - } - - const audioBuffer = await readResponseToBufferWithLimit(response, { - maxBytes: MAX_TTS_AUDIO_BYTES, - label: 'Deepgram TTS audio response', - }) - - let finalFormat: string = encoding - if (container === 'wav') { - finalFormat = 'wav' - } else if (container === 'ogg') { - finalFormat = 'ogg' - } - - const mimeType = getMimeType(finalFormat) - - return { - audioBuffer, - format: finalFormat, - mimeType, - } -} - -async function synthesizeWithElevenLabs( - params: ElevenLabsTtsUnifiedParams -): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> { - const { - text, - apiKey, - voiceId, - modelId = 'eleven_turbo_v2_5', - stability = 0.5, - similarityBoost = 0.8, - style, - useSpeakerBoost = true, - } = params - - const voiceSettings: any = { - stability: Math.max(0, Math.min(1, stability)), - similarity_boost: Math.max(0, Math.min(1, similarityBoost)), - use_speaker_boost: useSpeakerBoost, - } - - if (style !== undefined) { - voiceSettings.style = Math.max(0, Math.min(1, style)) - } - - const response = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`, { - method: 'POST', - headers: { - Accept: 'audio/mpeg', - 'Content-Type': 'application/json', - 'xi-api-key': apiKey, - }, - body: JSON.stringify({ - text, - model_id: modelId, - voice_settings: voiceSettings, - }), - }) - - if (!response.ok) { - const error = await readTtsErrorJson(response, 'ElevenLabs TTS error response') - const errorMessage = getTtsErrorMessage(error, response.statusText) - throw new Error(`ElevenLabs TTS API error: ${errorMessage}`) - } - - const audioBuffer = await readResponseToBufferWithLimit(response, { - maxBytes: MAX_TTS_AUDIO_BYTES, - label: 'ElevenLabs TTS audio response', - }) - - return { - audioBuffer, - format: 'mp3', - mimeType: 'audio/mpeg', - } -} - -async function synthesizeWithCartesia( - params: Partial -): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> { - const { - text, - apiKey, - modelId = 'sonic-3', - voice, - language = 'en', - outputFormat, - speed, - emotion, - } = params - - if (!text || !apiKey) { - throw new Error('text and apiKey are required for Cartesia') - } - - const requestBody: Record = { - model_id: modelId, - transcript: text, - language, - } - - if (voice) { - requestBody.voice = { - mode: 'id', - id: voice, - } - } - - const generationConfig: Record = {} - if (speed !== undefined) generationConfig.speed = speed - if (emotion !== undefined) generationConfig.emotion = emotion - if (Object.keys(generationConfig).length > 0) { - requestBody.generation_config = generationConfig - } - - if (outputFormat && typeof outputFormat === 'object') { - requestBody.output_format = outputFormat - } - - if (!requestBody.output_format) { - requestBody.output_format = { - container: 'wav', - encoding: 'pcm_s16le', - sample_rate: 24000, - } - } - - logger.info('Cartesia API request:', { - model_id: requestBody.model_id, - has_voice: !!requestBody.voice, - language: requestBody.language, - output_format: requestBody.output_format, - has_generation_config: !!requestBody.generation_config, - }) - - const response = await fetch('https://api.cartesia.ai/tts/bytes', { - method: 'POST', - headers: { - Authorization: `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - 'Cartesia-Version': '2025-04-16', - }, - body: JSON.stringify(requestBody), - }) - - if (!response.ok) { - const error = await readTtsErrorJson(response, 'Cartesia TTS error response') - const errorMessage = getTtsErrorMessage(error, response.statusText) - const errorDetail = typeof error.detail === 'string' ? error.detail : '' - logger.error('Cartesia API error details:', { - status: response.status, - error: errorMessage, - detail: errorDetail, - requestBody: JSON.stringify(requestBody), - }) - throw new Error( - `Cartesia TTS API error: ${errorMessage}${errorDetail ? ` - ${errorDetail}` : ''}` - ) - } - - const audioBuffer = await readResponseToBufferWithLimit(response, { - maxBytes: MAX_TTS_AUDIO_BYTES, - label: 'Cartesia TTS audio response', - }) - - const format = - outputFormat && typeof outputFormat === 'object' && 'container' in outputFormat - ? (outputFormat.container as string) - : 'mp3' - const mimeType = getMimeType(format) - - return { - audioBuffer, - format, - mimeType, - } -} - -async function synthesizeWithGoogle( - params: Partial -): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> { - const { - text, - apiKey, - voiceId, - languageCode, - gender, - audioEncoding = 'MP3', - speakingRate = 1.0, - pitch = 0.0, - volumeGainDb, - sampleRateHertz, - effectsProfileId, - } = params - - if (!text || !apiKey || !languageCode) { - throw new Error('text, apiKey, and languageCode are required for Google Cloud TTS') - } - - const clampedSpeakingRate = Math.max(0.25, Math.min(2.0, speakingRate)) - - const audioConfig: Record = { - audioEncoding, - speakingRate: clampedSpeakingRate, - pitch, - } - - if (volumeGainDb !== undefined) { - audioConfig.volumeGainDb = volumeGainDb - } - if (sampleRateHertz) { - audioConfig.sampleRateHertz = sampleRateHertz - } - if (effectsProfileId && effectsProfileId.length > 0) { - audioConfig.effectsProfileId = effectsProfileId - } - - // Build voice config based on what's provided - const voice: Record = { - languageCode, - } - - // If voiceId is provided, use it (it takes precedence over gender) - if (voiceId) { - voice.name = voiceId - } - - // Only include gender if specified (don't default to NEUTRAL as it's not supported) - if (gender) { - voice.ssmlGender = gender - } - - // If neither voiceId nor gender is provided, default to a specific voice - if (!voiceId && !gender) { - voice.name = 'en-US-Neural2-C' - } - - const requestBody: Record = { - input: { text }, - voice, - audioConfig, - } - - const response = await fetch( - `https://texttospeech.googleapis.com/v1/text:synthesize?key=${apiKey}`, - { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(requestBody), - } - ) - - if (!response.ok) { - const error = await readTtsErrorJson(response, 'Google TTS error response') - const errorMessage = getTtsErrorMessage(error, response.statusText) - throw new Error(`Google Cloud TTS API error: ${errorMessage}`) - } - - const data = await readResponseJsonWithLimit<{ audioContent?: string }>(response, { - maxBytes: MAX_TTS_JSON_BYTES, - label: 'Google TTS JSON response', - }) - const audioContent = data.audioContent - - if (!audioContent) { - throw new Error('No audio content returned from Google Cloud TTS') - } - - const audioBuffer = Buffer.from(audioContent, 'base64') - assertKnownSizeWithinLimit(audioBuffer.length, MAX_TTS_AUDIO_BYTES, 'Google TTS audio response') - - const format = audioEncoding.toLowerCase().replace('_', '') - const mimeType = getMimeType(format) - - return { - audioBuffer, - format, - mimeType, - } -} - -async function synthesizeWithAzure( - params: Partial -): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> { - const { - text, - apiKey, - voiceId = 'en-US-JennyNeural', - region = 'eastus', - outputFormat = 'audio-24khz-96kbitrate-mono-mp3', - rate, - pitch, - style, - styleDegree, - role, - } = params - - if (!text || !apiKey) { - throw new Error('text and apiKey are required for Azure TTS') - } - - const AZURE_REGION_RE = /^[a-z][a-z0-9-]{1,30}[a-z0-9]$/ - if (!AZURE_REGION_RE.test(region)) { - throw new Error( - 'Invalid Azure region: must match /^[a-z][a-z0-9-]{1,30}[a-z0-9]$/ (e.g. eastus, westeurope)' - ) - } - - let ssml = `` - - if (style) { - ssml += ` -): Promise<{ audioBuffer: Buffer; format: string; mimeType: string }> { - const { - text, - apiKey, - userId, - voice, - quality = 'standard', - outputFormat = 'mp3', - speed = 1.0, - temperature, - voiceGuidance, - textGuidance, - sampleRate, - } = params - - if (!text || !apiKey || !userId) { - throw new Error('text, apiKey, and userId are required for PlayHT') - } - - const requestBody: Record = { - text, - quality, - output_format: outputFormat, - speed, - } - - if (voice) requestBody.voice = voice - if (temperature !== undefined) requestBody.temperature = temperature - if (voiceGuidance !== undefined) requestBody.voice_guidance = voiceGuidance - if (textGuidance !== undefined) requestBody.text_guidance = textGuidance - if (sampleRate) requestBody.sample_rate = sampleRate - - const response = await fetch('https://api.play.ht/api/v2/tts/stream', { - method: 'POST', - headers: { - AUTHORIZATION: apiKey, - 'X-USER-ID': userId, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(requestBody), - }) - - if (!response.ok) { - const error = await readTtsErrorJson(response, 'PlayHT TTS error response') - const errorMessage = getTtsErrorMessage(error, response.statusText) - throw new Error(`PlayHT TTS API error: ${errorMessage}`) - } - - const audioBuffer = await readResponseToBufferWithLimit(response, { - maxBytes: MAX_TTS_AUDIO_BYTES, - label: 'PlayHT TTS audio response', - }) - - const format = outputFormat || 'mp3' - const mimeType = getMimeType(format) - - return { - audioBuffer, - format, - mimeType, - } -} diff --git a/apps/sim/app/api/tools/twilio/get-recording/route.ts b/apps/sim/app/api/tools/twilio/get-recording/route.ts deleted file mode 100644 index ddd0fc9350c..00000000000 --- a/apps/sim/app/api/tools/twilio/get-recording/route.ts +++ /dev/null @@ -1,246 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { twilioGetRecordingContract } from '@/lib/api/contracts/tools/communication/messaging' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getExtensionFromMimeType } from '@/lib/uploads/utils/file-utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('TwilioGetRecordingAPI') - -interface TwilioRecordingResponse { - sid?: string - call_sid?: string - duration?: string - status?: string - channels?: number - source?: string - price?: string - price_unit?: string - uri?: string - error_code?: number - message?: string - error_message?: string -} - -interface TwilioErrorResponse { - message?: string -} - -interface TwilioTranscription { - transcription_text?: string - status?: string - price?: string - price_unit?: string -} - -interface TwilioTranscriptionsResponse { - transcriptions?: TwilioTranscription[] -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Twilio get recording attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - const parsed = await parseRequest(twilioGetRecordingContract, request, {}) - if (!parsed.success) return parsed.response - const { accountSid, authToken, recordingSid } = parsed.data.body - - if (!accountSid.startsWith('AC')) { - return NextResponse.json( - { - success: false, - error: `Invalid Account SID format. Account SID must start with "AC" (you provided: ${accountSid.substring(0, 2)}...)`, - }, - { status: 400 } - ) - } - - const twilioAuth = Buffer.from(`${accountSid}:${authToken}`).toString('base64') - - logger.info(`[${requestId}] Getting recording info from Twilio`, { recordingSid }) - - const infoUrl = `https://api.twilio.com/2010-04-01/Accounts/${accountSid}/Recordings/${recordingSid}.json` - const infoUrlValidation = await validateUrlWithDNS(infoUrl, 'infoUrl') - if (!infoUrlValidation.isValid) { - return NextResponse.json({ success: false, error: infoUrlValidation.error }, { status: 400 }) - } - - const infoResponse = await secureFetchWithPinnedIP(infoUrl, infoUrlValidation.resolvedIP!, { - method: 'GET', - headers: { Authorization: `Basic ${twilioAuth}` }, - }) - - if (!infoResponse.ok) { - const errorData = (await infoResponse.json().catch(() => ({}))) as TwilioErrorResponse - logger.error(`[${requestId}] Twilio API error`, { - status: infoResponse.status, - error: errorData, - }) - return NextResponse.json( - { success: false, error: errorData.message || `Twilio API error: ${infoResponse.status}` }, - { status: 400 } - ) - } - - const data = (await infoResponse.json()) as TwilioRecordingResponse - - if (data.error_code) { - return NextResponse.json({ - success: false, - output: { - success: false, - error: data.message || data.error_message || 'Failed to retrieve recording', - }, - error: data.message || data.error_message || 'Failed to retrieve recording', - }) - } - - const baseUrl = 'https://api.twilio.com' - const mediaUrl = data.uri ? `${baseUrl}${data.uri.replace('.json', '')}` : undefined - - let transcriptionText: string | undefined - let transcriptionStatus: string | undefined - let transcriptionPrice: string | undefined - let transcriptionPriceUnit: string | undefined - let file: - | { - name: string - mimeType: string - data: string - size: number - } - | undefined - - try { - const transcriptionUrl = `https://api.twilio.com/2010-04-01/Accounts/${accountSid}/Transcriptions.json?RecordingSid=${data.sid}` - logger.info(`[${requestId}] Checking for transcriptions`) - - const transcriptionUrlValidation = await validateUrlWithDNS( - transcriptionUrl, - 'transcriptionUrl' - ) - if (transcriptionUrlValidation.isValid) { - const transcriptionResponse = await secureFetchWithPinnedIP( - transcriptionUrl, - transcriptionUrlValidation.resolvedIP!, - { - method: 'GET', - headers: { Authorization: `Basic ${twilioAuth}` }, - } - ) - - if (transcriptionResponse.ok) { - const transcriptionData = - (await transcriptionResponse.json()) as TwilioTranscriptionsResponse - - if (transcriptionData.transcriptions && transcriptionData.transcriptions.length > 0) { - const transcription = transcriptionData.transcriptions[0] - transcriptionText = transcription.transcription_text - transcriptionStatus = transcription.status - transcriptionPrice = transcription.price - transcriptionPriceUnit = transcription.price_unit - logger.info(`[${requestId}] Transcription found`, { - status: transcriptionStatus, - textLength: transcriptionText?.length, - }) - } - } - } - } catch (error) { - logger.warn(`[${requestId}] Failed to fetch transcription:`, error) - } - - if (mediaUrl) { - try { - const mediaUrlValidation = await validateUrlWithDNS(mediaUrl, 'mediaUrl') - if (mediaUrlValidation.isValid) { - const mediaResponse = await secureFetchWithPinnedIP( - mediaUrl, - mediaUrlValidation.resolvedIP!, - { - method: 'GET', - headers: { Authorization: `Basic ${twilioAuth}` }, - } - ) - - if (mediaResponse.ok) { - const contentType = - mediaResponse.headers.get('content-type') || 'application/octet-stream' - const extension = getExtensionFromMimeType(contentType) || 'dat' - const arrayBuffer = await mediaResponse.arrayBuffer() - const buffer = Buffer.from(arrayBuffer) - const fileName = `${data.sid || recordingSid}.${extension}` - - file = { - name: fileName, - mimeType: contentType, - data: buffer.toString('base64'), - size: buffer.length, - } - } - } - } catch (error) { - logger.warn(`[${requestId}] Failed to download recording media:`, error) - } - } - - logger.info(`[${requestId}] Twilio recording fetched successfully`, { - recordingSid: data.sid, - hasFile: !!file, - hasTranscription: !!transcriptionText, - }) - - return NextResponse.json({ - success: true, - output: { - success: true, - recordingSid: data.sid, - callSid: data.call_sid, - duration: data.duration ? Number.parseInt(data.duration, 10) : undefined, - status: data.status, - channels: data.channels, - source: data.source, - mediaUrl, - file, - price: data.price, - priceUnit: data.price_unit, - uri: data.uri, - transcriptionText, - transcriptionStatus, - transcriptionPrice, - transcriptionPriceUnit, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error fetching Twilio recording:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/typeform/files/route.ts b/apps/sim/app/api/tools/typeform/files/route.ts deleted file mode 100644 index f4ded92ff92..00000000000 --- a/apps/sim/app/api/tools/typeform/files/route.ts +++ /dev/null @@ -1,168 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { typeformFilesContract } from '@/lib/api/contracts/tools/typeform' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { - DEFAULT_MAX_ERROR_BODY_BYTES, - isPayloadSizeLimitError, - readResponseTextWithLimit, - readResponseToBufferWithLimit, -} from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { uploadCopilotFile } from '@/lib/uploads/contexts/copilot' -import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' - -const logger = createLogger('TypeformFilesAPI') -const MAX_TYPEFORM_FILE_BYTES = 10 * 1024 * 1024 - -export const dynamic = 'force-dynamic' - -function buildTypeformFileUrl({ - formId, - responseId, - fieldId, - filename, - inline, -}: { - formId: string - responseId: string - fieldId: string - filename: string - inline?: boolean -}): string { - const encodedFormId = encodeURIComponent(formId) - const encodedResponseId = encodeURIComponent(responseId) - const encodedFieldId = encodeURIComponent(fieldId) - const encodedFilename = encodeURIComponent(filename) - const url = new URL( - `https://api.typeform.com/forms/${encodedFormId}/responses/${encodedResponseId}/fields/${encodedFieldId}/files/${encodedFilename}` - ) - if (inline !== undefined) { - url.searchParams.set('inline', String(inline)) - } - return url.toString() -} - -function getFilename( - response: { headers: { get(name: string): string | null } }, - fallback: string -): string { - const contentDisposition = response.headers.get('content-disposition') || '' - const filenameMatch = contentDisposition.match(/filename="(.+?)"/) - return filenameMatch?.[1] || fallback || 'typeform-file' -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest( - typeformFilesContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Invalid request data') }, - { status: 400 } - ), - } - ) - if (!parsed.success) return parsed.response - - try { - const body = parsed.data.body - const fileUrl = buildTypeformFileUrl(body) - const urlValidation = await validateUrlWithDNS(fileUrl, 'typeformFileUrl') - if (!urlValidation.isValid) { - return NextResponse.json( - { success: false, error: urlValidation.error || 'Invalid Typeform file URL' }, - { status: 400 } - ) - } - - const response = await secureFetchWithPinnedIP(fileUrl, urlValidation.resolvedIP!, { - headers: { Authorization: `Bearer ${body.apiKey}` }, - maxResponseBytes: MAX_TYPEFORM_FILE_BYTES, - }) - - if (!response.ok) { - const errorText = await readResponseTextWithLimit(response, { - maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, - label: 'Typeform file error response', - }).catch(() => '') - return NextResponse.json( - { - success: false, - error: `Failed to download Typeform file: ${response.status} ${errorText}`, - }, - { status: response.status } - ) - } - - const buffer = await readResponseToBufferWithLimit(response, { - maxBytes: MAX_TYPEFORM_FILE_BYTES, - label: 'Typeform file download', - }) - const contentType = response.headers.get('content-type') || 'application/octet-stream' - const filename = getFilename(response, body.filename) - const executionContext = - body.workspaceId && body.workflowId && body.executionId - ? { - workspaceId: body.workspaceId, - workflowId: body.workflowId, - executionId: body.executionId, - } - : undefined - - if (executionContext) { - const file = await uploadExecutionFile( - executionContext, - buffer, - filename, - contentType, - authResult.userId - ) - return NextResponse.json({ - success: true, - output: { - fileUrl: file.url, - file: { ...file, mimeType: contentType }, - contentType, - filename, - }, - }) - } - - const file = await uploadCopilotFile({ - buffer, - fileName: filename, - contentType, - userId: authResult.userId, - }) - - return NextResponse.json({ - success: true, - output: { - fileUrl: file.url || fileUrl, - file, - contentType, - filename, - }, - }) - } catch (error) { - logger.error('Typeform file download failed', { error }) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Failed to download Typeform file') }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/uptimerobot/create-psp/route.ts b/apps/sim/app/api/tools/uptimerobot/create-psp/route.ts deleted file mode 100644 index 1d879c5735b..00000000000 --- a/apps/sim/app/api/tools/uptimerobot/create-psp/route.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { uptimeRobotCreatePspContract } from '@/lib/api/contracts/tools/uptimerobot' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { forwardPspRequest } from '@/app/api/tools/uptimerobot/server-utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('UptimeRobotCreatePspAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized UptimeRobot create-psp request: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(uptimeRobotCreatePspContract, request, {}) - if (!parsed.success) return parsed.response - const body = parsed.data.body - - return forwardPspRequest({ - apiKey: body.apiKey, - method: 'POST', - path: '/psps', - fields: body, - userId: authResult.userId, - requestId, - logger, - }) - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - logger.error(`[${requestId}] Unexpected error creating status page:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/uptimerobot/server-utils.ts b/apps/sim/app/api/tools/uptimerobot/server-utils.ts deleted file mode 100644 index 9c3c7d2f316..00000000000 --- a/apps/sim/app/api/tools/uptimerobot/server-utils.ts +++ /dev/null @@ -1,170 +0,0 @@ -import type { Logger } from '@sim/logger' -import { NextResponse } from 'next/server' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { mapPsp, UPTIMEROBOT_API_BASE } from '@/tools/uptimerobot/types' - -/** Fields shared by the PSP create and update routes (before the files). */ -interface PspFormFields { - friendlyName?: string | null - monitorIds?: string | null - status?: string | null - password?: string | null - customDomain?: string | null - hideUrlLinks?: boolean | null - noIndex?: boolean | null - logo?: unknown - icon?: unknown -} - -/** - * Appends a single optional image file (logo or icon) to the form after - * downloading it from storage and verifying the caller may access it. - * - * @returns an error `NextResponse` if the file is invalid or access is denied, - * otherwise `null`. - */ -async function appendPspImage( - form: FormData, - field: 'logo' | 'icon', - file: unknown, - userId: string, - requestId: string, - logger: Logger -): Promise { - const userFiles = processFilesToUserFiles([file as RawFileInput], requestId, logger) - if (userFiles.length === 0) { - // A file was supplied but could not be resolved to a stored UserFile (e.g. a - // bare string reference). Surface it rather than silently dropping the image. - return NextResponse.json( - { success: false, error: `Invalid ${field} file: expected an uploaded file reference` }, - { status: 400 } - ) - } - - const userFile = userFiles[0] - const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) - if (denied) return denied - - const { buffer, contentType } = await downloadServableFileFromStorage( - userFile, - requestId, - logger, - { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - } - ) - const mimeType = contentType || userFile.type || 'application/octet-stream' - form.append(field, new Blob([new Uint8Array(buffer)], { type: mimeType }), userFile.name) - return null -} - -/** - * Builds the multipart form for a PSP request, downloads any referenced - * logo/icon files, forwards the request to UptimeRobot, and returns a typed - * `{ success, output: { psp } }` envelope as a `NextResponse`. - */ -export async function forwardPspRequest(options: { - apiKey: string - method: 'POST' | 'PATCH' - path: string - fields: PspFormFields - userId: string - requestId: string - logger: Logger -}): Promise { - const { apiKey, method, path, fields, userId, requestId, logger } = options - - const form = new FormData() - if (fields.friendlyName) form.append('friendlyName', fields.friendlyName) - if (fields.status) form.append('status', fields.status) - if (fields.password) form.append('password', fields.password) - if (fields.customDomain) form.append('customDomain', fields.customDomain) - if (typeof fields.hideUrlLinks === 'boolean') { - form.append('hideUrlLinks', String(fields.hideUrlLinks)) - } - if (typeof fields.noIndex === 'boolean') form.append('noIndex', String(fields.noIndex)) - if (fields.monitorIds) { - for (const id of fields.monitorIds.split(',')) { - const trimmed = id.trim() - if (trimmed) form.append('monitorIds', trimmed) - } - } - - if (fields.logo) { - const denied = await appendPspImage(form, 'logo', fields.logo, userId, requestId, logger) - if (denied) return denied - } - if (fields.icon) { - const denied = await appendPspImage(form, 'icon', fields.icon, userId, requestId, logger) - if (denied) return denied - } - - const response = await fetch(`${UPTIMEROBOT_API_BASE}${path}`, { - method, - headers: { Authorization: `Bearer ${apiKey}`, Accept: 'application/json' }, - body: form, - }) - - const text = await response.text() - if (!response.ok) { - let message: string | undefined - try { - message = JSON.parse(text)?.message - } catch { - message = undefined - } - logger.error(`[${requestId}] UptimeRobot PSP request failed`, { - status: response.status, - body: text, - }) - return NextResponse.json( - { success: false, error: message || `UptimeRobot API error (HTTP ${response.status})` }, - { status: response.status } - ) - } - - // A successful PSP create/update must return the PspDto object. An empty or - // non-object body is unexpected — reject it rather than mapping a phantom PSP - // (id: 0, empty name, null images) back to the workflow. - if (!text) { - logger.error(`[${requestId}] UptimeRobot returned an empty PSP response`) - return NextResponse.json( - { success: false, error: 'UptimeRobot returned an unexpected response' }, - { status: 502 } - ) - } - - let data: Record - try { - const parsed = JSON.parse(text) - if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { - throw new Error('Expected a PSP object response') - } - data = parsed as Record - } catch { - logger.error(`[${requestId}] UptimeRobot returned an unexpected PSP response`, { body: text }) - return NextResponse.json( - { success: false, error: 'UptimeRobot returned an unexpected response' }, - { status: 502 } - ) - } - - // A real PspDto always carries a positive numeric `id` and a non-empty - // `friendlyName` (both spec-required). If they are absent, the body is a `{}` - // or metadata envelope, not a status page — surface the provider error rather - // than mapping a phantom PSP. - if (typeof data.id !== 'number' || data.id < 1 || !data.friendlyName) { - logger.error(`[${requestId}] UptimeRobot returned a PSP response without core fields`, { - body: text, - }) - return NextResponse.json( - { success: false, error: 'UptimeRobot returned an unexpected response' }, - { status: 502 } - ) - } - - return NextResponse.json({ success: true, output: { psp: mapPsp(data) } }) -} diff --git a/apps/sim/app/api/tools/uptimerobot/update-psp/route.ts b/apps/sim/app/api/tools/uptimerobot/update-psp/route.ts deleted file mode 100644 index 063a4252ca6..00000000000 --- a/apps/sim/app/api/tools/uptimerobot/update-psp/route.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { uptimeRobotUpdatePspContract } from '@/lib/api/contracts/tools/uptimerobot' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { forwardPspRequest } from '@/app/api/tools/uptimerobot/server-utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('UptimeRobotUpdatePspAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized UptimeRobot update-psp request: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(uptimeRobotUpdatePspContract, request, {}) - if (!parsed.success) return parsed.response - const body = parsed.data.body - - return forwardPspRequest({ - apiKey: body.apiKey, - method: 'PATCH', - path: `/psps/${body.pspId}`, - fields: body, - userId: authResult.userId, - requestId, - logger, - }) - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - logger.error(`[${requestId}] Unexpected error updating status page:`, error) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/vanta/download/route.ts b/apps/sim/app/api/tools/vanta/download/route.ts deleted file mode 100644 index 48a23ac94ae..00000000000 --- a/apps/sim/app/api/tools/vanta/download/route.ts +++ /dev/null @@ -1,160 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { vantaDownloadContract } from '@/lib/api/contracts/tools/vanta' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - buildVantaUrl, - extractVantaError, - fetchVantaWithAuth, - getVantaBaseUrl, - VANTA_READ_SCOPE, -} from '@/tools/vanta/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('VantaDownloadAPI') - -const MAX_DOWNLOAD_SIZE_BYTES = 100 * 1024 * 1024 - -function downloadSizeError(bytes: number): NextResponse { - const sizeMB = (bytes / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { success: false, error: `File size (${sizeMB}MB) exceeds download limit of 100MB` }, - { status: 400 } - ) -} - -/** - * Reads a response body incrementally, aborting as soon as the accumulated - * size exceeds the limit so oversized files are never fully buffered. - * Returns null when the limit is exceeded. - */ -async function readBodyWithLimit(response: Response, maxBytes: number): Promise { - const reader = response.body?.getReader() - if (!reader) { - const buffer = Buffer.from(await response.arrayBuffer()) - return buffer.length > maxBytes ? null : buffer - } - - const chunks: Uint8Array[] = [] - let total = 0 - while (true) { - const { done, value } = await reader.read() - if (done) break - total += value.byteLength - if (total > maxBytes) { - await reader.cancel() - return null - } - chunks.push(value) - } - return Buffer.concat(chunks) -} - -/** - * Extracts the filename from a Content-Disposition header, if present. - */ -function getFileNameFromContentDisposition(header: string | null): string | null { - if (!header) return null - const utf8Match = header.match(/filename\*=UTF-8''([^;]+)/i) - if (utf8Match) { - try { - return decodeURIComponent(utf8Match[1]) - } catch { - return null - } - } - const plainMatch = header.match(/filename="?([^";]+)"?/i) - return plainMatch ? plainMatch[1] : null -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Vanta download attempt`, { - error: authResult.error || 'Unauthorized', - }) - return NextResponse.json( - { success: false, error: authResult.error || 'Unauthorized' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(vantaDownloadContract, request, {}) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const mediaUrl = buildVantaUrl( - getVantaBaseUrl(params.region), - `/documents/${encodeURIComponent(params.documentId)}/uploads/${encodeURIComponent(params.uploadedFileId)}/media` - ) - - logger.info(`[${requestId}] Downloading Vanta document file`, { - documentId: params.documentId, - uploadedFileId: params.uploadedFileId, - }) - - const response = await fetchVantaWithAuth( - { - clientId: params.clientId, - clientSecret: params.clientSecret, - region: params.region, - scope: VANTA_READ_SCOPE, - }, - (accessToken) => - fetch(mediaUrl, { - method: 'GET', - headers: { Authorization: `Bearer ${accessToken}` }, - cache: 'no-store', - }) - ) - - if (!response.ok) { - const errorData: unknown = await response.json().catch(() => null) - const message = extractVantaError(errorData, 'Failed to download Vanta document file') - logger.error(`[${requestId}] Vanta download failed`, { status: response.status, message }) - return NextResponse.json({ success: false, error: message }, { status: response.status }) - } - - const contentLength = Number(response.headers.get('content-length')) - if (Number.isFinite(contentLength) && contentLength > MAX_DOWNLOAD_SIZE_BYTES) { - return downloadSizeError(contentLength) - } - - const buffer = await readBodyWithLimit(response, MAX_DOWNLOAD_SIZE_BYTES) - if (buffer === null) { - return NextResponse.json( - { success: false, error: 'File exceeds download limit of 100MB' }, - { status: 400 } - ) - } - - const mimeType = response.headers.get('content-type') || 'application/octet-stream' - const name = - getFileNameFromContentDisposition(response.headers.get('content-disposition')) || - `vanta-document-file-${params.uploadedFileId}` - - logger.info(`[${requestId}] Vanta download successful`, { name, size: buffer.length }) - - return NextResponse.json({ - success: true, - output: { - file: { name, mimeType, data: buffer.toString('base64'), size: buffer.length }, - name, - mimeType, - size: buffer.length, - }, - }) - } catch (error) { - const message = toError(error).message - logger.error(`[${requestId}] Vanta download failed`, { error: message }) - return NextResponse.json({ success: false, error: message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/vanta/query/route.ts b/apps/sim/app/api/tools/vanta/query/route.ts deleted file mode 100644 index 102c20af00f..00000000000 --- a/apps/sim/app/api/tools/vanta/query/route.ts +++ /dev/null @@ -1,423 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import type { VantaQueryBody } from '@/lib/api/contracts/tools/vanta' -import { vantaQueryContract } from '@/lib/api/contracts/tools/vanta' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - asVantaRecord, - buildVantaUrl, - extractVantaError, - fetchVantaWithAuth, - getVantaBaseUrl, - getVantaListResults, - normalizeVantaControl, - normalizeVantaControlDetail, - normalizeVantaDocument, - normalizeVantaDocumentDetail, - normalizeVantaFramework, - normalizeVantaFrameworkDetail, - normalizeVantaMonitoredComputer, - normalizeVantaPerson, - normalizeVantaPolicy, - normalizeVantaRiskScenario, - normalizeVantaTest, - normalizeVantaTestEntity, - normalizeVantaUploadedFile, - normalizeVantaVendor, - normalizeVantaVulnerability, - normalizeVantaVulnerabilityRemediation, - normalizeVantaVulnerableAsset, - splitVantaCommaList, - VANTA_READ_SCOPE, - VANTA_WRITE_SCOPE, -} from '@/tools/vanta/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('VantaQueryAPI') - -interface VantaApiRequest { - method: 'GET' | 'POST' - url: string -} - -/** - * Maps a validated query operation to the Vanta API request it performs. - */ -function buildVantaApiRequest(baseUrl: string, params: VantaQueryBody): VantaApiRequest { - const id = encodeURIComponent - - switch (params.operation) { - case 'vanta_list_frameworks': - return { - method: 'GET', - url: buildVantaUrl(baseUrl, '/frameworks', { - pageSize: params.pageSize, - pageCursor: params.pageCursor, - }), - } - case 'vanta_get_framework': - return { method: 'GET', url: buildVantaUrl(baseUrl, `/frameworks/${id(params.frameworkId)}`) } - case 'vanta_list_framework_controls': - return { - method: 'GET', - url: buildVantaUrl(baseUrl, `/frameworks/${id(params.frameworkId)}/controls`, { - pageSize: params.pageSize, - pageCursor: params.pageCursor, - }), - } - case 'vanta_list_controls': - return { - method: 'GET', - url: buildVantaUrl(baseUrl, '/controls', { - frameworkMatchesAny: splitVantaCommaList(params.frameworkMatchesAny), - pageSize: params.pageSize, - pageCursor: params.pageCursor, - }), - } - case 'vanta_get_control': - return { method: 'GET', url: buildVantaUrl(baseUrl, `/controls/${id(params.controlId)}`) } - case 'vanta_list_control_tests': - return { - method: 'GET', - url: buildVantaUrl(baseUrl, `/controls/${id(params.controlId)}/tests`, { - pageSize: params.pageSize, - pageCursor: params.pageCursor, - }), - } - case 'vanta_list_control_documents': - return { - method: 'GET', - url: buildVantaUrl(baseUrl, `/controls/${id(params.controlId)}/documents`, { - pageSize: params.pageSize, - pageCursor: params.pageCursor, - }), - } - case 'vanta_list_tests': - return { - method: 'GET', - url: buildVantaUrl(baseUrl, '/tests', { - statusFilter: params.statusFilter, - frameworkFilter: params.frameworkFilter, - integrationFilter: params.integrationFilter, - controlFilter: params.controlFilter, - ownerFilter: params.ownerFilter, - categoryFilter: params.categoryFilter, - isInRollout: params.isInRollout, - pageSize: params.pageSize, - pageCursor: params.pageCursor, - }), - } - case 'vanta_get_test': - return { method: 'GET', url: buildVantaUrl(baseUrl, `/tests/${id(params.testId)}`) } - case 'vanta_list_test_entities': - return { - method: 'GET', - url: buildVantaUrl(baseUrl, `/tests/${id(params.testId)}/entities`, { - entityStatus: params.entityStatus, - pageSize: params.pageSize, - pageCursor: params.pageCursor, - }), - } - case 'vanta_list_documents': - return { - method: 'GET', - url: buildVantaUrl(baseUrl, '/documents', { - frameworkMatchesAny: splitVantaCommaList(params.frameworkMatchesAny), - statusMatchesAny: splitVantaCommaList(params.statusMatchesAny), - pageSize: params.pageSize, - pageCursor: params.pageCursor, - }), - } - case 'vanta_get_document': - return { method: 'GET', url: buildVantaUrl(baseUrl, `/documents/${id(params.documentId)}`) } - case 'vanta_list_document_uploads': - return { - method: 'GET', - url: buildVantaUrl(baseUrl, `/documents/${id(params.documentId)}/uploads`, { - pageSize: params.pageSize, - pageCursor: params.pageCursor, - }), - } - case 'vanta_submit_document': - return { - method: 'POST', - url: buildVantaUrl(baseUrl, `/documents/${id(params.documentId)}/submit`), - } - case 'vanta_list_people': - return { - method: 'GET', - url: buildVantaUrl(baseUrl, '/people', { - emailAndNameFilter: params.emailAndNameFilter, - employmentStatus: params.employmentStatus, - groupIdsMatchesAny: splitVantaCommaList(params.groupIdsMatchesAny), - tasksSummaryStatusMatchesAny: splitVantaCommaList(params.tasksSummaryStatusMatchesAny), - taskTypeMatchesAny: splitVantaCommaList(params.taskTypeMatchesAny), - taskStatusMatchesAny: splitVantaCommaList(params.taskStatusMatchesAny), - pageSize: params.pageSize, - pageCursor: params.pageCursor, - }), - } - case 'vanta_get_person': - return { method: 'GET', url: buildVantaUrl(baseUrl, `/people/${id(params.personId)}`) } - case 'vanta_list_policies': - return { - method: 'GET', - url: buildVantaUrl(baseUrl, '/policies', { - pageSize: params.pageSize, - pageCursor: params.pageCursor, - }), - } - case 'vanta_get_policy': - return { method: 'GET', url: buildVantaUrl(baseUrl, `/policies/${id(params.policyId)}`) } - case 'vanta_list_vendors': - return { - method: 'GET', - url: buildVantaUrl(baseUrl, '/vendors', { - name: params.name, - statusMatchesAny: splitVantaCommaList(params.statusMatchesAny), - pageSize: params.pageSize, - pageCursor: params.pageCursor, - }), - } - case 'vanta_get_vendor': - return { method: 'GET', url: buildVantaUrl(baseUrl, `/vendors/${id(params.vendorId)}`) } - case 'vanta_list_monitored_computers': - return { - method: 'GET', - url: buildVantaUrl(baseUrl, '/monitored-computers', { - complianceStatusFilterMatchesAny: splitVantaCommaList( - params.complianceStatusFilterMatchesAny - ), - pageSize: params.pageSize, - pageCursor: params.pageCursor, - }), - } - case 'vanta_list_vulnerabilities': - return { - method: 'GET', - url: buildVantaUrl(baseUrl, '/vulnerabilities', { - q: params.q, - severity: params.severity, - isFixAvailable: params.isFixAvailable, - isDeactivated: params.isDeactivated, - includeVulnerabilitiesWithoutSlas: params.includeVulnerabilitiesWithoutSlas, - packageIdentifier: params.packageIdentifier, - externalVulnerabilityId: params.externalVulnerabilityId, - integrationId: params.integrationId, - vulnerableAssetId: params.vulnerableAssetId, - slaDeadlineAfterDate: params.slaDeadlineAfterDate, - slaDeadlineBeforeDate: params.slaDeadlineBeforeDate, - pageSize: params.pageSize, - pageCursor: params.pageCursor, - }), - } - case 'vanta_list_vulnerability_remediations': - return { - method: 'GET', - url: buildVantaUrl(baseUrl, '/vulnerability-remediations', { - integrationId: params.integrationId, - severity: params.severity, - isRemediatedOnTime: params.isRemediatedOnTime, - remediatedAfterDate: params.remediatedAfterDate, - remediatedBeforeDate: params.remediatedBeforeDate, - pageSize: params.pageSize, - pageCursor: params.pageCursor, - }), - } - case 'vanta_list_vulnerable_assets': - return { - method: 'GET', - url: buildVantaUrl(baseUrl, '/vulnerable-assets', { - q: params.q, - integrationId: params.integrationId, - assetType: params.assetType, - assetExternalAccountId: params.assetExternalAccountId, - pageSize: params.pageSize, - pageCursor: params.pageCursor, - }), - } - case 'vanta_get_vulnerable_asset': - return { - method: 'GET', - url: buildVantaUrl(baseUrl, `/vulnerable-assets/${id(params.vulnerableAssetId)}`), - } - case 'vanta_list_risk_scenarios': - return { - method: 'GET', - url: buildVantaUrl(baseUrl, '/risk-scenarios', { - searchString: params.searchString, - includeIgnored: params.includeIgnored, - type: params.type, - ownerMatchesAny: splitVantaCommaList(params.ownerMatchesAny), - categoryMatchesAny: splitVantaCommaList(params.categoryMatchesAny), - ciaCategoryMatchesAny: splitVantaCommaList(params.ciaCategoryMatchesAny), - treatmentTypeMatchesAny: splitVantaCommaList(params.treatmentTypeMatchesAny), - inherentScoreGroupMatchesAny: splitVantaCommaList(params.inherentScoreGroupMatchesAny), - residualScoreGroupMatchesAny: splitVantaCommaList(params.residualScoreGroupMatchesAny), - reviewStatusMatchesAny: splitVantaCommaList(params.reviewStatusMatchesAny), - orderBy: params.orderBy, - pageSize: params.pageSize, - pageCursor: params.pageCursor, - }), - } - case 'vanta_get_risk_scenario': - return { - method: 'GET', - url: buildVantaUrl(baseUrl, `/risk-scenarios/${id(params.riskScenarioId)}`), - } - } -} - -/** - * Normalizes a successful Vanta API response body into the operation's - * documented output shape. - */ -function buildVantaOutput(params: VantaQueryBody, data: unknown): Record { - switch (params.operation) { - case 'vanta_list_frameworks': { - const { data: items, pageInfo } = getVantaListResults(data) - return { frameworks: items.map(normalizeVantaFramework), pageInfo } - } - case 'vanta_get_framework': - return { framework: normalizeVantaFrameworkDetail(asVantaRecord(data)) } - case 'vanta_list_framework_controls': - case 'vanta_list_controls': { - const { data: items, pageInfo } = getVantaListResults(data) - return { controls: items.map(normalizeVantaControl), pageInfo } - } - case 'vanta_get_control': - return { control: normalizeVantaControlDetail(asVantaRecord(data)) } - case 'vanta_list_control_tests': - case 'vanta_list_tests': { - const { data: items, pageInfo } = getVantaListResults(data) - return { tests: items.map(normalizeVantaTest), pageInfo } - } - case 'vanta_get_test': - return { test: normalizeVantaTest(asVantaRecord(data)) } - case 'vanta_list_test_entities': { - const { data: items, pageInfo } = getVantaListResults(data) - return { entities: items.map(normalizeVantaTestEntity), pageInfo } - } - case 'vanta_list_control_documents': - case 'vanta_list_documents': { - const { data: items, pageInfo } = getVantaListResults(data) - return { documents: items.map(normalizeVantaDocument), pageInfo } - } - case 'vanta_get_document': - return { document: normalizeVantaDocumentDetail(asVantaRecord(data)) } - case 'vanta_list_document_uploads': { - const { data: items, pageInfo } = getVantaListResults(data) - return { uploads: items.map(normalizeVantaUploadedFile), pageInfo } - } - case 'vanta_submit_document': - return { documentId: params.documentId, submitted: true } - case 'vanta_list_people': { - const { data: items, pageInfo } = getVantaListResults(data) - return { people: items.map(normalizeVantaPerson), pageInfo } - } - case 'vanta_get_person': - return { person: normalizeVantaPerson(asVantaRecord(data)) } - case 'vanta_list_policies': { - const { data: items, pageInfo } = getVantaListResults(data) - return { policies: items.map(normalizeVantaPolicy), pageInfo } - } - case 'vanta_get_policy': - return { policy: normalizeVantaPolicy(asVantaRecord(data)) } - case 'vanta_list_vendors': { - const { data: items, pageInfo } = getVantaListResults(data) - return { vendors: items.map(normalizeVantaVendor), pageInfo } - } - case 'vanta_get_vendor': - return { vendor: normalizeVantaVendor(asVantaRecord(data)) } - case 'vanta_list_monitored_computers': { - const { data: items, pageInfo } = getVantaListResults(data) - return { computers: items.map(normalizeVantaMonitoredComputer), pageInfo } - } - case 'vanta_list_vulnerabilities': { - const { data: items, pageInfo } = getVantaListResults(data) - return { vulnerabilities: items.map(normalizeVantaVulnerability), pageInfo } - } - case 'vanta_list_vulnerability_remediations': { - const { data: items, pageInfo } = getVantaListResults(data) - return { remediations: items.map(normalizeVantaVulnerabilityRemediation), pageInfo } - } - case 'vanta_list_vulnerable_assets': { - const { data: items, pageInfo } = getVantaListResults(data) - return { assets: items.map(normalizeVantaVulnerableAsset), pageInfo } - } - case 'vanta_get_vulnerable_asset': - return { asset: normalizeVantaVulnerableAsset(asVantaRecord(data)) } - case 'vanta_list_risk_scenarios': { - const { data: items, pageInfo } = getVantaListResults(data) - return { riskScenarios: items.map(normalizeVantaRiskScenario), pageInfo } - } - case 'vanta_get_risk_scenario': - return { riskScenario: normalizeVantaRiskScenario(asVantaRecord(data)) } - } -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Vanta query attempt`, { - error: authResult.error || 'Unauthorized', - }) - return NextResponse.json( - { success: false, error: authResult.error || 'Unauthorized' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(vantaQueryContract, request, {}) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - const baseUrl = getVantaBaseUrl(params.region) - const scope = - params.operation === 'vanta_submit_document' ? VANTA_WRITE_SCOPE : VANTA_READ_SCOPE - - logger.info(`[${requestId}] Vanta query request`, { operation: params.operation }) - - const apiRequest = buildVantaApiRequest(baseUrl, params) - const response = await fetchVantaWithAuth( - { - clientId: params.clientId, - clientSecret: params.clientSecret, - region: params.region, - scope, - }, - (accessToken) => - fetch(apiRequest.url, { - method: apiRequest.method, - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - cache: 'no-store', - }) - ) - - if (!response.ok) { - const errorData: unknown = await response.json().catch(() => null) - return NextResponse.json( - { success: false, error: extractVantaError(errorData, 'Vanta request failed') }, - { status: response.status } - ) - } - - const data: unknown = response.status === 204 ? null : await response.json().catch(() => null) - return NextResponse.json({ success: true, output: buildVantaOutput(params, data) }) - } catch (error) { - const message = toError(error).message - logger.error(`[${requestId}] Vanta query failed`, { error: message }) - return NextResponse.json({ success: false, error: message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/vanta/upload/route.ts b/apps/sim/app/api/tools/vanta/upload/route.ts deleted file mode 100644 index 932b68658c7..00000000000 --- a/apps/sim/app/api/tools/vanta/upload/route.ts +++ /dev/null @@ -1,166 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { vantaUploadContract } from '@/lib/api/contracts/tools/vanta' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { - asVantaRecord, - buildVantaUrl, - extractVantaError, - fetchVantaWithAuth, - getVantaBaseUrl, - normalizeVantaUploadedFile, - VANTA_DOCUMENT_UPLOAD_SCOPE, -} from '@/tools/vanta/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('VantaUploadAPI') - -const MAX_UPLOAD_SIZE_BYTES = 100 * 1024 * 1024 - -function uploadSizeError(bytes: number): NextResponse { - const sizeMB = (bytes / (1024 * 1024)).toFixed(2) - return NextResponse.json( - { success: false, error: `File size (${sizeMB}MB) exceeds upload limit of 100MB` }, - { status: 400 } - ) -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Vanta upload attempt`, { - error: authResult.error || 'Missing userId', - }) - return NextResponse.json( - { success: false, error: authResult.error || 'Unauthorized' }, - { status: 401 } - ) - } - - const parsed = await parseRequest(vantaUploadContract, request, {}) - if (!parsed.success) return parsed.response - const params = parsed.data.body - - let fileBuffer: Buffer - let fileName: string - let mimeType: string - - if (params.file) { - const userFiles = processFilesToUserFiles([params.file as RawFileInput], requestId, logger) - if (userFiles.length === 0) { - return NextResponse.json({ success: false, error: 'Invalid file input' }, { status: 400 }) - } - - const userFile = userFiles[0] - const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) - if (denied) return denied - - if (userFile.size > MAX_UPLOAD_SIZE_BYTES) { - return uploadSizeError(userFile.size) - } - - try { - const resolved = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_UPLOAD_SIZE_BYTES, - }) - fileBuffer = resolved.buffer - fileName = params.fileName || userFile.name - mimeType = - resolved.contentType || userFile.type || params.mimeType || 'application/octet-stream' - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - if (isPayloadSizeLimitError(error)) { - return uploadSizeError(error.observedBytes ?? userFile.size) - } - logger.error(`[${requestId}] Failed to download Vanta upload file`, { - error: getErrorMessage(error), - }) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Failed to download file') }, - { status: 500 } - ) - } - } else if (params.fileContent) { - fileBuffer = Buffer.from(params.fileContent, 'base64') - fileName = params.fileName || 'file' - mimeType = params.mimeType || 'application/octet-stream' - } else { - return NextResponse.json({ success: false, error: 'File is required' }, { status: 400 }) - } - - if (fileBuffer.length > MAX_UPLOAD_SIZE_BYTES) { - return uploadSizeError(fileBuffer.length) - } - - logger.info(`[${requestId}] Uploading file to Vanta document`, { - documentId: params.documentId, - fileName, - size: fileBuffer.length, - }) - - const uploadUrl = buildVantaUrl( - getVantaBaseUrl(params.region), - `/documents/${encodeURIComponent(params.documentId)}/uploads` - ) - const response = await fetchVantaWithAuth( - { - clientId: params.clientId, - clientSecret: params.clientSecret, - region: params.region, - scope: VANTA_DOCUMENT_UPLOAD_SCOPE, - }, - (accessToken) => { - const formData = new FormData() - formData.append( - 'file', - new Blob([new Uint8Array(fileBuffer)], { type: mimeType }), - fileName - ) - if (params.description) { - formData.append('description', params.description) - } - if (params.effectiveAtDate) { - formData.append('effectiveAtDate', params.effectiveAtDate) - } - return fetch(uploadUrl, { - method: 'POST', - headers: { Authorization: `Bearer ${accessToken}` }, - body: formData, - cache: 'no-store', - }) - } - ) - - const data: unknown = await response.json().catch(() => null) - if (!response.ok) { - const message = extractVantaError(data, 'Failed to upload file to Vanta document') - logger.error(`[${requestId}] Vanta upload failed`, { status: response.status, message }) - return NextResponse.json({ success: false, error: message }, { status: response.status }) - } - - logger.info(`[${requestId}] Vanta upload successful`, { documentId: params.documentId }) - - return NextResponse.json({ - success: true, - output: { upload: normalizeVantaUploadedFile(asVantaRecord(data)) }, - }) - } catch (error) { - const message = toError(error).message - logger.error(`[${requestId}] Vanta upload failed`, { error: message }) - return NextResponse.json({ success: false, error: message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/video/route.test.ts b/apps/sim/app/api/tools/video/route.test.ts deleted file mode 100644 index 032bb5ab540..00000000000 --- a/apps/sim/app/api/tools/video/route.test.ts +++ /dev/null @@ -1,121 +0,0 @@ -/** - * @vitest-environment node - */ -import { createMockRequest, hybridAuthMockFns } from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' -import { PRIVATE_MODEL_INPUT_PROVENANCE_HEADER } from '@/lib/execution/model-input-provenance' -import { - RESOLVED_SECRET_PROVENANCE_FIELD, - RESOLVED_SECRET_PROVENANCE_METADATA_V1, -} from '@/lib/execution/private-tool-metadata' - -const { mockDownloadFileFromStorage, mockIsModelSafeWorkspaceFileKey } = vi.hoisted(() => ({ - mockDownloadFileFromStorage: vi.fn(), - mockIsModelSafeWorkspaceFileKey: vi.fn(), -})) - -vi.mock('@/app/api/files/authorization', () => ({ - assertToolFileAccess: vi.fn().mockResolvedValue(null), -})) -vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ - downloadFileFromStorage: mockDownloadFileFromStorage, -})) -vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({ - isModelSafeWorkspaceFileKey: mockIsModelSafeWorkspaceFileKey, - MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE: - 'File cannot be sent to a model because its secret provenance is unavailable', -})) - -import { POST } from '@/app/api/tools/video/route' - -const baseBody = { - provider: 'runway', - apiKey: 'test-api-key', - prompt: 'Generate a short test video', - visualReference: { - id: 'file-1', - name: 'reference.png', - size: 5, - type: 'image/png', - key: 'workspace/workspace-1/reference.png', - }, -} - -describe('POST /api/tools/video', () => { - beforeEach(() => { - vi.clearAllMocks() - hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ - success: true, - userId: 'user-1', - authType: 'internal_jwt', - }) - mockIsModelSafeWorkspaceFileKey.mockResolvedValue(true) - mockDownloadFileFromStorage.mockResolvedValue(Buffer.from('image')) - }) - - it.each(['luma', 'veo', 'falai', 'minimax'])( - 'keeps the headerless projected %s path independent of opaque provenance', - async (provider) => { - const response = await POST( - createMockRequest('POST', { - provider, - apiKey: 'test-api-key', - prompt: 'x', - }) - ) - - expect(response.status).toBe(400) - expect(await response.json()).toEqual({ - error: 'Prompt must be between 3 and 2000 characters', - }) - } - ) - - it('rejects an incomplete private provenance envelope before inspecting the file', async () => { - const response = await POST( - createMockRequest( - 'POST', - { - ...baseBody, - [RESOLVED_SECRET_PROVENANCE_FIELD]: { - version: 1, - complete: false, - entries: [], - }, - }, - { [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1 } - ) - ) - - expect(response.status).toBe(400) - expect(await response.json()).toEqual({ error: 'Model input provenance is unavailable' }) - expect(mockIsModelSafeWorkspaceFileKey).not.toHaveBeenCalled() - expect(mockDownloadFileFromStorage).not.toHaveBeenCalled() - }) - - it('rejects tracked unsafe files after accepting an exact empty opaque envelope', async () => { - mockIsModelSafeWorkspaceFileKey.mockResolvedValueOnce(false) - - const response = await POST( - createMockRequest( - 'POST', - { - ...baseBody, - [RESOLVED_SECRET_PROVENANCE_FIELD]: { - version: 1, - complete: true, - entries: [], - }, - }, - { [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1 } - ) - ) - - expect(response.status).toBe(400) - expect(await response.json()).toEqual({ - error: 'File cannot be sent to a model because its secret provenance is unavailable', - }) - expect(mockIsModelSafeWorkspaceFileKey).toHaveBeenCalledWith(baseBody.visualReference.key) - expect(mockDownloadFileFromStorage).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/app/api/tools/video/route.ts b/apps/sim/app/api/tools/video/route.ts deleted file mode 100644 index 9030c571ab6..00000000000 --- a/apps/sim/app/api/tools/video/route.ts +++ /dev/null @@ -1,1399 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { sleep } from '@sim/utils/helpers' -import { generateId } from '@sim/utils/id' -import { isRecordLike } from '@sim/utils/object' -import { type NextRequest, NextResponse } from 'next/server' -import { videoProviders, videoToolContract } from '@/lib/api/contracts/tools/media/video' -import { getValidationErrorMessage, parseRequest, validationErrorResponse } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' -import { - assertKnownSizeWithinLimit, - DEFAULT_MAX_ERROR_BODY_BYTES, - isPayloadSizeLimitError, - PayloadSizeLimitError, - readResponseJsonWithLimit, - readResponseTextWithLimit, - readResponseToBufferWithLimit, -} from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' -import { type FalAICostMetadata, getFalAICostMetadata } from '@/lib/tools/falai-pricing' -import { - isModelSafeWorkspaceFileKey, - MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, -} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' -import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import type { UserFile } from '@/executor/types' - -const logger = createLogger('VideoProxyAPI') -const MAX_VIDEO_OUTPUT_BYTES = 250 * 1024 * 1024 -const MAX_VIDEO_REFERENCE_IMAGE_BYTES = 25 * 1024 * 1024 -const MAX_VIDEO_JSON_BYTES = 2 * 1024 * 1024 - -export const dynamic = 'force-dynamic' -/** - * Mirrors the hosted workflow execution ceiling (7 days) used by - * `getMaxExecutionTimeout()` for the provider polling loops below. Next.js requires a - * static literal for `maxDuration`, so this value must be kept in sync with that source. - */ -export const maxDuration = 604800 - -async function readVideoResponseBuffer(response: Response, label: string): Promise { - return readResponseToBufferWithLimit(response, { - maxBytes: MAX_VIDEO_OUTPUT_BYTES, - label, - }) -} - -async function readVideoJson>( - response: Response, - label: string -): Promise { - return readResponseJsonWithLimit(response, { - maxBytes: MAX_VIDEO_JSON_BYTES, - label, - }) -} - -async function readVideoErrorText(response: Response, label: string): Promise { - return readResponseTextWithLimit(response, { - maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, - label, - }).catch(() => '') -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateId() - logger.info(`[${requestId}] Video generation request started`) - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest( - videoToolContract, - request, - {}, - { - validationErrorResponse: (error) => { - logger.warn(`[${requestId}] Invalid video request:`, error.issues) - return validationErrorResponse( - error, - getValidationErrorMessage(error, 'Invalid request data') - ) - }, - } - ) - if (!parsed.success) return parsed.response - - const body = parsed.data.body - const { provider, apiKey, model, prompt, duration, aspectRatio, resolution } = body - if (provider === 'runway') { - const modelInputProvenance = validateOpaqueModelInputProvenance({ - headers: request.headers, - payload: body, - isInternalRequest: true, - }) - if (!modelInputProvenance.success) { - return NextResponse.json( - { error: modelInputProvenance.error }, - { status: modelInputProvenance.status } - ) - } - } - - const validProviders = videoProviders - if (!validProviders.includes(provider as (typeof videoProviders)[number])) { - return NextResponse.json( - { error: `Invalid provider. Must be one of: ${validProviders.join(', ')}` }, - { status: 400 } - ) - } - - if (prompt.length < 3 || prompt.length > 2000) { - return NextResponse.json( - { error: 'Prompt must be between 3 and 2000 characters' }, - { status: 400 } - ) - } - - // Validate duration (provider-specific constraints) - if (provider === 'veo') { - if (duration !== undefined && ![4, 6, 8].includes(duration)) { - return NextResponse.json( - { error: 'Duration must be 4, 6, or 8 seconds for Veo' }, - { status: 400 } - ) - } - } else if (provider === 'minimax') { - if (duration !== undefined && ![6, 10].includes(duration)) { - return NextResponse.json( - { error: 'Duration must be 6 or 10 seconds for MiniMax' }, - { status: 400 } - ) - } - } else if (provider !== 'falai' && duration !== undefined && (duration < 5 || duration > 10)) { - // Fal.ai has variable duration constraints per model, skip validation - return NextResponse.json( - { error: 'Duration must be between 5 and 10 seconds' }, - { status: 400 } - ) - } - - if (provider !== 'falai') { - const validAspectRatios = provider === 'veo' ? ['16:9', '9:16'] : ['16:9', '9:16', '1:1'] - if (aspectRatio && !validAspectRatios.includes(aspectRatio)) { - return NextResponse.json( - { error: `Aspect ratio must be ${validAspectRatios.join(', ')}` }, - { status: 400 } - ) - } - } - - logger.info(`[${requestId}] Generating video with ${provider}, model: ${model || 'default'}`) - - let videoUrl: string - let videoBuffer: Buffer - let width: number | undefined - let height: number | undefined - let jobId: string | undefined - let actualDuration: number | undefined - let falaiCost: FalAICostMetadata | undefined - - if (body.visualReference) { - const denied = await assertToolFileAccess( - body.visualReference.key, - authResult.userId, - requestId, - logger - ) - if (denied) return denied - if (!(await isModelSafeWorkspaceFileKey(body.visualReference.key))) { - return NextResponse.json( - { error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE }, - { status: 400 } - ) - } - } - - try { - if (provider === 'runway') { - const result = await generateWithRunway( - apiKey, - model || 'gen-4', - prompt, - duration || 5, - aspectRatio || '16:9', - resolution || '1080p', - body.visualReference, - requestId, - logger - ) - videoBuffer = result.buffer - width = result.width - height = result.height - jobId = result.jobId - actualDuration = result.duration - } else if (provider === 'veo') { - const result = await generateWithVeo( - apiKey, - model || 'veo-3', - prompt, - duration || 8, // Default to 8 seconds (valid: 4, 6, or 8) - aspectRatio || '16:9', - resolution || '1080p', - requestId, - logger - ) - videoBuffer = result.buffer - width = result.width - height = result.height - jobId = result.jobId - actualDuration = result.duration - } else if (provider === 'luma') { - const result = await generateWithLuma( - apiKey, - model || 'ray-2', - prompt, - duration || 5, - aspectRatio || '16:9', - resolution || '1080p', - body.cameraControl, - requestId, - logger - ) - videoBuffer = result.buffer - width = result.width - height = result.height - jobId = result.jobId - actualDuration = result.duration - } else if (provider === 'minimax') { - const result = await generateWithMiniMax( - apiKey, - model || 'hailuo-2.3', - prompt, - duration || 6, - body.promptOptimizer !== false, - body.endpoint, - requestId, - logger - ) - videoBuffer = result.buffer - width = result.width - height = result.height - jobId = result.jobId - actualDuration = result.duration - } else if (provider === 'falai') { - if (!model) { - return NextResponse.json( - { error: 'Model is required for Fal.ai provider' }, - { status: 400 } - ) - } - const validationError = getFalAIValidationError(model, duration, aspectRatio, resolution) - if (validationError) { - return NextResponse.json({ error: validationError }, { status: 400 }) - } - const result = await generateWithFalAI( - apiKey, - model, - prompt, - duration, - aspectRatio, - resolution, - body.promptOptimizer, - body.generateAudio, - body.useHostedCostTracking === true, - requestId, - logger - ) - videoBuffer = result.buffer - width = result.width - height = result.height - jobId = result.jobId - actualDuration = result.duration - falaiCost = result.falaiCost - } else { - return NextResponse.json({ error: `Unknown provider: ${provider}` }, { status: 400 }) - } - } catch (error) { - logger.error(`[${requestId}] Video generation failed:`, error) - const errorMessage = getErrorMessage(error, 'Video generation failed') - return NextResponse.json( - { error: errorMessage }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } - - const executionContext = - body.workspaceId && body.workflowId && body.executionId - ? { - workspaceId: body.workspaceId, - workflowId: body.workflowId, - executionId: body.executionId, - } - : null - - logger.info(`[${requestId}] Storing video file, size: ${videoBuffer.length} bytes`) - - if (executionContext) { - const { uploadExecutionFile } = await import('@/lib/uploads/contexts/execution') - const timestamp = Date.now() - const fileName = `video-${provider}-${timestamp}.mp4` - - let videoFile - try { - videoFile = await uploadExecutionFile( - executionContext, - videoBuffer, - fileName, - 'video/mp4', - authResult.userId - ) - - logger.info(`[${requestId}] Video stored successfully:`, { - fileName, - size: videoFile.size, - executionId: body.executionId, - }) - } catch (error) { - logger.error(`[${requestId}] Failed to upload video file:`, error) - throw new Error(`Failed to store video: ${getErrorMessage(error, 'Unknown error')}`) - } - - return NextResponse.json({ - videoUrl: videoFile.url, - videoFile, - duration: actualDuration || duration, - width, - height, - provider, - model: model || 'default', - jobId, - __falaiCostDollars: falaiCost?.costDollars, - __falaiBilling: falaiCost, - }) - } - - const { StorageService } = await import('@/lib/uploads') - const { getBaseUrl } = await import('@/lib/core/utils/urls') - const timestamp = Date.now() - const fileName = `video-${provider}-${timestamp}.mp4` - - try { - const fileInfo = await StorageService.uploadFile({ - file: videoBuffer, - fileName, - contentType: 'video/mp4', - context: 'copilot', - }) - - videoUrl = `${getBaseUrl()}${fileInfo.path}` - } catch (error) { - logger.error(`[${requestId}] Failed to upload video file (fallback):`, error) - throw new Error(`Failed to store video: ${getErrorMessage(error, 'Unknown error')}`) - } - - logger.info(`[${requestId}] Video generation completed successfully`) - - return NextResponse.json({ - videoUrl, - duration: actualDuration || duration, - width, - height, - provider, - model: model || 'default', - jobId, - __falaiCostDollars: falaiCost?.costDollars, - __falaiBilling: falaiCost, - }) - } catch (error) { - logger.error(`[${requestId}] Video proxy error:`, error) - const errorMessage = getErrorMessage(error, 'Unknown error') - return NextResponse.json( - { error: errorMessage }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } -}) - -async function generateWithRunway( - apiKey: string, - model: string, - prompt: string, - duration: number, - aspectRatio: string, - resolution: string, - visualReference: UserFile | undefined, - requestId: string, - logger: ReturnType -): Promise<{ buffer: Buffer; width: number; height: number; jobId: string; duration: number }> { - logger.info(`[${requestId}] Starting Runway Gen-4 generation`) - - const dimensions = getVideoDimensions(aspectRatio, resolution) - - // Convert aspect ratio to resolution format for 2024-11-06 API version - const ratioMap: { [key: string]: string } = { - '16:9': '1280:720', // Landscape (720p) - '9:16': '720:1280', // Portrait (720p) - '1:1': '960:960', // Square - } - const runwayRatio = ratioMap[aspectRatio] || '1280:720' - - const createPayload: any = { - promptText: prompt, - duration, - ratio: runwayRatio, // Use resolution-based ratio for 2024-11-06 API - model: 'gen4_turbo', // Only gen4_turbo supports image-to-video // Use underscore - } - - if (visualReference) { - if (visualReference.size > MAX_VIDEO_REFERENCE_IMAGE_BYTES) { - throw new PayloadSizeLimitError({ - label: 'video visual reference', - maxBytes: MAX_VIDEO_REFERENCE_IMAGE_BYTES, - observedBytes: visualReference.size, - }) - } - const refBuffer = await downloadFileFromStorage(visualReference, requestId, logger, { - maxBytes: MAX_VIDEO_REFERENCE_IMAGE_BYTES, - }) - assertKnownSizeWithinLimit( - refBuffer.length, - MAX_VIDEO_REFERENCE_IMAGE_BYTES, - 'video visual reference' - ) - const refBase64 = refBuffer.toString('base64') - createPayload.promptImage = `data:${visualReference.type};base64,${refBase64}` // Use promptImage - } - - const createResponse = await fetch('https://api.dev.runwayml.com/v1/image_to_video', { - method: 'POST', - headers: { - Authorization: `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - 'X-Runway-Version': '2024-11-06', - }, - body: JSON.stringify(createPayload), - }) - - if (!createResponse.ok) { - const error = await readVideoErrorText(createResponse, 'Runway create error response') - throw new Error(`Runway API error: ${createResponse.status} - ${error}`) - } - - const createData = await readVideoJson<{ id: string }>(createResponse, 'Runway create response') - const taskId = createData.id - - logger.info(`[${requestId}] Runway task created: ${taskId}`) - - const pollIntervalMs = 5000 - const maxAttempts = Math.ceil(getMaxExecutionTimeout() / pollIntervalMs) - let attempts = 0 - - while (attempts < maxAttempts) { - await sleep(pollIntervalMs) - - const statusResponse = await fetch(`https://api.dev.runwayml.com/v1/tasks/${taskId}`, { - headers: { - Authorization: `Bearer ${apiKey}`, - 'X-Runway-Version': '2024-11-06', - }, - }) - - if (!statusResponse.ok) { - await readVideoErrorText(statusResponse, 'Runway status error response') - throw new Error(`Runway status check failed: ${statusResponse.status}`) - } - - const statusData = await readVideoJson<{ - status?: string - output?: string[] - failure?: string - }>(statusResponse, 'Runway status response') - - if (statusData.status === 'SUCCEEDED') { - logger.info(`[${requestId}] Runway generation completed after ${attempts * 5}s`) - - const videoUrl = statusData.output?.[0] - if (!videoUrl) { - throw new Error('No video URL in response') - } - - const videoResponse = await fetch(videoUrl) - if (!videoResponse.ok) { - await readVideoErrorText(videoResponse, 'Runway video error response') - throw new Error(`Failed to download video: ${videoResponse.status}`) - } - - return { - buffer: await readVideoResponseBuffer(videoResponse, 'Runway video response'), - width: dimensions.width, - height: dimensions.height, - jobId: taskId, - duration, - } - } - - if (statusData.status === 'FAILED') { - throw new Error(`Runway generation failed: ${statusData.failure || 'Unknown error'}`) - } - - attempts++ - } - - throw new Error('Runway generation timed out') -} - -async function generateWithVeo( - apiKey: string, - model: string, - prompt: string, - duration: number, - aspectRatio: string, - resolution: string, - requestId: string, - logger: ReturnType -): Promise<{ buffer: Buffer; width: number; height: number; jobId: string; duration: number }> { - logger.info(`[${requestId}] Starting Google Veo generation`) - - const dimensions = getVideoDimensions(aspectRatio, resolution) - - const modelNameMap: Record = { - 'veo-3': 'veo-3.0-generate-001', - 'veo-3-fast': 'veo-3.0-fast-generate-001', // Fixed: was incorrectly mapped to 3.1 - 'veo-3.1': 'veo-3.1-generate-preview', - } - const modelName = modelNameMap[model] || 'veo-3.1-generate-preview' - - const createPayload = { - instances: [ - { - prompt, - }, - ], - parameters: { - aspectRatio: aspectRatio, // Keep as "16:9", don't convert - resolution: resolution, - durationSeconds: duration, // Keep as number - }, - } - - const createResponse = await fetch( - `https://generativelanguage.googleapis.com/v1beta/models/${modelName}:predictLongRunning`, - { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'x-goog-api-key': apiKey, - }, - body: JSON.stringify(createPayload), - } - ) - - if (!createResponse.ok) { - const error = await readVideoErrorText(createResponse, 'Veo create error response') - throw new Error(`Veo API error: ${createResponse.status} - ${error}`) - } - - const createData = await readVideoJson<{ name: string }>(createResponse, 'Veo create response') - const operationName = createData.name - - logger.info(`[${requestId}] Veo operation created: ${operationName}`) - - const pollIntervalMs = 5000 - const maxAttempts = Math.ceil(getMaxExecutionTimeout() / pollIntervalMs) - let attempts = 0 - - while (attempts < maxAttempts) { - await sleep(pollIntervalMs) - - const statusResponse = await fetch( - `https://generativelanguage.googleapis.com/v1beta/${operationName}`, - { - headers: { - 'x-goog-api-key': apiKey, - }, - } - ) - - if (!statusResponse.ok) { - await readVideoErrorText(statusResponse, 'Veo status error response') - throw new Error(`Veo status check failed: ${statusResponse.status}`) - } - - const statusData = await readVideoJson<{ - done?: boolean - error?: { message?: string } - response?: { - generateVideoResponse?: { generatedSamples?: Array<{ video?: { uri?: string } }> } - } - }>(statusResponse, 'Veo status response') - - if (statusData.done) { - if (statusData.error) { - throw new Error(`Veo generation failed: ${statusData.error.message}`) - } - - logger.info(`[${requestId}] Veo generation completed after ${attempts * 5}s`) - - const videoUri = statusData.response?.generateVideoResponse?.generatedSamples?.[0]?.video?.uri - if (!videoUri) { - throw new Error('No video URI in response') - } - - const videoResponse = await fetch(videoUri, { - headers: { - 'x-goog-api-key': apiKey, - }, - }) - - if (!videoResponse.ok) { - await readVideoErrorText(videoResponse, 'Veo video error response') - throw new Error(`Failed to download video: ${videoResponse.status}`) - } - - return { - buffer: await readVideoResponseBuffer(videoResponse, 'Veo video response'), - width: dimensions.width, - height: dimensions.height, - jobId: operationName, - duration, - } - } - - attempts++ - } - - throw new Error('Veo generation timed out') -} - -async function generateWithLuma( - apiKey: string, - model: string, - prompt: string, - duration: number, - aspectRatio: string, - resolution: string, - cameraControl: any | undefined, - requestId: string, - logger: ReturnType -): Promise<{ buffer: Buffer; width: number; height: number; jobId: string; duration: number }> { - logger.info(`[${requestId}] Starting Luma Dream Machine generation`) - - const dimensions = getVideoDimensions(aspectRatio, resolution) - - const createPayload: any = { - prompt, - model: model || 'ray-2', - aspect_ratio: aspectRatio, - loop: false, - } - - if (duration) { - createPayload.duration = `${duration}s` - } - - if (resolution) { - createPayload.resolution = resolution - } - - if (cameraControl) { - createPayload.concepts = Array.isArray(cameraControl) ? cameraControl : [{ key: cameraControl }] - } - - const createResponse = await fetch('https://api.lumalabs.ai/dream-machine/v1/generations', { - method: 'POST', - headers: { - Authorization: `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(createPayload), - }) - - if (!createResponse.ok) { - const error = await readVideoErrorText(createResponse, 'Luma create error response') - throw new Error(`Luma API error: ${createResponse.status} - ${error}`) - } - - const createData = await readVideoJson<{ id: string }>(createResponse, 'Luma create response') - const generationId = createData.id - - logger.info(`[${requestId}] Luma generation created: ${generationId}`) - - const pollIntervalMs = 5000 - const maxAttempts = Math.ceil(getMaxExecutionTimeout() / pollIntervalMs) - let attempts = 0 - - while (attempts < maxAttempts) { - await sleep(pollIntervalMs) - - const statusResponse = await fetch( - `https://api.lumalabs.ai/dream-machine/v1/generations/${generationId}`, - { - headers: { - Authorization: `Bearer ${apiKey}`, - }, - } - ) - - if (!statusResponse.ok) { - await readVideoErrorText(statusResponse, 'Luma status error response') - throw new Error(`Luma status check failed: ${statusResponse.status}`) - } - - const statusData = await readVideoJson<{ - state?: string - failure_reason?: string - assets?: { video?: string } - }>(statusResponse, 'Luma status response') - - if (statusData.state === 'completed') { - logger.info(`[${requestId}] Luma generation completed after ${attempts * 5}s`) - - const videoUrl = statusData.assets?.video - if (!videoUrl) { - throw new Error('No video URL in response') - } - - const videoResponse = await fetch(videoUrl) - if (!videoResponse.ok) { - await readVideoErrorText(videoResponse, 'Luma video error response') - throw new Error(`Failed to download video: ${videoResponse.status}`) - } - - return { - buffer: await readVideoResponseBuffer(videoResponse, 'Luma video response'), - width: dimensions.width, - height: dimensions.height, - jobId: generationId, - duration, - } - } - - if (statusData.state === 'failed') { - throw new Error(`Luma generation failed: ${statusData.failure_reason || 'Unknown error'}`) - } - - attempts++ - } - - throw new Error('Luma generation timed out') -} - -async function generateWithMiniMax( - apiKey: string, - model: string, - prompt: string, - duration: number, - promptOptimizer: boolean, - endpoint: string | undefined, - requestId: string, - logger: ReturnType -): Promise<{ buffer: Buffer; width: number; height: number; jobId: string; duration: number }> { - logger.info(`[${requestId}] Starting MiniMax Hailuo generation via MiniMax Platform API`) - logger.info( - `[${requestId}] Request params - model: ${model}, duration: ${duration}, endpoint: ${endpoint || 'standard'}, promptOptimizer: ${promptOptimizer}` - ) - - const useProResolution = endpoint === 'pro' && duration === 6 - const resolution = useProResolution ? '1080P' : '768P' - const dimensions = useProResolution ? { width: 1920, height: 1080 } : { width: 1360, height: 768 } - - logger.info( - `[${requestId}] Using resolution: ${resolution}, dimensions: ${dimensions.width}x${dimensions.height}` - ) - - const minimaxModel = model === 'hailuo-02' ? 'MiniMax-Hailuo-02' : 'MiniMax-Hailuo-2.3' - - const createResponse = await fetch('https://api.minimax.io/v1/video_generation', { - method: 'POST', - headers: { - Authorization: `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - model: minimaxModel, - prompt: prompt, - duration: duration, - resolution: resolution, - prompt_optimizer: promptOptimizer, - }), - }) - - if (!createResponse.ok) { - const errorText = await readVideoErrorText(createResponse, 'MiniMax create error response') - if (createResponse.status === 401 || createResponse.status === 1004) { - throw new Error( - `MiniMax API authentication failed (${createResponse.status}). Please ensure you're using a valid MiniMax API key from platform.minimax.io. Error: ${errorText}` - ) - } - throw new Error(`MiniMax API error: ${createResponse.status} - ${errorText}`) - } - - const createData = await readVideoJson<{ - base_resp?: { status_code?: number; status_msg?: string } - task_id?: string - }>(createResponse, 'MiniMax create response') - - // Check for error in response - if (createData.base_resp?.status_code !== 0) { - throw new Error(`MiniMax API error: ${createData.base_resp?.status_msg || 'Unknown error'}`) - } - - const taskId = createData.task_id - if (!taskId) { - throw new Error('MiniMax response missing task_id') - } - - logger.info(`[${requestId}] MiniMax task created: ${taskId}`) - - const pollIntervalMs = 5000 - const maxAttempts = Math.ceil(getMaxExecutionTimeout() / pollIntervalMs) - let attempts = 0 - - while (attempts < maxAttempts) { - await sleep(pollIntervalMs) - - const statusResponse = await fetch( - `https://api.minimax.io/v1/query/video_generation?task_id=${taskId}`, - { - headers: { - Authorization: `Bearer ${apiKey}`, - }, - } - ) - - if (!statusResponse.ok) { - await readVideoErrorText(statusResponse, 'MiniMax status error response') - throw new Error(`MiniMax status check failed: ${statusResponse.status}`) - } - - const statusData = await readVideoJson<{ - base_resp?: { status_code?: number; status_msg?: string } - status?: string - file_id?: string - error?: string - }>(statusResponse, 'MiniMax status response') - - if ( - statusData.base_resp?.status_code !== 0 && - statusData.base_resp?.status_code !== undefined - ) { - throw new Error( - `MiniMax status query error: ${statusData.base_resp?.status_msg || 'Unknown error'}` - ) - } - - if (statusData.status === 'Success' || statusData.status === 'success') { - logger.info(`[${requestId}] MiniMax generation completed after ${attempts * 5}s`) - - const fileId = statusData.file_id - if (!fileId) { - throw new Error('No file_id in response') - } - - // Download the video using file_id - const fileResponse = await fetch( - `https://api.minimax.io/v1/files/retrieve?file_id=${fileId}`, - { - headers: { - Authorization: `Bearer ${apiKey}`, - }, - } - ) - - if (!fileResponse.ok) { - await readVideoErrorText(fileResponse, 'MiniMax file error response') - throw new Error(`Failed to download video: ${fileResponse.status}`) - } - - const fileData = await readVideoJson<{ file?: { download_url?: string } }>( - fileResponse, - 'MiniMax file response' - ) - const videoUrl = fileData.file?.download_url - - if (!videoUrl) { - throw new Error('No download URL in file response') - } - - // Download the actual video file - const videoResponse = await fetch(videoUrl) - if (!videoResponse.ok) { - await readVideoErrorText(videoResponse, 'MiniMax video error response') - throw new Error(`Failed to download video from URL: ${videoResponse.status}`) - } - - return { - buffer: await readVideoResponseBuffer(videoResponse, 'MiniMax video response'), - width: dimensions.width, - height: dimensions.height, - jobId: taskId, - duration, - } - } - - if (statusData.status === 'Failed' || statusData.status === 'failed') { - throw new Error(`MiniMax generation failed: ${statusData.error || 'Unknown error'}`) - } - - // Status is still "Processing" or "Queueing", continue polling - attempts++ - } - - throw new Error('MiniMax generation timed out') -} - -type FalAIDurationFormat = 'number' | 'seconds' | 'string' - -interface FalAIModelConfig { - endpoint: string - durationFormat?: FalAIDurationFormat - durationOptions?: readonly number[] - supportsAspectRatio?: boolean - aspectRatioOptions?: readonly string[] - supportsResolution?: boolean - resolutionOptions?: readonly string[] - supportsPromptOptimizer?: boolean - supportsGenerateAudio?: boolean -} - -interface FalAIRequestBody { - prompt: string - duration?: number | string - aspect_ratio?: string - resolution?: string - prompt_optimizer?: boolean - generate_audio?: boolean -} - -const FALAI_MODEL_CONFIGS: Record = { - 'veo-3.1': { - endpoint: 'fal-ai/veo3.1', - durationFormat: 'seconds', - durationOptions: [4, 6, 8], - supportsAspectRatio: true, - aspectRatioOptions: ['16:9', '9:16'], - supportsResolution: true, - resolutionOptions: ['720p', '1080p', '4k'], - supportsGenerateAudio: true, - }, - 'veo-3.1-fast': { - endpoint: 'fal-ai/veo3.1/fast', - durationFormat: 'seconds', - durationOptions: [4, 6, 8], - supportsAspectRatio: true, - aspectRatioOptions: ['16:9', '9:16'], - supportsResolution: true, - resolutionOptions: ['720p', '1080p', '4k'], - supportsGenerateAudio: true, - }, - 'sora-2': { - endpoint: 'fal-ai/sora-2/text-to-video', - durationFormat: 'number', - durationOptions: [4, 8, 12, 16, 20], - supportsAspectRatio: true, - aspectRatioOptions: ['16:9', '9:16'], - supportsResolution: true, - resolutionOptions: ['720p'], - }, - 'sora-2-pro': { - endpoint: 'fal-ai/sora-2/text-to-video/pro', - durationFormat: 'number', - durationOptions: [4, 8, 12, 16, 20], - supportsAspectRatio: true, - aspectRatioOptions: ['16:9', '9:16'], - supportsResolution: true, - resolutionOptions: ['720p', '1080p', 'true_1080p'], - }, - 'seedance-2.0': { - endpoint: 'bytedance/seedance-2.0/text-to-video', - durationFormat: 'string', - durationOptions: [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], - supportsAspectRatio: true, - aspectRatioOptions: ['auto', '21:9', '16:9', '4:3', '1:1', '3:4', '9:16'], - supportsResolution: true, - resolutionOptions: ['480p', '720p', '1080p'], - supportsGenerateAudio: true, - }, - 'seedance-2.0-fast': { - endpoint: 'bytedance/seedance-2.0/fast/text-to-video', - durationFormat: 'string', - durationOptions: [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], - supportsAspectRatio: true, - aspectRatioOptions: ['auto', '21:9', '16:9', '4:3', '1:1', '3:4', '9:16'], - supportsResolution: true, - resolutionOptions: ['480p', '720p'], - supportsGenerateAudio: true, - }, - 'kling-v3-pro': { - endpoint: 'fal-ai/kling-video/v3/pro/text-to-video', - durationFormat: 'string', - durationOptions: [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], - supportsAspectRatio: true, - aspectRatioOptions: ['16:9', '9:16', '1:1'], - supportsGenerateAudio: true, - }, - 'kling-v3-4k': { - endpoint: 'fal-ai/kling-video/v3/4k/text-to-video', - durationFormat: 'string', - durationOptions: [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], - supportsAspectRatio: true, - aspectRatioOptions: ['16:9', '9:16', '1:1'], - supportsGenerateAudio: true, - }, - 'kling-o3-pro': { - endpoint: 'fal-ai/kling-video/o3/pro/text-to-video', - durationFormat: 'string', - durationOptions: [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], - supportsAspectRatio: true, - aspectRatioOptions: ['16:9', '9:16', '1:1'], - supportsGenerateAudio: true, - }, - 'kling-o3-4k': { - endpoint: 'fal-ai/kling-video/o3/4k/text-to-video', - durationFormat: 'string', - durationOptions: [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], - supportsAspectRatio: true, - aspectRatioOptions: ['16:9', '9:16', '1:1'], - supportsGenerateAudio: true, - }, - 'kling-2.5-turbo-pro': { - endpoint: 'fal-ai/kling-video/v2.5-turbo/pro/text-to-video', - durationFormat: 'string', - supportsAspectRatio: true, - supportsResolution: true, - }, - 'kling-2.1-pro': { - endpoint: 'fal-ai/kling-video/v2.1/master/text-to-video', - durationFormat: 'string', - supportsAspectRatio: true, - supportsResolution: true, - }, - 'minimax-hailuo-2.3-pro': { - endpoint: 'fal-ai/minimax/hailuo-2.3/pro/text-to-video', - supportsPromptOptimizer: true, - }, - 'minimax-hailuo-2.3-standard': { - endpoint: 'fal-ai/minimax/hailuo-2.3/standard/text-to-video', - durationFormat: 'string', - durationOptions: [6, 10], - supportsPromptOptimizer: true, - }, - 'minimax-hailuo-02-pro': { - endpoint: 'fal-ai/minimax/hailuo-02/pro/text-to-video', - durationFormat: 'string', - supportsAspectRatio: true, - supportsResolution: true, - supportsPromptOptimizer: true, - }, - 'minimax-hailuo-02-standard': { - endpoint: 'fal-ai/minimax/hailuo-02/standard/text-to-video', - durationFormat: 'string', - supportsAspectRatio: true, - supportsResolution: true, - supportsPromptOptimizer: true, - }, - 'wan-2.2-a14b-turbo': { - endpoint: 'fal-ai/wan/v2.2-a14b/text-to-video/turbo', - supportsAspectRatio: true, - aspectRatioOptions: ['16:9', '9:16', '1:1'], - supportsResolution: true, - resolutionOptions: ['480p', '580p', '720p'], - }, - 'wan-2.1': { - endpoint: 'fal-ai/wan-t2v', - }, - 'ltx-2.3': { - endpoint: 'fal-ai/ltx-2.3/text-to-video', - durationFormat: 'number', - durationOptions: [6, 8, 10], - supportsAspectRatio: true, - aspectRatioOptions: ['16:9', '9:16'], - supportsResolution: true, - resolutionOptions: ['1080p', '1440p', '2160p'], - supportsGenerateAudio: true, - }, - 'ltx-2.3-fast': { - endpoint: 'fal-ai/ltx-2.3/text-to-video/fast', - durationFormat: 'number', - durationOptions: [6, 8, 10, 12, 14, 16, 18, 20], - supportsAspectRatio: true, - aspectRatioOptions: ['16:9', '9:16'], - supportsResolution: true, - resolutionOptions: ['1080p', '1440p', '2160p'], - supportsGenerateAudio: true, - }, - 'ltxv-0.9.8': { - endpoint: 'fal-ai/ltxv-13b-098-distilled', - }, -} - -function formatFalAIDuration( - format: FalAIDurationFormat | undefined, - duration: number | undefined -): string | number | undefined { - if (!format || duration === undefined) return undefined - - if (format === 'number') return duration - if (format === 'seconds') return `${duration}s` - return String(duration) -} - -function getStringProperty( - record: Record | undefined, - key: string -): string | undefined { - const value = record?.[key] - return typeof value === 'string' ? value : undefined -} - -function getNumberProperty( - record: Record | undefined, - key: string -): number | undefined { - const value = record?.[key] - return typeof value === 'number' ? value : undefined -} - -function formatAllowedValues(allowed: readonly (number | string)[]): string { - return allowed.map(String).join(', ') -} - -function getFalAIValidationError( - model: string, - duration: number | undefined, - aspectRatio: string | undefined, - resolution: string | undefined -): string | undefined { - const modelConfig = FALAI_MODEL_CONFIGS[model] - if (!modelConfig) { - return `Unknown Fal.ai model: ${model}` - } - - if ( - duration !== undefined && - modelConfig.durationOptions && - !modelConfig.durationOptions.includes(duration) - ) { - return `Invalid duration for Fal.ai model ${model}. Supported durations: ${formatAllowedValues(modelConfig.durationOptions)}` - } - - if (aspectRatio) { - if (!modelConfig.supportsAspectRatio) { - return `Fal.ai model ${model} does not support aspect ratio` - } - - if (modelConfig.aspectRatioOptions && !modelConfig.aspectRatioOptions.includes(aspectRatio)) { - return `Invalid aspect ratio for Fal.ai model ${model}. Supported aspect ratios: ${formatAllowedValues(modelConfig.aspectRatioOptions)}` - } - } - - if (resolution) { - if (!modelConfig.supportsResolution) { - return `Fal.ai model ${model} does not support resolution` - } - - if (modelConfig.resolutionOptions && !modelConfig.resolutionOptions.includes(resolution)) { - return `Invalid resolution for Fal.ai model ${model}. Supported resolutions: ${formatAllowedValues(modelConfig.resolutionOptions)}` - } - } - - if ( - model === 'ltx-2.3-fast' && - duration !== undefined && - duration > 10 && - resolution && - resolution !== '1080p' - ) { - return 'Fal.ai model ltx-2.3-fast only supports durations over 10 seconds with 1080p resolution' - } - - return undefined -} - -function getFalAIErrorMessage(error: unknown): string { - if (typeof error === 'string') return error - if (isRecordLike(error)) return getStringProperty(error, 'message') || JSON.stringify(error) - return 'Unknown error' -} - -function buildFalAIQueueUrl( - endpoint: string, - requestId: string, - path: 'response' | 'status' -): string { - return `https://queue.fal.run/${endpoint}/requests/${requestId}/${path}` -} - -async function generateWithFalAI( - apiKey: string, - model: string, - prompt: string, - duration: number | undefined, - aspectRatio: string | undefined, - resolution: string | undefined, - promptOptimizer: boolean | undefined, - generateAudio: boolean | undefined, - useHostedCostTracking: boolean, - requestId: string, - logger: ReturnType -): Promise<{ - buffer: Buffer - width: number - height: number - jobId: string - duration: number - falaiCost?: FalAICostMetadata -}> { - logger.info(`[${requestId}] Starting Fal.ai generation with model: ${model}`) - - const modelConfig = FALAI_MODEL_CONFIGS[model] - if (!modelConfig) { - throw new Error(`Unknown Fal.ai model: ${model}`) - } - - const requestBody: FalAIRequestBody = { prompt } - const formattedDuration = formatFalAIDuration(modelConfig.durationFormat, duration) - - if (formattedDuration !== undefined) { - requestBody.duration = formattedDuration - } - - if (modelConfig.supportsAspectRatio && aspectRatio) { - requestBody.aspect_ratio = aspectRatio - } - - if (modelConfig.supportsResolution && resolution) { - requestBody.resolution = resolution - } - - if (modelConfig.supportsPromptOptimizer && promptOptimizer !== undefined) { - requestBody.prompt_optimizer = promptOptimizer - } - - if (modelConfig.supportsGenerateAudio && generateAudio !== undefined) { - requestBody.generate_audio = generateAudio - } - - const createResponse = await fetch(`https://queue.fal.run/${modelConfig.endpoint}`, { - method: 'POST', - headers: { - Authorization: `Key ${apiKey}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(requestBody), - }) - - if (!createResponse.ok) { - const error = await readVideoErrorText(createResponse, 'Fal.ai create error response') - throw new Error(`Fal.ai API error: ${createResponse.status} - ${error}`) - } - - const createData = await readVideoJson(createResponse, 'Fal.ai queue response') - if (!isRecordLike(createData)) { - throw new Error('Invalid Fal.ai queue response') - } - - const requestIdFal = getStringProperty(createData, 'request_id') - if (!requestIdFal) { - throw new Error('Fal.ai queue response missing request_id') - } - - const statusUrl = - getStringProperty(createData, 'status_url') || - buildFalAIQueueUrl(modelConfig.endpoint, requestIdFal, 'status') - const responseUrl = - getStringProperty(createData, 'response_url') || - buildFalAIQueueUrl(modelConfig.endpoint, requestIdFal, 'response') - - logger.info(`[${requestId}] Fal.ai request created: ${requestIdFal}`) - - const pollIntervalMs = 5000 - const maxAttempts = Math.ceil(getMaxExecutionTimeout() / pollIntervalMs) - let attempts = 0 - - while (attempts < maxAttempts) { - await sleep(pollIntervalMs) - - const statusResponse = await fetch(statusUrl, { - headers: { - Authorization: `Key ${apiKey}`, - }, - }) - - if (!statusResponse.ok) { - await readVideoErrorText(statusResponse, 'Fal.ai status error response') - throw new Error(`Fal.ai status check failed: ${statusResponse.status}`) - } - - const statusData = await readVideoJson(statusResponse, 'Fal.ai status response') - if (!isRecordLike(statusData)) { - throw new Error('Invalid Fal.ai status response') - } - - if (getStringProperty(statusData, 'status') === 'COMPLETED') { - const statusError = statusData.error - if (statusError) { - throw new Error(`Fal.ai generation failed: ${getFalAIErrorMessage(statusError)}`) - } - - logger.info(`[${requestId}] Fal.ai generation completed after ${attempts * 5}s`) - - const resultResponse = await fetch( - getStringProperty(statusData, 'response_url') || responseUrl, - { - headers: { - Authorization: `Key ${apiKey}`, - }, - } - ) - - if (!resultResponse.ok) { - await readVideoErrorText(resultResponse, 'Fal.ai result error response') - throw new Error(`Failed to fetch result: ${resultResponse.status}`) - } - - const resultData = await readVideoJson(resultResponse, 'Fal.ai result response') - if (!isRecordLike(resultData)) { - throw new Error('Invalid Fal.ai result response') - } - - const videoOutput = isRecordLike(resultData.video) ? resultData.video : undefined - const fallbackOutput = isRecordLike(resultData.output) ? resultData.output : undefined - const videoUrl = - getStringProperty(videoOutput, 'url') || getStringProperty(fallbackOutput, 'url') - if (!videoUrl) { - throw new Error('No video URL in response') - } - - const videoResponse = await fetch(videoUrl) - if (!videoResponse.ok) { - await readVideoErrorText(videoResponse, 'Fal.ai video error response') - throw new Error(`Failed to download video: ${videoResponse.status}`) - } - - let width = getNumberProperty(videoOutput, 'width') || 1920 - let height = getNumberProperty(videoOutput, 'height') || 1080 - - if (!getNumberProperty(videoOutput, 'width') && aspectRatio?.includes(':')) { - const dims = getVideoDimensions(aspectRatio, resolution || '1080p') - width = dims.width - height = dims.height - } - - return { - buffer: await readVideoResponseBuffer(videoResponse, 'Fal.ai video response'), - width, - height, - jobId: requestIdFal, - duration: getNumberProperty(videoOutput, 'duration') || duration || 5, - falaiCost: useHostedCostTracking - ? await getFalAICostMetadata({ - apiKey, - endpointId: modelConfig.endpoint, - requestId: requestIdFal, - }) - : undefined, - } - } - - if (['ERROR', 'FAILED', 'CANCELLED'].includes(getStringProperty(statusData, 'status') || '')) { - throw new Error(`Fal.ai generation failed: ${getFalAIErrorMessage(statusData.error)}`) - } - - attempts++ - } - - throw new Error('Fal.ai generation timed out') -} - -function getVideoDimensions( - aspectRatio: string, - resolution: string -): { width: number; height: number } { - let height: number - if (resolution === '4k' || resolution === '2160p') { - height = 2160 - } else if (resolution === 'true_1080p') { - height = 1080 - } else { - const parsedHeight = Number.parseInt(resolution.replace('p', '')) - height = Number.isFinite(parsedHeight) ? parsedHeight : 1080 - } - - const [ratioW, ratioH] = aspectRatio.split(':').map(Number) - if (!Number.isFinite(ratioW) || !Number.isFinite(ratioH) || ratioH === 0) { - return { width: Math.round((height * 16) / 9), height } - } - - const width = Math.round((height * ratioW) / ratioH) - - return { width, height } -} diff --git a/apps/sim/app/api/tools/vision/analyze/route.ts b/apps/sim/app/api/tools/vision/analyze/route.ts deleted file mode 100644 index 74a0cb0560f..00000000000 --- a/apps/sim/app/api/tools/vision/analyze/route.ts +++ /dev/null @@ -1,415 +0,0 @@ -import { GoogleGenAI } from '@google/genai' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { visionAnalyzeContract } from '@/lib/api/contracts/tools/media/vision' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' -import { - isModelSafeWorkspaceFileKey, - MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, -} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import { - extractStorageKey, - isInternalFileUrl, - processSingleFileToUserFile, -} from '@/lib/uploads/utils/file-utils' -import { - downloadFileFromStorage, - resolveInternalFileUrl, -} from '@/lib/uploads/utils/file-utils.server' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { convertUsageMetadata, extractTextContent } from '@/providers/google/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('VisionAnalyzeAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized Vision analyze attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info(`[${requestId}] Authenticated Vision analyze request via ${authResult.authType}`, { - userId: authResult.userId, - }) - - const userId = authResult.userId - - const parsed = await parseRequest(visionAnalyzeContract, request, {}) - if (!parsed.success) return parsed.response - - const validatedData = parsed.data.body - const modelInputProvenance = validateOpaqueModelInputProvenance({ - headers: request.headers, - payload: validatedData, - isInternalRequest: true, - }) - if (!modelInputProvenance.success) { - return NextResponse.json( - { success: false, error: modelInputProvenance.error }, - { status: modelInputProvenance.status } - ) - } - - if (!validatedData.imageUrl && !validatedData.imageFile) { - return NextResponse.json( - { - success: false, - error: 'Either imageUrl or imageFile is required', - }, - { status: 400 } - ) - } - - logger.info(`[${requestId}] Analyzing image`, { - hasFile: !!validatedData.imageFile, - hasUrl: !!validatedData.imageUrl, - model: validatedData.model, - }) - - let imageSource: string = validatedData.imageUrl || '' - - if (validatedData.imageFile) { - const rawFile = validatedData.imageFile - logger.info(`[${requestId}] Processing image file: ${rawFile.name}`) - - let userFile - try { - userFile = processSingleFileToUserFile(rawFile, requestId, logger) - } catch (error) { - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Failed to process image file'), - }, - { status: 400 } - ) - } - - let base64 = userFile.base64 - let bufferLength = 0 - if (!base64) { - const denied = await assertToolFileAccess( - userFile.key, - authResult.userId, - requestId, - logger - ) - if (denied) return denied - if (!(await isModelSafeWorkspaceFileKey(userFile.key))) { - return NextResponse.json( - { - success: false, - error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, - }, - { status: 400 } - ) - } - // The three providers this route serves disagree too much for a single - // route-wide image limit to be right (Anthropic: 10MB base64 per image; - // OpenAI: 512MB total request payload), and picking the lowest would reject - // images the others accept. So bound the buffer we hold and let each provider - // reject what it will not take, with its own message. - const buffer = await downloadFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - }) - base64 = buffer.toString('base64') - bufferLength = buffer.length - } - const mimeType = userFile.type || 'image/jpeg' - imageSource = `data:${mimeType};base64,${base64}` - if (bufferLength > 0) { - logger.info(`[${requestId}] Converted image to base64 (${bufferLength} bytes)`) - } - } - - let imageUrlValidation: Awaited> | null = null - if (imageSource && !imageSource.startsWith('data:')) { - if (imageSource.startsWith('/') && !isInternalFileUrl(imageSource)) { - return NextResponse.json( - { - success: false, - error: 'Invalid file path. Only uploaded files are supported for internal paths.', - }, - { status: 400 } - ) - } - - if (isInternalFileUrl(imageSource)) { - if (!userId) { - return NextResponse.json( - { - success: false, - error: 'Authentication required for internal file access', - }, - { status: 401 } - ) - } - const resolution = await resolveInternalFileUrl(imageSource, userId, requestId, logger) - if (resolution.error) { - return NextResponse.json( - { - success: false, - error: resolution.error.message, - }, - { status: resolution.error.status } - ) - } - imageSource = resolution.fileUrl || imageSource - if (!(await isModelSafeWorkspaceFileKey(extractStorageKey(validatedData.imageUrl!)))) { - return NextResponse.json( - { - success: false, - error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, - }, - { status: 400 } - ) - } - } - - imageUrlValidation = await validateUrlWithDNS(imageSource, 'imageUrl') - if (!imageUrlValidation.isValid) { - return NextResponse.json( - { - success: false, - error: imageUrlValidation.error, - }, - { status: 400 } - ) - } - } - - const defaultPrompt = 'Please analyze this image and describe what you see in detail.' - const prompt = validatedData.prompt || defaultPrompt - - const isClaude = validatedData.model.startsWith('claude-') - const isGemini = validatedData.model.startsWith('gemini-') - const apiUrl = isClaude - ? 'https://api.anthropic.com/v1/messages' - : 'https://api.openai.com/v1/chat/completions' - - const headers: Record = { - 'Content-Type': 'application/json', - } - - if (isClaude) { - headers['x-api-key'] = validatedData.apiKey - headers['anthropic-version'] = '2023-06-01' - } else { - headers.Authorization = `Bearer ${validatedData.apiKey}` - } - - let requestBody: any - - if (isGemini) { - let base64Payload = imageSource - if (!base64Payload.startsWith('data:')) { - const urlValidation = - imageUrlValidation || (await validateUrlWithDNS(base64Payload, 'imageUrl')) - if (!urlValidation.isValid) { - return NextResponse.json({ success: false, error: urlValidation.error }, { status: 400 }) - } - - const response = await secureFetchWithPinnedIP(base64Payload, urlValidation.resolvedIP!, { - method: 'GET', - }) - if (!response.ok) { - await response.text().catch(() => {}) - return NextResponse.json( - { success: false, error: 'Failed to fetch image for Gemini' }, - { status: 400 } - ) - } - const contentType = - response.headers.get('content-type') || validatedData.imageFile?.type || 'image/jpeg' - const arrayBuffer = await response.arrayBuffer() - const base64 = Buffer.from(arrayBuffer).toString('base64') - base64Payload = `data:${contentType};base64,${base64}` - } - const base64Marker = ';base64,' - const markerIndex = base64Payload.indexOf(base64Marker) - if (!base64Payload.startsWith('data:') || markerIndex === -1) { - return NextResponse.json( - { success: false, error: 'Invalid base64 image format' }, - { status: 400 } - ) - } - const rawMimeType = base64Payload.slice('data:'.length, markerIndex) - const mediaType = rawMimeType.split(';')[0] || 'image/jpeg' - const base64Data = base64Payload.slice(markerIndex + base64Marker.length) - if (!base64Data) { - return NextResponse.json( - { success: false, error: 'Invalid base64 image format' }, - { status: 400 } - ) - } - - const ai = new GoogleGenAI({ apiKey: validatedData.apiKey }) - const geminiResponse = await ai.models.generateContent({ - model: validatedData.model, - contents: [ - { - role: 'user', - parts: [{ text: prompt }, { inlineData: { mimeType: mediaType, data: base64Data } }], - }, - ], - }) - - const content = extractTextContent(geminiResponse.candidates?.[0]) - const usage = convertUsageMetadata(geminiResponse.usageMetadata) - - return NextResponse.json({ - success: true, - output: { - content, - model: validatedData.model, - tokens: usage.totalTokenCount || undefined, - }, - }) - } - - if (isClaude) { - if (imageSource.startsWith('data:')) { - const base64Match = imageSource.match(/^data:([^;]+);base64,(.+)$/) - if (!base64Match) { - return NextResponse.json( - { success: false, error: 'Invalid base64 image format' }, - { status: 400 } - ) - } - const [, mediaType, base64Data] = base64Match - - requestBody = { - model: validatedData.model, - max_tokens: 1024, - messages: [ - { - role: 'user', - content: [ - { type: 'text', text: prompt }, - { - type: 'image', - source: { - type: 'base64', - media_type: mediaType, - data: base64Data, - }, - }, - ], - }, - ], - } - } else { - requestBody = { - model: validatedData.model, - max_tokens: 1024, - messages: [ - { - role: 'user', - content: [ - { type: 'text', text: prompt }, - { - type: 'image', - source: { type: 'url', url: imageSource }, - }, - ], - }, - ], - } - } - } else { - requestBody = { - model: validatedData.model, - messages: [ - { - role: 'user', - content: [ - { type: 'text', text: prompt }, - { - type: 'image_url', - image_url: { - url: imageSource, - }, - }, - ], - }, - ], - max_completion_tokens: 1000, - } - } - - logger.info(`[${requestId}] Sending request to ${isClaude ? 'Anthropic' : 'OpenAI'} API`) - const response = await fetch(apiUrl, { - method: 'POST', - headers, - body: JSON.stringify(requestBody), - }) - - if (!response.ok) { - const errorData = await response.json().catch(() => ({})) - logger.error(`[${requestId}] Vision API error:`, errorData) - return NextResponse.json( - { - success: false, - error: errorData.error?.message || errorData.message || 'Failed to analyze image', - }, - { status: response.status } - ) - } - - const data = await response.json() - const result = data.content?.[0]?.text || data.choices?.[0]?.message?.content - - logger.info(`[${requestId}] Image analyzed successfully`) - - return NextResponse.json({ - success: true, - output: { - content: result, - model: data.model, - tokens: data.content - ? (data.usage?.input_tokens || 0) + (data.usage?.output_tokens || 0) - : data.usage?.total_tokens, - usage: data.usage - ? { - input_tokens: data.usage.input_tokens, - output_tokens: data.usage.output_tokens, - total_tokens: - data.usage.total_tokens || - (data.usage.input_tokens || 0) + (data.usage.output_tokens || 0), - } - : undefined, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error analyzing image:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/wealthbox/item/route.ts b/apps/sim/app/api/tools/wealthbox/item/route.ts deleted file mode 100644 index da8ad62b91f..00000000000 --- a/apps/sim/app/api/tools/wealthbox/item/route.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { wealthboxItemContract } from '@/lib/api/contracts/selectors/wealthbox' -import { parseRequest } from '@/lib/api/server' -import { authorizeCredentialUse } from '@/lib/auth/credential-access' -import { validatePathSegment } from '@/lib/core/security/input-validation' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('WealthboxItemAPI') - -export const GET = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const parsed = await parseRequest(wealthboxItemContract, request, {}) - if (!parsed.success) return parsed.response - const { credentialId, itemId, type } = parsed.data.query - - const itemIdValidation = validatePathSegment(itemId, { - paramName: 'itemId', - maxLength: 100, - allowHyphens: true, - allowUnderscores: true, - allowDots: false, - }) - if (!itemIdValidation.isValid) { - logger.warn(`[${requestId}] Invalid itemId format: ${itemId}`) - return NextResponse.json({ error: itemIdValidation.error }, { status: 400 }) - } - - const credentialIdValidation = validatePathSegment(credentialId, { - paramName: 'credentialId', - maxLength: 100, - allowHyphens: true, - allowUnderscores: true, - allowDots: false, - }) - if (!credentialIdValidation.isValid) { - logger.warn(`[${requestId}] Invalid credentialId format: ${credentialId}`) - return NextResponse.json({ error: credentialIdValidation.error }, { status: 400 }) - } - - const credAccess = await authorizeCredentialUse(request, { - credentialId, - requireWorkflowIdForInternal: false, - }) - if (!credAccess.ok || !credAccess.credentialOwnerUserId) { - logger.warn(`[${requestId}] Credential access denied`, { error: credAccess.error }) - return NextResponse.json({ error: credAccess.error || 'Unauthorized' }, { status: 401 }) - } - - const accessToken = await refreshAccessTokenIfNeeded( - credentialId, - credAccess.credentialOwnerUserId, - requestId - ) - - if (!accessToken) { - logger.error(`[${requestId}] Failed to obtain valid access token`) - return NextResponse.json({ error: 'Failed to obtain valid access token' }, { status: 401 }) - } - - const endpoints = { - note: 'notes', - contact: 'contacts', - task: 'tasks', - } - const endpoint = endpoints[type as keyof typeof endpoints] - - logger.info(`[${requestId}] Fetching ${type} ${itemId} from Wealthbox`) - - const response = await fetch(`https://api.crmworkspace.com/v1/${endpoint}/${itemId}`, { - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json', - }, - }) - - if (!response.ok) { - const errorText = await response.text() - logger.error( - `[${requestId}] Wealthbox API error: ${response.status} ${response.statusText}`, - { - error: errorText, - endpoint, - itemId, - } - ) - - if (response.status === 404) { - return NextResponse.json({ error: 'Item not found' }, { status: 404 }) - } - - return NextResponse.json( - { error: `Failed to fetch ${type} from Wealthbox` }, - { status: response.status } - ) - } - - const data = (await response.json()) as Record - - const firstName = typeof data.first_name === 'string' ? data.first_name : '' - const lastName = typeof data.last_name === 'string' ? data.last_name : '' - const item = { - id: data.id?.toString() || itemId, - name: - (typeof data.content === 'string' && data.content) || - (typeof data.name === 'string' && data.name) || - `${firstName} ${lastName}`.trim() || - `${type} ${data.id}`, - type, - content: typeof data.content === 'string' ? data.content : '', - createdAt: typeof data.created_at === 'string' ? data.created_at : '', - updatedAt: typeof data.updated_at === 'string' ? data.updated_at : '', - } - - logger.info(`[${requestId}] Successfully fetched ${type} ${itemId} from Wealthbox`) - - return NextResponse.json({ item }, { status: 200 }) - } catch (error) { - logger.error(`[${requestId}] Error fetching Wealthbox item`, error) - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/whatsapp/get-media/route.ts b/apps/sim/app/api/tools/whatsapp/get-media/route.ts deleted file mode 100644 index 30433c71f4c..00000000000 --- a/apps/sim/app/api/tools/whatsapp/get-media/route.ts +++ /dev/null @@ -1,217 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { - type WhatsAppGetMediaRouteResponse, - whatsappGetMediaContract, - whatsappGetMediaOutputSchema, -} from '@/lib/api/contracts/tools/whatsapp' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { - isPayloadSizeLimitError, - readResponseJsonWithLimit, - readResponseToBufferWithLimit, -} from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { uploadCopilotFile } from '@/lib/uploads/contexts/copilot' -import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' -import { getExtensionFromMimeType } from '@/lib/uploads/utils/file-utils' -import { sanitizeFileName } from '@/executor/constants' -import type { UserFile } from '@/executor/types' -import { - buildMediaUrl, - extractWhatsAppErrorMessage, - WHATSAPP_MEDIA_MAX_BYTES, -} from '@/tools/whatsapp/utils' - -export const dynamic = 'force-dynamic' -export const maxDuration = 300 - -const logger = createLogger('WhatsAppGetMediaAPI') - -const MAX_GRAPH_METADATA_BYTES = 256 * 1024 - -/** - * Meta's CDN is reported to reject requests without a conventional User-Agent. - * This is not documented behavior, so it is sent defensively rather than relied upon. - */ -const DOWNLOAD_USER_AGENT = 'SimWhatsAppMedia/1.0' - -function failureResponse(error: string, status: number) { - const body = { success: false, error } satisfies WhatsAppGetMediaRouteResponse - return NextResponse.json(body, { status }) -} - -interface WhatsAppMediaMetadata { - url: string - mimeType: string - fileSize: number | null - sha256: string | null - id: string -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized WhatsApp media download attempt: ${authResult.error}`) - return failureResponse(authResult.error || 'Authentication required', 401) - } - - const parsed = await parseRequest( - whatsappGetMediaContract, - request, - {}, - { - validationErrorResponse: (error) => - failureResponse(getValidationErrorMessage(error, 'Invalid request data'), 400), - } - ) - if (!parsed.success) return parsed.response - - const { accessToken, mediaId, phoneNumberId, workspaceId, workflowId, executionId } = - parsed.data.body - const authorization = `Bearer ${accessToken.trim()}` - - try { - const metadataResponse = await fetch(buildMediaUrl(mediaId, phoneNumberId), { - headers: { Authorization: authorization }, - signal: request.signal, - }) - - const metadataBody = await readResponseJsonWithLimit>( - metadataResponse, - { - maxBytes: MAX_GRAPH_METADATA_BYTES, - label: `WhatsApp media ${mediaId} metadata`, - signal: request.signal, - } - ) - - if (!metadataResponse.ok) { - const message = extractWhatsAppErrorMessage(metadataBody, metadataResponse.status) - logger.error(`[${requestId}] WhatsApp media lookup failed`, { - status: metadataResponse.status, - }) - return failureResponse( - message, - metadataResponse.status >= 400 && metadataResponse.status < 500 - ? metadataResponse.status - : 502 - ) - } - - const url = typeof metadataBody.url === 'string' ? metadataBody.url : undefined - if (!url) { - return failureResponse('WhatsApp media metadata did not include a download URL', 502) - } - - // file_size comes back as a string in some responses, so coerce before comparing. - const parsedSize = Number(metadataBody.file_size) - const metadata: WhatsAppMediaMetadata = { - url, - mimeType: - typeof metadataBody.mime_type === 'string' && metadataBody.mime_type.length > 0 - ? metadataBody.mime_type - : 'application/octet-stream', - fileSize: Number.isFinite(parsedSize) ? parsedSize : null, - sha256: typeof metadataBody.sha256 === 'string' ? metadataBody.sha256 : null, - id: typeof metadataBody.id === 'string' ? metadataBody.id : mediaId, - } - - // Reject oversized media from the declared size before spending bandwidth on it. - if (metadata.fileSize !== null && metadata.fileSize > WHATSAPP_MEDIA_MAX_BYTES) { - return failureResponse( - `WhatsApp media is ${(metadata.fileSize / (1024 * 1024)).toFixed(2)} MB, which exceeds the 100 MB download limit`, - 413 - ) - } - - // The media URL points at Meta's CDN and still requires the bearer token. - const urlValidation = await validateUrlWithDNS(metadata.url, 'mediaUrl') - if (!urlValidation.isValid) { - return failureResponse(`Invalid WhatsApp media URL: ${urlValidation.error}`, 502) - } - - const mediaResponse = await secureFetchWithPinnedIP(metadata.url, urlValidation.resolvedIP!, { - method: 'GET', - headers: { - Authorization: authorization, - 'User-Agent': DOWNLOAD_USER_AGENT, - }, - maxResponseBytes: WHATSAPP_MEDIA_MAX_BYTES, - stripAuthOnRedirect: true, - signal: request.signal, - }) - - if (!mediaResponse.ok) { - logger.error(`[${requestId}] WhatsApp media download failed`, { - status: mediaResponse.status, - }) - return failureResponse( - mediaResponse.status === 404 - ? 'WhatsApp media not found or its download URL expired (URLs are valid for 5 minutes)' - : `Failed to download WhatsApp media (${mediaResponse.status})`, - mediaResponse.status >= 400 && mediaResponse.status < 500 ? mediaResponse.status : 502 - ) - } - - const buffer = await readResponseToBufferWithLimit(mediaResponse, { - maxBytes: WHATSAPP_MEDIA_MAX_BYTES, - label: 'WhatsApp media download', - }) - - const extension = getExtensionFromMimeType(metadata.mimeType) ?? 'bin' - const fileName = sanitizeFileName(`whatsapp-${metadata.id}.${extension}`) - - const file: UserFile = - workspaceId && workflowId && executionId - ? await uploadExecutionFile( - { workspaceId, workflowId, executionId }, - buffer, - fileName, - metadata.mimeType, - authResult.userId - ) - : await uploadCopilotFile({ - buffer, - fileName, - contentType: metadata.mimeType, - userId: authResult.userId, - }) - - logger.info(`[${requestId}] WhatsApp media downloaded`, { - mediaId: metadata.id, - mimeType: metadata.mimeType, - size: buffer.length, - }) - - const output = whatsappGetMediaOutputSchema.parse({ - file, - mediaId: metadata.id, - mimeType: metadata.mimeType, - fileSize: buffer.length, - sha256: metadata.sha256, - }) - - return NextResponse.json({ - success: true, - output, - } satisfies WhatsAppGetMediaRouteResponse) - } catch (error) { - logger.error(`[${requestId}] WhatsApp media download failed`, { error }) - - if (isPayloadSizeLimitError(error)) { - return failureResponse('WhatsApp media exceeds the 100 MB download limit', 413) - } - - return failureResponse(getErrorMessage(error, 'Failed to download WhatsApp media'), 500) - } -}) diff --git a/apps/sim/app/api/tools/whatsapp/send-media/route.ts b/apps/sim/app/api/tools/whatsapp/send-media/route.ts deleted file mode 100644 index cd611dc809c..00000000000 --- a/apps/sim/app/api/tools/whatsapp/send-media/route.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { - type WhatsAppSendMediaRouteResponse, - whatsappSendMediaContract, -} from '@/lib/api/contracts/tools/whatsapp' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { uploadWhatsAppMedia } from '@/app/api/tools/whatsapp/upload.server' -import { - buildAuthHeaders, - buildMediaMessageBody, - buildMessagesUrl, - transformWhatsAppSendResponse, -} from '@/tools/whatsapp/utils' - -export const dynamic = 'force-dynamic' -export const maxDuration = 300 - -const logger = createLogger('WhatsAppSendMediaAPI') - -function failureResponse(error: string, status: number) { - const body = { success: false, error } satisfies WhatsAppSendMediaRouteResponse - return NextResponse.json(body, { status }) -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized WhatsApp media send attempt: ${authResult.error}`) - return failureResponse(authResult.error || 'Authentication required', 401) - } - - const parsed = await parseRequest( - whatsappSendMediaContract, - request, - {}, - { - validationErrorResponse: (error) => - failureResponse(getValidationErrorMessage(error, 'Invalid request data'), 400), - } - ) - if (!parsed.success) return parsed.response - - const body = parsed.data.body - const suppliedSources = [body.file, body.mediaId, body.mediaLink].filter(Boolean).length - if (suppliedSources === 0) { - return failureResponse('Provide a file, a media ID, or a media link', 400) - } - if (suppliedSources > 1) { - return failureResponse('Provide only one of file, media ID, or media link', 400) - } - - try { - // A dropped file has no WhatsApp identity yet, so upload it first and send the - // resulting media ID. Media persists 30 days, so the ID is returned for reuse. - let uploadedMediaId: string | undefined - let filename = body.filename ?? undefined - - if (body.file) { - const uploaded = await uploadWhatsAppMedia({ - file: body.file, - accessToken: body.accessToken, - phoneNumberId: body.phoneNumberId, - userId: authResult.userId, - requestId, - logger, - signal: request.signal, - }) - - if (!uploaded.ok) { - return 'response' in uploaded - ? uploaded.response - : failureResponse(uploaded.error, uploaded.status) - } - - uploadedMediaId = uploaded.media.mediaId - filename = filename ?? uploaded.media.fileName - } - - const messageBody = buildMediaMessageBody({ - phoneNumber: body.phoneNumber, - mediaType: body.mediaType, - mediaId: uploadedMediaId ?? body.mediaId ?? undefined, - mediaLink: body.mediaLink ?? undefined, - caption: body.caption ?? undefined, - filename, - }) - - const response = await fetch(buildMessagesUrl(body.phoneNumberId), { - method: 'POST', - headers: buildAuthHeaders(body.accessToken), - body: JSON.stringify(messageBody), - signal: request.signal, - }) - - const sendResult = await transformWhatsAppSendResponse(response) - - // transformWhatsAppSendResponse throws on a non-OK send, so reaching here means success. - return NextResponse.json({ - success: true, - output: { - ...sendResult.output, - success: true, - messageId: sendResult.output.messageId ?? '', - inputPhoneNumber: sendResult.output.inputPhoneNumber ?? null, - whatsappUserId: sendResult.output.whatsappUserId ?? null, - contacts: sendResult.output.contacts ?? [], - ...(uploadedMediaId ? { mediaId: uploadedMediaId } : {}), - }, - } satisfies WhatsAppSendMediaRouteResponse) - } catch (error) { - logger.error(`[${requestId}] WhatsApp media send failed`, { error }) - return failureResponse( - getErrorMessage(error, 'Failed to send WhatsApp media'), - isPayloadSizeLimitError(error) ? 413 : 500 - ) - } -}) diff --git a/apps/sim/app/api/tools/whatsapp/upload-media/route.ts b/apps/sim/app/api/tools/whatsapp/upload-media/route.ts deleted file mode 100644 index befa0f34d22..00000000000 --- a/apps/sim/app/api/tools/whatsapp/upload-media/route.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { - type WhatsAppUploadMediaRouteResponse, - whatsappUploadMediaContract, -} from '@/lib/api/contracts/tools/whatsapp' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { uploadWhatsAppMedia } from '@/app/api/tools/whatsapp/upload.server' - -export const dynamic = 'force-dynamic' -export const maxDuration = 300 - -const logger = createLogger('WhatsAppUploadMediaAPI') - -function failureResponse(error: string, status: number) { - const body = { success: false, error } satisfies WhatsAppUploadMediaRouteResponse - return NextResponse.json(body, { status }) -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized WhatsApp media upload attempt: ${authResult.error}`) - return failureResponse(authResult.error || 'Authentication required', 401) - } - - const parsed = await parseRequest( - whatsappUploadMediaContract, - request, - {}, - { - validationErrorResponse: (error) => - failureResponse(getValidationErrorMessage(error, 'Invalid request data'), 400), - } - ) - if (!parsed.success) return parsed.response - - const { accessToken, phoneNumberId, file } = parsed.data.body - - try { - const result = await uploadWhatsAppMedia({ - file, - accessToken, - phoneNumberId, - userId: authResult.userId, - requestId, - logger, - signal: request.signal, - }) - - if (!result.ok) { - return 'response' in result ? result.response : failureResponse(result.error, result.status) - } - - return NextResponse.json({ - success: true, - output: result.media, - } satisfies WhatsAppUploadMediaRouteResponse) - } catch (error) { - logger.error(`[${requestId}] WhatsApp media upload failed`, { error }) - return failureResponse( - getErrorMessage(error, 'Failed to upload media to WhatsApp'), - isPayloadSizeLimitError(error) ? 413 : 500 - ) - } -}) diff --git a/apps/sim/app/api/tools/whatsapp/upload.server.ts b/apps/sim/app/api/tools/whatsapp/upload.server.ts deleted file mode 100644 index ea84d2d5ab5..00000000000 --- a/apps/sim/app/api/tools/whatsapp/upload.server.ts +++ /dev/null @@ -1,148 +0,0 @@ -import type { Logger } from '@sim/logger' -import type { NextResponse } from 'next/server' -import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' -import type { RawFileInput } from '@/lib/uploads/utils/file-utils' -import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { - buildMediaUploadUrl, - extractWhatsAppErrorMessage, - whatsappMediaLimitFor, -} from '@/tools/whatsapp/utils' - -/** WhatsApp error and upload envelopes are small; cap the read so a hostile body cannot balloon memory. */ -const MAX_GRAPH_RESPONSE_BYTES = 256 * 1024 - -export interface UploadedWhatsAppMedia { - mediaId: string - fileName: string - mimeType: string - size: number -} - -export type UploadWhatsAppMediaResult = - | { ok: true; media: UploadedWhatsAppMedia } - | { ok: false; error: string; status: number } - | { ok: false; response: NextResponse } - -/** - * Pull a stored user file, enforce WhatsApp's documented per-type size ceiling, and - * upload it to `/{phone-number-id}/media`. Shared by the standalone Upload Media - * operation and the file path of Send Media. - */ -export async function uploadWhatsAppMedia({ - file, - accessToken, - phoneNumberId, - userId, - requestId, - logger, - signal, -}: { - file: RawFileInput - accessToken: string - phoneNumberId: string - userId: string - requestId: string - logger: Logger - signal?: AbortSignal -}): Promise { - const userFile = processSingleFileToUserFile(file, requestId, logger) - if (!userFile) { - return { ok: false, error: 'No valid file provided for upload', status: 400 } - } - - const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) - if (denied) return { ok: false, response: denied } - - // Check the declared size against WhatsApp's ceiling before pulling bytes, then cap - // the storage read itself so a mis-declared size cannot blow past it. - const declaredMimeType = userFile.type || 'application/octet-stream' - const declaredLimit = whatsappMediaLimitFor(declaredMimeType) - if (userFile.size > declaredLimit.maxBytes) { - return { - ok: false, - error: `${userFile.name} is ${(userFile.size / (1024 * 1024)).toFixed(2)} MB, which exceeds WhatsApp's limit for ${declaredLimit.label}`, - status: 413, - } - } - - let buffer: Buffer - let contentType: string - try { - const downloaded = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: declaredLimit.maxBytes, - signal, - }) - buffer = downloaded.buffer - contentType = downloaded.contentType - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return { ok: false, response: notReady } - throw error - } - - // downloadServableFileFromStorage can swap an AI-generated doc for its compiled - // artifact, so re-resolve the limit against the type actually being sent. - const resolvedMimeType = contentType || declaredMimeType - const resolvedLimit = whatsappMediaLimitFor(resolvedMimeType) - if (buffer.length > resolvedLimit.maxBytes) { - return { - ok: false, - error: `${userFile.name} is ${(buffer.length / (1024 * 1024)).toFixed(2)} MB, which exceeds WhatsApp's limit for ${resolvedLimit.label}`, - status: 413, - } - } - - const formData = new FormData() - formData.append('messaging_product', 'whatsapp') - formData.append('type', resolvedMimeType) - formData.append( - 'file', - new Blob([new Uint8Array(buffer)], { type: resolvedMimeType }), - userFile.name - ) - - logger.info(`[${requestId}] Uploading media to WhatsApp`, { - fileName: userFile.name, - mimeType: resolvedMimeType, - size: buffer.length, - }) - - // Content-Type is intentionally omitted so fetch sets the multipart boundary. - const response = await fetch(buildMediaUploadUrl(phoneNumberId), { - method: 'POST', - headers: { Authorization: `Bearer ${accessToken.trim()}` }, - body: formData, - signal, - }) - - const data = await readResponseJsonWithLimit>(response, { - maxBytes: MAX_GRAPH_RESPONSE_BYTES, - label: 'WhatsApp media upload response', - signal, - }) - - if (!response.ok) { - logger.error(`[${requestId}] WhatsApp media upload failed`, { status: response.status }) - return { - ok: false, - error: extractWhatsAppErrorMessage(data, response.status), - status: response.status >= 400 && response.status < 500 ? response.status : 502, - } - } - - const mediaId = typeof data.id === 'string' ? data.id : undefined - if (!mediaId) { - return { ok: false, error: 'WhatsApp upload response did not include a media ID', status: 502 } - } - - logger.info(`[${requestId}] WhatsApp media uploaded`, { mediaId }) - - return { - ok: true, - media: { mediaId, fileName: userFile.name, mimeType: resolvedMimeType, size: buffer.length }, - } -} diff --git a/apps/sim/app/api/tools/windchill/route.test.ts b/apps/sim/app/api/tools/windchill/route.test.ts deleted file mode 100644 index 7504e7e606c..00000000000 --- a/apps/sim/app/api/tools/windchill/route.test.ts +++ /dev/null @@ -1,836 +0,0 @@ -/** - * @vitest-environment node - */ -import { createMockRequest as createTestingRequest, resetEnvMock } from '@sim/testing' -import { NextResponse } from 'next/server' -import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' -import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' - -const { - MockInvalidBindingError, - MockWindchillProviderError, - mockAssertToolFileAccess, - mockBindDelegation, - mockCreateWindchillSession, - mockDownloadServableFileFromStorage, - mockDownloadWindchillContent, - mockGetSession, - mockResolveWindchillContentUrl, - mockProcessFilesToUserFiles, - mockUploadCopilotFile, - mockUploadExecutionFile, - mockUploadWindchillContent, - mockWindchillMutationRequest, -} = vi.hoisted(() => { - class MockInvalidBindingError extends Error {} - class MockWindchillProviderError extends Error { - constructor( - message: string, - readonly status: number - ) { - super(message) - this.name = 'WindchillProviderError' - } - } - - return { - MockInvalidBindingError, - MockWindchillProviderError, - mockAssertToolFileAccess: vi.fn(), - mockBindDelegation: vi.fn(), - mockCreateWindchillSession: vi.fn(), - mockDownloadServableFileFromStorage: vi.fn(), - mockDownloadWindchillContent: vi.fn(), - mockGetSession: vi.fn(), - mockResolveWindchillContentUrl: vi.fn(), - mockProcessFilesToUserFiles: vi.fn(), - mockUploadCopilotFile: vi.fn(), - mockUploadExecutionFile: vi.fn(), - mockUploadWindchillContent: vi.fn(), - mockWindchillMutationRequest: vi.fn(), - } -}) - -vi.mock('@/lib/auth', () => ({ getSession: mockGetSession })) -vi.mock('@/lib/auth/internal-delegation', () => ({ - bindInternalExecutorDelegation: mockBindDelegation, - InvalidInternalDelegationBindingError: MockInvalidBindingError, -})) -vi.unmock('@/lib/auth/internal') - -vi.mock('@/app/api/files/authorization', () => ({ - assertToolFileAccess: mockAssertToolFileAccess, -})) -vi.mock('@/lib/uploads/utils/file-utils', () => ({ - processFilesToUserFiles: mockProcessFilesToUserFiles, -})) -vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ - downloadServableFileFromStorage: mockDownloadServableFileFromStorage, -})) -vi.mock('@/lib/uploads/utils/servable-file-response', () => ({ - docNotReadyResponse: vi.fn().mockReturnValue(null), -})) -vi.mock('@/lib/uploads/contexts/copilot', () => ({ - uploadCopilotFile: mockUploadCopilotFile, -})) -vi.mock('@/lib/uploads/contexts/execution', () => ({ - uploadExecutionFile: mockUploadExecutionFile, -})) -vi.mock('@/tools/windchill/utils.server', () => ({ - createWindchillSession: mockCreateWindchillSession, - downloadWindchillContent: mockDownloadWindchillContent, - resolveWindchillContentUrl: mockResolveWindchillContentUrl, - sanitizeWindchillError: (message: string) => message.replace(/https?:\/\/\S+/g, '[redacted URL]'), - uploadWindchillContent: mockUploadWindchillContent, - windchillDocumentUrl: (baseUrl: string, documentOid: string) => - `${baseUrl}/DocMgmt/Documents('${encodeURIComponent(documentOid)}')`, - windchillMutationRequest: mockWindchillMutationRequest, - WindchillProviderError: MockWindchillProviderError, -})) - -import { generateInternalDelegationToken, generateInternalToken } from '@/lib/auth/internal' -import { POST } from '@/app/api/tools/windchill/route' - -const BASE_BODY = { - baseUrl: 'https://windchill.example.com/Windchill/servlet/odata/v6', - username: 'windchill-user', - password: 'not-a-real-password', -} - -const DOCUMENT_OID = 'OR:wt.doc.WTDocument:1' -const SECOND_DOCUMENT_OID = 'OR:wt.doc.WTDocument:2' -let delegationToken = '' -let legacyInternalToken = '' - -function createMockRequest(method: string, body: unknown, headers: Record = {}) { - return createTestingRequest(method, body, { - authorization: `Bearer ${delegationToken}`, - ...headers, - }) -} - -const MUTATION_CASES = [ - { - operation: 'windchill_create_document', - input: { name: 'Specification', containerOid: 'OR:wt.pdmlink.PDMLinkProduct:1' }, - url: '/DocMgmt/Documents', - method: 'POST', - }, - { - operation: 'windchill_create_documents', - input: { - documents: [{ name: 'Specification', containerOid: 'OR:wt.pdmlink.PDMLinkProduct:1' }], - }, - url: '/DocMgmt/CreateDocuments', - method: 'POST', - }, - { - operation: 'windchill_update_document', - input: { documentOid: DOCUMENT_OID, attributes: { Title: 'Updated' } }, - url: '/DocMgmt/Documents(', - method: 'PATCH', - }, - { - operation: 'windchill_update_documents', - input: { documents: [{ id: DOCUMENT_OID, attributes: { Title: 'Updated' } }] }, - url: '/DocMgmt/UpdateDocuments', - method: 'POST', - }, - { - operation: 'windchill_update_common_properties', - input: { documentOid: DOCUMENT_OID, commonProperties: { Name: 'Renamed' } }, - url: '/PTC.DocMgmt.UpdateCommonProperties', - method: 'POST', - }, - { - operation: 'windchill_delete_document', - input: { documentOid: DOCUMENT_OID }, - url: '/DocMgmt/Documents(', - method: 'DELETE', - }, - { - operation: 'windchill_delete_documents', - input: { documentOids: [DOCUMENT_OID, SECOND_DOCUMENT_OID] }, - url: '/DocMgmt/DeleteDocuments', - method: 'POST', - }, - { - operation: 'windchill_check_out_document', - input: { documentOid: DOCUMENT_OID, checkOutNote: 'Editing' }, - url: '/PTC.DocMgmt.CheckOut', - method: 'POST', - }, - { - operation: 'windchill_check_out_documents', - input: { documentOids: [DOCUMENT_OID], checkOutNote: 'Editing' }, - url: '/DocMgmt/CheckOutDocuments', - method: 'POST', - }, - { - operation: 'windchill_check_in_document', - input: { documentOid: DOCUMENT_OID, checkInNote: 'Done', keepCheckedOut: false }, - url: '/PTC.DocMgmt.CheckIn', - method: 'POST', - }, - { - operation: 'windchill_check_in_documents', - input: { documentOids: [DOCUMENT_OID], checkInNote: 'Done' }, - url: '/DocMgmt/CheckInDocuments', - method: 'POST', - }, - { - operation: 'windchill_undo_check_out_document', - input: { documentOid: DOCUMENT_OID }, - url: '/PTC.DocMgmt.UndoCheckOut', - method: 'POST', - }, - { - operation: 'windchill_undo_check_out_documents', - input: { documentOids: [DOCUMENT_OID] }, - url: '/DocMgmt/UndoCheckOutDocuments', - method: 'POST', - }, - { - operation: 'windchill_revise_document', - input: { documentOid: DOCUMENT_OID, versionId: 'B' }, - url: '/PTC.DocMgmt.Revise', - method: 'POST', - }, - { - operation: 'windchill_revise_documents', - input: { documentOids: [DOCUMENT_OID] }, - url: '/DocMgmt/ReviseDocuments', - method: 'POST', - }, - { - operation: 'windchill_set_lifecycle_state', - input: { documentOid: DOCUMENT_OID, stateValue: 'RELEASED', stateDisplay: 'Released' }, - url: '/PTC.DocMgmt.SetState', - method: 'POST', - }, - { - operation: 'windchill_update_document_security_labels', - input: { - securityLabelUpdates: [{ id: DOCUMENT_OID, labels: { EXPORT_CONTROL: 'L1' } }], - }, - url: '/DocMgmt/EditDocumentsSecurityLabels', - method: 'POST', - }, -] as const - -const MUTATION_PAYLOAD_CASES = [ - { - operation: 'windchill_check_out_documents', - input: { documentOids: [DOCUMENT_OID], checkOutNote: 'Editing' }, - body: { Documents: [{ ID: DOCUMENT_OID }], CheckOutNote: 'Editing' }, - }, - { - operation: 'windchill_check_in_document', - input: { - documentOid: DOCUMENT_OID, - checkInNote: 'Done', - keepCheckedOut: false, - checkOutNote: 'Continue editing', - }, - body: { - CheckInNote: 'Done', - KeepCheckedOut: false, - CheckOutNote: 'Continue editing', - }, - }, - { - operation: 'windchill_revise_document', - input: { documentOid: DOCUMENT_OID, versionId: 'B' }, - body: { VersionId: 'B' }, - }, - { - operation: 'windchill_update_common_properties', - input: { - documentOid: DOCUMENT_OID, - commonProperties: { Name: 'Renamed', Number: 'DOC-001' }, - }, - body: { Updates: { Name: 'Renamed', Number: 'DOC-001' } }, - }, - { - operation: 'windchill_revise_documents', - input: { documentOids: [DOCUMENT_OID] }, - body: { Documents: [{ ID: DOCUMENT_OID }] }, - }, - { - operation: 'windchill_set_lifecycle_state', - input: { documentOid: DOCUMENT_OID, stateValue: 'RELEASED', stateDisplay: 'Released' }, - body: { State: { Display: 'Released', Value: 'RELEASED' } }, - }, - { - operation: 'windchill_update_document_security_labels', - input: { - securityLabelUpdates: [{ id: DOCUMENT_OID, labels: { EXPORT_CONTROL: 'L1' } }], - }, - body: { Documents: [{ EXPORT_CONTROL: 'L1', ID: DOCUMENT_OID }] }, - }, -] as const - -beforeAll(async () => { - delegationToken = await generateInternalDelegationToken({ - subjectUserId: 'user-1', - workflowId: '550e8400-e29b-41d4-a716-446655440001', - }) - legacyInternalToken = await generateInternalToken() -}) - -afterAll(resetEnvMock) - -beforeEach(() => { - vi.clearAllMocks() - mockGetSession.mockResolvedValue(null) - mockBindDelegation.mockImplementation(async (delegation, options) => ({ - kind: 'delegated', - serviceId: 'executor', - subjectUserId: delegation.subjectUserId, - workspaceId: '550e8400-e29b-41d4-a716-446655440000', - delegationId: delegation.delegationId, - audience: options.audience, - issuedAt: delegation.issuedAt, - expiresAt: delegation.expiresAt, - delegationContext: { - kind: 'workflow_execution', - workflowId: delegation.workflowId, - executionId: delegation.executionId, - }, - })) - mockCreateWindchillSession.mockResolvedValue({ - nonceHeader: 'CSRF_NONCE', - nonceValue: 'nonce-value', - cookie: 'JSESSIONID=session-value', - }) - mockWindchillMutationRequest.mockResolvedValue({ value: [{ ID: DOCUMENT_OID }] }) - mockAssertToolFileAccess.mockResolvedValue(null) - mockProcessFilesToUserFiles.mockReturnValue([ - { - key: 'workspace/workspace-1/specification.pdf', - name: 'specification.pdf', - size: 3, - type: 'application/pdf', - }, - ]) - mockDownloadServableFileFromStorage.mockResolvedValue({ - buffer: Buffer.from('pdf'), - contentType: 'application/pdf', - }) - mockUploadWindchillContent.mockResolvedValue(['specification.pdf']) - mockResolveWindchillContentUrl.mockImplementation( - async ({ contentPath }: { contentPath: string }) => - `https://windchill.example.com/Windchill/servlet/WindchillGW/download?from=${encodeURIComponent(contentPath)}` - ) - mockDownloadWindchillContent.mockResolvedValue({ - buffer: Buffer.from('pdf'), - contentType: 'application/pdf', - contentDisposition: 'attachment; filename="specification.pdf"', - }) - mockUploadCopilotFile.mockResolvedValue({ - id: 'file-1', - name: 'specification.pdf', - url: '/api/files/serve?key=copilot/specification.pdf', - size: 3, - type: 'application/pdf', - key: 'copilot/specification.pdf', - }) -}) - -describe('POST /api/tools/windchill', () => { - it('authenticates before parsing the request body', async () => { - const response = await POST(createTestingRequest('POST', { operation: 'not-valid' })) - - expect(response.status).toBe(401) - expect(await response.json()).toEqual({ success: false, error: 'Unauthorized' }) - expect(mockCreateWindchillSession).not.toHaveBeenCalled() - }) - - it('binds executor identity and scope through the canonical delegation path', async () => { - const response = await POST( - createMockRequest('POST', { - ...BASE_BODY, - operation: 'windchill_update_document', - documentOid: DOCUMENT_OID, - attributes: { Title: 'Updated' }, - }) - ) - - expect(response.status).toBe(200) - expect(mockBindDelegation).toHaveBeenCalledWith(expect.any(Object), { - audience: 'sim:windchill', - resourceScope: undefined, - }) - }) - - it('rejects browser sessions and legacy internal tokens', async () => { - mockGetSession.mockResolvedValueOnce({ - user: { id: 'user-1' }, - session: { id: 'session-1' }, - }) - - const sessionResponse = await POST(createTestingRequest('POST', BASE_BODY)) - const legacyResponse = await POST( - createTestingRequest('POST', BASE_BODY, { - authorization: `Bearer ${legacyInternalToken}`, - }) - ) - - expect(sessionResponse.status).toBe(401) - expect(legacyResponse.status).toBe(401) - expect(mockBindDelegation).not.toHaveBeenCalled() - }) - - it('rejects malformed operation inputs at the shared contract boundary', async () => { - const response = await POST( - createMockRequest('POST', { - ...BASE_BODY, - operation: 'windchill_update_document', - documentOid: DOCUMENT_OID, - attributes: {}, - }) - ) - - expect(response.status).toBe(400) - expect(mockCreateWindchillSession).not.toHaveBeenCalled() - }) - - it('rejects an invalid service root before reading a protected upload', async () => { - const response = await POST( - createMockRequest('POST', { - ...BASE_BODY, - baseUrl: `${BASE_BODY.baseUrl}?token=secret`, - operation: 'windchill_upload_primary_content', - documentOid: DOCUMENT_OID, - primaryFile: { - key: 'workspace/workspace-1/specification.pdf', - name: 'specification.pdf', - size: 3, - type: 'application/pdf', - }, - }) - ) - - expect(response.status).toBe(400) - expect(mockAssertToolFileAccess).not.toHaveBeenCalled() - expect(mockDownloadServableFileFromStorage).not.toHaveBeenCalled() - }) - - it.each(MUTATION_CASES)( - 'dispatches $operation through one CSRF-protected transaction', - async ({ operation, input, url, method }) => { - const response = await POST(createMockRequest('POST', { ...BASE_BODY, operation, ...input })) - - expect(response.status).toBe(200) - expect((await response.json()).success).toBe(true) - expect(mockCreateWindchillSession).toHaveBeenCalledTimes(1) - expect(mockWindchillMutationRequest).toHaveBeenCalledTimes(1) - expect(mockWindchillMutationRequest.mock.calls[0][0].url).toContain(url) - expect(mockWindchillMutationRequest.mock.calls[0][0].method).toBe(method) - } - ) - - it.each(MUTATION_PAYLOAD_CASES)( - 'encodes the exact $operation action payload', - async ({ operation, input, body }) => { - const response = await POST( - createMockRequest('POST', { - ...BASE_BODY, - operation, - ...input, - }) - ) - - expect(response.status).toBe(200) - expect(mockWindchillMutationRequest.mock.calls[0][0].body).toEqual(body) - } - ) - - it('maps create bindings and custom attributes without allowing them to replace bindings', async () => { - const response = await POST( - createMockRequest('POST', { - ...BASE_BODY, - operation: 'windchill_create_document', - name: 'Specification', - containerOid: 'OR:wt.pdmlink.PDMLinkProduct:1', - folderOid: 'OR:wt.folder.SubFolder:2', - attributes: { CustomString: 'value' }, - }) - ) - - expect(response.status).toBe(200) - expect((await response.json()).output.affectedIds).toEqual([DOCUMENT_OID]) - expect(mockWindchillMutationRequest.mock.calls[0][0].body).toEqual({ - CustomString: 'value', - Name: 'Specification', - 'Context@odata.bind': "Containers('OR%3Awt.pdmlink.PDMLinkProduct%3A1')", - 'Folder@odata.bind': "Folders('OR%3Awt.folder.SubFolder%3A2')", - }) - }) - - it('returns operation-specific single, bulk, and delete mutation shapes', async () => { - const singleResponse = await POST( - createMockRequest('POST', { - ...BASE_BODY, - operation: 'windchill_update_document', - documentOid: DOCUMENT_OID, - attributes: { Title: 'Updated' }, - }) - ) - const singleOutput = (await singleResponse.json()).output - expect(singleOutput.document).toMatchObject({ id: DOCUMENT_OID }) - expect(singleOutput).not.toHaveProperty('documents') - - const bulkResponse = await POST( - createMockRequest('POST', { - ...BASE_BODY, - operation: 'windchill_update_documents', - documents: [{ id: DOCUMENT_OID, attributes: { Title: 'Updated' } }], - }) - ) - const bulkOutput = (await bulkResponse.json()).output - expect(bulkOutput.documents).toEqual([expect.objectContaining({ id: DOCUMENT_OID })]) - expect(bulkOutput).not.toHaveProperty('document') - - const deleteResponse = await POST( - createMockRequest('POST', { - ...BASE_BODY, - operation: 'windchill_delete_document', - documentOid: DOCUMENT_OID, - }) - ) - const deleteOutput = (await deleteResponse.json()).output - expect(deleteOutput.affectedIds).toEqual([DOCUMENT_OID]) - expect(deleteOutput).not.toHaveProperty('document') - expect(deleteOutput).not.toHaveProperty('documents') - }) - - it('authorizes and reads a UserFile before starting the upload transaction', async () => { - const response = await POST( - createMockRequest('POST', { - ...BASE_BODY, - operation: 'windchill_upload_primary_content', - documentOid: DOCUMENT_OID, - primaryFile: { - key: 'workspace/workspace-1/specification.pdf', - name: 'specification.pdf', - size: 3, - type: 'application/pdf', - }, - }) - ) - - expect(response.status).toBe(200) - expect(mockAssertToolFileAccess).toHaveBeenCalledWith( - 'workspace/workspace-1/specification.pdf', - 'user-1', - expect.any(String), - expect.anything() - ) - expect(mockDownloadServableFileFromStorage).toHaveBeenCalledTimes(1) - expect(mockUploadWindchillContent).toHaveBeenCalledWith( - expect.objectContaining({ - documentOid: DOCUMENT_OID, - primaryContent: true, - files: [ - expect.objectContaining({ - name: 'specification.pdf', - mimeType: 'application/pdf', - size: 3, - }), - ], - }) - ) - }) - - it('uploads multiple authorized files as attachments', async () => { - mockProcessFilesToUserFiles.mockReturnValueOnce([ - { - key: 'workspace/workspace-1/one.txt', - name: 'one.txt', - size: 3, - type: 'text/plain', - }, - { - key: 'workspace/workspace-1/two.txt', - name: 'two.txt', - size: 3, - type: 'text/plain', - }, - ]) - mockDownloadServableFileFromStorage.mockResolvedValue({ - buffer: Buffer.from('txt'), - contentType: 'text/plain', - }) - mockUploadWindchillContent.mockResolvedValueOnce(['one.txt', 'two.txt']) - - const response = await POST( - createMockRequest('POST', { - ...BASE_BODY, - operation: 'windchill_upload_attachments', - documentOid: DOCUMENT_OID, - attachmentFiles: [ - { key: 'workspace/workspace-1/one.txt', name: 'one.txt', size: 3 }, - { key: 'workspace/workspace-1/two.txt', name: 'two.txt', size: 3 }, - ], - }) - ) - - expect(response.status).toBe(200) - expect(mockAssertToolFileAccess).toHaveBeenCalledTimes(2) - expect(mockDownloadServableFileFromStorage).toHaveBeenCalledTimes(2) - expect(mockUploadWindchillContent).toHaveBeenCalledWith( - expect.objectContaining({ primaryContent: false }) - ) - expect(mockDownloadServableFileFromStorage.mock.calls[0][3]).toEqual({ - maxBytes: MAX_FILE_SIZE, - }) - expect(mockDownloadServableFileFromStorage.mock.calls[1][3]).toEqual({ - maxBytes: MAX_FILE_SIZE - 3, - }) - }) - - it('rejects attachment counts above the contract limit before reading storage', async () => { - const response = await POST( - createMockRequest('POST', { - ...BASE_BODY, - operation: 'windchill_upload_attachments', - documentOid: DOCUMENT_OID, - attachmentFiles: Array.from({ length: 11 }, (_, index) => ({ - key: `workspace/workspace-1/${index}.txt`, - name: `${index}.txt`, - size: 1, - })), - }) - ) - - expect(response.status).toBe(400) - expect(mockAssertToolFileAccess).not.toHaveBeenCalled() - expect(mockDownloadServableFileFromStorage).not.toHaveBeenCalled() - }) - - it('rejects declared aggregate upload size before reading storage', async () => { - mockProcessFilesToUserFiles.mockReturnValueOnce([ - { - key: 'workspace/workspace-1/oversized.bin', - name: 'oversized.bin', - size: MAX_FILE_SIZE + 1, - type: 'application/octet-stream', - }, - ]) - - const response = await POST( - createMockRequest('POST', { - ...BASE_BODY, - operation: 'windchill_upload_primary_content', - documentOid: DOCUMENT_OID, - primaryFile: { - key: 'workspace/workspace-1/oversized.bin', - name: 'oversized.bin', - size: MAX_FILE_SIZE + 1, - }, - }) - ) - - expect(response.status).toBe(413) - expect(mockAssertToolFileAccess).not.toHaveBeenCalled() - expect(mockDownloadServableFileFromStorage).not.toHaveBeenCalled() - }) - - it('stops an under-reported upload at the remaining aggregate byte budget', async () => { - mockProcessFilesToUserFiles.mockReturnValueOnce([ - { key: 'workspace/workspace-1/one.txt', name: 'one.txt', size: 1, type: 'text/plain' }, - { key: 'workspace/workspace-1/two.txt', name: 'two.txt', size: 1, type: 'text/plain' }, - { - key: 'workspace/workspace-1/three.txt', - name: 'three.txt', - size: 1, - type: 'text/plain', - }, - ]) - mockDownloadServableFileFromStorage - .mockResolvedValueOnce({ buffer: Buffer.from('one'), contentType: 'text/plain' }) - .mockRejectedValueOnce( - new PayloadSizeLimitError({ - label: 'Uploaded file', - maxBytes: MAX_FILE_SIZE - 3, - observedBytes: MAX_FILE_SIZE - 2, - }) - ) - - const response = await POST( - createMockRequest('POST', { - ...BASE_BODY, - operation: 'windchill_upload_attachments', - documentOid: DOCUMENT_OID, - attachmentFiles: [ - { key: 'workspace/workspace-1/one.txt', name: 'one.txt', size: 1 }, - { key: 'workspace/workspace-1/two.txt', name: 'two.txt', size: 1 }, - { key: 'workspace/workspace-1/three.txt', name: 'three.txt', size: 1 }, - ], - }) - ) - - expect(response.status).toBe(413) - expect(mockDownloadServableFileFromStorage).toHaveBeenCalledTimes(2) - expect(mockDownloadServableFileFromStorage.mock.calls[1][3]).toEqual({ - maxBytes: MAX_FILE_SIZE - 3, - }) - expect(mockUploadWindchillContent).not.toHaveBeenCalled() - }) - - it('stops before storage or Windchill when file ownership is denied', async () => { - mockAssertToolFileAccess.mockResolvedValueOnce( - NextResponse.json({ success: false, error: 'File not found' }, { status: 404 }) - ) - - const response = await POST( - createMockRequest('POST', { - ...BASE_BODY, - operation: 'windchill_upload_primary_content', - documentOid: DOCUMENT_OID, - primaryFile: { - key: 'workspace/other/specification.pdf', - name: 'specification.pdf', - size: 3, - type: 'application/pdf', - }, - }) - ) - - expect(response.status).toBe(404) - expect(mockDownloadServableFileFromStorage).not.toHaveBeenCalled() - expect(mockUploadWindchillContent).not.toHaveBeenCalled() - }) - - it('stores downloads as a UserFile instead of returning inline bytes', async () => { - const response = await POST( - createMockRequest('POST', { - ...BASE_BODY, - operation: 'windchill_download_primary_content', - documentOid: DOCUMENT_OID, - }) - ) - const data = await response.json() - - expect(response.status).toBe(200) - expect(mockResolveWindchillContentUrl).toHaveBeenCalledWith( - expect.objectContaining({ - contentPath: expect.stringContaining('/PrimaryContent'), - }) - ) - expect(mockResolveWindchillContentUrl.mock.calls[0][0].contentPath).not.toContain('$value') - expect(mockDownloadWindchillContent).toHaveBeenCalledWith( - expect.objectContaining({ - url: expect.stringContaining('/WindchillGW/download'), - }) - ) - expect(mockUploadCopilotFile).toHaveBeenCalledWith( - expect.objectContaining({ - buffer: Buffer.from('pdf'), - fileName: 'specification.pdf', - contentType: 'application/pdf', - userId: 'user-1', - }) - ) - expect(data.output.file).toMatchObject({ key: 'copilot/specification.pdf' }) - expect(data.output.content).toBeUndefined() - }) - - it('downloads an attachment through its document-scoped content path', async () => { - const response = await POST( - createMockRequest('POST', { - ...BASE_BODY, - operation: 'windchill_download_attachment', - documentOid: DOCUMENT_OID, - attachmentOid: 'OR:wt.content.ApplicationData:2', - }) - ) - - expect(response.status).toBe(200) - expect(mockResolveWindchillContentUrl).toHaveBeenCalledWith( - expect.objectContaining({ - contentPath: expect.stringContaining("/Attachments('OR%3Awt.content.ApplicationData%3A2')"), - }) - ) - expect(mockDownloadWindchillContent).toHaveBeenCalledWith( - expect.objectContaining({ - url: expect.stringContaining('/WindchillGW/download'), - }) - ) - }) - - it('uses execution storage derived from the bound delegation principal', async () => { - mockBindDelegation.mockResolvedValueOnce({ - kind: 'delegated', - serviceId: 'executor', - subjectUserId: 'user-1', - workspaceId: '550e8400-e29b-41d4-a716-446655440000', - delegationId: 'delegation-1', - audience: 'sim:windchill', - issuedAt: new Date('2026-01-01T00:00:00.000Z'), - expiresAt: new Date('2027-01-01T00:00:00.000Z'), - delegationContext: { - kind: 'workflow_execution', - workflowId: '550e8400-e29b-41d4-a716-446655440001', - executionId: 'execution-1', - }, - }) - mockUploadExecutionFile.mockResolvedValueOnce({ - id: 'file-2', - name: 'specification.pdf', - url: '/api/files/serve?key=execution/specification.pdf', - size: 3, - type: 'application/pdf', - key: 'execution/specification.pdf', - }) - - const response = await POST( - createMockRequest('POST', { - ...BASE_BODY, - operation: 'windchill_download_primary_content', - documentOid: DOCUMENT_OID, - workspaceId: 'forged-workspace', - workflowId: 'forged-workflow', - executionId: 'forged-execution', - }) - ) - - expect(response.status).toBe(200) - expect(mockUploadExecutionFile).toHaveBeenCalledWith( - { - workspaceId: '550e8400-e29b-41d4-a716-446655440000', - workflowId: '550e8400-e29b-41d4-a716-446655440001', - executionId: 'execution-1', - }, - Buffer.from('pdf'), - 'specification.pdf', - 'application/pdf', - 'user-1' - ) - expect(mockUploadCopilotFile).not.toHaveBeenCalled() - }) - - it('preserves sanitized provider status codes', async () => { - mockWindchillMutationRequest.mockRejectedValueOnce( - new MockWindchillProviderError('Windchill rejected the transition', 409) - ) - - const response = await POST( - createMockRequest('POST', { - ...BASE_BODY, - operation: 'windchill_set_lifecycle_state', - documentOid: DOCUMENT_OID, - stateValue: 'RELEASED', - stateDisplay: 'Released', - }) - ) - - expect(response.status).toBe(409) - expect(await response.json()).toEqual({ - success: false, - error: 'Windchill rejected the transition', - }) - }) -}) diff --git a/apps/sim/app/api/tools/windchill/route.ts b/apps/sim/app/api/tools/windchill/route.ts deleted file mode 100644 index a12ed6a0f31..00000000000 --- a/apps/sim/app/api/tools/windchill/route.ts +++ /dev/null @@ -1,648 +0,0 @@ -import { - type BoundWorkflowExecutionDelegatedPrincipal, - requirePrincipalSubjectUserId, -} from '@sim/auth/principal' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import type { - WindchillOperationBody, - WindchillOperationResponse, -} from '@/lib/api/contracts/tools/windchill' -import { windchillOperationContract } from '@/lib/api/contracts/tools/windchill' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { - createInternalSessionOrExecutorAuth, - InternalUnauthenticatedError, -} from '@/lib/api/server/routes' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { uploadCopilotFile } from '@/lib/uploads/contexts/copilot' -import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' -import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' -import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' -import { assertToolFileAccess } from '@/app/api/files/authorization' -import { sanitizeFileName } from '@/executor/constants' -import type { UserFile } from '@/executor/types' -import { - encodeWindchillOid, - normalizeServiceRoot, - normalizeWindchillDocument, - normalizeWindchillDocuments, - sanitizeWindchillError, -} from '@/tools/windchill/utils' -import { - createWindchillSession, - downloadWindchillContent, - resolveWindchillContentUrl, - uploadWindchillContent, - WindchillProviderError, - type WindchillUploadFile, - windchillDocumentUrl, - windchillMutationRequest, -} from '@/tools/windchill/utils.server' - -export const dynamic = 'force-dynamic' -export const maxDuration = 900 - -const logger = createLogger('WindchillAPI') -const windchillSessionOrExecutorAuth = createInternalSessionOrExecutorAuth({ - audience: 'sim:windchill', -}) - -async function authenticateWindchillExecutor( - request: NextRequest -): Promise { - const principal = await windchillSessionOrExecutorAuth.authenticate(request, {}) - if ( - principal.kind !== 'delegated' || - principal.serviceId !== 'executor' || - !principal.delegationContext - ) { - throw new InternalUnauthenticatedError('Authentication required') - } - return { ...principal, delegationContext: principal.delegationContext } -} - -type WindchillRouteOutput = Extract['output'] -type MutationOperation = Exclude< - WindchillRouteOutput['operation'], - | 'windchill_download_attachment' - | 'windchill_download_primary_content' - | 'windchill_upload_attachments' - | 'windchill_upload_primary_content' -> - -const BULK_RESULT_OPERATIONS = [ - 'windchill_create_documents', - 'windchill_update_documents', - 'windchill_check_out_documents', - 'windchill_check_in_documents', - 'windchill_undo_check_out_documents', - 'windchill_revise_documents', - 'windchill_update_document_security_labels', -] as const satisfies readonly MutationOperation[] - -const DELETE_OPERATIONS = [ - 'windchill_delete_document', - 'windchill_delete_documents', -] as const satisfies readonly MutationOperation[] - -type BulkResultOperation = (typeof BULK_RESULT_OPERATIONS)[number] -type DeleteOperation = (typeof DELETE_OPERATIONS)[number] - -function isBulkResultOperation(operation: MutationOperation): operation is BulkResultOperation { - return BULK_RESULT_OPERATIONS.includes(operation as BulkResultOperation) -} - -function isDeleteOperation(operation: MutationOperation): operation is DeleteOperation { - return DELETE_OPERATIONS.includes(operation as DeleteOperation) -} - -function successResponse(output: WindchillRouteOutput) { - const body = { success: true, output } satisfies WindchillOperationResponse - return NextResponse.json(body) -} - -function failureResponse(error: string, status: number) { - const body = { - success: false, - error: sanitizeWindchillError(error), - } satisfies WindchillOperationResponse - return NextResponse.json(body, { status }) -} - -function documentsById(documentOids: string[]) { - return documentOids.map((ID) => ({ ID })) -} - -/** Keeps the media type and drops any `; charset=...` parameters Windchill cannot use. */ -function safeMimeType(value: string | undefined): string { - const mediaType = value?.split(';', 1)[0]?.trim() - if (mediaType && /^[A-Za-z0-9!#$&^_.+-]+\/[A-Za-z0-9!#$&^_.+-]+$/.test(mediaType)) { - return mediaType - } - return 'application/octet-stream' -} - -function mutationOutput( - operation: MutationOperation, - data: unknown, - fallbackIds: string[] -): WindchillRouteOutput { - const documents = normalizeWindchillDocuments(data) - const document = documents[0] ?? normalizeWindchillDocument(data) - const collectionIds = documents - .map((item) => item.id) - .filter((id): id is string => typeof id === 'string') - const returnedIds = - document?.id && !collectionIds.includes(document.id) - ? [document.id, ...collectionIds] - : collectionIds - const affectedIds = returnedIds.length > 0 ? returnedIds : fallbackIds - if (isDeleteOperation(operation)) return { operation, affectedIds: fallbackIds } - if (isBulkResultOperation(operation)) { - return { - operation, - affectedIds, - ...(documents.length > 0 ? { documents } : {}), - } - } - return { - operation, - affectedIds, - ...(document ? { document } : {}), - } -} - -async function executeMutation( - body: Exclude< - WindchillOperationBody, - | { operation: 'windchill_download_primary_content' } - | { operation: 'windchill_upload_primary_content' } - | { operation: 'windchill_download_attachment' } - | { operation: 'windchill_upload_attachments' } - >, - signal: AbortSignal -): Promise { - const session = await createWindchillSession(body, signal) - const root = normalizeServiceRoot(body.baseUrl) - - switch (body.operation) { - case 'windchill_create_document': { - const data = await windchillMutationRequest({ - params: body, - session, - url: `${root}/DocMgmt/Documents`, - method: 'POST', - body: { - ...(body.attributes ?? {}), - Name: body.name, - ...(body.number ? { Number: body.number } : {}), - ...(body.title ? { Title: body.title } : {}), - ...(body.description ? { Description: body.description } : {}), - 'Context@odata.bind': `Containers('${encodeWindchillOid(body.containerOid)}')`, - ...(body.folderOid - ? { 'Folder@odata.bind': `Folders('${encodeWindchillOid(body.folderOid)}')` } - : {}), - }, - signal, - }) - return mutationOutput(body.operation, data, []) - } - case 'windchill_create_documents': { - const data = await windchillMutationRequest({ - params: body, - session, - url: `${root}/DocMgmt/CreateDocuments`, - method: 'POST', - body: { - Documents: body.documents.map((document) => ({ - ...(document.attributes ?? {}), - Name: document.name, - ...(document.number ? { Number: document.number } : {}), - ...(document.title ? { Title: document.title } : {}), - ...(document.description ? { Description: document.description } : {}), - 'Context@odata.bind': `Containers('${encodeWindchillOid(document.containerOid)}')`, - ...(document.folderOid - ? { 'Folder@odata.bind': `Folders('${encodeWindchillOid(document.folderOid)}')` } - : {}), - })), - }, - signal, - }) - return mutationOutput(body.operation, data, []) - } - case 'windchill_update_document': { - const data = await windchillMutationRequest({ - params: body, - session, - url: windchillDocumentUrl(root, body.documentOid), - method: 'PATCH', - body: body.attributes, - signal, - }) - return mutationOutput(body.operation, data, [body.documentOid]) - } - case 'windchill_update_common_properties': { - const data = await windchillMutationRequest({ - params: body, - session, - url: `${windchillDocumentUrl(root, body.documentOid)}/PTC.DocMgmt.UpdateCommonProperties`, - method: 'POST', - body: { Updates: body.commonProperties }, - signal, - }) - return mutationOutput(body.operation, data, [body.documentOid]) - } - case 'windchill_update_documents': { - const data = await windchillMutationRequest({ - params: body, - session, - url: `${root}/DocMgmt/UpdateDocuments`, - method: 'POST', - body: { - Documents: body.documents.map((document) => ({ - ...document.attributes, - ID: document.id, - })), - }, - signal, - }) - return mutationOutput( - body.operation, - data, - body.documents.map((document) => document.id) - ) - } - case 'windchill_delete_document': { - const data = await windchillMutationRequest({ - params: body, - session, - url: windchillDocumentUrl(root, body.documentOid), - method: 'DELETE', - signal, - }) - return mutationOutput(body.operation, data, [body.documentOid]) - } - case 'windchill_delete_documents': { - const data = await windchillMutationRequest({ - params: body, - session, - url: `${root}/DocMgmt/DeleteDocuments`, - method: 'POST', - body: { Documents: documentsById(body.documentOids) }, - signal, - }) - return mutationOutput(body.operation, data, body.documentOids) - } - case 'windchill_check_out_document': { - const data = await windchillMutationRequest({ - params: body, - session, - url: `${windchillDocumentUrl(root, body.documentOid)}/PTC.DocMgmt.CheckOut`, - method: 'POST', - body: { ...(body.checkOutNote ? { CheckOutNote: body.checkOutNote } : {}) }, - signal, - }) - return mutationOutput(body.operation, data, [body.documentOid]) - } - case 'windchill_check_out_documents': { - const data = await windchillMutationRequest({ - params: body, - session, - url: `${root}/DocMgmt/CheckOutDocuments`, - method: 'POST', - body: { - Documents: documentsById(body.documentOids), - ...(body.checkOutNote ? { CheckOutNote: body.checkOutNote } : {}), - }, - signal, - }) - return mutationOutput(body.operation, data, body.documentOids) - } - case 'windchill_check_in_document': { - const data = await windchillMutationRequest({ - params: body, - session, - url: `${windchillDocumentUrl(root, body.documentOid)}/PTC.DocMgmt.CheckIn`, - method: 'POST', - body: { - ...(body.checkInNote ? { CheckInNote: body.checkInNote } : {}), - ...(body.keepCheckedOut !== undefined ? { KeepCheckedOut: body.keepCheckedOut } : {}), - ...(body.checkOutNote ? { CheckOutNote: body.checkOutNote } : {}), - }, - signal, - }) - return mutationOutput(body.operation, data, [body.documentOid]) - } - case 'windchill_check_in_documents': { - const data = await windchillMutationRequest({ - params: body, - session, - url: `${root}/DocMgmt/CheckInDocuments`, - method: 'POST', - body: { - Documents: documentsById(body.documentOids), - ...(body.checkInNote ? { CheckInNote: body.checkInNote } : {}), - ...(body.keepCheckedOut !== undefined ? { KeepCheckedOut: body.keepCheckedOut } : {}), - ...(body.checkOutNote ? { CheckOutNote: body.checkOutNote } : {}), - }, - signal, - }) - return mutationOutput(body.operation, data, body.documentOids) - } - case 'windchill_undo_check_out_document': { - const data = await windchillMutationRequest({ - params: body, - session, - url: `${windchillDocumentUrl(root, body.documentOid)}/PTC.DocMgmt.UndoCheckOut`, - method: 'POST', - body: {}, - signal, - }) - return mutationOutput(body.operation, data, [body.documentOid]) - } - case 'windchill_undo_check_out_documents': { - const data = await windchillMutationRequest({ - params: body, - session, - url: `${root}/DocMgmt/UndoCheckOutDocuments`, - method: 'POST', - body: { Documents: documentsById(body.documentOids) }, - signal, - }) - return mutationOutput(body.operation, data, body.documentOids) - } - case 'windchill_revise_document': { - const data = await windchillMutationRequest({ - params: body, - session, - url: `${windchillDocumentUrl(root, body.documentOid)}/PTC.DocMgmt.Revise`, - method: 'POST', - body: { ...(body.versionId ? { VersionId: body.versionId } : {}) }, - signal, - }) - return mutationOutput(body.operation, data, [body.documentOid]) - } - case 'windchill_revise_documents': { - const data = await windchillMutationRequest({ - params: body, - session, - url: `${root}/DocMgmt/ReviseDocuments`, - method: 'POST', - body: { - Documents: documentsById(body.documentOids), - }, - signal, - }) - return mutationOutput(body.operation, data, body.documentOids) - } - case 'windchill_set_lifecycle_state': { - const data = await windchillMutationRequest({ - params: body, - session, - url: `${windchillDocumentUrl(root, body.documentOid)}/PTC.DocMgmt.SetState`, - method: 'POST', - body: { State: { Display: body.stateDisplay, Value: body.stateValue } }, - signal, - }) - return mutationOutput(body.operation, data, [body.documentOid]) - } - case 'windchill_update_document_security_labels': { - const data = await windchillMutationRequest({ - params: body, - session, - url: `${root}/DocMgmt/EditDocumentsSecurityLabels`, - method: 'POST', - body: { - Documents: body.securityLabelUpdates.map((update) => ({ - ...update.labels, - ID: update.id, - })), - }, - signal, - }) - return mutationOutput( - body.operation, - data, - body.securityLabelUpdates.map((update) => update.id) - ) - } - } -} - -async function loadUploadFiles( - inputs: RawFileInput[], - userId: string, - requestId: string -): Promise { - let userFiles: UserFile[] - try { - userFiles = processFilesToUserFiles(inputs, requestId, logger) - } catch (error) { - return failureResponse(getErrorMessage(error, 'Invalid file input'), 400) - } - if (userFiles.length !== inputs.length) return failureResponse('Invalid file input', 400) - - const declaredTotal = userFiles.reduce((total, file) => total + file.size, 0) - if (declaredTotal > MAX_FILE_SIZE) { - return failureResponse('Combined Windchill upload exceeds the maximum file size', 413) - } - - const files: WindchillUploadFile[] = [] - let actualTotal = 0 - for (const userFile of userFiles) { - const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) - if (denied) return denied - try { - const servable = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_FILE_SIZE - actualTotal, - }) - actualTotal += servable.buffer.length - if (actualTotal > MAX_FILE_SIZE) { - return failureResponse('Combined Windchill upload exceeds the maximum file size', 413) - } - files.push({ - name: sanitizeFileName(userFile.name), - mimeType: safeMimeType(servable.contentType || userFile.type), - size: servable.buffer.length, - buffer: servable.buffer, - }) - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - return failureResponse( - getErrorMessage(error, 'Failed to read uploaded file'), - isPayloadSizeLimitError(error) ? 413 : 400 - ) - } - } - return files -} - -function contentDispositionFileName(value: string | null): string | null { - if (!value) return null - const encoded = value.match(/filename\*=UTF-8''([^;]+)/i)?.[1] - if (encoded) { - try { - return decodeURIComponent(encoded) - } catch { - return encoded - } - } - return ( - value.match(/filename\s*=\s*"([^"]+)"/i)?.[1] ?? - value.match(/filename\s*=\s*([^;]+)/i)?.[1]?.trim() ?? - null - ) -} - -async function storeDownloadedFile({ - principal, - buffer, - fileName, - contentType, -}: { - principal: BoundWorkflowExecutionDelegatedPrincipal - buffer: Buffer - fileName: string - contentType: string -}): Promise { - const { workflowId, executionId } = principal.delegationContext - if (executionId) { - return uploadExecutionFile( - { - workspaceId: principal.workspaceId, - workflowId, - executionId, - }, - buffer, - fileName, - contentType, - requirePrincipalSubjectUserId(principal) - ) - } - return uploadCopilotFile({ - buffer, - fileName, - contentType, - userId: requirePrincipalSubjectUserId(principal), - }) -} - -async function executeDownload( - body: Extract< - WindchillOperationBody, - | { operation: 'windchill_download_primary_content' } - | { operation: 'windchill_download_attachment' } - >, - principal: BoundWorkflowExecutionDelegatedPrincipal, - signal: AbortSignal -): Promise { - const documentUrl = windchillDocumentUrl(body.baseUrl, body.documentOid) - const contentPath = - body.operation === 'windchill_download_primary_content' - ? `${documentUrl}/PrimaryContent` - : `${documentUrl}/Attachments('${encodeWindchillOid(body.attachmentOid)}')` - const contentUrl = await resolveWindchillContentUrl({ - params: body, - contentPath, - signal, - }) - const downloaded = await downloadWindchillContent({ - params: body, - url: contentUrl, - maxBytes: MAX_FILE_SIZE, - signal, - }) - const fallback = - body.operation === 'windchill_download_primary_content' - ? 'windchill-primary-content.bin' - : 'windchill-attachment.bin' - const fileName = sanitizeFileName( - body.fileName || contentDispositionFileName(downloaded.contentDisposition) || fallback - ) - const mimeType = safeMimeType(downloaded.contentType) - const file = await storeDownloadedFile({ - principal, - buffer: downloaded.buffer, - fileName, - contentType: mimeType, - }) - return { - operation: body.operation, - file: { ...file }, - fileName, - mimeType, - } -} - -export const POST = withRouteHandler( - async (request: NextRequest) => { - const requestId = generateRequestId() - let principal: BoundWorkflowExecutionDelegatedPrincipal - try { - principal = await authenticateWindchillExecutor(request) - } catch (error) { - if (error instanceof InternalUnauthenticatedError) { - return failureResponse(error.message, 401) - } - throw error - } - - const parsed = await parseRequest( - windchillOperationContract, - request, - {}, - { - validationErrorResponse: (error) => - failureResponse(getValidationErrorMessage(error, 'Invalid Windchill request'), 400), - invalidJsonResponse: () => - failureResponse('Windchill request body must be valid JSON', 400), - payloadTooLargeResponse: () => failureResponse('Windchill request body is too large', 413), - } - ) - if (!parsed.success) return parsed.response - const body = parsed.data.body - - try { - if ( - body.operation === 'windchill_download_primary_content' || - body.operation === 'windchill_download_attachment' - ) { - return successResponse(await executeDownload(body, principal, request.signal)) - } - - if ( - body.operation === 'windchill_upload_primary_content' || - body.operation === 'windchill_upload_attachments' - ) { - const inputs = - body.operation === 'windchill_upload_primary_content' - ? [body.primaryFile] - : body.attachmentFiles - const files = await loadUploadFiles( - inputs, - requirePrincipalSubjectUserId(principal), - requestId - ) - if (files instanceof NextResponse) return files - const uploadedFileNames = await uploadWindchillContent({ - params: body, - documentOid: body.documentOid, - files, - primaryContent: body.operation === 'windchill_upload_primary_content', - signal: request.signal, - }) - return successResponse({ - operation: body.operation, - affectedIds: [body.documentOid], - uploadedFileNames, - }) - } - - return successResponse(await executeMutation(body, request.signal)) - } catch (error) { - logger.error('Windchill operation failed', { - operation: body.operation, - error: sanitizeWindchillError(getErrorMessage(error, 'Windchill operation failed')), - }) - if (error instanceof WindchillProviderError) { - const status = error.status >= 400 && error.status <= 599 ? error.status : 502 - return failureResponse(error.message, status) - } - return failureResponse( - getErrorMessage(error, 'Windchill operation failed'), - isPayloadSizeLimitError(error) ? 413 : 500 - ) - } - }, - { - unhandledErrorResponse: () => failureResponse('Windchill operation failed', 500), - } -) diff --git a/apps/sim/app/api/tools/wordpress/upload/route.ts b/apps/sim/app/api/tools/wordpress/upload/route.ts deleted file mode 100644 index b7b9ba8c2a1..00000000000 --- a/apps/sim/app/api/tools/wordpress/upload/route.ts +++ /dev/null @@ -1,215 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { wordpressUploadContract } from '@/lib/api/contracts/storage-transfer' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import { - getFileExtension, - getMimeTypeFromExtension, - processSingleFileToUserFile, -} from '@/lib/uploads/utils/file-utils' -import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' -import { assertToolFileAccess } from '@/app/api/files/authorization' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('WordPressUploadAPI') - -const WORDPRESS_COM_API_BASE = 'https://public-api.wordpress.com/wp/v2/sites' - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success || !authResult.userId) { - logger.warn(`[${requestId}] Unauthorized WordPress upload attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - logger.info( - `[${requestId}] Authenticated WordPress upload request via ${authResult.authType}`, - { - userId: authResult.userId, - } - ) - - const parsed = await parseRequest(wordpressUploadContract, request, {}) - if (!parsed.success) return parsed.response - const validatedData = parsed.data.body - - logger.info(`[${requestId}] Uploading file to WordPress`, { - siteId: validatedData.siteId, - filename: validatedData.filename, - hasFile: !!validatedData.file, - }) - - if (!validatedData.file) { - return NextResponse.json( - { - success: false, - error: 'No file provided. Please upload a file.', - }, - { status: 400 } - ) - } - - const fileData = validatedData.file - - let userFile - try { - userFile = processSingleFileToUserFile(fileData, requestId, logger) - } catch (error) { - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Failed to process file'), - }, - { status: 400 } - ) - } - - const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) - if (denied) return denied - - logger.info(`[${requestId}] Downloading file from storage`, { - fileName: userFile.name, - key: userFile.key, - size: userFile.size, - }) - - let fileBuffer: Buffer - let resolvedContentType: string - - try { - const servable = await downloadServableFileFromStorage(userFile, requestId, logger, { - maxBytes: MAX_BUFFERED_TRANSFER_BYTES, - }) - fileBuffer = servable.buffer - resolvedContentType = servable.contentType - } catch (error) { - const notReady = docNotReadyResponse(error) - if (notReady) return notReady - logger.error(`[${requestId}] Failed to download file:`, error) - return NextResponse.json( - { - success: false, - error: `Failed to download file: ${getErrorMessage(error, 'Unknown error')}`, - }, - { status: isPayloadSizeLimitError(error) ? 413 : 500 } - ) - } - - const filename = validatedData.filename || userFile.name - const mimeType = - resolvedContentType || userFile.type || getMimeTypeFromExtension(getFileExtension(filename)) - - logger.info(`[${requestId}] Uploading to WordPress`, { - siteId: validatedData.siteId, - filename, - mimeType, - size: fileBuffer.length, - }) - - const formData = new FormData() - const uint8Array = new Uint8Array(fileBuffer) - const blob = new Blob([uint8Array], { type: mimeType }) - formData.append('file', blob, filename) - - if (validatedData.title) { - formData.append('title', validatedData.title) - } - if (validatedData.caption) { - formData.append('caption', validatedData.caption) - } - if (validatedData.altText) { - formData.append('alt_text', validatedData.altText) - } - if (validatedData.description) { - formData.append('description', validatedData.description) - } - - const uploadResponse = await fetch(`${WORDPRESS_COM_API_BASE}/${validatedData.siteId}/media`, { - method: 'POST', - headers: { - Authorization: `Bearer ${validatedData.accessToken}`, - }, - body: formData, - }) - - if (!uploadResponse.ok) { - const errorText = await uploadResponse.text() - let errorMessage = `WordPress API error: ${uploadResponse.statusText}` - - try { - const errorJson = JSON.parse(errorText) - errorMessage = errorJson.message || errorJson.error || errorMessage - } catch { - // Use default error message - } - - logger.error(`[${requestId}] WordPress API error:`, { - status: uploadResponse.status, - statusText: uploadResponse.statusText, - error: errorText, - }) - return NextResponse.json( - { - success: false, - error: errorMessage, - }, - { status: uploadResponse.status } - ) - } - - const uploadData = await uploadResponse.json() - - logger.info(`[${requestId}] File uploaded successfully`, { - mediaId: uploadData.id, - sourceUrl: uploadData.source_url, - }) - - return NextResponse.json({ - success: true, - output: { - media: { - id: uploadData.id, - date: uploadData.date, - slug: uploadData.slug, - type: uploadData.type, - link: uploadData.link, - title: uploadData.title, - caption: uploadData.caption, - alt_text: uploadData.alt_text, - media_type: uploadData.media_type, - mime_type: uploadData.mime_type, - source_url: uploadData.source_url, - media_details: uploadData.media_details, - }, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error uploading file to WordPress:`, error) - - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Internal server error'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/workday/assign-onboarding/route.ts b/apps/sim/app/api/tools/workday/assign-onboarding/route.ts deleted file mode 100644 index 96879dcc14c..00000000000 --- a/apps/sim/app/api/tools/workday/assign-onboarding/route.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { workdayAssignOnboardingContract } from '@/lib/api/contracts/tools/workday' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createWorkdaySoapClient, extractRefId, wdRef } from '@/tools/workday/soap' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('WorkdayAssignOnboardingAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(workdayAssignOnboardingContract, request, {}) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - const client = await createWorkdaySoapClient( - data.tenantUrl, - data.tenant, - 'humanResources', - data.username, - data.password - ) - - const [result] = await client.Put_Onboarding_Plan_AssignmentAsync({ - Onboarding_Plan_Assignment_Data: { - Onboarding_Plan_Reference: wdRef('Onboarding_Plan_ID', data.onboardingPlanId), - Person_Reference: wdRef('WID', data.workerId), - Action_Event_Reference: wdRef('WID', data.actionEventId), - Assignment_Effective_Moment: new Date().toISOString(), - Active: true, - }, - }) - - return NextResponse.json({ - success: true, - output: { - assignmentId: extractRefId(result?.Onboarding_Plan_Assignment_Reference), - workerId: data.workerId, - planId: data.onboardingPlanId, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Workday assign onboarding failed`, { error }) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/workday/change-job/route.ts b/apps/sim/app/api/tools/workday/change-job/route.ts deleted file mode 100644 index 4af97d796ed..00000000000 --- a/apps/sim/app/api/tools/workday/change-job/route.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { workdayChangeJobContract } from '@/lib/api/contracts/tools/workday' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createWorkdaySoapClient, extractRefId, wdRef } from '@/tools/workday/soap' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('WorkdayChangeJobAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(workdayChangeJobContract, request, {}) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - const changeJobDetailData: Record = { - Reason_Reference: wdRef('Change_Job_Subcategory_ID', data.reason), - } - if (data.newSupervisoryOrgId) { - changeJobDetailData.Supervisory_Organization_Reference = wdRef( - 'Organization_Reference_ID', - data.newSupervisoryOrgId - ) - } - if (data.newPositionId) { - changeJobDetailData.Proposed_Position_Reference = wdRef('Position_ID', data.newPositionId) - } - const jobDetailsData: Record = {} - if (data.newJobProfileId) { - jobDetailsData.Job_Profile_Reference = wdRef('Job_Profile_ID', data.newJobProfileId) - } - if (data.newLocationId) { - jobDetailsData.Location_Reference = wdRef('Location_ID', data.newLocationId) - } - if (Object.keys(jobDetailsData).length > 0) { - changeJobDetailData.Job_Details_Data = jobDetailsData - } - - const client = await createWorkdaySoapClient( - data.tenantUrl, - data.tenant, - 'staffing', - data.username, - data.password - ) - - const [result] = await client.Change_JobAsync({ - Business_Process_Parameters: { - Auto_Complete: true, - Run_Now: true, - }, - Change_Job_Data: { - Worker_Reference: wdRef('Employee_ID', data.workerId), - Effective_Date: data.effectiveDate, - Change_Job_Detail_Data: changeJobDetailData, - }, - }) - - const eventRef = result?.Event_Reference - - return NextResponse.json({ - success: true, - output: { - eventId: extractRefId(eventRef), - workerId: data.workerId, - effectiveDate: data.effectiveDate, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Workday change job failed`, { error }) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/workday/create-prehire/route.ts b/apps/sim/app/api/tools/workday/create-prehire/route.ts deleted file mode 100644 index 7ed61b992f8..00000000000 --- a/apps/sim/app/api/tools/workday/create-prehire/route.ts +++ /dev/null @@ -1,126 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { workdayCreatePrehireContract } from '@/lib/api/contracts/tools/workday' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createWorkdaySoapClient, extractRefId, wdRef } from '@/tools/workday/soap' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('WorkdayCreatePrehireAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(workdayCreatePrehireContract, request, {}) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - if (!data.email && !data.phoneNumber && !data.address) { - return NextResponse.json( - { - success: false, - error: 'At least one contact method (email, phone, or address) is required', - }, - { status: 400 } - ) - } - - const parts = data.legalName.trim().split(/\s+/) - const firstName = parts[0] ?? '' - const lastName = parts.length > 1 ? parts.slice(1).join(' ') : '' - - if (!lastName) { - return NextResponse.json( - { success: false, error: 'Legal name must include both a first name and last name' }, - { status: 400 } - ) - } - - const client = await createWorkdaySoapClient( - data.tenantUrl, - data.tenant, - 'recruiting', - data.username, - data.password - ) - - const contactData: Record = {} - if (data.email) { - contactData.Email_Address_Data = [ - { - Email_Address: data.email, - Usage_Data: { - Type_Data: { Type_Reference: wdRef('Communication_Usage_Type_ID', 'WORK') }, - Public: true, - }, - }, - ] - } - if (data.phoneNumber) { - contactData.Phone_Data = [ - { - Phone_Number: data.phoneNumber, - Phone_Device_Type_Reference: wdRef('Phone_Device_Type_ID', 'Landline'), - Usage_Data: { - Type_Data: { Type_Reference: wdRef('Communication_Usage_Type_ID', 'WORK') }, - Public: true, - }, - }, - ] - } - if (data.address) { - contactData.Address_Data = [ - { - Formatted_Address: data.address, - Usage_Data: { - Type_Data: { Type_Reference: wdRef('Communication_Usage_Type_ID', 'WORK') }, - Public: true, - }, - }, - ] - } - - const [result] = await client.Put_ApplicantAsync({ - Applicant_Data: { - Personal_Data: { - Name_Data: { - Legal_Name_Data: { - Name_Detail_Data: { - Country_Reference: wdRef('ISO_3166-1_Alpha-2_Code', data.countryCode ?? 'US'), - First_Name: firstName, - Last_Name: lastName, - }, - }, - }, - Contact_Information_Data: contactData, - }, - }, - }) - - const applicantRef = result?.Applicant_Reference - - return NextResponse.json({ - success: true, - output: { - preHireId: extractRefId(applicantRef), - descriptor: applicantRef?.attributes?.Descriptor ?? null, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Workday create prehire failed`, { error }) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/workday/get-compensation/route.ts b/apps/sim/app/api/tools/workday/get-compensation/route.ts deleted file mode 100644 index eb57a04418e..00000000000 --- a/apps/sim/app/api/tools/workday/get-compensation/route.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { workdayGetCompensationContract } from '@/lib/api/contracts/tools/workday' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createWorkdaySoapClient, - extractRefId, - normalizeSoapArray, - parseSoapNumber, - type WorkdayCompensationDataSoap, - type WorkdayCompensationPlanSoap, - type WorkdayWorkerSoap, -} from '@/tools/workday/soap' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('WorkdayGetCompensationAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(workdayGetCompensationContract, request, {}) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - const client = await createWorkdaySoapClient( - data.tenantUrl, - data.tenant, - 'humanResources', - data.username, - data.password - ) - - const [result] = await client.Get_WorkersAsync({ - Request_References: { - Worker_Reference: { - ID: { attributes: { 'wd:type': 'Employee_ID' }, $value: data.workerId }, - }, - }, - Response_Group: { - Include_Reference: true, - Include_Compensation: true, - }, - }) - - const worker = - normalizeSoapArray( - result?.Response_Data?.Worker as WorkdayWorkerSoap | WorkdayWorkerSoap[] | undefined - )[0] ?? null - const compensationData = worker?.Worker_Data?.Compensation_Data - - const mapPlan = (p: WorkdayCompensationPlanSoap) => ({ - id: extractRefId(p.Compensation_Plan_Reference) ?? null, - planName: p.Compensation_Plan_Reference?.attributes?.Descriptor ?? null, - amount: - parseSoapNumber(p.Amount) ?? - parseSoapNumber(p.Per_Unit_Amount) ?? - parseSoapNumber(p.Individual_Target_Amount) ?? - null, - currency: extractRefId(p.Currency_Reference) ?? null, - frequency: extractRefId(p.Frequency_Reference) ?? null, - }) - - const planTypeKeys: (keyof WorkdayCompensationDataSoap)[] = [ - 'Employee_Base_Pay_Plan_Assignment_Data', - 'Employee_Salary_Unit_Plan_Assignment_Data', - 'Employee_Bonus_Plan_Assignment_Data', - 'Employee_Allowance_Plan_Assignment_Data', - 'Employee_Commission_Plan_Assignment_Data', - 'Employee_Stock_Plan_Assignment_Data', - 'Employee_Period_Salary_Plan_Assignment_Data', - ] - - const compensationPlans: ReturnType[] = [] - for (const key of planTypeKeys) { - for (const plan of normalizeSoapArray(compensationData?.[key])) { - compensationPlans.push(mapPlan(plan)) - } - } - - return NextResponse.json({ - success: true, - output: { compensationPlans }, - }) - } catch (error) { - logger.error(`[${requestId}] Workday get compensation failed`, { error }) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/workday/get-organizations/route.ts b/apps/sim/app/api/tools/workday/get-organizations/route.ts deleted file mode 100644 index 7758ec415bd..00000000000 --- a/apps/sim/app/api/tools/workday/get-organizations/route.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { workdayGetOrganizationsContract } from '@/lib/api/contracts/tools/workday' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createWorkdaySoapClient, - extractRefId, - normalizeSoapArray, - parseSoapBoolean, - parseSoapNumber, - type WorkdayOrganizationSoap, -} from '@/tools/workday/soap' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('WorkdayGetOrganizationsAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(workdayGetOrganizationsContract, request, {}) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - const client = await createWorkdaySoapClient( - data.tenantUrl, - data.tenant, - 'humanResources', - data.username, - data.password - ) - - const limit = data.limit ?? 20 - const offset = data.offset ?? 0 - const page = offset > 0 ? Math.floor(offset / limit) + 1 : 1 - - const [result] = await client.Get_OrganizationsAsync({ - Response_Filter: { Page: page, Count: limit }, - Request_Criteria: data.type - ? { - Organization_Type_Reference: { - ID: { - attributes: { 'wd:type': 'Organization_Type_ID' }, - $value: data.type, - }, - }, - } - : undefined, - Response_Group: { Include_Hierarchy_Data: true }, - }) - - const orgsArray = normalizeSoapArray( - result?.Response_Data?.Organization as - | WorkdayOrganizationSoap - | WorkdayOrganizationSoap[] - | undefined - ) - - const organizations = orgsArray.map((o) => { - const inactive = parseSoapBoolean(o.Organization_Data?.Inactive) - return { - id: extractRefId(o.Organization_Reference) ?? null, - descriptor: o.Organization_Descriptor ?? null, - type: extractRefId(o.Organization_Data?.Organization_Type_Reference) ?? null, - subtype: extractRefId(o.Organization_Data?.Organization_Subtype_Reference) ?? null, - isActive: inactive == null ? null : !inactive, - } - }) - - const total = parseSoapNumber(result?.Response_Results?.Total_Results) ?? organizations.length - - return NextResponse.json({ - success: true, - output: { organizations, total }, - }) - } catch (error) { - logger.error(`[${requestId}] Workday get organizations failed`, { error }) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/workday/get-worker/route.ts b/apps/sim/app/api/tools/workday/get-worker/route.ts deleted file mode 100644 index 96fe04b884b..00000000000 --- a/apps/sim/app/api/tools/workday/get-worker/route.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { workdayGetWorkerContract } from '@/lib/api/contracts/tools/workday' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createWorkdaySoapClient, - extractRefId, - normalizeSoapArray, - type WorkdayWorkerSoap, -} from '@/tools/workday/soap' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('WorkdayGetWorkerAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(workdayGetWorkerContract, request, {}) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - const client = await createWorkdaySoapClient( - data.tenantUrl, - data.tenant, - 'humanResources', - data.username, - data.password - ) - - const [result] = await client.Get_WorkersAsync({ - Request_References: { - Worker_Reference: { - ID: { attributes: { 'wd:type': 'Employee_ID' }, $value: data.workerId }, - }, - }, - Response_Group: { - Include_Reference: true, - Include_Personal_Information: true, - Include_Employment_Information: true, - Include_Compensation: true, - Include_Organizations: true, - }, - }) - - const worker = - normalizeSoapArray( - result?.Response_Data?.Worker as WorkdayWorkerSoap | WorkdayWorkerSoap[] | undefined - )[0] ?? null - - return NextResponse.json({ - success: true, - output: { - worker: worker - ? { - id: extractRefId(worker.Worker_Reference) ?? null, - descriptor: worker.Worker_Descriptor ?? null, - personalData: worker.Worker_Data?.Personal_Data ?? null, - employmentData: worker.Worker_Data?.Employment_Data ?? null, - compensationData: worker.Worker_Data?.Compensation_Data ?? null, - organizationData: worker.Worker_Data?.Organization_Data ?? null, - } - : null, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Workday get worker failed`, { error }) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/workday/hire/route.ts b/apps/sim/app/api/tools/workday/hire/route.ts deleted file mode 100644 index 9bc24a6b745..00000000000 --- a/apps/sim/app/api/tools/workday/hire/route.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { workdayHireContract } from '@/lib/api/contracts/tools/workday' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createWorkdaySoapClient, extractRefId, wdRef } from '@/tools/workday/soap' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('WorkdayHireAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(workdayHireContract, request, {}) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - const client = await createWorkdaySoapClient( - data.tenantUrl, - data.tenant, - 'staffing', - data.username, - data.password - ) - - const [result] = await client.Hire_EmployeeAsync({ - Business_Process_Parameters: { - Auto_Complete: true, - Run_Now: true, - }, - Hire_Employee_Data: { - Applicant_Reference: wdRef('Applicant_ID', data.preHireId), - Position_Reference: wdRef('Position_ID', data.positionId), - Hire_Date: data.hireDate, - Hire_Employee_Event_Data: { - Employee_Type_Reference: wdRef('Employee_Type_ID', data.employeeType ?? 'Regular'), - First_Day_of_Work: data.hireDate, - }, - }, - }) - - const employeeRef = result?.Employee_Reference - const eventRef = result?.Event_Reference - - return NextResponse.json({ - success: true, - output: { - workerId: extractRefId(employeeRef), - employeeId: extractRefId(employeeRef), - eventId: extractRefId(eventRef), - hireDate: data.hireDate, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Workday hire employee failed`, { error }) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/workday/list-workers/route.ts b/apps/sim/app/api/tools/workday/list-workers/route.ts deleted file mode 100644 index 878a4048f15..00000000000 --- a/apps/sim/app/api/tools/workday/list-workers/route.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { workdayListWorkersContract } from '@/lib/api/contracts/tools/workday' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createWorkdaySoapClient, - extractRefId, - normalizeSoapArray, - parseSoapNumber, - type WorkdayWorkerSoap, -} from '@/tools/workday/soap' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('WorkdayListWorkersAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(workdayListWorkersContract, request, {}) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - const client = await createWorkdaySoapClient( - data.tenantUrl, - data.tenant, - 'humanResources', - data.username, - data.password - ) - - const limit = data.limit ?? 20 - const offset = data.offset ?? 0 - const page = offset > 0 ? Math.floor(offset / limit) + 1 : 1 - - const [result] = await client.Get_WorkersAsync({ - Response_Filter: { Page: page, Count: limit }, - Response_Group: { - Include_Reference: true, - Include_Personal_Information: true, - Include_Employment_Information: true, - }, - }) - - const workersArray = normalizeSoapArray( - result?.Response_Data?.Worker as WorkdayWorkerSoap | WorkdayWorkerSoap[] | undefined - ) - - const workers = workersArray.map((w) => ({ - id: extractRefId(w.Worker_Reference) ?? null, - descriptor: w.Worker_Descriptor ?? null, - personalData: w.Worker_Data?.Personal_Data ?? null, - employmentData: w.Worker_Data?.Employment_Data ?? null, - })) - - const total = parseSoapNumber(result?.Response_Results?.Total_Results) ?? workers.length - - return NextResponse.json({ - success: true, - output: { workers, total }, - }) - } catch (error) { - logger.error(`[${requestId}] Workday list workers failed`, { error }) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/workday/terminate/route.ts b/apps/sim/app/api/tools/workday/terminate/route.ts deleted file mode 100644 index 9548af4467b..00000000000 --- a/apps/sim/app/api/tools/workday/terminate/route.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { workdayTerminateContract } from '@/lib/api/contracts/tools/workday' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createWorkdaySoapClient, extractRefId, wdRef } from '@/tools/workday/soap' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('WorkdayTerminateAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(workdayTerminateContract, request, {}) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - const client = await createWorkdaySoapClient( - data.tenantUrl, - data.tenant, - 'staffing', - data.username, - data.password - ) - - const [result] = await client.Terminate_EmployeeAsync({ - Business_Process_Parameters: { - Auto_Complete: true, - Run_Now: true, - }, - Terminate_Employee_Data: { - Employee_Reference: wdRef('Employee_ID', data.workerId), - Termination_Date: data.terminationDate, - Terminate_Event_Data: { - Primary_Reason_Reference: wdRef('Termination_Subcategory_ID', data.reason), - Last_Day_of_Work: data.lastDayOfWork ?? data.terminationDate, - Notification_Date: data.notificationDate ?? data.terminationDate, - }, - }, - }) - - const eventRef = result?.Event_Reference - - return NextResponse.json({ - success: true, - output: { - eventId: extractRefId(eventRef), - workerId: data.workerId, - terminationDate: data.terminationDate, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Workday terminate employee failed`, { error }) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/workday/update-worker/route.ts b/apps/sim/app/api/tools/workday/update-worker/route.ts deleted file mode 100644 index e3e5f7757dc..00000000000 --- a/apps/sim/app/api/tools/workday/update-worker/route.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { workdayUpdateWorkerContract } from '@/lib/api/contracts/tools/workday' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createWorkdaySoapClient, extractRefId, wdRef } from '@/tools/workday/soap' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('WorkdayUpdateWorkerAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(workdayUpdateWorkerContract, request, {}) - if (!parsed.success) return parsed.response - const data = parsed.data.body - - const client = await createWorkdaySoapClient( - data.tenantUrl, - data.tenant, - 'humanResources', - data.username, - data.password - ) - - const [result] = await client.Change_Personal_InformationAsync({ - Business_Process_Parameters: { - Auto_Complete: true, - Run_Now: true, - }, - Change_Personal_Information_Business_Process_Data: { - Person_Reference: wdRef('Employee_ID', data.workerId), - Personal_Information_Data: data.fields, - }, - }) - - return NextResponse.json({ - success: true, - output: { - eventId: extractRefId(result?.Personal_Information_Change_Event_Reference), - workerId: data.workerId, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Workday update worker failed`, { error }) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Unknown error') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/zoho_desk/attachment/route.ts b/apps/sim/app/api/tools/zoho_desk/attachment/route.ts deleted file mode 100644 index 0104619fad9..00000000000 --- a/apps/sim/app/api/tools/zoho_desk/attachment/route.ts +++ /dev/null @@ -1,143 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { zohoDeskGetAttachmentContract } from '@/lib/api/contracts/tools/zoho-desk' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' -import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { isZohoHost } from '@/tools/zoho_desk/host-allowlist' -import { - buildZohoDeskHeaders, - deriveAttachmentName, - getZohoDeskApiBase, - resolveZohoAttachmentUrl, -} from '@/tools/zoho_desk/utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('ZohoDeskAttachmentAPI') - -/** - * Ceiling on a downloaded attachment. - * - * This route returns the file base64-encoded inside its JSON body, and the - * executor reads an internal tool response through `readToolResponseBody`, which - * caps at `MAX_TOOL_RESPONSE_BODY_BYTES` (10 MB). Base64 inflates by 4/3, so the - * largest attachment that can actually survive the round trip is ~7.5 MB of raw - * bytes. A larger ceiling here is not merely useless - it is actively harmful: - * the route would download, encode, and serialize the whole file (peaking around - * 250 MB of live allocation for a 50 MB attachment, with nothing limiting - * concurrent downloads) only for the executor to reject the oversized body - * afterwards. Capping at the reachable size makes the transport limit enforce - * itself early, while the bytes are still being streamed. - * - * Raising this requires uploading in the route and returning a file reference - * instead of inline base64, the way the WhatsApp media and Typeform file routes - * do - not a bigger number here. - */ -const MAX_ATTACHMENT_BYTES = 7 * 1024 * 1024 - -export const POST = withRouteHandler(async (request: NextRequest) => { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - return NextResponse.json({ success: false, error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest(zohoDeskGetAttachmentContract, request, {}) - if (!parsed.success) return parsed.response - const { accessToken, apiDomain, orgId, href, fileName } = parsed.data.body - - let downloadUrl: URL - try { - downloadUrl = resolveZohoAttachmentUrl( - href, - getZohoDeskApiBase({ apiDomain: apiDomain ?? undefined }) - ) - } catch { - return NextResponse.json({ success: false, error: 'Invalid attachment href' }, { status: 400 }) - } - - if (downloadUrl.protocol !== 'https:' || !isZohoHost(downloadUrl.hostname)) { - return NextResponse.json( - { success: false, error: 'Attachment href must be an https Zoho URL' }, - { status: 400 } - ) - } - - try { - // Even though the initial host is allowlisted, the download URL is - // user/LLM-influenced and Zoho may redirect. secureFetchWithValidation pins - // the resolved IP, blocks private/reserved targets on every hop, and - // (stripAuthOnRedirect) drops the OAuth token if a redirect leaves the - // original origin, so the credential never reaches an untrusted host. - // maxResponseBytes enforces the size cap while streaming. - const response = await secureFetchWithValidation(downloadUrl.toString(), { - method: 'GET', - headers: buildZohoDeskHeaders({ accessToken, orgId }), - timeout: 30_000, - maxResponseBytes: MAX_ATTACHMENT_BYTES, - stripAuthOnRedirect: true, - }) - - if (!response.ok) { - logger.warn('Failed to download Zoho Desk attachment', { status: response.status }) - return NextResponse.json( - { success: false, error: `Failed to download attachment (HTTP ${response.status})` }, - { status: response.status >= 400 && response.status < 500 ? response.status : 502 } - ) - } - - // A 204 (or any non-200 success) carries no body, so arrayBuffer() would - // yield zero bytes and the route would report success with an empty file. - if (response.status !== 200) { - return NextResponse.json( - { success: false, error: `Attachment returned no content (HTTP ${response.status})` }, - { status: 502 } - ) - } - - const arrayBuffer = await response.arrayBuffer() - - // ToolFileData (consumed by FileToolProcessor) keys the file name as `name`. - const name = deriveAttachmentName( - fileName, - response.headers.get('content-disposition'), - downloadUrl.pathname - ) - const mimeType = response.headers.get('content-type') || 'application/octet-stream' - - return NextResponse.json({ - success: true, - output: { - file: { - data: Buffer.from(arrayBuffer).toString('base64'), - mimeType, - name, - }, - }, - }) - } catch (error) { - // An oversized attachment is a client-visible limit, not a server fault - - // surface it as 413 with the actual ceiling, mirroring the WhatsApp media - // route, instead of collapsing it into a generic 500. - if (isPayloadSizeLimitError(error)) { - logger.warn('Zoho Desk attachment exceeds the download limit', { - maxBytes: MAX_ATTACHMENT_BYTES, - }) - return NextResponse.json( - { - success: false, - error: `Attachment exceeds the ${Math.floor(MAX_ATTACHMENT_BYTES / (1024 * 1024))} MB download limit`, - }, - { status: 413 } - ) - } - logger.error('Error downloading Zoho Desk attachment', { error: getErrorMessage(error) }) - return NextResponse.json( - { success: false, error: getErrorMessage(error, 'Failed to download attachment') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/zoom/get-recordings/route.ts b/apps/sim/app/api/tools/zoom/get-recordings/route.ts deleted file mode 100644 index 12da2faa429..00000000000 --- a/apps/sim/app/api/tools/zoom/get-recordings/route.ts +++ /dev/null @@ -1,211 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { zoomGetRecordingsContract } from '@/lib/api/contracts/tools/zoom' -import { parseRequest } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getExtensionFromMimeType } from '@/lib/uploads/utils/file-utils' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('ZoomGetRecordingsAPI') - -interface ZoomRecordingFile { - id?: string - meeting_id?: string - recording_start?: string - recording_end?: string - file_type?: string - file_extension?: string - file_size?: number - play_url?: string - download_url?: string - status?: string - recording_type?: string -} - -interface ZoomRecordingsResponse { - uuid?: string - id?: string | number - account_id?: string - host_id?: string - topic?: string - type?: number - start_time?: string - duration?: number - total_size?: number - recording_count?: number - share_url?: string - recording_files?: ZoomRecordingFile[] -} - -interface ZoomErrorResponse { - message?: string - code?: number -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized Zoom get recordings attempt: ${authResult.error}`) - return NextResponse.json( - { - success: false, - error: authResult.error || 'Authentication required', - }, - { status: 401 } - ) - } - - const parsed = await parseRequest(zoomGetRecordingsContract, request, {}) - if (!parsed.success) return parsed.response - - const { accessToken, meetingId, includeFolderItems, ttl, downloadFiles } = parsed.data.body - - const baseUrl = `https://api.zoom.us/v2/meetings/${encodeURIComponent(meetingId)}/recordings` - const queryParams = new URLSearchParams() - - if (includeFolderItems != null) { - queryParams.append('include_folder_items', String(includeFolderItems)) - } - if (ttl) { - queryParams.append('ttl', String(ttl)) - } - - const queryString = queryParams.toString() - const apiUrl = queryString ? `${baseUrl}?${queryString}` : baseUrl - - logger.info(`[${requestId}] Fetching recordings from Zoom`, { meetingId }) - - const urlValidation = await validateUrlWithDNS(apiUrl, 'apiUrl') - if (!urlValidation.isValid) { - return NextResponse.json({ success: false, error: urlValidation.error }, { status: 400 }) - } - - const response = await secureFetchWithPinnedIP(apiUrl, urlValidation.resolvedIP!, { - method: 'GET', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - - if (!response.ok) { - const errorData = (await response.json().catch(() => ({}))) as ZoomErrorResponse - logger.error(`[${requestId}] Zoom API error`, { - status: response.status, - error: errorData, - }) - return NextResponse.json( - { success: false, error: errorData.message || `Zoom API error: ${response.status}` }, - { status: 400 } - ) - } - - const data = (await response.json()) as ZoomRecordingsResponse - const files: Array<{ - name: string - mimeType: string - data: string - size: number - }> = [] - - if (downloadFiles && Array.isArray(data.recording_files)) { - for (const file of data.recording_files) { - if (!file?.download_url) continue - - try { - const fileUrlValidation = await validateUrlWithDNS(file.download_url, 'downloadUrl') - if (!fileUrlValidation.isValid) continue - - const downloadResponse = await secureFetchWithPinnedIP( - file.download_url, - fileUrlValidation.resolvedIP!, - { - method: 'GET', - headers: { Authorization: `Bearer ${accessToken}` }, - } - ) - - if (!downloadResponse.ok) continue - - const contentType = - downloadResponse.headers.get('content-type') || 'application/octet-stream' - const arrayBuffer = await downloadResponse.arrayBuffer() - const buffer = Buffer.from(arrayBuffer) - const extension = - file.file_extension?.toString().toLowerCase() || - getExtensionFromMimeType(contentType) || - 'dat' - const fileName = `zoom-recording-${file.id || file.recording_start || Date.now()}.${extension}` - - files.push({ - name: fileName, - mimeType: contentType, - data: buffer.toString('base64'), - size: buffer.length, - }) - } catch (error) { - logger.warn(`[${requestId}] Failed to download recording file:`, error) - } - } - } - - logger.info(`[${requestId}] Zoom recordings fetched successfully`, { - recordingCount: data.recording_files?.length || 0, - downloadedCount: files.length, - }) - - return NextResponse.json({ - success: true, - output: { - recording: { - uuid: data.uuid, - id: data.id, - account_id: data.account_id, - host_id: data.host_id, - topic: data.topic, - type: data.type, - start_time: data.start_time, - duration: data.duration, - total_size: data.total_size, - recording_count: data.recording_count, - share_url: data.share_url, - recording_files: (data.recording_files || []).map((file: ZoomRecordingFile) => ({ - id: file.id, - meeting_id: file.meeting_id, - recording_start: file.recording_start, - recording_end: file.recording_end, - file_type: file.file_type, - file_extension: file.file_extension, - file_size: file.file_size, - play_url: file.play_url, - download_url: file.download_url, - status: file.status, - recording_type: file.recording_type, - })), - }, - files: files.length > 0 ? files : undefined, - }, - }) - } catch (error) { - logger.error(`[${requestId}] Error fetching Zoom recordings:`, error) - return NextResponse.json( - { - success: false, - error: getErrorMessage(error, 'Unknown error occurred'), - }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/tools/zoominfo/proxy/route.ts b/apps/sim/app/api/tools/zoominfo/proxy/route.ts deleted file mode 100644 index 510fe9b2d77..00000000000 --- a/apps/sim/app/api/tools/zoominfo/proxy/route.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { getValidationErrorMessage, isZodError } from '@/lib/api/server' -import { checkInternalAuth } from '@/lib/auth/hybrid' -import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' -import { generateRequestId } from '@/lib/core/utils/request' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - assertSafeZoomInfoUrl, - extractZoomInfoError, - fetchZoomInfoAccessToken, - ZOOMINFO_API_BASE, - ZOOMINFO_OUTBOUND_FETCH_TIMEOUT_MS, - type ZoomInfoProxyRequest, - ZoomInfoProxyRequestSchema, -} from '@/app/api/tools/zoominfo/shared' - -export const dynamic = 'force-dynamic' - -const logger = createLogger('ZoomInfoProxyAPI') - -function buildApiUrl(req: ZoomInfoProxyRequest): string { - const subPath = req.path.startsWith('/') ? req.path : `/${req.path}` - const url = `${ZOOMINFO_API_BASE}${subPath}` - - if (!req.query || Object.keys(req.query).length === 0) { - return url - } - const search = new URLSearchParams() - for (const [key, value] of Object.entries(req.query)) { - if (value === undefined || value === null) continue - search.append(key, String(value)) - } - const queryString = search.toString() - if (!queryString) return url - return url.includes('?') ? `${url}&${queryString}` : `${url}?${queryString}` -} - -interface Invocation { - status: number - body: unknown -} - -async function callZoomInfo(req: ZoomInfoProxyRequest, accessToken: string): Promise { - const url = assertSafeZoomInfoUrl(buildApiUrl(req), 'apiUrl').toString() - const hasBody = req.body !== undefined && req.body !== null - const headers: Record = { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - } - if (hasBody) headers['Content-Type'] = 'application/json' - - const response = await secureFetchWithValidation( - url, - { - method: req.method, - headers, - body: hasBody - ? typeof req.body === 'string' - ? req.body - : JSON.stringify(req.body) - : undefined, - timeout: ZOOMINFO_OUTBOUND_FETCH_TIMEOUT_MS, - }, - 'apiUrl' - ) - - const raw = await response.text() - let parsed: unknown = null - if (raw.length > 0) { - try { - parsed = JSON.parse(raw) - } catch { - parsed = raw - } - } - return { status: response.status, body: parsed } -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - const requestId = generateRequestId() - - try { - const authResult = await checkInternalAuth(request, { requireWorkflowId: false }) - if (!authResult.success) { - logger.warn(`[${requestId}] Unauthorized ZoomInfo proxy request: ${authResult.error}`) - return NextResponse.json( - { success: false, error: authResult.error || 'Authentication required' }, - { status: 401 } - ) - } - - // boundary-raw-json: internal proxy envelope validated by ZoomInfoProxyRequestSchema below; not a public boundary - const json = await request.json() - const proxyReq = ZoomInfoProxyRequestSchema.parse(json) - - const accessToken = await fetchZoomInfoAccessToken(proxyReq, requestId) - const invocation = await callZoomInfo(proxyReq, accessToken) - - if (invocation.status >= 200 && invocation.status < 300) { - const data = invocation.status === 204 ? null : invocation.body - return NextResponse.json({ success: true, output: { status: invocation.status, data } }) - } - - const message = extractZoomInfoError(invocation.body, invocation.status) - logger.warn( - `[${requestId}] ZoomInfo API error (${invocation.status}) ${proxyReq.path}: ${message}` - ) - return NextResponse.json( - { success: false, error: message, status: invocation.status }, - { status: invocation.status } - ) - } catch (error) { - if (isZodError(error)) { - logger.warn(`[${requestId}] Validation error:`, error.issues) - return NextResponse.json( - { success: false, error: getValidationErrorMessage(error, 'Validation failed') }, - { status: 400 } - ) - } - logger.error(`[${requestId}] Unexpected ZoomInfo proxy error:`, error) - return NextResponse.json({ success: false, error: toError(error).message }, { status: 500 }) - } -}) diff --git a/apps/sim/app/api/tools/zoominfo/shared.ts b/apps/sim/app/api/tools/zoominfo/shared.ts deleted file mode 100644 index 15a3092327f..00000000000 --- a/apps/sim/app/api/tools/zoominfo/shared.ts +++ /dev/null @@ -1,190 +0,0 @@ -import { createHash } from 'node:crypto' -import { createLogger } from '@sim/logger' -import { isPrivateIpHost } from '@sim/security/ssrf' -import { z } from 'zod' -import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' - -const logger = createLogger('ZoomInfoShared') - -export const ZOOMINFO_API_BASE = 'https://api.zoominfo.com/gtm' -export const ZOOMINFO_TOKEN_URL = `${ZOOMINFO_API_BASE}/oauth/v1/token` -export const ZOOMINFO_OUTBOUND_FETCH_TIMEOUT_MS = 30_000 - -export const ZoomInfoAuthSchema = z.object({ - clientId: z.string().min(1, 'clientId is required'), - clientSecret: z.string().min(1, 'clientSecret is required'), -}) - -export type ZoomInfoAuth = z.infer - -export const ZoomInfoHttpMethod = z.enum(['GET', 'POST', 'PUT', 'PATCH', 'DELETE']) - -export const ZoomInfoProxyPath = z - .string() - .min(1, 'path is required') - .refine( - (p) => - !p.split(/[/\\]/).some((seg) => seg === '..' || seg === '.') && - !p.includes('#') && - !/%(?:2[eEfF]|5[cC]|23)/.test(p), - { - message: - 'path must not contain ".." or "." segments, "#", or percent-encoded path/fragment characters', - } - ) - -export const ZoomInfoProxyRequestSchema = ZoomInfoAuthSchema.extend({ - path: ZoomInfoProxyPath, - method: ZoomInfoHttpMethod.default('POST'), - query: z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])).optional(), - body: z.unknown().optional(), -}) - -export type ZoomInfoProxyRequest = z.infer - -const FORBIDDEN_HOSTS = new Set([ - 'localhost', - '0.0.0.0', - '127.0.0.1', - '169.254.169.254', - 'metadata.google.internal', - 'metadata', - '[::1]', - '[::]', -]) - -export function assertSafeZoomInfoUrl(rawUrl: string, label: string): URL { - let parsed: URL - try { - parsed = new URL(rawUrl) - } catch { - throw new Error(`${label} must be a valid URL`) - } - if (parsed.protocol !== 'https:') { - throw new Error(`${label} must use https://`) - } - const host = parsed.hostname.toLowerCase() - if (FORBIDDEN_HOSTS.has(host)) { - throw new Error(`${label} host is not allowed`) - } - if (isPrivateIpHost(host)) { - throw new Error(`${label} host is not allowed (private/loopback range)`) - } - if (host !== 'api.zoominfo.com') { - throw new Error(`${label} host must be api.zoominfo.com`) - } - return parsed -} - -interface CachedToken { - accessToken: string - expiresAt: number -} - -const TOKEN_CACHE = new Map() -const TOKEN_CACHE_MAX_ENTRIES = 500 -const TOKEN_SAFETY_WINDOW_MS = 60_000 - -function tokenCacheKey(auth: ZoomInfoAuth): string { - const secretHash = createHash('sha256').update(auth.clientSecret).digest('hex').slice(0, 16) - return `${auth.clientId}::${secretHash}` -} - -function rememberToken(key: string, token: CachedToken): void { - if (TOKEN_CACHE.has(key)) TOKEN_CACHE.delete(key) - TOKEN_CACHE.set(key, token) - while (TOKEN_CACHE.size > TOKEN_CACHE_MAX_ENTRIES) { - const oldestKey = TOKEN_CACHE.keys().next().value - if (oldestKey === undefined) break - TOKEN_CACHE.delete(oldestKey) - } -} - -export async function fetchZoomInfoAccessToken( - auth: ZoomInfoAuth, - requestId: string -): Promise { - const cacheKey = tokenCacheKey(auth) - const cached = TOKEN_CACHE.get(cacheKey) - if (cached && cached.expiresAt - TOKEN_SAFETY_WINDOW_MS > Date.now()) { - return cached.accessToken - } - - const tokenUrl = assertSafeZoomInfoUrl(ZOOMINFO_TOKEN_URL, 'tokenUrl').toString() - const basic = Buffer.from(`${auth.clientId}:${auth.clientSecret}`).toString('base64') - - const params = new URLSearchParams() - params.set('grant_type', 'client_credentials') - - const response = await secureFetchWithValidation( - tokenUrl, - { - method: 'POST', - headers: { - Authorization: `Basic ${basic}`, - 'Content-Type': 'application/x-www-form-urlencoded', - Accept: 'application/json', - }, - body: params.toString(), - timeout: ZOOMINFO_OUTBOUND_FETCH_TIMEOUT_MS, - }, - 'tokenUrl' - ) - - if (!response.ok) { - const text = await response.text().catch(() => '') - logger.warn(`[${requestId}] ZoomInfo token fetch failed (${response.status}): ${text}`) - throw new Error(`ZoomInfo token request failed: HTTP ${response.status}`) - } - - const data = (await response.json()) as { - access_token?: string - expires_in?: number - } - - if (!data.access_token) { - throw new Error('ZoomInfo token response missing access_token') - } - - const expiresInMs = (data.expires_in ?? 3300) * 1000 - rememberToken(cacheKey, { - accessToken: data.access_token, - expiresAt: Date.now() + expiresInMs, - }) - return data.access_token -} - -export function extractZoomInfoError(body: unknown, status: number): string { - if (body && typeof body === 'object') { - const obj = body as Record - if (obj.error && typeof obj.error === 'object') { - const eo = obj.error as Record - const message = typeof eo.message === 'string' ? eo.message : '' - const code = typeof eo.code === 'string' ? eo.code : '' - if (message) return code ? `[${code}] ${message}` : message - } - if (typeof obj.error === 'string' && obj.error.length > 0) { - const desc = typeof obj.error_description === 'string' ? `: ${obj.error_description}` : '' - return `${obj.error}${desc}` - } - if (typeof obj.message === 'string' && obj.message.length > 0) { - return obj.message - } - if (Array.isArray(obj.errors) && obj.errors.length > 0) { - return obj.errors - .map((e) => { - if (e && typeof e === 'object') { - const eo = e as Record - const title = typeof eo.title === 'string' ? eo.title : '' - const detail = typeof eo.detail === 'string' ? `: ${eo.detail}` : '' - return `${title}${detail}`.trim() - } - return String(e) - }) - .filter(Boolean) - .join('; ') - } - } - if (typeof body === 'string' && body.length > 0) return body - return `ZoomInfo request failed with HTTP ${status}` -} diff --git a/apps/sim/app/api/users/me/usage-logs/export/route.test.ts b/apps/sim/app/api/users/me/usage-logs/export/route.test.ts index 87854ca15c7..fd4f01a2588 100644 --- a/apps/sim/app/api/users/me/usage-logs/export/route.test.ts +++ b/apps/sim/app/api/users/me/usage-logs/export/route.test.ts @@ -7,6 +7,7 @@ import { apportionCredits } from '@/lib/billing/credits/conversion' const { mockGetUserUsageLogs, mockGetUsageCreditsByLogId } = vi.hoisted(() => ({ mockGetUserUsageLogs: vi.fn(), + /** Still mocked because the module exports it; the export route must never call it. */ mockGetUsageCreditsByLogId: vi.fn(), })) @@ -47,7 +48,6 @@ describe('GET /api/users/me/usage-logs/export', () => { summary: { totalCost: 0.5, bySource: { copilot: 0.5 } }, pagination: { hasMore: false }, }) - mockGetUsageCreditsByLogId.mockResolvedValue(apportionCredits([{ key: 'log-1', dollars: 0.5 }])) const response = await GET(createMockRequest('GET')) const csv = await response.text() @@ -108,10 +108,6 @@ describe('GET /api/users/me/usage-logs/export', () => { 'user-1', expect.objectContaining({ source: ['copilot', 'workspace-chat'] }) ) - expect(mockGetUsageCreditsByLogId).toHaveBeenCalledWith( - 'user-1', - expect.objectContaining({ source: ['copilot', 'workspace-chat'] }) - ) }) it('names the specific workflow for workflow-sourced rows', async () => { @@ -208,7 +204,9 @@ describe('GET /api/users/me/usage-logs/export', () => { expect(csv.split('\n')).toHaveLength(3) }) - it('apportions credits once over the whole filtered set, not per page', async () => { + it('apportions across pages so the printed rows reconcile to the printed total', async () => { + // 0.002 + 0.002 = 0.004 -> 0.8 credits -> 1 credit for the whole file. Rounding each + // row alone would print 0 and 0; the largest-remainder split prints 1 and 0. mockGetUserUsageLogs .mockResolvedValueOnce({ logs: [ @@ -224,16 +222,35 @@ describe('GET /api/users/me/usage-logs/export', () => { summary: { totalCost: 0, bySource: {} }, pagination: { hasMore: false }, }) - mockGetUsageCreditsByLogId.mockResolvedValue( - apportionCredits([ - { key: 'log-1', dollars: 0.002 }, - { key: 'log-2', dollars: 0.002 }, - ]) + + const csv = await (await GET(createMockRequest('GET'))).text() + const printed = csv + .split('\n') + .slice(1) + .map((line) => Number(line.split(',')[2])) + + expect(printed.reduce((sum, credits) => sum + credits, 0)).toBe( + Object.values( + apportionCredits([ + { key: 'log-1', dollars: 0.002 }, + { key: 'log-2', dollars: 0.002 }, + ]) + ).reduce((sum, credits) => sum + credits, 0) ) + }) + + it('never issues the whole-filter apportionment read, so the safety cap actually bounds it', async () => { + // The cap stops the paging loop; a second unbounded read beside it would make that + // cap meaningless, and with `period=all` it is a full lifetime scan. + mockGetUserUsageLogs.mockResolvedValueOnce({ + logs: [{ id: 'log-1', createdAt: '2026-07-01T00:00:00.000Z', source: 'copilot', cost: 0.5 }], + summary: { totalCost: 0, bySource: {} }, + pagination: { hasMore: false }, + }) await GET(createMockRequest('GET')) - expect(mockGetUsageCreditsByLogId).toHaveBeenCalledTimes(1) + expect(mockGetUsageCreditsByLogId).not.toHaveBeenCalled() }) it('stops at exactly the safety cap without an extra wasted page fetch', async () => { diff --git a/apps/sim/app/api/users/me/usage-logs/export/route.ts b/apps/sim/app/api/users/me/usage-logs/export/route.ts index e9c20256455..97fc7e3bffd 100644 --- a/apps/sim/app/api/users/me/usage-logs/export/route.ts +++ b/apps/sim/app/api/users/me/usage-logs/export/route.ts @@ -3,7 +3,8 @@ import { type NextRequest, NextResponse } from 'next/server' import { exportUsageLogsContract } from '@/lib/api/contracts/user' import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' -import { getUsageCreditsByLogId, getUserUsageLogs } from '@/lib/billing/core/usage-log' +import { getUserUsageLogs } from '@/lib/billing/core/usage-log' +import { apportionCredits } from '@/lib/billing/credits/conversion' import { BILLING_USAGE_LOG_SOURCE_LABELS, toBillingUsageLogSource, @@ -70,7 +71,23 @@ export const GET = withRouteHandler(async (request: NextRequest) => { cursorCreatedAt = lastRow ? new Date(lastRow.createdAt) : undefined } - const creditsByLogId = await getUsageCreditsByLogId(auth.userId, filter) + /** + * Apportioned over the rows this file actually contains, not over a second + * unbounded read of the whole filter. + * + * `getUsageCreditsByLogId` exists for the paginated list, where a row must show the + * same value on every page and only the full set can guarantee that. An export has + * no pages: it already holds every row it will print. Re-reading the filter here + * made {@link EXPORT_SAFETY_CAP} illusory — the loop stopped at the cap and the very + * next statement loaded every matching row anyway, which with `period=all` is an + * unbounded lifetime scan. + * + * The values are unchanged in the ordinary case, because an untruncated export's row + * set *is* the whole filter. When it truncates they are strictly better: the printed + * rows now reconcile to the printed total instead of to one that includes rows the + * file does not contain. + */ + const creditsByLogId = apportionCredits(rows.map((log) => ({ key: log.id, dollars: log.cost }))) if (truncated) { logger.error('Usage log export hit the safety cap — investigate this account', { diff --git a/apps/sim/app/api/v1/admin/organizations/route.ts b/apps/sim/app/api/v1/admin/organizations/route.ts index ecc446207d3..497d0cad173 100644 --- a/apps/sim/app/api/v1/admin/organizations/route.ts +++ b/apps/sim/app/api/v1/admin/organizations/route.ts @@ -19,6 +19,10 @@ * - ownerId: string - User ID of the organization owner (required) * * Response: AdminSingleResponse + * + * Creates the organization and its owner membership, and nothing else. Attaching + * or creating a workspace for it is deliberately not done here — see the note on + * `adminV1CreateOrganizationContract`. */ import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' diff --git a/apps/sim/app/api/v2/knowledge/search/route.ts b/apps/sim/app/api/v2/knowledge/search/route.ts index 9ee4f6c9244..4cb0bb11edb 100644 --- a/apps/sim/app/api/v2/knowledge/search/route.ts +++ b/apps/sim/app/api/v2/knowledge/search/route.ts @@ -7,11 +7,7 @@ import { searchKnowledge } from '@/lib/knowledge/application/search' export const dynamic = 'force-dynamic' export const revalidate = 0 -/** - * Mirrors the internal Knowledge-search cap in `app/api/knowledge/search/route.ts` - * so the public surface is never more permissive than the internal one. Kept as a - * literal because the internal route declares the same literal inline. - */ +/** Keeps public Knowledge search request materialization bounded to 2 MiB. */ export const V2_KNOWLEDGE_SEARCH_MAX_BODY_BYTES = 2 * 1024 * 1024 /** POST /api/v2/knowledge/search — Vector / tag search across knowledge bases. */ diff --git a/apps/sim/app/api/workspaces/[id]/permissions/route.test.ts b/apps/sim/app/api/workspaces/[id]/permissions/route.test.ts index b163b2bb428..36683982480 100644 --- a/apps/sim/app/api/workspaces/[id]/permissions/route.test.ts +++ b/apps/sim/app/api/workspaces/[id]/permissions/route.test.ts @@ -73,7 +73,7 @@ function queuePersonalWorkspace( ) { const workspaceRow = { ownerId: OWNER_ID, billedAccountUserId, organizationId: null } queueTableRows(schemaMock.workspace, [workspaceRow]) - /** The in-transaction re-read of the same row, taken `FOR UPDATE`. */ + /** The in-transaction re-read of the same row, taken `FOR NO KEY UPDATE`. */ permissionsMockFns.mockGetWorkspaceWithOwner.mockResolvedValue({ id: WORKSPACE_ID, ...workspaceRow, diff --git a/apps/sim/app/f/[token]/public-file-view.tsx b/apps/sim/app/f/[token]/public-file-view.tsx index 8247cfe24d7..5f4cd049cb2 100644 --- a/apps/sim/app/f/[token]/public-file-view.tsx +++ b/apps/sim/app/f/[token]/public-file-view.tsx @@ -114,6 +114,7 @@ export function PublicFileView({ contentSource={source} canEdit={false} readOnly + enableFind /> diff --git a/apps/sim/app/organization/[organizationId]/settings/[section]/page.tsx b/apps/sim/app/organization/[organizationId]/settings/[section]/page.tsx deleted file mode 100644 index ab2f2fce4ae..00000000000 --- a/apps/sim/app/organization/[organizationId]/settings/[section]/page.tsx +++ /dev/null @@ -1,72 +0,0 @@ -import type { Metadata } from 'next' -import { notFound, redirect } from 'next/navigation' -import { - getOrganizationSettingsFeatures, - getSettingsSectionMeta, - isOrganizationSettingsSectionAvailable, - ORGANIZATION_SETTINGS_ITEMS, - ORGANIZATION_SETTINGS_PATH_ALIASES, - parseSettingsPathSection, -} from '@/components/settings/navigation' -import { OrganizationSettingsRenderer } from '@/components/settings/organization-settings-renderer' -import { SettingsUnavailable } from '@/components/settings/settings-unavailable' -import { getSession } from '@/lib/auth' -import { isOrganizationOnEnterprisePlan } from '@/lib/billing' -import { canOpenOrganizationSettingsSection } from '@/lib/organizations/settings-access' - -interface OrganizationSettingsSectionPageProps { - params: Promise<{ organizationId: string; section: string }> -} - -export async function generateMetadata({ - params, -}: OrganizationSettingsSectionPageProps): Promise { - const { section } = await params - const parsed = parseSettingsPathSection({ - path: section, - items: ORGANIZATION_SETTINGS_ITEMS, - defaultSection: null, - aliases: ORGANIZATION_SETTINGS_PATH_ALIASES, - }) - const meta = parsed ? getSettingsSectionMeta('organization', parsed) : null - return { title: meta ? `${meta.label} - Organization settings` : 'Organization settings' } -} - -export default async function OrganizationSettingsSectionPage({ - params, -}: OrganizationSettingsSectionPageProps) { - const session = await getSession() - if (!session?.user) redirect('/login') - - const { organizationId, section } = await params - const parsed = parseSettingsPathSection({ - path: section, - items: ORGANIZATION_SETTINGS_ITEMS, - defaultSection: null, - aliases: ORGANIZATION_SETTINGS_PATH_ALIASES, - }) - if (!parsed) notFound() - - const canOpen = await canOpenOrganizationSettingsSection(organizationId, session.user.id, parsed) - if (!canOpen) return - const hasEnterprisePlan = - parsed !== 'members' && - parsed !== 'billing' && - (await isOrganizationOnEnterprisePlan(organizationId)) - if ( - !isOrganizationSettingsSectionAvailable( - parsed, - getOrganizationSettingsFeatures(hasEnterprisePlan) - ) - ) { - return ( - - ) - } - - return -} diff --git a/apps/sim/app/organization/[organizationId]/settings/layout.tsx b/apps/sim/app/organization/[organizationId]/settings/layout.tsx deleted file mode 100644 index 12dce2f96e3..00000000000 --- a/apps/sim/app/organization/[organizationId]/settings/layout.tsx +++ /dev/null @@ -1,35 +0,0 @@ -import { redirect } from 'next/navigation' -import { SettingsUnavailable } from '@/components/settings/settings-unavailable' -import { StandaloneSettingsShell } from '@/components/settings/standalone-settings-shell' -import { getSession } from '@/lib/auth' -import { isOrganizationOnEnterprisePlan } from '@/lib/billing' -import { getOrganizationSettingsAccess } from '@/lib/organizations/settings-access' - -interface OrganizationSettingsLayoutProps { - children: React.ReactNode - params: Promise<{ organizationId: string }> -} - -export default async function OrganizationSettingsLayout({ - children, - params, -}: OrganizationSettingsLayoutProps) { - const session = await getSession() - if (!session?.user) redirect('/login') - - const { organizationId } = await params - const access = await getOrganizationSettingsAccess(organizationId, session.user.id) - if (!access.isMember) return - const hasEnterprisePlan = access.isAdmin && (await isOrganizationOnEnterprisePlan(organizationId)) - - return ( - - {children} - - ) -} diff --git a/apps/sim/app/organization/[organizationId]/settings/page.tsx b/apps/sim/app/organization/[organizationId]/settings/page.tsx deleted file mode 100644 index c0d5c917a48..00000000000 --- a/apps/sim/app/organization/[organizationId]/settings/page.tsx +++ /dev/null @@ -1,11 +0,0 @@ -import { redirect } from 'next/navigation' -import { getOrganizationSettingsHref } from '@/components/settings/navigation' - -interface OrganizationSettingsPageProps { - params: Promise<{ organizationId: string }> -} - -export default async function OrganizationSettingsPage({ params }: OrganizationSettingsPageProps) { - const { organizationId } = await params - redirect(getOrganizationSettingsHref(organizationId, 'members')) -} diff --git a/apps/sim/app/organization/[organizationId]/settings/unavailable/page.tsx b/apps/sim/app/organization/[organizationId]/settings/unavailable/page.tsx deleted file mode 100644 index 9e4bd26f677..00000000000 --- a/apps/sim/app/organization/[organizationId]/settings/unavailable/page.tsx +++ /dev/null @@ -1,5 +0,0 @@ -import { SettingsUnavailable } from '@/components/settings/settings-unavailable' - -export default function OrganizationSettingsUnavailablePage() { - return -} diff --git a/apps/sim/app/workspace/[workspaceId]/components/find-bar/use-find-shortcut.ts b/apps/sim/app/workspace/[workspaceId]/components/find-bar/use-find-shortcut.ts new file mode 100644 index 00000000000..f0bd842f6c9 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/components/find-bar/use-find-shortcut.ts @@ -0,0 +1,46 @@ +'use client' + +import type React from 'react' +import { useEffect } from 'react' + +interface UseFindShortcutOptions { + /** + * Whether this surface currently owns Cmd/Ctrl+F. Every find surface binds its own listener, so + * exactly one owner may be enabled at a time — the surfaces arbitrate by mounting (the Files list + * disables itself while a file is open, and the file editor enables itself only where the document + * is the page), by an embed flag (the table grid), or by DOM containment (the browser session). + * Two enabled owners mounted at once would race, and first-registered would win. + */ + enabled: boolean + /** The find bar's input, focused and selected once the bar opens. */ + inputRef: React.RefObject + onOpen: () => void +} + +/** + * Binds Cmd/Ctrl+F to open a find bar, overriding the browser's own find. + * + * Listens on the document rather than a container so the shortcut answers before anything inside the + * surface has been focused — a file that has only been opened, never clicked into, still responds. + * A press another surface already consumed is left alone (`defaultPrevented`), and any chord with a + * further modifier falls through to the browser, so Cmd+Shift+F and Cmd+Alt+F keep their meanings. + */ +export function useFindShortcut({ enabled, inputRef, onOpen }: UseFindShortcutOptions): void { + useEffect(() => { + if (!enabled) return + const handleFindShortcut = (event: KeyboardEvent) => { + if (!(event.metaKey || event.ctrlKey) || event.altKey || event.shiftKey) return + if (event.key.toLowerCase() !== 'f') return + if (event.defaultPrevented) return + event.preventDefault() + onOpen() + // After the open has painted the bar, so there is an input to focus. + requestAnimationFrame(() => { + inputRef.current?.focus() + inputRef.current?.select() + }) + } + document.addEventListener('keydown', handleFindShortcut) + return () => document.removeEventListener('keydown', handleFindShortcut) + }, [enabled, inputRef, onOpen]) +} diff --git a/apps/sim/app/workspace/[workspaceId]/components/index.ts b/apps/sim/app/workspace/[workspaceId]/components/index.ts index 95adab9b7bb..9791199d107 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/components/index.ts @@ -8,6 +8,7 @@ export type { ErrorBoundaryProps, ErrorStateProps } from './error' export { ErrorShell, ErrorState } from './error' export type { FindBarProps } from './find-bar/find-bar' export { FindBar } from './find-bar/find-bar' +export { useFindShortcut } from './find-bar/use-find-shortcut' export { InlineRenameInput } from './inline-rename-input' export { IntegrationTabsHeader } from './integration-tabs-header' export { MessageActions } from './message-actions' diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx index 9b878147309..d41df8897e2 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx @@ -127,6 +127,13 @@ interface FileViewerProps { * untitled, so the caller can name the file after it. Only wired for the editable markdown editor. */ onDeriveTitleFromHeading?: (headingText: string) => void + /** + * Let an open markdown file claim Cmd/Ctrl+F for find-in-document. Set wherever the file is the + * whole pane the user is reading — the Files page, the mothership file view, the public share + * page. Left off for the streaming-file preview, which is a pane beside a conversation that owns + * its own find. See {@link RichMarkdownEditorProps.enableFind}. + */ + enableFind?: boolean } export function FileViewer(props: FileViewerProps) { @@ -165,6 +172,7 @@ function FileViewerContent({ previewContextKey, collaborative, onDeriveTitleFromHeading, + enableFind = false, }: FileViewerProps) { const category = resolveFileCategory(file.type, file.name) @@ -181,7 +189,13 @@ function FileViewerContent({ // the bubble menu, and every other editing affordance. if (isMarkdownFile(file)) { return ( - + ) } return @@ -212,6 +226,7 @@ function FileViewerContent({ previewContextKey={previewContextKey} collaborative={collaborative} onDeriveTitleFromHeading={onDeriveTitleFromHeading} + enableFind={enableFind} /> ) } diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions.ts index ef9f7f6cf5d..771a242ab2e 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/editor-extensions.ts @@ -15,6 +15,7 @@ import { } from './collaboration/caret-presence' import { LinkEmbed } from './embed/link-embed' import { createMarkdownContentExtensions } from './extensions' +import { RichMarkdownFind } from './find' import { ResizableImage } from './image' import { RichMarkdownKeymap } from './keymap' import { MarkdownPaste } from './markdown-paste' @@ -48,8 +49,8 @@ interface MarkdownEditorExtensionOptions { * The full extension set for the live editor: the content extensions with their React node-view nodes * injected (code-block language picker, resizable image, mention chip) plus the UI-only extensions — * `CodeBlockHighlight` (Prism), `SlashCommand` (the `/` block menu), `Mention` (the `@` menu), - * `RichMarkdownKeymap`, `MarkdownPaste`, `Placeholder`, and — when `embeds` is set — `LinkEmbed` - * (media players for standalone links). + * `RichMarkdownKeymap`, `MarkdownPaste`, `Placeholder`, `RichMarkdownFind` (the Cmd/Ctrl+F match + * highlights), and — when `embeds` is set — `LinkEmbed` (media players for standalone links). * * Kept separate from `extensions.ts` so those node views (and the block registry the mention chip pulls * in for brand icons) stay out of the headless round-trip path, which only needs the schema. @@ -94,6 +95,7 @@ export function createMarkdownEditorExtensions({ ] : []), CodeBlockHighlight, + RichMarkdownFind, SlashCommand, Mention, RichMarkdownKeymap, diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.test.ts new file mode 100644 index 00000000000..2f730b40771 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.test.ts @@ -0,0 +1,132 @@ +/** + * @vitest-environment jsdom + */ +import { Editor } from '@tiptap/core' +import { undoDepth } from '@tiptap/pm/history' +import { afterEach, describe, expect, it } from 'vitest' +import { createMarkdownContentExtensions } from '../extensions' +import { getFindTally, RichMarkdownFind, setFindQuery, stepFindMatch } from './find-extension' + +let editor: Editor | null = null +afterEach(() => { + editor?.destroy() + editor = null +}) + +function mountEditor(markdown: string): Editor { + const element = document.createElement('div') + document.body.append(element) + editor = new Editor({ + element, + extensions: [...createMarkdownContentExtensions(), RichMarkdownFind], + }) + editor.commands.setContent(markdown, { contentType: 'markdown' }) + return editor +} + +/** The painted highlights, in document order, with the active one marked. */ +function paintedMatches(instance: Editor): string[] { + return Array.from(instance.view.dom.querySelectorAll('.rich-find-match')).map((element) => + element.classList.contains('rich-find-match-active') + ? `[${element.textContent}]` + : (element.textContent ?? '') + ) +} + +describe('RichMarkdownFind', () => { + it('paints nothing until a term is set', () => { + const instance = mountEditor('alpha beta alpha') + expect(paintedMatches(instance)).toEqual([]) + expect(getFindTally(instance.state).matches).toHaveLength(0) + }) + + it('paints every match and marks the first one active', () => { + const instance = mountEditor('alpha beta alpha') + setFindQuery(instance, 'alpha') + expect(paintedMatches(instance)).toEqual(['[alpha]', 'alpha']) + }) + + it('steps the active match forward and backward, wrapping at both ends', () => { + const instance = mountEditor('one one one') + setFindQuery(instance, 'one') + + stepFindMatch(instance, 1) + expect(paintedMatches(instance)).toEqual(['one', '[one]', 'one']) + + stepFindMatch(instance, 1) + expect(paintedMatches(instance)).toEqual(['one', 'one', '[one]']) + + // Past the end wraps to the first, and back past the start wraps to the last. + stepFindMatch(instance, 1) + expect(paintedMatches(instance)).toEqual(['[one]', 'one', 'one']) + stepFindMatch(instance, -1) + expect(paintedMatches(instance)).toEqual(['one', 'one', '[one]']) + }) + + it('re-searches when the document changes under a live search', () => { + const instance = mountEditor('alpha') + setFindQuery(instance, 'alpha') + expect(getFindTally(instance.state).matches).toHaveLength(1) + + instance.commands.insertContentAt(instance.state.doc.content.size, ' and alpha again') + expect(getFindTally(instance.state).matches).toHaveLength(2) + expect(paintedMatches(instance)).toEqual(['[alpha]', 'alpha']) + }) + + it('drops a match the document no longer contains, without leaving a stale highlight', () => { + const instance = mountEditor('alpha beta') + setFindQuery(instance, 'beta') + expect(paintedMatches(instance)).toEqual(['[beta]']) + + instance.commands.setContent('alpha only', { contentType: 'markdown' }) + expect(paintedMatches(instance)).toEqual([]) + expect(getFindTally(instance.state).matches).toHaveLength(0) + }) + + it('clamps the active index when an edit shrinks the match set', () => { + const instance = mountEditor('x x x') + setFindQuery(instance, 'x') + stepFindMatch(instance, 2) + expect(getFindTally(instance.state).activeIndex).toBe(2) + + instance.commands.setContent('x', { contentType: 'markdown' }) + const tally = getFindTally(instance.state) + expect(tally.matches).toHaveLength(1) + expect(tally.activeIndex).toBe(0) + expect(paintedMatches(instance)).toEqual(['[x]']) + }) + + it('searches a term applied before any other transaction', () => { + // The hook re-applies a pending term the moment the editor exists; setting a query as the very + // first thing that happens to a fresh editor must land, not wait for a later transaction. + const instance = mountEditor('alpha beta') + setFindQuery(instance, 'beta') + expect(getFindTally(instance.state).matches).toHaveLength(1) + expect(paintedMatches(instance)).toEqual(['[beta]']) + }) + + it('clears every highlight when the term is emptied', () => { + const instance = mountEditor('alpha') + setFindQuery(instance, 'alpha') + expect(paintedMatches(instance)).toEqual(['[alpha]']) + + setFindQuery(instance, '') + expect(paintedMatches(instance)).toEqual([]) + }) + + it('never writes to the document, the selection, or the undo history', () => { + const instance = mountEditor('alpha beta alpha') + const before = instance.getMarkdown() + const selectionBefore = instance.state.selection.from + const undoBefore = undoDepth(instance.state) + + setFindQuery(instance, 'alpha') + stepFindMatch(instance, 1) + + expect(instance.getMarkdown()).toBe(before) + expect(instance.state.selection.from).toBe(selectionBefore) + // A search that added an undo step would make the user's next Cmd+Z undo the search + // instead of their real last edit. + expect(undoDepth(instance.state)).toBe(undoBefore) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.ts new file mode 100644 index 00000000000..e2ac3f0a20a --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-extension.ts @@ -0,0 +1,151 @@ +import type { Editor } from '@tiptap/core' +import { Extension } from '@tiptap/core' +import type { EditorState } from '@tiptap/pm/state' +import { Plugin, PluginKey } from '@tiptap/pm/state' +import { Decoration, DecorationSet } from '@tiptap/pm/view' +import { EMPTY_FIND_RESULT, type FindMatch, findMatches } from './find-matches' + +/** Class on every match. The active one carries {@link ACTIVE_MATCH_CLASS} as well. */ +const MATCH_CLASS = 'rich-find-match' + +/** Class on the one match the bar is currently pointing at. Also how the hook finds it to scroll to. */ +export const ACTIVE_MATCH_CLASS = 'rich-find-match-active' + +interface RichMarkdownFindState { + query: string + matches: readonly FindMatch[] + truncated: boolean + /** 0-based index into `matches`. Meaningless, but still 0, when there are none. */ + activeIndex: number + /** + * The rendered highlights, built here rather than in the `decorations` prop. ProseMirror asks for + * decorations on every view update — including caret moves and remote cursor traffic — so building + * them there would rebuild all 500 for transactions that changed nothing about the search. + */ + decorations: DecorationSet +} + +/** What the surface reads back off the plugin. */ +export type FindTally = Pick + +/** Transaction meta the surface sets to drive the search. Absent fields keep their current value. */ +interface FindMeta { + query?: string + /** Any integer; wrapped into range against the match count, so stepping past either end cycles. */ + activeIndex?: number +} + +const RICH_FIND_PLUGIN_KEY = new PluginKey('richMarkdownFind') + +const INITIAL_STATE: RichMarkdownFindState = { + query: '', + matches: EMPTY_FIND_RESULT.matches, + truncated: false, + activeIndex: 0, + decorations: DecorationSet.empty, +} + +function wrapIndex(index: number, length: number): number { + if (length === 0) return 0 + return ((index % length) + length) % length +} + +function buildDecorations( + doc: EditorState['doc'], + matches: readonly FindMatch[], + activeIndex: number +): DecorationSet { + if (matches.length === 0) return DecorationSet.empty + return DecorationSet.create( + doc, + matches.map((match, index) => + Decoration.inline(match.from, match.to, { + class: index === activeIndex ? `${MATCH_CLASS} ${ACTIVE_MATCH_CLASS}` : MATCH_CLASS, + }) + ) + ) +} + +/** + * Renders the find highlights over the document, and owns the match set they are built from. + * + * The surface never passes matches in — it sets only the term and the active index as transaction + * meta ({@link setFindQuery}, {@link stepFindMatch}) and reads the resulting state back with + * {@link getFindTally}. Keeping the search here is what makes it survive editing: the plugin + * re-searches on any transaction that changed the document, so a highlight can never be left + * pointing at a position the edit moved or deleted (a stale `Decoration` on a removed range throws). + * + * Decorations are inline and add no node to the document, so the search leaves the markdown — and + * the collaborative Y.Doc behind it — completely untouched. Nothing here dispatches, and with no + * term set every branch short-circuits to the untouched previous state, so the plugin is inert + * until a find bar opens. + */ +export const RichMarkdownFind = Extension.create({ + name: 'richMarkdownFind', + + addProseMirrorPlugins() { + return [ + new Plugin({ + key: RICH_FIND_PLUGIN_KEY, + state: { + init: () => INITIAL_STATE, + apply(transaction, value, _oldState, newState) { + const meta = transaction.getMeta(RICH_FIND_PLUGIN_KEY) as FindMeta | undefined + const query = meta?.query ?? value.query + const requestedIndex = meta?.activeIndex ?? value.activeIndex + const unsearched = query.trim().length === 0 + if (unsearched && value.matches.length === 0) { + return query === value.query && value.activeIndex === 0 + ? value + : { ...INITIAL_STATE, query } + } + if (!transaction.docChanged && query === value.query) { + const activeIndex = wrapIndex(requestedIndex, value.matches.length) + if (activeIndex === value.activeIndex) return value + return { + ...value, + activeIndex, + decorations: buildDecorations(newState.doc, value.matches, activeIndex), + } + } + const { matches, truncated } = unsearched + ? EMPTY_FIND_RESULT + : findMatches(newState.doc, query) + const activeIndex = wrapIndex(requestedIndex, matches.length) + return { + query, + matches, + truncated, + activeIndex, + decorations: buildDecorations(newState.doc, matches, activeIndex), + } + }, + }, + props: { + decorations: (state) => RICH_FIND_PLUGIN_KEY.getState(state)?.decorations ?? null, + }, + }), + ] + }, +}) + +/** The match set, cap flag and active index the find bar renders from. */ +export function getFindTally(state: EditorState): FindTally { + return RICH_FIND_PLUGIN_KEY.getState(state) ?? INITIAL_STATE +} + +function dispatchFindMeta(editor: Editor, meta: FindMeta): void { + // `setMeta` alone leaves the transaction with no steps, so this never touches the document, + // the undo history, or the collaborative document. + editor.view.dispatch(editor.state.tr.setMeta(RICH_FIND_PLUGIN_KEY, meta)) +} + +/** Searches for `query` and makes its first match active. An empty term clears the highlights. */ +export function setFindQuery(editor: Editor, query: string): void { + dispatchFindMeta(editor, { query, activeIndex: 0 }) +} + +/** Moves the active match by `delta`, cycling past either end. */ +export function stepFindMatch(editor: Editor, delta: number): void { + dispatchFindMeta(editor, { activeIndex: getFindTally(editor.state).activeIndex + delta }) +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.test.ts new file mode 100644 index 00000000000..b62f392cea7 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.test.ts @@ -0,0 +1,87 @@ +/** + * @vitest-environment jsdom + */ +import { Editor } from '@tiptap/core' +import { afterEach, describe, expect, it } from 'vitest' +import { createMarkdownContentExtensions } from '../extensions' +import { FIND_MATCH_LIMIT, findMatches } from './find-matches' + +let editor: Editor | null = null +afterEach(() => { + editor?.destroy() + editor = null +}) + +/** Parses markdown through the real schema, so matches are checked against real document positions. */ +function docFor(markdown: string) { + editor = new Editor({ extensions: createMarkdownContentExtensions() }) + editor.commands.setContent(markdown, { contentType: 'markdown' }) + return editor.state.doc +} + +/** The text each match actually covers — the only assertion that proves the positions are right. */ +function matchedText(markdown: string, query: string): string[] { + const doc = docFor(markdown) + return findMatches(doc, query).matches.map((match) => doc.textBetween(match.from, match.to)) +} + +describe('findMatches', () => { + it('finds every occurrence across blocks, case-insensitively', () => { + const doc = docFor('# Report\n\nthe report is ready') + const { matches, truncated } = findMatches(doc, 'report') + expect(matches).toHaveLength(2) + expect(truncated).toBe(false) + expect(matches.map((m) => doc.textBetween(m.from, m.to))).toEqual(['Report', 'report']) + }) + + it('returns nothing for an empty, whitespace-only, or unmatched term', () => { + expect(findMatches(docFor('hello'), '').matches).toHaveLength(0) + expect(findMatches(docFor('hello'), ' ').matches).toHaveLength(0) + expect(findMatches(docFor('hello'), 'zzz').matches).toHaveLength(0) + }) + + it('folds whitespace the way the rest of the app\u2019s search does', () => { + // Inherited from `forEachSearchOccurrence`: a typed space matches a non-breaking one, so a + // term copied out of agent-written prose still finds itself. + expect(matchedText('one\u00a0two', 'one two')).toEqual(['one\u00a0two']) + }) + + it('keeps positions correct after a code point that lowercases to two characters', () => { + expect(matchedText('\u0130stanbul and target', 'target')).toEqual(['target']) + }) + + it('matches across a mark boundary within a block', () => { + // `he**llo**` is two text nodes in one paragraph; a per-text-node search would miss it. + expect(matchedText('he**llo** world', 'hello')).toEqual(['hello']) + }) + + it('never matches across a block boundary', () => { + expect(matchedText('ab\n\ncd', 'abcd')).toEqual([]) + }) + + it('never matches across an inline atom', () => { + // The image between them occupies a position; joining `a` to `b` would be a phantom match. + expect(matchedText('a![alt](https://x.com/i.png)b', 'ab')).toEqual([]) + }) + + it('keeps positions correct after an inline atom', () => { + expect(matchedText('![alt](https://x.com/i.png) target', 'target')).toEqual(['target']) + }) + + it('does not overlap matches of a self-overlapping term', () => { + expect(matchedText('aaaa', 'aa')).toEqual(['aa', 'aa']) + }) + + it('caps the match set and reports it as truncated', () => { + const doc = docFor(Array.from({ length: FIND_MATCH_LIMIT + 10 }, () => 'x').join(' ')) + const { matches, truncated } = findMatches(doc, 'x') + expect(matches).toHaveLength(FIND_MATCH_LIMIT) + expect(truncated).toBe(true) + }) + + it('honors a caller-supplied limit', () => { + const { matches, truncated } = findMatches(docFor('x x x x'), 'x', 2) + expect(matches).toHaveLength(2) + expect(truncated).toBe(true) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.ts new file mode 100644 index 00000000000..5133557d52f --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/find-matches.ts @@ -0,0 +1,108 @@ +import { forEachSearchOccurrence } from '@sim/utils/string' +import type { Node as ProseMirrorNode } from '@tiptap/pm/model' + +/** One match, as a document position range that `Decoration.inline` can be built from. */ +export interface FindMatch { + from: number + to: number +} + +export interface FindResult { + matches: readonly FindMatch[] + /** More matches existed than the cap allowed; the tail was dropped. */ + truncated: boolean +} + +/** + * Cap on matches collected per search. A find bar is a navigation affordance, not a report: past a + * few hundred hits the tally stops meaning anything, while the decoration set it would build grows + * with the document. The bar renders the cap as `500+` so the number on screen is never a lie. + */ +export const FIND_MATCH_LIMIT = 500 + +export const EMPTY_FIND_RESULT: FindResult = { matches: [], truncated: false } + +/** + * Stands in for one position of a non-text inline node (an image, a mention chip) so a match can + * never span one — searching `ab` must not join the `a` before an image to the `b` after it. U+FFFF + * is a permanent Unicode non-character, so no query can contain it and match the placeholder itself, + * and it is not whitespace, so the shared scan's whitespace folding leaves it alone. + */ +const ATOM_PLACEHOLDER = '￿' + +/** A run of the flattened block text, and the document position its first character sits at. */ +interface TextSegment { + textStart: number + docStart: number +} + +/** + * Case-insensitive, non-overlapping search of `query` across every textblock in `doc`, returning + * document position ranges. + * + * What counts as an occurrence is not decided here — {@link forEachSearchOccurrence} owns that for + * the whole app (workflow search, the canvas Note card, and this), including the whitespace fold + * that makes a typed space match a non-breaking one. This module only supplies the text to scan and + * maps the indices back to ProseMirror positions. + * + * Text is flattened per textblock rather than per text node, so a term still matches when it runs + * across a mark boundary (`he**llo**`), and never across a block boundary — which no on-screen line + * does. Each block flattens to a string whose length equals the block's content size, so a string + * index maps back to a document position by walking the segment it falls in. + */ +export function findMatches( + doc: ProseMirrorNode, + query: string, + limit: number = FIND_MATCH_LIMIT +): FindResult { + if (query.trim().length === 0) return EMPTY_FIND_RESULT + + const matches: FindMatch[] = [] + let truncated = false + + doc.descendants((node, pos) => { + if (truncated) return false + if (!node.isTextblock) return true + + // The common paragraph is a single text node, where the mapping is a constant offset and the + // segment table is pure garbage. Only a block mixing marks or atoms needs one built. + const soleChild = node.childCount === 1 ? node.firstChild : null + const soleText = soleChild?.isText ? (soleChild.text ?? null) : null + + let text = soleText ?? '' + let segments: TextSegment[] | null = null + if (soleText === null) { + const built: TextSegment[] = [] + node.forEach((child, offset) => { + built.push({ textStart: text.length, docStart: pos + 1 + offset }) + text += child.isText && child.text ? child.text : ATOM_PLACEHOLDER.repeat(child.nodeSize) + }) + segments = built + } + + let segmentIndex = 0 + forEachSearchOccurrence(text, query, (start, end) => { + if (truncated) return + if (matches.length >= limit) { + truncated = true + return + } + if (!segments) { + matches.push({ from: pos + 1 + start, to: pos + 1 + end }) + return + } + // Segments are ordered and occurrences arrive left to right, so the cursor only moves forward. + while (segmentIndex + 1 < segments.length && segments[segmentIndex + 1].textStart <= start) { + segmentIndex++ + } + const segment = segments[segmentIndex] + const from = segment.docStart + (start - segment.textStart) + matches.push({ from, to: from + (end - start) }) + }) + + // Textblocks do not nest, so there is nothing below one to search. + return false + }) + + return matches.length === 0 && !truncated ? EMPTY_FIND_RESULT : { matches, truncated } +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/index.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/index.ts new file mode 100644 index 00000000000..8f9b8b15704 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/index.ts @@ -0,0 +1,2 @@ +export { RichMarkdownFind } from './find-extension' +export { type MarkdownFindController, useMarkdownFind } from './use-markdown-find' diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/use-markdown-find.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/use-markdown-find.ts new file mode 100644 index 00000000000..b0a97d5a6f6 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/find/use-markdown-find.ts @@ -0,0 +1,168 @@ +'use client' + +import type React from 'react' +import { useCallback, useEffect, useRef, useState } from 'react' +import type { Editor } from '@tiptap/react' +import { useFindShortcut } from '@/app/workspace/[workspaceId]/components' +import { ACTIVE_MATCH_CLASS, getFindTally, setFindQuery, stepFindMatch } from './find-extension' + +/** What the surface hands `FindBar`, plus the open state the shortcut drives. */ +export interface MarkdownFindController { + isOpen: boolean + query: string + count: number + currentIndex: number + truncated: boolean + inputRef: React.RefObject + setQuery: (query: string) => void + next: () => void + prev: () => void + close: () => void +} + +/** The only three values the bar renders, mirrored out of the plugin. */ +interface FindTally { + count: number + currentIndex: number + truncated: boolean +} + +const EMPTY_TALLY: FindTally = { count: 0, currentIndex: 0, truncated: false } + +interface UseMarkdownFindOptions { + editor: Editor | null + /** + * Whether this editor claims Cmd/Ctrl+F. Off wherever the component renders inside a preview or + * embedded pane, where the surface around it — not the document — owns the shortcut. See + * {@link useFindShortcut} for how the surfaces arbitrate. + */ + enabled: boolean +} + +/** + * Find-in-document for the rich markdown editor: owns the term, the tally and the stepping that + * `FindBar` renders, and reveals each match as it becomes active. + * + * The match set itself lives in the ProseMirror plugin (`./find-extension`), which re-searches on + * every document change. This subscribes to the editor's transactions while the bar is open and + * mirrors only the three numbers the bar shows, so a keystroke that leaves the tally identical + * re-renders nothing — the editor is configured not to re-render React on transactions, and this + * must not undo that. + */ +export function useMarkdownFind({ + editor, + enabled, +}: UseMarkdownFindOptions): MarkdownFindController { + const [isOpen, setIsOpen] = useState(false) + const [query, setQueryState] = useState('') + const [tally, setTally] = useState(EMPTY_TALLY) + const inputRef = useRef(null) + const editorRef = useRef(editor) + editorRef.current = editor + + /** + * Scrolls the active highlight into view. Read from the DOM rather than mapped from the position + * so it lands on what is actually painted — a match inside a node view (a code block, a table + * cell) is rendered by that view, and its own scroll container is the one that has to move. + */ + const revealActiveMatch = useCallback(() => { + requestAnimationFrame(() => { + editorRef.current?.view.dom + .querySelector(`.${ACTIVE_MATCH_CLASS}`) + ?.scrollIntoView({ block: 'center' }) + }) + }, []) + + /** + * The single point where plugin state becomes React state. Driven by the editor's own + * `transaction` event, which TipTap emits synchronously from every dispatch — including the ones + * `setQuery` and `step` make below, so neither needs to sync by hand. + */ + const syncTally = useCallback(() => { + const current = editorRef.current + if (!current) return + const { matches, activeIndex, truncated } = getFindTally(current.state) + setTally((previous) => + previous.count === matches.length && + previous.currentIndex === activeIndex && + previous.truncated === truncated + ? previous + : { count: matches.length, currentIndex: activeIndex, truncated } + ) + }, []) + + /** + * Keeps the tally honest while the bar is open — the document can change underneath a live search + * from the user's own typing, a collaborator, or a streaming agent edit, and the plugin re-searches + * on each of those. Not subscribed while the bar is closed, so typing costs nothing then. + */ + useEffect(() => { + if (!editor || !isOpen) return + syncTally() + editor.on('transaction', syncTally) + return () => { + editor.off('transaction', syncTally) + } + }, [editor, isOpen, syncTally]) + + /** + * Re-applies the live term to a newly arrived editor. `useEditor` returns null on the first render, + * so a term typed into the bar before the editor mounts would be held in React and never searched, + * leaving the bar at "No results" until the next keystroke pushed it through. + */ + const queryRef = useRef(query) + queryRef.current = query + useEffect(() => { + if (!editor || queryRef.current.length === 0) return + setFindQuery(editor, queryRef.current) + }, [editor]) + + const setQuery = useCallback( + (next: string) => { + setQueryState(next) + const current = editorRef.current + if (!current) return + setFindQuery(current, next) + revealActiveMatch() + }, + [revealActiveMatch] + ) + + const step = useCallback( + (delta: number) => { + const current = editorRef.current + if (!current) return + stepFindMatch(current, delta) + revealActiveMatch() + }, + [revealActiveMatch] + ) + + const next = useCallback(() => step(1), [step]) + const prev = useCallback(() => step(-1), [step]) + + /** Closing ends the search: term, highlights and active match all go. */ + const close = useCallback(() => { + setIsOpen(false) + setQueryState('') + setTally(EMPTY_TALLY) + const current = editorRef.current + if (current) setFindQuery(current, '') + }, []) + + const open = useCallback(() => setIsOpen(true), []) + useFindShortcut({ enabled, inputRef, onOpen: open }) + + return { + isOpen, + query, + count: tally.count, + currentIndex: tally.currentIndex, + truncated: tally.truncated, + inputRef, + setQuery, + next, + prev, + close, + } +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css index b148b2904f2..a936b509ffe 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css @@ -585,3 +585,24 @@ border-radius: 2px; pointer-events: none; } + +/* Cmd/Ctrl+F match highlights. Every hit carries the same tint the rest of the app + * paints a search match with. */ +.rich-markdown-nodes .rich-find-match { + background-color: var(--highlight-match-bg); + color: var(--highlight-match-text); + border-radius: 2px; +} + +/* The active hit is a stronger fill of the same hue, never a ring: an outline drawn + * around a run of text traces the line box, so on a heading it reads as a stray + * rectangle rather than as emphasis. Filling instead keeps the two states one family. + * + * The ink is fixed rather than tokenized because the fill is: `--brand-secondary` is + * the same blue in both themes, so a theme-flipping text token would go white on light + * blue in dark mode. Same reasoning as the note card's active search mark, which pairs + * its solid fill with a fixed dark ink for exactly this reason. */ +.rich-markdown-nodes .rich-find-match-active { + background-color: var(--brand-secondary); + color: #000; +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index 9684f1bab84..e9c171c59fb 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -16,6 +16,7 @@ import { } from '@/lib/copilot/chat/selection-context' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { extractEmbeddedFileRef, extractImgSrcs } from '@/lib/uploads/utils/embedded-image-ref' +import { FindBar } from '@/app/workspace/[workspaceId]/components' import { isUntitledName } from '@/app/workspace/[workspaceId]/files/untitled-title' import { useUploadWorkspaceFile } from '@/hooks/queries/workspace-files' import { useAddToChat } from '@/hooks/use-add-to-chat' @@ -39,6 +40,7 @@ import { import { nextCollabReadiness } from './collaboration/readiness' import { useFileDocCollaboration } from './collaboration/use-file-doc-collaboration' import { createMarkdownEditorExtensions } from './editor-extensions' +import { useMarkdownFind } from './find' import { findHeadingPos } from './heading-anchors' import { moveDraggedImageNode } from './image-drag-move' import { extractImageFiles, findHostedImageAttrs, shouldSkipFileUpload } from './image-paste' @@ -159,6 +161,12 @@ interface RichMarkdownEditorProps { * {@link isUntitledName}. */ onDeriveTitleFromHeading?: (headingText: string) => void + /** + * Claim Cmd/Ctrl+F for find-in-document. Off by default, because this editor also renders as a + * preview pane beside something that owns the shortcut itself. Every find surface binds its own + * listener, so only one may be enabled at a time — see {@link useFindShortcut}. + */ + enableFind?: boolean } /** Inline WYSIWYG markdown editor: agent output streams in read-only, then the same instance becomes editable on settle. */ @@ -180,6 +188,7 @@ export const RichMarkdownEditor = memo(function RichMarkdownEditor({ disableTagging, collaborative = false, onDeriveTitleFromHeading, + enableFind = false, }: RichMarkdownEditorProps) { const { data: session, isPending: isSessionPending } = useSession() const userId = session?.user?.id ?? '' @@ -253,6 +262,7 @@ export const RichMarkdownEditor = memo(function RichMarkdownEditor({ onSaveShortcut={saveImmediately} onCollabReadyChange={setCollabReady} onDeriveTitleFromHeading={onDeriveTitleFromHeading} + enableFind={enableFind} /> ) }) @@ -283,6 +293,8 @@ interface LoadedRichMarkdownEditorProps { onCollabReadyChange: (ready: boolean) => void /** See {@link RichMarkdownEditorProps.onDeriveTitleFromHeading}. */ onDeriveTitleFromHeading?: (headingText: string) => void + /** See {@link RichMarkdownEditorProps.enableFind}. */ + enableFind: boolean } interface SettledContent { @@ -314,6 +326,7 @@ export function LoadedRichMarkdownEditor({ onSaveShortcut, onCollabReadyChange, onDeriveTitleFromHeading, + enableFind, }: LoadedRichMarkdownEditorProps) { /** Whether this editor mounted mid-stream — if so it starts empty and syncs streamed chunks until settle. */ const streamingAtMountRef = useRef(isStreaming) @@ -1208,43 +1221,70 @@ export function LoadedRichMarkdownEditor({ // shows the base content until the seed swaps it in, avoiding both a blank frame and a garbled merge. const showPlaceholder = collaborationEnabled && !collabReady + /** + * Find is off while the placeholder is up. The text on screen then belongs to the placeholder's own + * editor, not to `editor` — which is still empty and hidden — so searching `editor` would answer + * "No results" for text the user can see. Declining the shortcut hands it back to the browser, whose + * native find reads the rendered placeholder correctly; it becomes ours once the seed lands. + */ + const find = useMarkdownFind({ editor, enabled: enableFind && !showPlaceholder }) + return ( -
- {editor && ( - + {find.isOpen && ( + )} - {editor && } - {editor && } - { - const input = event.currentTarget - const images = Array.from(input.files ?? []).filter((f) => f.type.startsWith('image/')) - const at = - pendingImagePosRef.current ?? editorInstanceRef.current?.state.selection.from ?? 0 - pendingImagePosRef.current = null - input.value = '' - if (images.length > 0) void insertImagesRef.current(images, at) - }} - /> - {showPlaceholder && placeholderContent && ( - - )} - +
+ {editor && ( + + )} + {editor && } + {editor && } + { + const input = event.currentTarget + const images = Array.from(input.files ?? []).filter((f) => f.type.startsWith('image/')) + const at = + pendingImagePosRef.current ?? editorInstanceRef.current?.state.selection.from ?? 0 + pendingImagePosRef.current = null + input.value = '' + if (images.length > 0) void insertImagesRef.current(images, at) + }} + /> + {showPlaceholder && placeholderContent && ( + + )} + +
) } diff --git a/apps/sim/app/workspace/[workspaceId]/files/files.tsx b/apps/sim/app/workspace/[workspaceId]/files/files.tsx index f81334b9512..dc7d32ec991 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/files.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/files.tsx @@ -70,6 +70,7 @@ import { resourceListState, selectionLabel, timeCell, + useFindShortcut, useResourceRowSelection, } from '@/app/workspace/[workspaceId]/components' import type { @@ -1584,25 +1585,14 @@ export function Files() { /** * Overrides the browser's Cmd/Ctrl+F with the in-list find while the list is - * showing. Skipped when a file is open — its editor owns the shortcut there — - * and when another surface already claimed the press. + * showing. Handed to the open file's editor instead once one is open. */ - useEffect(() => { - const handleFindShortcut = (e: KeyboardEvent) => { - if (!(e.metaKey || e.ctrlKey) || e.altKey || e.shiftKey) return - if (e.key.toLowerCase() !== 'f') return - if (fileIdFromRouteRef.current) return - if (e.defaultPrevented) return - e.preventDefault() - setFindOpen(true) - requestAnimationFrame(() => { - findInputRef.current?.focus() - findInputRef.current?.select() - }) - } - document.addEventListener('keydown', handleFindShortcut) - return () => document.removeEventListener('keydown', handleFindShortcut) - }, []) + const handleFindOpen = useCallback(() => setFindOpen(true), []) + useFindShortcut({ + enabled: !fileIdFromRoute, + inputRef: findInputRef, + onOpen: handleFindOpen, + }) const handleCyclePreviewMode = useCallback(() => { setPreviewMode((prev) => { @@ -2138,6 +2128,7 @@ export function Files() { discardRef={discardRef} collaborative onDeriveTitleFromHeading={handleDeriveTitleFromHeading} + enableFind /> ) diff --git a/apps/sim/app/workspace/[workspaceId]/logs/components/dashboard/components/index.ts b/apps/sim/app/workspace/[workspaceId]/logs/components/dashboard/components/index.ts index 7f3469b912e..85d51971767 100644 --- a/apps/sim/app/workspace/[workspaceId]/logs/components/dashboard/components/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/logs/components/dashboard/components/index.ts @@ -1,3 +1,2 @@ -export { LineChart, type LineChartMultiSeries, type LineChartPoint } from './line-chart' export { StatusBar, type StatusBarSegment } from './status-bar' export { type WorkflowExecutionItem, WorkflowsList } from './workflows-list' diff --git a/apps/sim/app/workspace/[workspaceId]/logs/components/dashboard/components/line-chart/index.ts b/apps/sim/app/workspace/[workspaceId]/logs/components/dashboard/components/line-chart/index.ts deleted file mode 100644 index be4131ebb5a..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/logs/components/dashboard/components/line-chart/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { LineChart, type LineChartMultiSeries, type LineChartPoint } from './line-chart' diff --git a/apps/sim/app/workspace/[workspaceId]/logs/components/dashboard/components/line-chart/line-chart.tsx b/apps/sim/app/workspace/[workspaceId]/logs/components/dashboard/components/line-chart/line-chart.tsx deleted file mode 100644 index 9717ab254e7..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/logs/components/dashboard/components/line-chart/line-chart.tsx +++ /dev/null @@ -1,712 +0,0 @@ -import { memo, useEffect, useMemo, useRef, useState } from 'react' -import { Button, cn } from '@sim/emcn' -import { generateShortId } from '@sim/utils/id' -import { formatDate, formatLatency } from '@/app/workspace/[workspaceId]/logs/utils' - -export interface LineChartPoint { - timestamp: string - value: number -} - -export interface LineChartMultiSeries { - id?: string - label: string - color: string - data: LineChartPoint[] - dashed?: boolean -} - -function LineChartComponent({ - data, - label, - color, - unit, - series, -}: { - data: LineChartPoint[] - label: string - color: string - unit?: string - series?: LineChartMultiSeries[] -}) { - const containerRef = useRef(null) - const uniqueId = useRef(`chart-${generateShortId(7)}`).current - const [containerWidth, setContainerWidth] = useState(null) - const width = containerWidth ?? 0 - const height = 166 - const padding = { top: 16, right: 28, bottom: 26, left: 26 } - useEffect(() => { - if (!containerRef.current) return - const element = containerRef.current - const ro = new ResizeObserver((entries) => { - const entry = entries[0] - if (entry?.contentRect && entry.contentRect.width > 0) { - const w = Math.max(280, Math.floor(entry.contentRect.width)) - setContainerWidth(w) - } - }) - ro.observe(element) - const rect = element.getBoundingClientRect() - if (rect?.width && rect.width > 0) setContainerWidth(Math.max(280, Math.floor(rect.width))) - return () => ro.disconnect() - }, []) - const chartWidth = width - padding.left - padding.right - const chartHeight = height - padding.top - padding.bottom - const [hoverIndex, setHoverIndex] = useState(null) - const [isDark, setIsDark] = useState(true) - const [hoverSeriesId, setHoverSeriesId] = useState(null) - const [activeSeriesId, setActiveSeriesId] = useState(null) - const [hoverPos, setHoverPos] = useState<{ x: number; y: number } | null>(null) - const [resolvedColors, setResolvedColors] = useState>({}) - - useEffect(() => { - if (typeof window === 'undefined') return - const el = document.documentElement - const update = () => setIsDark(el.classList.contains('dark')) - update() - const observer = new MutationObserver(update) - observer.observe(el, { attributes: true, attributeFilter: ['class'] }) - return () => observer.disconnect() - }, []) - - useEffect(() => { - if (typeof window === 'undefined') return - - const resolveColor = (c: string): string => { - if (!c.startsWith('var(')) return c - - const tempEl = document.createElement('div') - tempEl.style.color = c - document.body.appendChild(tempEl) - const computed = window.getComputedStyle(tempEl).color - document.body.removeChild(tempEl) - return computed - } - - const colorMap: Record = { base: resolveColor(color) } - const allSeriesToResolve = Array.isArray(series) && series.length > 0 ? series : [] - - for (const s of allSeriesToResolve) { - const id = s.id || s.label || '' - if (id) colorMap[id] = resolveColor(s.color) - } - - setResolvedColors(colorMap) - }, [color, series]) - - const hasExternalWrapper = !label || label === '' - - const allSeries = useMemo( - () => - (Array.isArray(series) && series.length > 0 - ? [{ id: 'base', label, color, data }, ...series] - : [{ id: 'base', label, color, data }] - ).map((s, idx) => ({ ...s, id: s.id || s.label || String(idx) })), - [series, label, color, data] - ) - - const { maxValue, minValue, valueRange } = useMemo(() => { - const flatValues = allSeries.flatMap((s) => s.data.map((d) => d.value)) - const rawMax = Math.max(...flatValues, 1) - const rawMin = Math.min(...flatValues, 0) - const paddedMax = rawMax === 0 ? 1 : rawMax * 1.1 - const paddedMin = Math.min(0, rawMin) - const unitSuffixPre = (unit || '').trim().toLowerCase() - let maxVal = Math.ceil(paddedMax) - let minVal = Math.floor(paddedMin) - if (unitSuffixPre === 'ms' || unitSuffixPre === 'latency') { - minVal = 0 - if (paddedMax < 10) { - maxVal = Math.ceil(paddedMax) - } else if (paddedMax < 100) { - maxVal = Math.ceil(paddedMax / 10) * 10 - } else if (paddedMax < 1000) { - maxVal = Math.ceil(paddedMax / 50) * 50 - } else if (paddedMax < 10000) { - maxVal = Math.ceil(paddedMax / 500) * 500 - } else { - maxVal = Math.ceil(paddedMax / 1000) * 1000 - } - } - return { - maxValue: maxVal, - minValue: minVal, - valueRange: maxVal - minVal || 1, - } - }, [allSeries, unit]) - - const yMin = padding.top + 3 - const yMax = padding.top + chartHeight - 3 - - const scaledPoints = useMemo( - () => - data.map((d, i) => { - const usableW = Math.max(1, chartWidth) - const x = padding.left + (i / (data.length - 1 || 1)) * usableW - const rawY = padding.top + chartHeight - ((d.value - minValue) / valueRange) * chartHeight - const y = Math.max(yMin, Math.min(yMax, rawY)) - return { x, y } - }), - [data, chartWidth, chartHeight, minValue, valueRange, yMin, yMax, padding.left, padding.top] - ) - - const scaledSeries = useMemo( - () => - allSeries.map((s) => { - const pts = s.data.map((d, i) => { - const usableW = Math.max(1, chartWidth) - const x = padding.left + (i / (s.data.length - 1 || 1)) * usableW - const rawY = padding.top + chartHeight - ((d.value - minValue) / valueRange) * chartHeight - const y = Math.max(yMin, Math.min(yMax, rawY)) - return { x, y } - }) - return { ...s, pts } - }), - [ - allSeries, - chartWidth, - chartHeight, - minValue, - valueRange, - yMin, - yMax, - padding.left, - padding.top, - ] - ) - - const getSeriesById = (id?: string | null) => scaledSeries.find((s) => s.id === id) - const visibleSeries = useMemo( - () => (activeSeriesId ? scaledSeries.filter((s) => s.id === activeSeriesId) : scaledSeries), - [activeSeriesId, scaledSeries] - ) - - const pathD = useMemo(() => { - if (scaledPoints.length <= 1) return '' - const p = scaledPoints - const tension = 0.2 - let d = `M ${p[0].x} ${p[0].y}` - for (let i = 0; i < p.length - 1; i++) { - const p0 = p[i - 1] || p[i] - const p1 = p[i] - const p2 = p[i + 1] - const p3 = p[i + 2] || p[i + 1] - const cp1x = p1.x + ((p2.x - p0.x) / 6) * tension - let cp1y = p1.y + ((p2.y - p0.y) / 6) * tension - const cp2x = p2.x - ((p3.x - p1.x) / 6) * tension - let cp2y = p2.y - ((p3.y - p1.y) / 6) * tension - cp1y = Math.max(yMin, Math.min(yMax, cp1y)) - cp2y = Math.max(yMin, Math.min(yMax, cp2y)) - d += ` C ${cp1x} ${cp1y}, ${cp2x} ${cp2y}, ${p2.x} ${p2.y}` - } - return d - }, [scaledPoints, yMin, yMax]) - - const getCompactDateLabel = (timestamp?: string) => { - if (!timestamp) return '' - try { - const f = formatDate(timestamp) - return `${f.compactDate} ${f.compactTime}` - } catch (e) { - const d = new Date(timestamp) - if (Number.isNaN(d.getTime())) return '' - return d.toLocaleString('en-US', { - month: 'short', - day: 'numeric', - hour: '2-digit', - minute: '2-digit', - hour12: false, - }) - } - } - - const currentHoverDate = - hoverIndex !== null && data[hoverIndex] ? getCompactDateLabel(data[hoverIndex].timestamp) : '' - - if (containerWidth === null) { - return ( -
- ) - } - - if (data.length === 0) { - return ( -
-

No data

-
- ) - } - - return ( -
- {!hasExternalWrapper && ( -
-

{label}

- {allSeries.length > 1 && ( -
- {scaledSeries.slice(1).map((s) => { - const isActive = activeSeriesId ? activeSeriesId === s.id : true - const isHovered = hoverSeriesId === s.id - const dimmed = activeSeriesId ? !isActive : false - return ( - - ) - })} -
- )} -
- )} -
- { - if (scaledPoints.length === 0) return - const rect = (e.currentTarget as SVGSVGElement).getBoundingClientRect() - const x = e.clientX - rect.left - const clamped = Math.max(padding.left, Math.min(width - padding.right, x)) - const ratio = (clamped - padding.left) / (chartWidth || 1) - const i = Math.round(ratio * (scaledPoints.length - 1)) - setHoverIndex(i) - setHoverPos({ x: clamped, y: e.clientY - rect.top }) - const cursorY = e.clientY - rect.top - if (activeSeriesId) { - setHoverSeriesId(activeSeriesId) - } else { - let best: { id: string | null; dy: number } = { - id: null, - dy: Number.POSITIVE_INFINITY, - } - for (const s of scaledSeries.slice(1)) { - const pt = s.pts[i] - if (!pt) continue - const dy = Math.abs(pt.y - cursorY) - if (dy < best.dy) best = { id: s.id || null, dy } - } - setHoverSeriesId(best.dy <= 12 ? best.id : null) - } - }} - onMouseLeave={() => { - setHoverIndex(null) - setHoverPos(null) - setHoverSeriesId(null) - }} - > - - - - - - - - - - - - - {[0.25, 0.5, 0.75].map((p) => ( - - ))} - - {!activeSeriesId && scaledPoints.length > 1 && ( - - )} - - {!activeSeriesId && - scaledPoints.length === 1 && - (() => { - const strokeWidth = isDark ? 1.7 : 2.0 - const capExtension = strokeWidth / 2 - return ( - - ) - })()} - - {visibleSeries.map((s, idx) => { - const isActive = activeSeriesId ? activeSeriesId === s.id : true - const isHovered = hoverSeriesId ? hoverSeriesId === s.id : false - const baseOpacity = isActive ? 1 : 0.12 - const strokeOpacity = isHovered ? 1 : baseOpacity - const sw = (() => { - switch ((s.id || '').toLowerCase()) { - case 'p50': - return isDark ? 1.5 : 1.7 - case 'p90': - return isDark ? 1.9 : 2.1 - case 'p99': - return isDark ? 2.3 : 2.5 - default: - return isDark ? 1.7 : 2.0 - } - })() - if (s.pts.length <= 1) { - const y = s.pts[0]?.y - if (y === undefined) return null - return ( - - ) - } - const p = (() => { - const p = s.pts - const tension = 0.2 - let d = `M ${p[0].x} ${p[0].y}` - for (let i = 0; i < p.length - 1; i++) { - const p0 = p[i - 1] || p[i] - const p1 = p[i] - const p2 = p[i + 1] - const p3 = p[i + 2] || p[i + 1] - const cp1x = p1.x + ((p2.x - p0.x) / 6) * tension - let cp1y = p1.y + ((p2.y - p0.y) / 6) * tension - const cp2x = p2.x - ((p3.x - p1.x) / 6) * tension - let cp2y = p2.y - ((p3.y - p1.y) / 6) * tension - cp1y = Math.max(yMin, Math.min(yMax, cp1y)) - cp2y = Math.max(yMin, Math.min(yMax, cp2y)) - d += ` C ${cp1x} ${cp1y}, ${cp2x} ${cp2y}, ${p2.x} ${p2.y}` - } - return d - })() - return ( - setActiveSeriesId((prev) => (prev === s.id ? null : s.id || null))} - /> - ) - })} - - {hoverIndex !== null && - scaledPoints[hoverIndex] && - scaledPoints.length > 1 && - (() => { - const guideSeries = - getSeriesById(activeSeriesId) || getSeriesById(hoverSeriesId) || scaledSeries[0] - const active = guideSeries - const pt = active.pts[hoverIndex] || scaledPoints[hoverIndex] - return ( - - - {activeSeriesId && - (() => { - const s = getSeriesById(activeSeriesId) - const spt = s?.pts?.[hoverIndex] - if (!s || !spt) return null - return ( - - ) - })()} - - ) - })()} - - {(() => { - if (data.length < 2) return null - const usableW = Math.max(1, chartWidth) - const firstTs = new Date(data[0].timestamp) - const lastTs = new Date(data[data.length - 1].timestamp) - const spanMs = Math.abs(lastTs.getTime() - firstTs.getTime()) - - const approxLabelWidth = 64 - const desired = Math.min(8, Math.max(3, Math.floor(usableW / approxLabelWidth))) - const rawIdx = Array.from({ length: desired }, (_, i) => - Math.round((i * (data.length - 1)) / Math.max(1, desired - 1)) - ) - const seen = new Set() - const idx = rawIdx.filter((i) => { - if (seen.has(i)) return false - seen.add(i) - return true - }) - - const formatTick = (d: Date) => { - if (spanMs <= 36 * 60 * 60 * 1000) { - return d.toLocaleTimeString('en-US', { - hour: '2-digit', - minute: '2-digit', - hour12: false, - }) - } - if (spanMs <= 90 * 24 * 60 * 60 * 1000) { - return d.toLocaleString('en-US', { month: 'short', day: 'numeric' }) - } - return d.toLocaleString('en-US', { month: 'short', year: 'numeric' }) - } - - return idx.map((i) => { - const x = padding.left + (i / (data.length - 1 || 1)) * usableW - const tsSource = data[i]?.timestamp - if (!tsSource) return null - const ts = new Date(tsSource) - const labelStr = Number.isNaN(ts.getTime()) ? '' : formatTick(ts) - return ( - - {labelStr} - - ) - }) - })()} - - {(() => { - const unitSuffix = (unit || '').trim() - const showInTicks = unitSuffix === '%' - const isLatency = unitSuffix.toLowerCase() === 'latency' - const fmtCompact = (v: number) => { - if (isLatency) { - if (v === 0) return '0' - return formatLatency(v) - } - return new Intl.NumberFormat('en-US', { - notation: 'compact', - maximumFractionDigits: 1, - }) - .format(v) - .toLowerCase() - } - return ( - <> - - {fmtCompact(maxValue)} - {showInTicks && !isLatency ? unit : ''} - - - {fmtCompact(minValue)} - {showInTicks && !isLatency ? unit : ''} - - - ) - })()} - - - - - {hoverIndex !== null && - scaledPoints[hoverIndex] && - (() => { - const active = - getSeriesById(activeSeriesId) || getSeriesById(hoverSeriesId) || scaledSeries[0] - const pt = active.pts[hoverIndex] || scaledPoints[hoverIndex] - const toDisplay = activeSeriesId - ? [getSeriesById(activeSeriesId)!] - : scaledSeries.length > 1 - ? scaledSeries.slice(1) - : [scaledSeries[0]] - - const fmt = (v?: number) => { - if (typeof v !== 'number' || !Number.isFinite(v)) return '—' - const u = unit || '' - if (u.includes('%')) return `${v.toFixed(1)}%` - if (u.toLowerCase() === 'latency') return formatLatency(v) - if (u.toLowerCase().includes('ms')) return `${Math.round(v)}ms` - if (u.toLowerCase().includes('exec')) return `${Math.round(v)}` - return `${Math.round(v)}${u}` - } - - const longest = toDisplay.reduce((m, s) => { - const seriesIndex = allSeries.findIndex((x) => x.id === s.id) - const v = allSeries[seriesIndex]?.data?.[hoverIndex]?.value - const valueStr = fmt(v) - const labelStr = s.label || String(s.id || '') - const len = `${labelStr} ${valueStr}`.length - return Math.max(m, len) - }, 0) - const tooltipMaxW = Math.min(220, Math.max(80, 7 * longest + 24)) - const anchorX = hoverPos?.x ?? pt.x - const margin = 10 - const preferRight = anchorX + margin + tooltipMaxW <= width - padding.right - const left = preferRight - ? Math.max( - padding.left, - Math.min(anchorX + margin, width - padding.right - tooltipMaxW) - ) - : Math.max( - padding.left, - Math.min(anchorX - margin - tooltipMaxW, width - padding.right - tooltipMaxW) - ) - const anchorY = hoverPos?.y ?? pt.y - const top = Math.min(Math.max(anchorY - 26, padding.top), height - padding.bottom - 18) - return ( -
- {currentHoverDate && ( -
- {currentHoverDate} -
- )} - {toDisplay.map((s) => { - const seriesIndex = allSeries.findIndex((x) => x.id === s.id) - const val = allSeries[seriesIndex]?.data?.[hoverIndex]?.value - const seriesLabel = s.label || s.id - const showLabel = - seriesLabel && seriesLabel !== 'base' && seriesLabel.trim() !== '' - return ( -
- - {showLabel && ( - {seriesLabel} - )} - {fmt(val)} -
- ) - })} -
- ) - })()} -
-
- ) -} - -/** - * Memoized LineChart component to prevent re-renders when parent updates. - */ -export const LineChart = memo(LineChartComponent) diff --git a/apps/sim/app/workspace/[workspaceId]/logs/components/dashboard/dashboard.tsx b/apps/sim/app/workspace/[workspaceId]/logs/components/dashboard/dashboard.tsx index 83a53d724c0..44b145c8b54 100644 --- a/apps/sim/app/workspace/[workspaceId]/logs/components/dashboard/dashboard.tsx +++ b/apps/sim/app/workspace/[workspaceId]/logs/components/dashboard/dashboard.tsx @@ -3,6 +3,7 @@ import { memo, useCallback, useMemo, useRef, useState } from 'react' import { Loader } from '@sim/emcn' import { useParams } from 'next/navigation' +import { LineChart } from '@/components/charts' import { DashboardSegmentsContext, type SegmentSelectionMode, @@ -11,7 +12,7 @@ import { useLogFilters } from '@/app/workspace/[workspaceId]/logs/hooks/use-log- import { formatLatency } from '@/app/workspace/[workspaceId]/logs/utils' import type { DashboardStatsResponse, WorkflowStats } from '@/hooks/queries/logs' import { useWorkflows } from '@/hooks/queries/workflows' -import { LineChart, WorkflowsList } from './components' +import { WorkflowsList } from './components' interface WorkflowExecution { workflowId: string diff --git a/apps/sim/app/workspace/[workspaceId]/logs/utils.ts b/apps/sim/app/workspace/[workspaceId]/logs/utils.ts index 4736430b5b2..b45d859aee4 100644 --- a/apps/sim/app/workspace/[workspaceId]/logs/utils.ts +++ b/apps/sim/app/workspace/[workspaceId]/logs/utils.ts @@ -1,7 +1,8 @@ import React from 'react' import { Badge } from '@sim/emcn' -import { formatDuration, formatRelativeTime } from '@sim/utils/formatting' +import { formatRelativeTime } from '@sim/utils/formatting' import { format } from 'date-fns' +import { formatChartLatency } from '@/components/charts/chart-format' import { getIntegrationMetadata } from '@/lib/logs/get-trigger-options' import { getBlock } from '@/blocks/registry' import { CORE_TRIGGER_TYPES } from '@/stores/logs/filters/types' @@ -199,13 +200,16 @@ export function parseDuration(log: LogWithDuration): number | null { } /** - * Format latency value for display in dashboard UI + * Format latency value for display in dashboard UI. + * + * Delegates so the axis ticks and the surrounding table can never disagree about + * what a duration reads as. + * * @param ms - Latency in milliseconds (number) * @returns Formatted latency string */ export function formatLatency(ms: number): string { - if (!Number.isFinite(ms) || ms <= 0) return '—' - return formatDuration(ms, { precision: 2 }) ?? '—' + return formatChartLatency(ms) } export const formatDate = (dateString: string) => { diff --git a/apps/sim/app/workspace/[workspaceId]/settings/[section]/page.test.tsx b/apps/sim/app/workspace/[workspaceId]/settings/[section]/page.test.tsx index 26e408c6af1..205c416042e 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/[section]/page.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/[section]/page.test.tsx @@ -42,6 +42,19 @@ vi.mock('@/components/settings/navigation', () => ({ getOrganizationSettingsFeatures: vi.fn(() => ({})), isOrganizationSettingsSectionAvailable: mockIsOrganizationSettingsSectionAvailable, resolveWorkspaceNavigation: mockResolveWorkspaceNavigation, + /** Mirrors the registry-derived map; a section absent here gets no organization gate. */ + UNIFIED_TO_ORGANIZATION_SECTION: { + organization: 'members', + billing: 'billing', + 'access-control': 'access-control', + 'audit-logs': 'audit-logs', + sso: 'sso', + sessions: 'sessions', + 'data-retention': 'data-retention', + 'data-drains': 'data-drains', + usage: 'usage', + whitelabeling: 'whitelabeling', + }, workspaceSectionUsesPermissionConfig: vi.fn((section: string) => ['secrets', 'api-keys', 'inbox', 'mcp', 'custom-tools'].includes(section) ), diff --git a/apps/sim/app/workspace/[workspaceId]/settings/[section]/page.tsx b/apps/sim/app/workspace/[workspaceId]/settings/[section]/page.tsx index 0cf8a89a781..ca0c7426125 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/[section]/page.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/[section]/page.tsx @@ -5,8 +5,8 @@ import { notFound, redirect } from 'next/navigation' import { getOrganizationSettingsFeatures, isOrganizationSettingsSectionAvailable, - type OrganizationSettingsSection, resolveWorkspaceNavigation, + UNIFIED_TO_ORGANIZATION_SECTION, type WorkspaceSettingsSection, workspaceSectionUsesPermissionConfig, } from '@/components/settings/navigation' @@ -48,18 +48,6 @@ const WORKSPACE_SECTION_MAP: Partial> = { - organization: 'members', - billing: 'billing', - 'access-control': 'access-control', - 'audit-logs': 'audit-logs', - sso: 'sso', - sessions: 'sessions', - 'data-retention': 'data-retention', - 'data-drains': 'data-drains', - whitelabeling: 'whitelabeling', -} - /** * Settings availability varies across workspaces, so a preserved section may * need to land on the destination workspace's universally available page. @@ -163,7 +151,7 @@ export default async function WorkspaceSettingsSectionPage({ } } - const organizationSection = ORGANIZATION_SECTION_MAP[parsed] + const organizationSection = UNIFIED_TO_ORGANIZATION_SECTION[parsed] if (organizationSection) { if (!isBillingEnabled && (parsed === 'billing' || parsed === 'organization')) { redirectToGeneralSettings(workspaceId) @@ -173,7 +161,12 @@ export default async function WorkspaceSettingsSectionPage({ redirectToGeneralSettings(workspaceId) } } else { - if (!hostContext.viewer.isHostOrganizationAdmin) { + /** + * The roster is the one organization section a plain member may open, read-only + * (`resolveOrganizationSectionAccess` returns `'view'` for it below). Everything + * else acts on the organization and stays admin-only. + */ + if (organizationSection !== 'members' && !hostContext.viewer.isHostOrganizationAdmin) { redirectToGeneralSettings(workspaceId) } /** diff --git a/apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx b/apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx index edb01ca9214..8295fa0b230 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx @@ -104,6 +104,9 @@ const DataRetentionSettings = dynamic(() => const DataDrainsSettings = dynamic(() => import('@/ee/data-drains/components/data-drains-settings').then((m) => m.DataDrainsSettings) ) +const UsageMonitoring = dynamic(() => + import('@/ee/organization-usage/components/usage-monitoring').then((m) => m.UsageMonitoring) +) const Desktop = dynamic(() => import('@/app/workspace/[workspaceId]/settings/components/desktop/desktop').then((m) => m.Desktop) ) @@ -172,6 +175,13 @@ export function SettingsPage({ section }: SettingsPageProps) { {effectiveSection === 'audit-logs' && organizationId && ( )} + {effectiveSection === 'usage' && organizationId && ( + + )} {effectiveSection === 'apikeys' && } {isBillingEnabled && effectiveSection === 'billing' && ( + {/* + Text with a numeric input mode rather than `type='number'`: the native stepper + is all that type buys and it does not fit the chip chrome. The minimum is + enforced on commit below, where it can explain itself, rather than by a `min` + attribute the browser enforces silently. + */} setDraft(e.target.value)} placeholder={ @@ -135,7 +139,6 @@ export function UsageLimitField({ : String(dollarsToCredits(currentLimit)) } disabled={!canEdit} - inputClassName='[appearance:textfield] [&::-webkit-inner-spin-button]:appearance-none [&::-webkit-outer-spin-button]:appearance-none' /> ) diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/team-management/components/manage-credits-modal/index.ts b/apps/sim/app/workspace/[workspaceId]/settings/components/manage-credits-modal/index.ts similarity index 100% rename from apps/sim/app/workspace/[workspaceId]/settings/components/team-management/components/manage-credits-modal/index.ts rename to apps/sim/app/workspace/[workspaceId]/settings/components/manage-credits-modal/index.ts diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/team-management/components/manage-credits-modal/manage-credits-modal.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/manage-credits-modal/manage-credits-modal.tsx similarity index 84% rename from apps/sim/app/workspace/[workspaceId]/settings/components/team-management/components/manage-credits-modal/manage-credits-modal.tsx rename to apps/sim/app/workspace/[workspaceId]/settings/components/manage-credits-modal/manage-credits-modal.tsx index 5a2f8f09b7e..0654a34d997 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/team-management/components/manage-credits-modal/manage-credits-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/manage-credits-modal/manage-credits-modal.tsx @@ -19,7 +19,12 @@ import { export interface ManageCreditsTarget { userId: string name: string - email: string + /** + * Optional: only ever read as a fallback when `name` is blank. Callers that + * already resolved a display name — the usage panel resolves name-or-email + * server-side — have nothing to add here. + */ + email?: string } interface ManageCreditsModalProps { @@ -104,9 +109,17 @@ export function ManageCreditsModal({ value={isLoading ? 'Loading…' : creditsUsed} copyLabel='Copy credits used' /> + {/* + Text with a numeric input mode, not `inputType='number'` — the same choice + the retry settings field documents. The native stepper is all the number + type buys, and it paints browser chrome inside a flat chip surface. It also + reports `''` for anything the browser considers invalid, so a typo arrived + here indistinguishable from a cleared field and saved as "no limit"; as text + it reaches the `Number.isInteger` check below and is refused. + */} Credit limit diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/segmented-meter/index.ts b/apps/sim/app/workspace/[workspaceId]/settings/components/segmented-meter/index.ts new file mode 100644 index 00000000000..8b883fad513 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/segmented-meter/index.ts @@ -0,0 +1 @@ +export { SegmentedMeter } from '@/app/workspace/[workspaceId]/settings/components/segmented-meter/segmented-meter' diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/segmented-meter/segmented-meter.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/segmented-meter/segmented-meter.tsx new file mode 100644 index 00000000000..8651571619c --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/segmented-meter/segmented-meter.tsx @@ -0,0 +1,56 @@ +import { cn } from '@sim/emcn' + +interface SegmentedMeterProps { + /** How much of the allowance is consumed, in the same unit as `total`. */ + used: number + /** The allowance. Consumption beyond it renders in the overage tone. */ + total: number + /** + * How many segments to draw. Seats use one pill per seat so the meter is + * countable; a credit allowance is in the tens of thousands, so it passes a fixed + * count and reads as a percentage instead. + */ + segments: number + className?: string +} + +/** + * The settings allowance meter. + * + * Extracted so the seat meter and the usage meter cannot drift — they are the same + * affordance answering the same question, and previously only one existed. + */ +export function SegmentedMeter({ used, total, segments, className }: SegmentedMeterProps) { + /** + * Both counts are measured against the larger of the two, so an overage has + * somewhere to render. Scaling the fill by `total` and clamping it meant + * `filledSegments` and the allowance were both `segments` whenever usage exceeded + * the limit, and the overage tone below could never be reached — the meter simply + * showed full. + */ + const scale = Math.max(total, used) + const filledSegments = scale > 0 ? Math.min(segments, Math.round((used / scale) * segments)) : 0 + const allowedSegments = scale > 0 ? Math.round((total / scale) * segments) : 0 + + return ( +
-
- {Array.from({ length: pillCount }).map((_, i) => { - const isFilled = i < usedSeats - const isOverage = i >= totalSeats - return ( -
- ) - })} -
+

{isOverLimit diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/team-management/team-management.test.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/team-management/team-management.test.tsx index 94c191fb855..ffa5a03c2ff 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/team-management/team-management.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/team-management/team-management.test.tsx @@ -94,7 +94,11 @@ describe('TeamManagement organization errors', () => { isLoading: false, }) - act(() => root.render()) + act(() => + root.render( + + ) + ) expect(container.textContent).toContain('Organization request failed') expect(container.textContent).not.toContain('no-organization-view') diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/team-management/team-management.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/team-management/team-management.tsx index b62f1e0b292..8efc23f7ba5 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/team-management/team-management.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/team-management/team-management.tsx @@ -35,13 +35,14 @@ const logger = createLogger('TeamManagement') interface TeamManagementProps { organizationId: string - billingHref?: string + /** + * Required: organization billing is reached only through a workspace, so the + * caller — which knows the workspace — is the only thing that can build it. + */ + billingHref: string } -export function TeamManagement({ - organizationId, - billingHref = `/organization/${organizationId}/settings/billing`, -}: TeamManagementProps) { +export function TeamManagement({ organizationId, billingHref }: TeamManagementProps) { const { data: session } = useSession() const { isInvitationsDisabled } = usePermissionConfig() const [memberQuery, setMemberQuery] = useSettingsSearch() diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/workflow-mcp-servers/workflow-mcp-servers.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/workflow-mcp-servers/workflow-mcp-servers.tsx index d635b71894b..9942803fc68 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/workflow-mcp-servers/workflow-mcp-servers.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/workflow-mcp-servers/workflow-mcp-servers.tsx @@ -76,7 +76,7 @@ interface ServerDetailViewProps { isDeleting: boolean } -type McpClientType = 'sim' | 'cursor' | 'claude-code' | 'claude-desktop' | 'vscode' +type McpClientType = 'sim' | 'cursor' | 'codex' | 'claude-code' | 'claude-desktop' | 'vscode' function ServerDetailView({ canManage, @@ -278,6 +278,14 @@ function ServerDetailView({ return `claude mcp add "${safeName}" --url "${mcpServerUrl}" --header "X-API-Key:$SIM_API_KEY"` } + if (client === 'codex') { + return [ + `[mcp_servers."${safeName}"]`, + `url = "${mcpServerUrl}"`, + ...(isPublic ? [] : ['env_http_headers = { "X-API-Key" = "SIM_API_KEY" }']), + ].join('\n') + } + if (client === 'cursor') { const cursorConfig = isPublic ? { url: mcpServerUrl } @@ -502,6 +510,7 @@ function ServerDetailView({ onValueChange={(v) => setActiveConfigTab(v as McpClientType)} > Cursor + Codex Claude Code Claude Desktop VS Code @@ -589,7 +598,13 @@ function ServerDetailView({

@@ -606,9 +621,21 @@ function ServerDetailView({ )}
+ {activeConfigTab === 'codex' && server.isPublic && ( +

+ Add this to ~/.codex/config.toml. +

+ )} {!server.isPublic && (

- Replace $SIM_API_KEY with your API key + {activeConfigTab === 'codex' ? ( + <> + Add this to ~/.codex/config.toml and + set the SIM_API_KEY environment variable with an existing API key + + ) : ( + 'Replace $SIM_API_KEY with your API key' + )} {canManage && ( <> , or{' '} @@ -621,6 +648,7 @@ function ServerDetailView({ )} + {activeConfigTab === 'codex' && '.'}

)}
diff --git a/apps/sim/app/workspace/[workspaceId]/settings/navigation.test.ts b/apps/sim/app/workspace/[workspaceId]/settings/navigation.test.ts index b3c64e62c0d..f304c1e6574 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/navigation.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/settings/navigation.test.ts @@ -31,6 +31,7 @@ describe('unified settings navigation', () => { { id: 'billing', label: 'Subscription', section: 'account' }, { id: 'teammates', label: 'Teammates', section: 'workspace' }, { id: 'organization', label: 'Members', section: 'organization' }, + { id: 'usage', label: 'Usage tracking', section: 'organization' }, { id: 'secrets', label: 'Secrets', section: 'workspace' }, { id: 'credential-groups', label: 'Credential groups', section: 'workspace' }, { id: 'custom-tools', label: 'Custom tools', section: 'workspace' }, @@ -82,6 +83,7 @@ describe('unified settings navigation', () => { ]) expect(idsForSection('organization')).toEqual([ 'organization', + 'usage', 'custom-blocks', 'forks', 'access-control', diff --git a/apps/sim/app/workspace/[workspaceId]/settings/usage/events/layout.tsx b/apps/sim/app/workspace/[workspaceId]/settings/usage/events/layout.tsx new file mode 100644 index 00000000000..5886a0c78b5 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/settings/usage/events/layout.tsx @@ -0,0 +1,18 @@ +import { + SettingsHeaderProvider, + SettingsHeaderShell, +} from '@/app/workspace/[workspaceId]/settings/components/settings-header/settings-header' + +/** + * Usage events is a static route outside `[section]`, so it inherits none of + * `SettingsSectionLayout`'s chrome. Its body renders through `SettingsPanel`, which + * only registers header config into `SettingsHeaderProvider` — without this shell the + * page would have no header bar, title, or scroll region. + */ +export default function UsageEventsLayout({ children }: { children: React.ReactNode }) { + return ( + + {children} + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/settings/usage/events/loading.tsx b/apps/sim/app/workspace/[workspaceId]/settings/usage/events/loading.tsx new file mode 100644 index 00000000000..1e9fd7ecb72 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/settings/usage/events/loading.tsx @@ -0,0 +1,22 @@ +'use client' + +import { ArrowLeft } from '@sim/emcn/icons' +import { useParams, useRouter } from 'next/navigation' +import { SettingsPanel } from '@/app/workspace/[workspaceId]/settings/components/settings-panel' + +export default function UsageEventsLoading() { + const router = useRouter() + const { workspaceId } = useParams<{ workspaceId: string }>() + + return ( + router.push(`/workspace/${workspaceId}/settings/usage`), + }} + title='Usage events' + description="Every credit-consuming event across your organization's workspaces." + /> + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/settings/usage/events/page.tsx b/apps/sim/app/workspace/[workspaceId]/settings/usage/events/page.tsx new file mode 100644 index 00000000000..36c0777ef07 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/settings/usage/events/page.tsx @@ -0,0 +1,47 @@ +import { Suspense } from 'react' +import type { Metadata } from 'next' +import { redirect } from 'next/navigation' +import { getSession } from '@/lib/auth' +import { isOrganizationFeatureEntitled } from '@/lib/billing/core/subscription' +import { isUsageMonitoringEnabled } from '@/lib/core/config/env-flags' +import { getWorkspaceHostContextForViewer } from '@/lib/workspaces/host-context' +import UsageEventsLoading from '@/app/workspace/[workspaceId]/settings/usage/events/loading' +import { UsageEventsView } from '@/ee/organization-usage/components/usage-events-view' + +export const metadata: Metadata = { + title: 'Usage events', +} + +interface UsageEventsPageProps { + params: Promise<{ workspaceId: string }> +} + +/** + * This route sits outside `[section]`, so it inherits none of that page's gates and + * has to repeat them: a host organization, an org-admin viewer, and the enterprise + * entitlement. The API refuses regardless; this keeps a deep link from rendering a + * shell the viewer can never fill. + */ +export default async function UsageEventsPage({ params }: UsageEventsPageProps) { + const session = await getSession() + if (!session?.user) redirect('/login') + + const { workspaceId } = await params + const hostContext = await getWorkspaceHostContextForViewer(workspaceId, session.user.id) + const organizationId = hostContext?.hostOrganizationId + if (!hostContext || !organizationId || !hostContext.viewer.isHostOrganizationAdmin) { + redirect(`/workspace/${workspaceId}/settings/general`) + } + if (!(await isOrganizationFeatureEntitled(organizationId, isUsageMonitoringEnabled))) { + redirect(`/workspace/${workspaceId}/settings/general`) + } + + return ( + }> + + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar/column-types.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar/column-types.ts index 2f235137f17..a6ea0ba2ac1 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar/column-types.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar/column-types.ts @@ -8,9 +8,9 @@ import { ALL_COLUMN_TYPES } from '@/lib/table/column-types' * "+ New column" dropdown to spawn a workflow group; the resulting columns are * stored as scalar types under the hood (none carry `'workflow'`). */ -export type SidebarColumnType = ColumnDefinition['type'] | 'workflow' +type SidebarColumnType = ColumnDefinition['type'] | 'workflow' -export interface ColumnTypeOption { +interface ColumnTypeOption { type: SidebarColumnType label: string icon: React.ComponentType<{ className?: string }> diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar/index.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar/index.ts index e458001136d..0308447977f 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar/index.ts @@ -1,8 +1,3 @@ export type { ColumnConfig } from './column-config-sidebar' export { ColumnConfigSidebar } from './column-config-sidebar' -export { - COLUMN_TYPE_OPTIONS, - type ColumnTypeOption, - PLAIN_COLUMN_TYPE_OPTIONS, - type SidebarColumnType, -} from './column-types' +export { COLUMN_TYPE_OPTIONS, PLAIN_COLUMN_TYPE_OPTIONS } from './column-types' diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-dropdown/column-dropdown.test.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-dropdown/column-dropdown.test.tsx new file mode 100644 index 00000000000..6409eb7f513 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-dropdown/column-dropdown.test.tsx @@ -0,0 +1,58 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { COLUMN_TYPE_OPTIONS } from '@/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar' +import { ColumnDropdown } from '@/app/workspace/[workspaceId]/tables/[tableId]/components/column-dropdown/column-dropdown' + +let container: HTMLDivElement +let root: Root + +beforeEach(() => { + globalThis.IS_REACT_ACT_ENVIRONMENT = true + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) +}) + +afterEach(() => { + act(() => root.unmount()) + container.remove() +}) + +describe('ColumnDropdown', () => { + it('lists Enrichments as a regular entry after the column options', () => { + const onPickEnrichment = vi.fn() + + act(() => { + root.render( + + ) + }) + act(() => { + container + .querySelector('button') + ?.dispatchEvent(new MouseEvent('pointerdown', { bubbles: true, button: 0 })) + }) + + const items = [...document.body.querySelectorAll('[role="menuitem"]')] + expect(items.map((item) => item.textContent)).toEqual([ + ...COLUMN_TYPE_OPTIONS.map((option) => option.label), + 'Enrichments', + ]) + expect(document.body.querySelector('[role="separator"]')).toBeNull() + + act(() => items.at(-1)?.click()) + expect(onPickEnrichment).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-dropdown/column-dropdown.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-dropdown/column-dropdown.tsx new file mode 100644 index 00000000000..c1829b3febe --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-dropdown/column-dropdown.tsx @@ -0,0 +1,113 @@ +'use client' + +import { + ChipChevronDown, + chipContentIconClass, + chipContentLabelClass, + chipVariants, + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, + Plus, +} from '@sim/emcn' +import { Sparkles } from '@sim/emcn/icons' +import type { ColumnDefinition } from '@/lib/table' +import { COLUMN_TYPE_OPTIONS } from '../column-config-sidebar' + +const CELL_HEADER = + 'border-[var(--border)] border-r border-b bg-[var(--bg)] px-2 py-[7px] text-left align-middle' + +interface ColumnDropdownProps { + /** `'header'` renders the page-header trigger (subtle Button); `'inline-header'` renders + * the in-table column-header `` trigger. Same dropdown content either way. */ + trigger: 'header' | 'inline-header' + disabled: boolean + onPickType: (type: ColumnDefinition['type']) => void + onPickWorkflow: () => void + onPickEnrichment: () => void + /** + * When true, the trigger stays visible and clickable but opens nothing — it + * calls {@link onBlocked} instead. Used when the table is schema-locked: + * hiding the control leaves the user guessing, so it stays and explains. + * Paired required so `blocked` can never be set without a handler. + */ + blocked: boolean + onBlocked: () => void +} + +/** + * "+ New column" dropdown — the single entry point for creating a column. + * Lists every column type plus "Workflow" and "Enrichments"; picking a type + * opens the right sidebar pre-seeded. + */ +export function ColumnDropdown({ + trigger, + disabled, + onPickType, + onPickWorkflow, + onPickEnrichment, + blocked, + onBlocked, +}: ColumnDropdownProps) { + const triggerButton = + trigger === 'header' ? ( + + ) : ( + + ) + + if (blocked) { + return trigger === 'inline-header' ? ( + {triggerButton} + ) : ( + triggerButton + ) + } + + const menu = ( + + {triggerButton} + + {COLUMN_TYPE_OPTIONS.map((option) => { + const Icon = option.icon + const onSelect = + option.type === 'workflow' + ? onPickWorkflow + : () => onPickType(option.type as ColumnDefinition['type']) + return ( + + + {option.label} + + ) + })} + + + Enrichments + + + + ) + + // The in-table trigger lives inside a `` so it must be a ``. The + // header trigger lives in the page header so it sits inline. + return trigger === 'inline-header' ? {menu} : menu +} diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-dropdown/index.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-dropdown/index.ts new file mode 100644 index 00000000000..02681f50420 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-dropdown/index.ts @@ -0,0 +1 @@ +export { ColumnDropdown } from './column-dropdown' diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/index.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/index.ts index 8307081376a..a50073f74a9 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/index.ts @@ -1,10 +1,10 @@ export * from './column-config-sidebar' +export * from './column-dropdown' export * from './columns-menu' export * from './context-menu' export * from './enrichment-details' export * from './enrichments-sidebar' export * from './lock-settings-modal' -export * from './new-column-dropdown' export * from './row-modal' export * from './run-status-control' export * from './save-view-modal' diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/new-column-dropdown/index.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/new-column-dropdown/index.ts deleted file mode 100644 index 026d9ff58f1..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/new-column-dropdown/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { NewColumnDropdown } from './new-column-dropdown' diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/new-column-dropdown/new-column-dropdown.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/new-column-dropdown/new-column-dropdown.tsx deleted file mode 100644 index 2e9b21332dc..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/new-column-dropdown/new-column-dropdown.tsx +++ /dev/null @@ -1,121 +0,0 @@ -'use client' - -import { - ChipChevronDown, - chipContentIconClass, - chipContentLabelClass, - chipVariants, - DropdownMenu, - DropdownMenuContent, - DropdownMenuItem, - DropdownMenuSeparator, - DropdownMenuTrigger, - Plus, -} from '@sim/emcn' -import { Sparkles } from '@sim/emcn/icons' -import type { ColumnDefinition } from '@/lib/table' -import { COLUMN_TYPE_OPTIONS } from '../column-config-sidebar' - -const CELL_HEADER = - 'border-[var(--border)] border-r border-b bg-[var(--bg)] px-2 py-[7px] text-left align-middle' - -interface NewColumnDropdownProps { - /** `'header'` renders the page-header trigger (subtle Button); `'inline-header'` renders - * the in-table column-header `` trigger. Same dropdown content either way. */ - trigger: 'header' | 'inline-header' - disabled: boolean - onPickType: (type: ColumnDefinition['type']) => void - onPickWorkflow: () => void - onPickEnrichment: () => void - /** - * When true, the trigger stays visible and clickable but opens nothing — it - * calls {@link onBlocked} instead. Used when the table is schema-locked: - * hiding the control leaves the user guessing, so it stays and explains. - * Paired required so `blocked` can never be set without a handler. - */ - blocked: boolean - onBlocked: () => void -} - -/** - * "+ New column" dropdown — the single entry point for creating a column. - * Lists every column type plus "Workflow" and "Enrichments"; picking a type - * opens the right sidebar pre-seeded. - */ -export function NewColumnDropdown({ - trigger, - disabled, - onPickType, - onPickWorkflow, - onPickEnrichment, - blocked, - onBlocked, -}: NewColumnDropdownProps) { - const triggerButton = - trigger === 'header' ? ( - - ) : ( - - ) - - if (blocked) { - return trigger === 'inline-header' ? ( - {triggerButton} - ) : ( - triggerButton - ) - } - - const menu = ( - - {triggerButton} - {/* Taller than the 240px shared default: the full type list is 9 items - (295px with its separator and padding), so the default cut the last - two off behind a scrollbar. Sized here rather than in the shared - component, which every other dropdown in the app relies on. */} - - <> - - - Enrichments - - - - {COLUMN_TYPE_OPTIONS.map((option) => { - const Icon = option.icon - const onSelect = - option.type === 'workflow' - ? onPickWorkflow - : () => onPickType(option.type as ColumnDefinition['type']) - return ( - - - {option.label} - - ) - })} - - - ) - - // The in-table trigger lives inside a `` so it must be a ``. The - // header trigger lives in the page header so it sits inline. - return trigger === 'inline-header' ? {menu} : menu -} diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx index 5758689092d..63c97d55518 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx @@ -57,8 +57,8 @@ import { useContextMenu, useTable } from '../../hooks' import type { EditingCell, QueryOptions, SaveReason } from '../../types' import { cleanCellValue, generateColumnName as sharedGenerateColumnName } from '../../utils' import type { ColumnConfig } from '../column-config-sidebar' +import { ColumnDropdown } from '../column-dropdown' import { ContextMenu } from '../context-menu' -import { NewColumnDropdown } from '../new-column-dropdown' import type { WorkflowConfig } from '../workflow-sidebar' import { ExpandedCellPopover } from './cells' import { ADD_COL_WIDTH, COL_WIDTH, SELECTION_TINT_BG } from './constants' @@ -898,7 +898,11 @@ export function TableGrid({ * so solo editing never pays the map build. */ const columnIndexById = useMemo(() => { const map = new Map() - if (remoteSelections.length > 0) displayColumns.forEach((col, index) => map.set(col.key, index)) + if (remoteSelections.length > 0) { + displayColumns.forEach((col, index) => { + map.set(col.key, index) + }) + } return map }, [displayColumns, remoteSelections.length]) @@ -907,7 +911,11 @@ export function TableGrid({ * solo editing never pays the O(n) map build on a refetch. */ const rowIndexById = useMemo(() => { const map = new Map() - if (remoteSelections.length > 0) rows.forEach((row, index) => map.set(row.id, index)) + if (remoteSelections.length > 0) { + rows.forEach((row, index) => { + map.set(row.id, index) + }) + } return map }, [rows, remoteSelections.length]) @@ -2251,7 +2259,9 @@ export function TableGrid({ const draggedGid = colByName.get(dragged)?.workflowGroupId const orderIndex = new Map() - currentOrder.forEach((n, i) => orderIndex.set(n, i)) + currentOrder.forEach((n, i) => { + orderIndex.set(n, i) + }) // Compute the contiguous run covering the dragged column. For a plain // column this is just [fromIndex, fromIndex]. For a group member it spans @@ -4853,7 +4863,7 @@ export function TableGrid({ ) })} {userPermissions.canEdit && ( - { ) }) + it.each([ + { + name: 'workspace API key', + serializedPrincipal: { + version: 1 as const, + principal: { + kind: 'workspace_api_key' as const, + workspaceId: 'workspace-1', + keyId: 'workspace-key-1', + }, + }, + isPublicApiAccess: false, + }, + { + name: 'public API system', + serializedPrincipal: { + version: 1 as const, + principal: { + kind: 'system' as const, + serviceId: 'public_api' as const, + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + }, + }, + isPublicApiAccess: true, + }, + ])( + 'restores the exact serialized $name principal before Trigger worker execution', + async ({ serializedPrincipal, isPublicApiAccess }) => { + mockPreprocessExecution.mockResolvedValueOnce({ + success: true, + actorUserId: 'actor-1', + workflowRecord: { + id: 'workflow-1', + userId: 'owner-1', + workspaceId: 'workspace-1', + variables: {}, + }, + billingAttribution, + executionTimeout: {}, + }) + mockExecuteWorkflowCore.mockResolvedValueOnce({ + success: true, + status: 'success', + output: { ok: true }, + metadata: { duration: 10, userId: 'actor-1' }, + }) + + await executeWorkflowJob({ + principal: serializedPrincipal, + workflowId: 'workflow-1', + userId: 'actor-1', + workspaceId: 'workspace-1', + billingAttribution, + triggerType: 'api', + executionId: `execution-${serializedPrincipal.principal.kind}`, + requestId: `request-${serializedPrincipal.principal.kind}`, + isPublicApiAccess, + }) + + const executionMetadata = mockExecutionSnapshot.mock.calls[0]?.[0] + expect(executionMetadata.userId).toBe('actor-1') + expect(executionMetadata.principal).toEqual(serializedPrincipal.principal) + expect(executionMetadata.isPublicApiAccess).toBe(isPublicApiAccess) + expect(executionMetadata.principal).not.toHaveProperty('userId') + } + ) + it('restores a legacy authenticated workflow job as its recorded user actor', async () => { mockPreprocessExecution.mockResolvedValueOnce({ success: true, @@ -545,6 +613,14 @@ describe('async preprocessing correlation threading', () => { loggingSession, }) ) + const executionMetadata = mockExecutionSnapshot.mock.calls[0]?.[0] + expect(executionMetadata.userId).toBe('actor-2') + expect(executionMetadata.principal).toEqual({ + kind: 'system', + serviceId: 'schedule', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + }) }) it('passes workflow correlation into preprocessing', async () => { diff --git a/apps/sim/background/webhook-execution.test.ts b/apps/sim/background/webhook-execution.test.ts index 1de01f40684..b81cc566dba 100644 --- a/apps/sim/background/webhook-execution.test.ts +++ b/apps/sim/background/webhook-execution.test.ts @@ -346,6 +346,51 @@ describe('executeWebhookJob fault vs error handling', () => { ) }) + it('restores the exact serialized webhook principal without substituting the billing actor', async () => { + const serializedPrincipal = { + version: 1 as const, + principal: { + kind: 'system' as const, + serviceId: 'webhook' as const, + webhookId: 'webhook-1', + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + provider: 'slack', + subject: { + kind: 'external_user' as const, + provider: 'slack', + tenantId: 'team-1', + subjectId: 'slack-user-1', + }, + }, + } + mockExecuteWorkflowCore.mockResolvedValueOnce({ + success: true, + status: 'completed', + output: {}, + logs: [], + executionState: { + blockStates: {}, + executedBlocks: [], + blockLogs: [], + decisions: {}, + completedLoops: [], + activeExecutionPath: [], + }, + }) + + await executeWebhookJob({ + ...payload, + provider: 'slack', + principal: serializedPrincipal, + }) + + const executionMetadata = mockExecutionSnapshot.mock.calls[0]?.[0] + expect(executionMetadata.userId).toBe('user-1') + expect(executionMetadata.principal).toEqual(serializedPrincipal.principal) + expect(executionMetadata.principal).not.toHaveProperty('userId') + }) + it('persists the reconstructed legacy principal on setup retries', async () => { executionPreprocessingMockFns.mockPreprocessExecution.mockResolvedValueOnce({ success: false, diff --git a/apps/sim/blocks/blocks/file.test.ts b/apps/sim/blocks/blocks/file.test.ts index 807ebc1f4f2..a5f3b46c4a3 100644 --- a/apps/sim/blocks/blocks/file.test.ts +++ b/apps/sim/blocks/blocks/file.test.ts @@ -16,7 +16,7 @@ describe('FileV4Block', () => { }, }) ).toMatchObject({ - filePath: 'https://example.com/image.jpg', + fileUrl: 'https://example.com/image.jpg', workspaceId: 'workspace-1', workflowId: 'workflow-1', executionId: 'execution-1', diff --git a/apps/sim/blocks/blocks/file.ts b/apps/sim/blocks/blocks/file.ts index 87712034751..e566a746dbd 100644 --- a/apps/sim/blocks/blocks/file.ts +++ b/apps/sim/blocks/blocks/file.ts @@ -827,8 +827,7 @@ export const FileV4Block: BlockConfig = { const fileUrl = resolveHttpFileUrl(params.fileUrl) return { - filePath: fileUrl, - fileType: params.fileType || 'auto', + fileUrl, headers: params.headers, workspaceId: params._context?.workspaceId, workflowId: params._context?.workflowId, @@ -1357,8 +1356,7 @@ export const FileV5Block: BlockConfig = { const fileUrl = resolveHttpFileUrl(params.fileUrl) return { - filePath: fileUrl, - fileType: params.fileType || 'auto', + fileUrl, headers: params.headers, workspaceId: params._context?.workspaceId, workflowId: params._context?.workflowId, diff --git a/apps/sim/blocks/blocks/google_drive.test.ts b/apps/sim/blocks/blocks/google_drive.test.ts new file mode 100644 index 00000000000..958076f2c23 --- /dev/null +++ b/apps/sim/blocks/blocks/google_drive.test.ts @@ -0,0 +1,97 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('@/triggers', () => ({ + getTrigger: () => ({ subBlocks: [] }), +})) + +import { GoogleDriveBlock } from '@/blocks/blocks/google_drive' +import { listTool } from '@/tools/google_drive/list' +import { listCommentsTool } from '@/tools/google_drive/list_comments' +import { listPermissionsTool } from '@/tools/google_drive/list_permissions' +import { listRevisionsTool } from '@/tools/google_drive/list_revisions' +import { searchTool } from '@/tools/google_drive/search' + +const paginationCases = [ + { operation: 'list', subBlockId: 'pageToken', tool: listTool }, + { operation: 'search', subBlockId: 'searchPageToken', tool: searchTool }, + { operation: 'list_permissions', subBlockId: 'permissionsPageToken', tool: listPermissionsTool }, + { operation: 'list_revisions', subBlockId: 'revisionsPageToken', tool: listRevisionsTool }, + { operation: 'list_comments', subBlockId: 'commentsPageToken', tool: listCommentsTool }, +] as const + +describe('GoogleDriveBlock pagination', () => { + const buildParams = GoogleDriveBlock.tools.config.params! + + describe.each(paginationCases)('$operation', ({ operation, subBlockId, tool }) => { + it('exposes a page token field scoped to the operation', () => { + expect(GoogleDriveBlock.subBlocks.find(({ id }) => id === subBlockId)).toMatchObject({ + type: 'short-input', + mode: 'advanced', + condition: { field: 'operation', value: operation }, + }) + }) + + /** + * `pageToken` is the canonical tool param, so the `list` case would forward + * through `...rest` even without the mapper. The per-operation ids are the + * ones the mapper has to translate, and none of them may survive as-is. + */ + it('forwards the page token to the tool under its own id', () => { + const params = buildParams({ operation, [subBlockId]: 'token-abc' }, undefined as never) + + expect(params).toMatchObject({ pageToken: 'token-abc' }) + if (subBlockId !== 'pageToken') expect(params[subBlockId]).toBeUndefined() + }) + + it('lets an agent feed a nextPageToken back in', () => { + expect(tool.params.pageToken?.visibility).toBe('user-or-llm') + }) + }) + + it('does not leak a page token into operations that do not paginate', () => { + expect( + buildParams({ operation: 'get_file', pageToken: 'token-abc' }, undefined as never).pageToken + ).toBeUndefined() + }) + + /** + * `shouldSerializeSubBlock` short-circuits for `advanced` fields in basic display + * mode without evaluating `condition`, so a page token typed under one operation + * genuinely reaches `inputs` after the user switches to another. The mapper must + * pick the token belonging to the operation being run and drop the rest. + */ + describe.each(paginationCases.filter(({ subBlockId }) => subBlockId !== 'pageToken'))( + '$subBlockId left over from a previous operation', + ({ subBlockId }) => { + it.each(['upload', 'get_file', 'list'])('is dropped under %s', (operation) => { + const params = buildParams({ operation, [subBlockId]: 'stale' }, undefined as never) + + expect(params.pageToken).toBeUndefined() + expect(params[subBlockId]).toBeUndefined() + }) + } + ) + + it('prefers the operation-owned token when a stale sibling is also present', () => { + const params = buildParams( + { + operation: 'search', + searchPageToken: 'search-token', + commentsPageToken: 'stale', + pageToken: 'stale-canonical', + }, + undefined as never + ) + + expect(params.pageToken).toBe('search-token') + expect(params.commentsPageToken).toBeUndefined() + expect(params.searchPageToken).toBeUndefined() + }) + + it('declares pageToken as a block input', () => { + expect(GoogleDriveBlock.inputs.pageToken).toBeDefined() + }) +}) diff --git a/apps/sim/blocks/blocks/google_drive.ts b/apps/sim/blocks/blocks/google_drive.ts index 7b470d505a5..3658b9be3c8 100644 --- a/apps/sim/blocks/blocks/google_drive.ts +++ b/apps/sim/blocks/blocks/google_drive.ts @@ -463,6 +463,14 @@ Return ONLY the query string - no explanations, no quotes around the whole thing placeholder: 'Number of results (default: 100, max: 100)', condition: { field: 'operation', value: 'list' }, }, + { + id: 'pageToken', + title: 'Page Token', + type: 'short-input', + placeholder: 'Token from a previous nextPageToken', + mode: 'advanced', + condition: { field: 'operation', value: 'list' }, + }, // Download File Fields - File Selector (basic mode) { id: 'downloadFileSelector', @@ -905,6 +913,14 @@ Return ONLY the message text - no subject line, no greetings/signatures, no extr condition: { field: 'operation', value: 'list_permissions' }, required: true, }, + { + id: 'permissionsPageToken', + title: 'Page Token', + type: 'short-input', + placeholder: 'Token from a previous nextPageToken', + mode: 'advanced', + condition: { field: 'operation', value: 'list_permissions' }, + }, // Get File Content Fields { id: 'getContentFileSelector', @@ -1073,6 +1089,14 @@ Return ONLY the query string - no explanations, no quotes around the whole thing mode: 'advanced', condition: { field: 'operation', value: 'search' }, }, + { + id: 'searchPageToken', + title: 'Page Token', + type: 'short-input', + placeholder: 'Token from a previous nextPageToken', + mode: 'advanced', + condition: { field: 'operation', value: 'search' }, + }, // Untrash File Fields { id: 'untrashFileSelector', @@ -1191,6 +1215,14 @@ Return ONLY the query string - no explanations, no quotes around the whole thing mode: 'advanced', condition: { field: 'operation', value: 'list_revisions' }, }, + { + id: 'revisionsPageToken', + title: 'Page Token', + type: 'short-input', + placeholder: 'Token from a previous nextPageToken', + mode: 'advanced', + condition: { field: 'operation', value: 'list_revisions' }, + }, { id: 'getRevisionFileSelector', title: 'Select File', @@ -1255,6 +1287,14 @@ Return ONLY the query string - no explanations, no quotes around the whole thing mode: 'advanced', condition: { field: 'operation', value: 'list_comments' }, }, + { + id: 'commentsPageToken', + title: 'Page Token', + type: 'short-input', + placeholder: 'Token from a previous nextPageToken', + mode: 'advanced', + condition: { field: 'operation', value: 'list_comments' }, + }, { id: 'includeDeleted', title: 'Include Deleted Comments', @@ -1473,6 +1513,11 @@ Return ONLY the comment text - no explanations, no quotes, no extra formatting.` searchPageSize, revisionsPageSize, commentsPageSize, + pageToken, + searchPageToken, + permissionsPageToken, + revisionsPageToken, + commentsPageToken, getContentExportMimeType, exportMimeType, ...rest @@ -1586,6 +1631,13 @@ Return ONLY the comment text - no explanations, no quotes, no extra formatting.` else if (params.operation === 'list_revisions') effectivePageSize = revisionsPageSize else if (params.operation === 'list_comments') effectivePageSize = commentsPageSize + let effectivePageToken: string | undefined = pageToken + if (params.operation === 'search') effectivePageToken = searchPageToken + else if (params.operation === 'list_permissions') effectivePageToken = permissionsPageToken + else if (params.operation === 'list_revisions') effectivePageToken = revisionsPageToken + else if (params.operation === 'list_comments') effectivePageToken = commentsPageToken + else if (params.operation !== 'list') effectivePageToken = undefined + const effectiveQuery = params.operation === 'search' ? searchQuery : query const effectiveMimeType = params.operation === 'get_content' @@ -1603,6 +1655,7 @@ Return ONLY the comment text - no explanations, no quotes, no extra formatting.` pageSize: effectivePageSize ? Number.parseInt(effectivePageSize as string, 10) : undefined, + pageToken: effectivePageToken?.trim() || undefined, query: effectiveQuery, mimeType: effectiveMimeType === 'auto' ? undefined : effectiveMimeType, type: shareType, // Map shareType to type for share tool @@ -1660,6 +1713,7 @@ Return ONLY the comment text - no explanations, no quotes, no extra formatting.` // List operation inputs query: { type: 'string', description: 'Search query' }, pageSize: { type: 'number', description: 'Results per page' }, + pageToken: { type: 'string', description: 'Pagination token from a previous nextPageToken' }, // Copy operation inputs newName: { type: 'string', description: 'New name for copied file' }, // Update operation inputs diff --git a/apps/sim/blocks/blocks/jira_service_management.test.ts b/apps/sim/blocks/blocks/jira_service_management.test.ts index 291644f55d8..89195c9505c 100644 --- a/apps/sim/blocks/blocks/jira_service_management.test.ts +++ b/apps/sim/blocks/blocks/jira_service_management.test.ts @@ -29,17 +29,17 @@ import { jsmGetSlaTool, jsmGetTransitionsTool, } from '@/tools/jsm' -import type { ToolConfig, ToolResponse } from '@/tools/types' +import type { InternalToolConfig, ToolResponse } from '@/tools/types' const DOMAIN = 'example.atlassian.net' -/** Injected by the executor from the OAuth credential before the tool's `body` runs. */ +/** Injected by the executor from the OAuth credential before the tool's operation input runs. */ const ACCESS_TOKEN = 'token-123' interface PaginatedCase { operation: string toolId: string - /** The tool's own `request.body`, captured at its concrete param type by `paginatedCase`. */ - buildBody: (params: Record) => Record + /** The tool's operation input, captured at its concrete param type by `paginatedCase`. */ + buildInput: (params: Record) => Record schema: z.ZodType extraInputs: Record } @@ -50,28 +50,24 @@ interface PaginatedCase { */ function paginatedCase( operation: string, - tool: ToolConfig, + tool: InternalToolConfig, schema: z.ZodType, extraInputs: Record = {} ): PaginatedCase { return { operation, toolId: tool.id, - buildBody: (params) => { - const bodyFn = tool.request.body - if (!bodyFn) throw new Error(`${tool.id} is missing request.body`) - return bodyFn(params as P) as Record - }, + buildInput: (params) => tool.operation.input(params as P) as Record, schema, extraInputs, } } /** - * Every paginated JSM operation, wired to the tool it resolves to and the contract its route - * parses the body with. This walks the real chain — block `tools.config.params` → the tool's - * `request.body` → the route contract — which is exactly where `jsm_get_comments` broke: the - * tools declare `start`/`limit` as `type: 'number'` while the contract demanded strings. + * Every paginated JSM operation, wired to the tool it resolves to and the schema its direct + * handler parses the operation input with. This walks the real chain — block + * `tools.config.params` → the tool's `operation.input` → direct-handler validation — where the + * tools declare `start`/`limit` as numbers and the provider operation normalizes them to strings. */ const PAGINATED_CASES: PaginatedCase[] = [ paginatedCase('get_service_desks', jsmGetServiceDesksTool, jsmServiceDesksBodySchema), @@ -101,9 +97,9 @@ const PAGINATED_CASES: PaginatedCase[] = [ }), ] -/** Run a set of block inputs through `tools.config.params`, then through the tool's request body. */ -function buildRequestBody( - { operation, buildBody, extraInputs }: PaginatedCase, +/** Run block inputs through `tools.config.params`, then through the tool's operation input. */ +function buildOperationInput( + { operation, buildInput, extraInputs }: PaginatedCase, pagination: Record ) { const paramsFn = JiraServiceManagementBlock.tools.config?.params @@ -117,7 +113,7 @@ function buildRequestBody( ...pagination, }) - return buildBody({ ...toolParams, accessToken: ACCESS_TOKEN, domain: DOMAIN }) + return buildInput({ ...toolParams, accessToken: ACCESS_TOKEN, domain: DOMAIN }) } describe.each(PAGINATED_CASES.map((testCase) => [testCase.operation, testCase] as const))( @@ -129,28 +125,31 @@ describe.each(PAGINATED_CASES.map((testCase) => [testCase.operation, testCase] a expect(JiraServiceManagementBlock.tools.access).toContain(testCase.toolId) }) - it('sends a body its route contract accepts when pagination is filled in', () => { - const body = buildRequestBody(testCase, { startIndex: '50', maxResults: '25' }) + it('builds direct operation input accepted by the handler when pagination is filled in', () => { + const input = buildOperationInput(testCase, { startIndex: '50', maxResults: '25' }) - expect(body.start).toBe(50) - expect(body.limit).toBe(25) - expect(testCase.schema.parse(body)).toMatchObject({ start: '50', limit: '25' }) + expect(input.start).toBe(50) + expect(input.limit).toBe(25) + expect(testCase.schema.parse(input)).toMatchObject({ start: '50', limit: '25' }) }) - it('sends a body its route contract accepts when pagination is left blank', () => { - const body = buildRequestBody(testCase, {}) + it('builds direct operation input accepted by the handler when pagination is blank', () => { + const input = buildOperationInput(testCase, {}) - expect(body.start).toBeUndefined() - expect(body.limit).toBeUndefined() - expect(() => testCase.schema.parse(body)).not.toThrow() + expect(input.start).toBeUndefined() + expect(input.limit).toBeUndefined() + expect(() => testCase.schema.parse(input)).not.toThrow() }) it('drops non-numeric pagination input instead of sending NaN', () => { - const body = buildRequestBody(testCase, { startIndex: 'not-a-number', maxResults: '' }) - - expect(body.start).toBeUndefined() - expect(body.limit).toBeUndefined() - expect(() => testCase.schema.parse(body)).not.toThrow() + const input = buildOperationInput(testCase, { + startIndex: 'not-a-number', + maxResults: '', + }) + + expect(input.start).toBeUndefined() + expect(input.limit).toBeUndefined() + expect(() => testCase.schema.parse(input)).not.toThrow() }) } ) diff --git a/apps/sim/blocks/blocks/pulse.ts b/apps/sim/blocks/blocks/pulse.ts index d4b98bceaad..30382333e96 100644 --- a/apps/sim/blocks/blocks/pulse.ts +++ b/apps/sim/blocks/blocks/pulse.ts @@ -141,7 +141,11 @@ export const PulseBlock: BlockConfig = { bounding_boxes: { type: 'json', description: 'Bounding box layout information' }, extraction_url: { type: 'string', description: 'URL for extraction results (large documents)' }, html: { type: 'string', description: 'HTML content if requested' }, - structured_output: { type: 'json', description: 'Structured output if schema was provided' }, + structured_output: { + type: 'json', + description: + 'Structured output; Sim exposes no input for supplying a schema, so this is always null', + }, chunks: { type: 'json', description: 'Chunked content if chunking was enabled' }, figures: { type: 'json', description: 'Extracted figures if figure extraction was enabled' }, }, diff --git a/apps/sim/blocks/blocks/sap_concur.ts b/apps/sim/blocks/blocks/sap_concur.ts index 46809164f5c..d39d475326d 100644 --- a/apps/sim/blocks/blocks/sap_concur.ts +++ b/apps/sim/blocks/blocks/sap_concur.ts @@ -2,7 +2,7 @@ import { SapConcurIcon } from '@/components/icons' import type { BlockConfig, BlockMeta } from '@/blocks/types' import { AuthMode, IntegrationType } from '@/blocks/types' import { normalizeFileInput } from '@/blocks/utils' -import type { SapConcurProxyResponse, UserFileLike } from '@/tools/sap_concur/types' +import type { SapConcurResponse, UserFileLike } from '@/tools/sap_concur/types' const toBool = (v: unknown): boolean | undefined => { if (v === undefined || v === null || v === '') return undefined @@ -192,7 +192,7 @@ const BODY_OPS = [ /** Canonical receipt pair: basic upload, advanced file reference. */ const RECEIPT_FIELD = ['receiptFile', 'receiptFileRef'] as const -export const SapConcurBlock: BlockConfig = { +export const SapConcurBlock: BlockConfig = { type: 'sap_concur', name: 'SAP Concur', description: 'Manage expense reports, travel requests, cash advances, and more in SAP Concur', diff --git a/apps/sim/blocks/blocks/sap_s4hana.ts b/apps/sim/blocks/blocks/sap_s4hana.ts index e43cd5bdc8f..4d1eddfa3a0 100644 --- a/apps/sim/blocks/blocks/sap_s4hana.ts +++ b/apps/sim/blocks/blocks/sap_s4hana.ts @@ -1,7 +1,7 @@ import { SapS4HanaIcon } from '@/components/icons' import type { BlockConfig, BlockMeta, CanvasSentence } from '@/blocks/types' import { AuthMode, IntegrationType } from '@/blocks/types' -import type { SapProxyResponse } from '@/tools/sap_s4hana/types' +import type { SapS4HanaResponse } from '@/tools/sap_s4hana/types' /** * Whichever name a new business partner carries: an organization has @@ -37,7 +37,7 @@ function updateSentence(noun: string, keyField: string): CanvasSentence { ] } -export const SapS4HanaBlock: BlockConfig = { +export const SapS4HanaBlock: BlockConfig = { type: 'sap_s4hana', name: 'SAP S4HANA', description: 'Read and write SAP S4HANA Cloud business data via OData', diff --git a/apps/sim/blocks/blocks/tinyfish.ts b/apps/sim/blocks/blocks/tinyfish.ts index 798fef37d40..53e96533e47 100644 --- a/apps/sim/blocks/blocks/tinyfish.ts +++ b/apps/sim/blocks/blocks/tinyfish.ts @@ -38,7 +38,7 @@ export const TinyFishBlock: BlockConfig = { docsLink: 'https://docs.sim.ai/integrations/tinyfish', category: 'tools', integrationType: IntegrationType.AI, - bgColor: '#FF6700', + bgColor: '#FFFFFF', icon: TinyFishIcon, canvasPresentation: { defaultTitle: 'TinyFish', diff --git a/apps/sim/blocks/blocks/vanta.ts b/apps/sim/blocks/blocks/vanta.ts index 674c5545df0..20746de67f5 100644 --- a/apps/sim/blocks/blocks/vanta.ts +++ b/apps/sim/blocks/blocks/vanta.ts @@ -286,14 +286,6 @@ export const VantaBlock: BlockConfig = { condition: { field: 'operation', value: 'upload_document_file' }, mode: 'advanced', }, - { - id: 'uploadMimeType', - title: 'MIME Type', - type: 'short-input', - placeholder: 'e.g., application/pdf (used when the file has no type of its own)', - condition: { field: 'operation', value: 'upload_document_file' }, - mode: 'advanced', - }, { id: 'uploadDescription', title: 'Description', @@ -930,7 +922,6 @@ export const VantaBlock: BlockConfig = { const normalizedFile = normalizeFileInput(rest.file, { single: true }) if (normalizedFile) result.file = normalizedFile result.fileName = optionalString(rest.uploadFileName) - result.mimeType = optionalString(rest.uploadMimeType) result.description = optionalString(rest.uploadDescription) result.effectiveAtDate = optionalString(rest.effectiveAtDate) break @@ -993,10 +984,6 @@ export const VantaBlock: BlockConfig = { uploadedFileId: { type: 'string', description: 'Uploaded file ID' }, file: { type: 'json', description: 'Evidence file to upload' }, uploadFileName: { type: 'string', description: 'Optional file name override' }, - uploadMimeType: { - type: 'string', - description: 'MIME type override used when the uploaded content has no type of its own', - }, uploadDescription: { type: 'string', description: 'Description of the uploaded evidence' }, effectiveAtDate: { type: 'string', description: 'Effective date of the document (ISO 8601)' }, frameworkMatchesAny: { type: 'string', description: 'Comma-separated framework ID filters' }, diff --git a/apps/sim/components/charts/bar-chart.tsx b/apps/sim/components/charts/bar-chart.tsx new file mode 100644 index 00000000000..17acfbe4cb9 --- /dev/null +++ b/apps/sim/components/charts/bar-chart.tsx @@ -0,0 +1,376 @@ +'use client' + +import { memo, useId, useMemo, useState } from 'react' +import { cn } from '@sim/emcn' +import { + formatChartCompactNumber, + formatChartLatency, + formatChartTimestamp, +} from '@/components/charts/chart-format' +import { + CHART_AXIS_LABEL_GAP, + CHART_DEFAULT_HEIGHT, + CHART_GRID_FRACTIONS, + CHART_TICK_FILL, + CHART_TICK_FONT_SIZE, + chartPlotBand, + formatTimeTick, + resolveChartPadding, + resolveSpanMs, + resolveTimeTickIndices, +} from '@/components/charts/chart-geometry' +import { + ChartTooltip, + ChartTooltipRow, + estimateTooltipHeight, + estimateTooltipWidth, + positionChartTooltip, +} from '@/components/charts/chart-tooltip' +import { + useChartWidth, + useIsDarkTheme, + useResolvedChartColors, +} from '@/components/charts/use-chart-theme' + +export interface BarChartPoint { + timestamp: string + value: number +} + +interface BarChartProps { + data: BarChartPoint[] + /** Pass `''` for the caller-owned-wrapper form, mirroring {@link LineChart}. */ + label: string + color: string + /** `''` | `'%'` | `'ms'` | `'latency'` | `'credits'` — drives tick and tooltip formatting. */ + unit?: string + height?: number + /** Bucket drawn at full opacity, e.g. the period in progress. */ + highlightIndex?: number +} + +/** Tick and tooltip text for a bucket's value, in the caller's unit. */ +function formatBarValue(value: number | undefined, unit: string | undefined): string { + if (typeof value !== 'number' || !Number.isFinite(value)) return '—' + const suffix = (unit ?? '').toLowerCase() + if (suffix.includes('%')) return `${value.toFixed(1)}%` + if (suffix === 'latency') return formatChartLatency(value) + if (suffix.includes('ms')) return `${Math.round(value)}ms` + if (suffix === 'credits') return formatChartCompactNumber(value) + return `${Math.round(value)}${unit ?? ''}` +} + +/** + * Discrete time buckets as bars. + * + * The sibling of {@link LineChart}, and deliberately built from the same geometry, + * tooltip, and theme modules: a smoothed line implies a continuous signal between + * samples, which is wrong for a calendar bucket like a day's spend, but the two must + * still line up pixel-for-pixel when stacked in one card. + */ +function BarChartComponent({ + data, + label, + color, + unit, + height = CHART_DEFAULT_HEIGHT, + highlightIndex, +}: BarChartProps) { + /* + `useId`, not `useRef(generateShortId())`: a ref initializer is evaluated on + every render and all but the first result thrown away, and React already has + a hook whose whole job is a stable unique id. + */ + const uniqueId = useId().replace(/:/g, '') + const [containerRef, containerWidth] = useChartWidth() + const width = containerWidth ?? 0 + const { yMin, yMax } = chartPlotBand(height) + const isDark = useIsDarkTheme() + const [hoverPos, setHoverPos] = useState<{ x: number; y: number } | null>(null) + + const resolvedColors = useResolvedChartColors({ base: color }) + const resolvedColor = resolvedColors.base || color + + const hasExternalWrapper = !label + + /** + * The track is read against its own background, so its opacity is per-theme + * rather than one shared value. `--border` is the platform's neutral track + * token — the same one the proportional row meters use — but it resolves to + * `#444` on dark and `#d8d8d8` on light, and a strength that reads as a column + * on near-black is a half-percent delta on white. Hover keeps the same ratio. + */ + const trackOpacity = isDark ? 0.12 : 0.3 + const trackHoverOpacity = isDark ? 0.22 : 0.5 + + const maxValue = useMemo(() => { + const peak = Math.max(...data.map((d) => d.value), 0) + return peak <= 0 ? 1 : peak * 1.1 + }, [data]) + + const padding = resolveChartPadding([formatBarValue(maxValue, unit), '0']) + const chartWidth = width - padding.left - padding.right + const chartHeight = height - padding.top - padding.bottom + + /** Slot geometry: every bucket owns an equal slice, with the bar centred in it. */ + const slot = data.length > 0 ? Math.max(1, chartWidth) / data.length : 0 + const barWidth = Math.max(1, Math.min(24, slot * 0.7)) + + /** + * Bars own a slot, so the hovered bucket is which slot the cursor is in — not the + * nearest sample, which is how a line chart resolves it. Derived, so a resize + * mid-hover cannot leave an index disagreeing with the slot geometry. + */ + const hoverIndex = + hoverPos === null || data.length === 0 || slot <= 0 + ? null + : Math.max(0, Math.min(data.length - 1, Math.floor((hoverPos.x - padding.left) / slot))) + + const bars = useMemo( + () => + data.map((point, index) => { + const x = padding.left + slot * index + (slot - barWidth) / 2 + const rawY = padding.top + chartHeight - (point.value / maxValue) * chartHeight + const y = Math.max(yMin, Math.min(yMax, rawY)) + return { + x, + y, + /* + * A zero bucket draws nothing. The clamp above keeps a *drawn* bar off the + * axis rule, but applied to zero it floored the bar at the 3px band and + * every empty day rendered as a small amount of usage — the densified zeros + * this chart exists to show honestly. Only the track represents an empty + * bucket. + */ + height: point.value > 0 ? Math.max(0, height - padding.bottom - y) : 0, + point, + } + }), + [data, slot, barWidth, maxValue, chartHeight, height, padding.left, padding.top, yMin, yMax] + ) + + if (containerWidth === null) { + return ( +
+ ) + } + + if (data.length === 0) { + return ( + // Keeps the measurement ref: dropping it here left the observer watching a + // detached node, so a resize while empty was never seen and the next non-empty + // render laid out at the stale width. +
+

No data

+
+ ) + } + + const spanMs = resolveSpanMs(data) + const tickIndices = resolveTimeTickIndices(data.length, Math.max(1, chartWidth)) + + return ( +
+ {!hasExternalWrapper && ( +
+

{label}

+
+ )} +
+ { + if (bars.length === 0 || slot <= 0) return + const rect = (e.currentTarget as SVGSVGElement).getBoundingClientRect() + setHoverPos({ x: e.clientX - rect.left, y: e.clientY - rect.top }) + }} + onMouseLeave={() => setHoverPos(null)} + > + + + + + + + + + + {CHART_GRID_FRACTIONS.map((fraction) => ( + + ))} + + {/* + A full-height track keeps an empty bucket visible and gives every slot + the same hover target, so a run of zero days reads as zero rather than + as missing data. + + Drawn outside the blend group below: the bars want `screen` on dark so + the gradient stays luminous, but a track composited that way is only + legible against a dark background, and on white it disappears. + */} + + {bars.map((bar, index) => ( + + ))} + + + + {bars.map( + (bar, index) => + bar.height > 0 && ( + + ) + )} + + + {tickIndices.map((index) => { + const timestamp = data[index]?.timestamp + if (!timestamp) return null + const date = new Date(timestamp) + return ( + + {Number.isNaN(date.getTime()) ? '' : formatTimeTick(date, spanMs)} + + ) + })} + + + {/* Same formatter the tooltip uses, or the axis and the hover disagree + about what the numbers mean on any non-`credits` unit. */} + {formatBarValue(maxValue, unit)} + + + 0 + + + + + + {hoverIndex !== null && + bars[hoverIndex] && + (() => { + const bar = bars[hoverIndex] + const value = formatBarValue(bar.point.value, unit) + const date = formatChartTimestamp(bar.point.timestamp) + const { left, top } = positionChartTooltip({ + anchorX: hoverPos?.x ?? bar.x, + anchorY: hoverPos?.y ?? bar.y, + width, + height, + tooltipMaxWidth: estimateTooltipWidth(value.length), + tooltipHeight: estimateTooltipHeight(1, Boolean(date)), + padding, + }) + return ( + + + + ) + })()} +
+
+ ) +} + +export const BarChart = memo(BarChartComponent) diff --git a/apps/sim/components/charts/chart-format.ts b/apps/sim/components/charts/chart-format.ts new file mode 100644 index 00000000000..c5b7151814b --- /dev/null +++ b/apps/sim/components/charts/chart-format.ts @@ -0,0 +1,31 @@ +import { formatDuration } from '@sim/utils/formatting' +import { format } from 'date-fns' + +/** + * Value and tick formatting shared by the chart family. + * + * These live here rather than in the logs feature's `utils.ts` because that module + * imports the block registry, and a chart that reached for it would drag the whole + * executable registry into every consumer's bundle. + */ + +/** Duration for an axis tick or tooltip. `—` for a missing or non-positive value. */ +export function formatChartLatency(ms: number): string { + if (!Number.isFinite(ms) || ms <= 0) return '—' + return formatDuration(ms, { precision: 2 }) ?? '—' +} + +/** The tooltip's header line: `MAR 4 3:05 PM`. Empty for an unparseable timestamp. */ +export function formatChartTimestamp(timestamp?: string): string { + if (!timestamp) return '' + const date = new Date(timestamp) + if (Number.isNaN(date.getTime())) return '' + return `${format(date, 'MMM d').toUpperCase()} ${format(date, 'h:mm a')}` +} + +/** Compact axis magnitude — `1.2k`, `3.4m`. */ +export function formatChartCompactNumber(value: number): string { + return new Intl.NumberFormat('en-US', { notation: 'compact', maximumFractionDigits: 1 }) + .format(value) + .toLowerCase() +} diff --git a/apps/sim/components/charts/chart-geometry.test.ts b/apps/sim/components/charts/chart-geometry.test.ts new file mode 100644 index 00000000000..6108dc3386b --- /dev/null +++ b/apps/sim/components/charts/chart-geometry.test.ts @@ -0,0 +1,120 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + CHART_AXIS_LABEL_GAP, + CHART_PADDING, + chartPlotBand, + estimateAxisLabelWidth, + formatTimeTick, + resolveChartPadding, + resolveSpanMs, + resolveTimeTickIndices, +} from '@/components/charts/chart-geometry' + +describe('resolveTimeTickIndices', () => { + it('budgets roughly one tick per 64px, clamped to 3..8', () => { + expect(resolveTimeTickIndices(100, 100).length).toBe(3) + expect(resolveTimeTickIndices(100, 320).length).toBe(5) + expect(resolveTimeTickIndices(100, 4000).length).toBe(8) + }) + + it('dedupes the collisions rounding produces on a short series', () => { + // 2 points across a wide chart wants 8 ticks but only has indices 0 and 1. + const indices = resolveTimeTickIndices(2, 4000) + expect(indices).toEqual([...new Set(indices)]) + expect(indices.every((index) => index >= 0 && index < 2)).toBe(true) + }) + + it('always spans the full series, first index to last', () => { + const indices = resolveTimeTickIndices(50, 640) + expect(indices[0]).toBe(0) + expect(indices[indices.length - 1]).toBe(49) + }) +}) + +describe('formatTimeTick', () => { + const date = new Date('2026-03-04T15:05:00.000Z') + + it('shows clock time within a day and a half', () => { + expect(formatTimeTick(date, 36 * 60 * 60 * 1000)).toMatch(/^\d{2}:\d{2}$/) + }) + + it('shows a calendar day within a quarter', () => { + expect(formatTimeTick(date, 40 * 24 * 60 * 60 * 1000)).toMatch(/^[A-Z][a-z]{2} \d{1,2}$/) + }) + + it('shows month and year beyond a quarter', () => { + expect(formatTimeTick(date, 400 * 24 * 60 * 60 * 1000)).toMatch(/^[A-Z][a-z]{2} \d{4}$/) + }) +}) + +describe('resolveSpanMs', () => { + it('measures first to last', () => { + expect( + resolveSpanMs([ + { timestamp: '2026-03-04T00:00:00.000Z' }, + { timestamp: '2026-03-05T00:00:00.000Z' }, + ]) + ).toBe(24 * 60 * 60 * 1000) + }) + + it('returns 0 for a degenerate or unparseable series rather than NaN', () => { + expect(resolveSpanMs([])).toBe(0) + expect(resolveSpanMs([{ timestamp: '2026-03-04T00:00:00.000Z' }])).toBe(0) + expect(resolveSpanMs([{ timestamp: 'nope' }, { timestamp: 'also nope' }])).toBe(0) + }) +}) + +describe('chartPlotBand', () => { + it('insets the band so strokes clear the axis rules', () => { + // The line and bar charts both clamp to this band, which is what keeps them + // aligned when stacked in the same card. + expect(chartPlotBand(166)).toEqual({ + yMin: CHART_PADDING.top + 3, + yMax: CHART_PADDING.top + (166 - CHART_PADDING.top - CHART_PADDING.bottom) - 3, + }) + }) + + it('tracks a caller-supplied height', () => { + expect(chartPlotBand(240).yMax).toBeGreaterThan(chartPlotBand(166).yMax) + }) +}) + +describe('resolveChartPadding', () => { + it('widens the gutter until the longest label fits beside the axis', () => { + const { left } = resolveChartPadding(['7.3k', '0']) + expect(left).toBeGreaterThanOrEqual(estimateAxisLabelWidth('7.3k') + CHART_AXIS_LABEL_GAP) + }) + + it('never narrows below the shared padding', () => { + expect(resolveChartPadding(['0', '0']).left).toBeGreaterThanOrEqual(CHART_PADDING.left) + expect(resolveChartPadding([]).left).toBeGreaterThanOrEqual(CHART_PADDING.left) + }) + + /** + * Three charts sit side by side on the logs dashboard. A gutter derived exactly from + * each one's own labels put their plot origins at 26, 27 and 32 — visibly ragged + * across a row that used to share one origin. + */ + it('resolves labels of similar width to the same gutter', () => { + const gutters = [['5'], ['1.2s'], ['12.3k'], ['0'], ['7.3k']].map( + (labels) => resolveChartPadding(labels).left + ) + expect(new Set(gutters).size).toBe(1) + }) + + it('still grows for a genuinely wider label', () => { + expect(resolveChartPadding(['123456.7m']).left).toBeGreaterThan( + resolveChartPadding(['7.3k']).left + ) + }) + + it('leaves the other three sides on the shared constant', () => { + const padding = resolveChartPadding(['123.4m']) + expect(padding.top).toBe(CHART_PADDING.top) + expect(padding.right).toBe(CHART_PADDING.right) + expect(padding.bottom).toBe(CHART_PADDING.bottom) + }) +}) diff --git a/apps/sim/components/charts/chart-geometry.ts b/apps/sim/components/charts/chart-geometry.ts new file mode 100644 index 00000000000..3494543c3c2 --- /dev/null +++ b/apps/sim/components/charts/chart-geometry.ts @@ -0,0 +1,130 @@ +/** + * Geometry shared by every chart in the family. + * + * Extracted so a sibling chart cannot drift: bar and line charts read the same + * padding, the same clamps, the same gridlines, and resolve their x ticks the same + * way, so two charts stacked in one card line up on the pixel. + * + * Pure — no React, no DOM — so a server module can read the constants. + */ + +export const CHART_PADDING = { top: 16, right: 28, bottom: 26, left: 26 } as const + +export type ChartPadding = { top: number; right: number; bottom: number; left: number } + +/** Matches the loader placeholders callers size themselves against. */ +export const CHART_DEFAULT_HEIGHT = 166 + +/** + * Below this the axis labels collide, so the chart scrolls rather than compresses. + * + * Consumers pair `overflow-x-auto` with `overflow-y-hidden`: a computed `overflow-x` + * other than `visible` promotes `overflow-y: visible` to `auto`, so the tooltip's + * shadow reaching the foot of the box raised a vertical scrollbar over the chart + * whenever the cursor neared the axis. + */ +export const CHART_MIN_WIDTH = 280 + +export const CHART_TICK_FILL = 'var(--text-tertiary)' +export const CHART_TICK_FONT_SIZE = 9 +export const CHART_GRID_FRACTIONS = [0.25, 0.5, 0.75] as const + +/** Punctuation and whitespace, which sit near half the width of a digit or letter. */ +const NARROW_GLYPH = /[.,:\s]/ + +/** Gap between a y-axis tick label's right edge and the axis rule. */ +export const CHART_AXIS_LABEL_GAP = 8 + +/** + * The gutter is rounded up to a multiple of this. + * + * Charts are read side by side — the logs dashboard puts three in one row — and a + * gutter derived exactly from each chart's own labels made `5`, `1.2s` and `12.3k` + * resolve to 26, 27 and 32, so three plots that used to share an origin no longer + * did. Quantizing collapses differences this small to one value while still growing + * for a genuinely wider label, and it turns the sub-pixel slack that `Math.ceil` + * alone left into several pixels. + */ +const CHART_AXIS_GUTTER_STEP = 8 + +/** + * Rendered width of a right-anchored y-axis tick label. + * + * SVG `` cannot be measured before layout, so the gutter that has to hold it + * is estimated from the glyphs instead. The ratios are for the UI sans at + * {@link CHART_TICK_FONT_SIZE}: digits and letters sit near 0.58em, punctuation and + * spaces near 0.3em. Deliberately generous — an over-wide gutter costs a couple of + * plot pixels, an under-wide one clips the label against the container's edge. + */ +export function estimateAxisLabelWidth(text: string): number { + let width = 0 + for (const character of text) { + width += NARROW_GLYPH.test(character) ? 0.3 : 0.58 + } + return width * CHART_TICK_FONT_SIZE +} + +/** + * {@link CHART_PADDING} with a left gutter wide enough for the chart's own y-axis + * labels. + * + * The fixed 26px gutter left 18px of drawable width once the label gap is taken out, + * which fits four narrow glyphs — so any tick past `7.3k` was cut off at the left edge + * of the container. Both charts resolve their gutter through this one function from + * the labels they are about to draw, so a bar and a line chart showing comparable + * magnitudes still line up when stacked in one card, and neither can clip. + */ +export function resolveChartPadding(yAxisLabels: readonly string[]): ChartPadding { + const widest = yAxisLabels.reduce((max, label) => Math.max(max, estimateAxisLabelWidth(label)), 0) + const required = Math.max(CHART_PADDING.left, widest + CHART_AXIS_LABEL_GAP) + return { + ...CHART_PADDING, + left: Math.ceil(required / CHART_AXIS_GUTTER_STEP) * CHART_AXIS_GUTTER_STEP, + } +} + +/** Vertical clamp for plotted geometry, keeping strokes off the axis rules. */ +export function chartPlotBand(height: number): { yMin: number; yMax: number } { + const chartHeight = height - CHART_PADDING.top - CHART_PADDING.bottom + return { yMin: CHART_PADDING.top + 3, yMax: CHART_PADDING.top + chartHeight - 3 } +} + +/** + * Evenly spaced point indices to label, budgeting ~64px per tick and deduping the + * collisions that rounding produces on short series. + */ +export function resolveTimeTickIndices(pointCount: number, usableWidth: number): number[] { + const approxLabelWidth = 64 + const desired = Math.min(8, Math.max(3, Math.floor(usableWidth / approxLabelWidth))) + const seen = new Set() + return Array.from({ length: desired }, (_, i) => + Math.round((i * (pointCount - 1)) / Math.max(1, desired - 1)) + ).filter((index) => { + if (seen.has(index)) return false + seen.add(index) + return true + }) +} + +/** + * Tick label whose precision follows the window: clock time within a day and a half, + * calendar day within a quarter, month beyond that. + */ +export function formatTimeTick(date: Date, spanMs: number): string { + if (spanMs <= 36 * 60 * 60 * 1000) { + return date.toLocaleTimeString('en-US', { hour: '2-digit', minute: '2-digit', hour12: false }) + } + if (spanMs <= 90 * 24 * 60 * 60 * 1000) { + return date.toLocaleString('en-US', { month: 'short', day: 'numeric' }) + } + return date.toLocaleString('en-US', { month: 'short', year: 'numeric' }) +} + +/** Milliseconds between the first and last timestamp, or 0 for a degenerate series. */ +export function resolveSpanMs(points: ReadonlyArray<{ timestamp: string }>): number { + if (points.length < 2) return 0 + const first = new Date(points[0].timestamp).getTime() + const last = new Date(points[points.length - 1].timestamp).getTime() + if (Number.isNaN(first) || Number.isNaN(last)) return 0 + return Math.abs(last - first) +} diff --git a/apps/sim/components/charts/chart-layout.test.tsx b/apps/sim/components/charts/chart-layout.test.tsx new file mode 100644 index 00000000000..3ab1c4e4bc8 --- /dev/null +++ b/apps/sim/components/charts/chart-layout.test.tsx @@ -0,0 +1,239 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { BarChart } from '@/components/charts/bar-chart' +import { CHART_PADDING } from '@/components/charts/chart-geometry' +import { RadarChart } from '@/components/charts/radar-chart' + +/** + * Rendered-geometry guards for the chart family. + * + * These assert against the real SVG the components emit rather than against the + * geometry helpers in isolation: the two clipping bugs this file exists for — a + * y-axis label cut off at the container's left edge, and a radar caption painting + * over the section beside it — were both invisible to a unit test of the maths, + * because each came from a *callsite* combining correct helpers wrongly. + */ + +let container: HTMLDivElement +let root: Root + +/** jsdom lays nothing out, so the width the chart measures has to be supplied. */ +function mountAtWidth(width: number, element: React.ReactElement): SVGSVGElement { + container = document.createElement('div') + document.body.appendChild(container) + vi.spyOn(HTMLElement.prototype, 'getBoundingClientRect').mockReturnValue({ + width, + height: 0, + top: 0, + left: 0, + right: width, + bottom: 0, + x: 0, + y: 0, + toJSON: () => ({}), + } as DOMRect) + root = createRoot(container) + act(() => root.render(element)) + const svg = container.querySelector('svg') + if (!svg) throw new Error('chart did not render an svg') + return svg +} + +/** Right-anchored SVG text at 9px, measured the way the chart's own estimator does. */ +function textExtent(text: string): number { + let width = 0 + for (const character of text) width += /[.,:\s]/.test(character) ? 0.3 : 0.58 + return width * 9 +} + +beforeEach(() => { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + globalThis.ResizeObserver = class { + observe() {} + unobserve() {} + disconnect() {} + } as unknown as typeof ResizeObserver +}) + +afterEach(() => { + act(() => root?.unmount()) + container?.remove() + vi.restoreAllMocks() +}) + +function dailySeries(count: number, peak: number) { + return Array.from({ length: count }, (_, index) => ({ + timestamp: new Date(Date.UTC(2026, 0, 1 + index)).toISOString(), + value: index === 0 ? peak : peak / 10, + })) +} + +describe('BarChart rendered geometry', () => { + const widths = [280, 420, 680, 1024] + const peaks = [7300, 173_000, 1_234_567] + + it.each(widths.flatMap((width) => peaks.map((peak) => [width, peak] as const)))( + 'keeps the y-axis labels inside the box at width %i, peak %i', + (width, peak) => { + const svg = mountAtWidth( + width, + + ) + const labels = [...svg.querySelectorAll('text')].filter( + (node) => node.getAttribute('text-anchor') === 'end' + ) + expect(labels.length).toBe(2) + for (const label of labels) { + const anchorX = Number(label.getAttribute('x')) + // Right-anchored: the glyphs run leftward from the anchor. + expect(anchorX - textExtent(label.textContent ?? '')).toBeGreaterThanOrEqual(0) + } + } + ) + + it.each(widths)('keeps every bar inside the plot area at width %i', (width) => { + const svg = mountAtWidth( + width, + + ) + const bars = [...svg.querySelectorAll('rect')] + expect(bars.length).toBeGreaterThan(0) + const svgWidth = Number(svg.getAttribute('width')) + for (const bar of bars) { + const x = Number(bar.getAttribute('x')) + const right = x + Number(bar.getAttribute('width')) + expect(x).toBeGreaterThanOrEqual(CHART_PADDING.left) + expect(right).toBeLessThanOrEqual(svgWidth - CHART_PADDING.right + 0.01) + } + }) + + it('keeps the first and last x-axis tick label inside the box', () => { + const width = 680 + const svg = mountAtWidth( + width, + + ) + const ticks = [...svg.querySelectorAll('text')].filter( + (node) => node.getAttribute('text-anchor') === 'middle' + ) + expect(ticks.length).toBeGreaterThan(1) + for (const tick of ticks) { + const centre = Number(tick.getAttribute('x')) + const half = textExtent(tick.textContent ?? '') / 2 + expect(centre - half).toBeGreaterThanOrEqual(0) + expect(centre + half).toBeLessThanOrEqual(width) + } + }) +}) + +describe('RadarChart rendered geometry', () => { + const LONG = 'Knowledge Base Sync' + + /** + * Every caption long, not just the first. + * + * The first axis sits at twelve o'clock, where a caption is centred and has the + * whole half-width to spend — the one position that cannot overflow horizontally. + * A fixture that only made that one long proved nothing about the axes that + * actually run out of room. + */ + function axesOf(count: number) { + return Array.from({ length: count }, (_, index) => ({ + label: `${LONG} ${index}`, + value: 100 * (index + 1), + display: String(100 * (index + 1)), + })) + } + + it.each([ + [280, 3], + [280, 6], + [320, 4], + [420, 5], + [420, 6], + [520, 7], + [680, 6], + ])('keeps every axis caption inside the box at width %i with %i axes', (width, axisCount) => { + const svg = mountAtWidth(width, ) + const height = Number(svg.getAttribute('height')) + const captions = [...svg.querySelectorAll('text')] + expect(captions.length).toBe(axisCount) + + for (const caption of captions) { + const x = Number(caption.getAttribute('x')) + const y = Number(caption.getAttribute('y')) + const anchor = caption.getAttribute('text-anchor') + const extent = textExtent(caption.textContent ?? '') + const left = anchor === 'start' ? x : anchor === 'end' ? x - extent : x - extent / 2 + const right = left + extent + expect(left).toBeGreaterThanOrEqual(0) + expect(right).toBeLessThanOrEqual(width) + + // An 'auto' baseline sits the glyphs above y; 'middle' centres them on it. + const capHeight = 9 + const top = + caption.getAttribute('dominant-baseline') === 'middle' ? y - capHeight / 2 : y - capHeight + const bottom = top + capHeight + expect(top).toBeGreaterThanOrEqual(0) + expect(bottom).toBeLessThanOrEqual(height) + } + }) + + it('draws a positive-radius web rather than collapsing at the narrow floor', () => { + const svg = mountAtWidth(280, ) + const rings = [...svg.querySelectorAll('polygon')].filter( + (node) => node.getAttribute('fill') === 'none' + ) + expect(rings.length).toBeGreaterThan(0) + const outer = rings[rings.length - 1] + const points = (outer.getAttribute('points') ?? '') + .split(' ') + .map((pair) => pair.split(',').map(Number)) + const xs = points.map(([x]) => x) + const ys = points.map(([, y]) => y) + expect(Math.max(...xs) - Math.min(...xs)).toBeGreaterThan(40) + expect(Math.max(...ys) - Math.min(...ys)).toBeGreaterThan(40) + }) + + it('renders the empty state rather than a degenerate polygon below three axes', () => { + container = document.createElement('div') + document.body.appendChild(container) + vi.spyOn(HTMLElement.prototype, 'getBoundingClientRect').mockReturnValue({ + width: 420, + height: 0, + top: 0, + left: 0, + right: 420, + bottom: 0, + x: 0, + y: 0, + toJSON: () => ({}), + } as DOMRect) + root = createRoot(container) + act(() => root.render()) + expect(container.querySelector('svg')).toBeNull() + expect(container.textContent).toContain('No data') + }) +}) diff --git a/apps/sim/components/charts/chart-tooltip.test.ts b/apps/sim/components/charts/chart-tooltip.test.ts new file mode 100644 index 00000000000..0f414b14a5f --- /dev/null +++ b/apps/sim/components/charts/chart-tooltip.test.ts @@ -0,0 +1,114 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { CHART_PADDING, resolveChartPadding } from '@/components/charts/chart-geometry' +import { + estimateTooltipHeight, + estimateTooltipWidth, + positionChartTooltip, +} from '@/components/charts/chart-tooltip' + +const WIDTH = 800 +const HEIGHT = 166 + +function place(anchorY: number, rows = 1, hasDate = true) { + const tooltipHeight = estimateTooltipHeight(rows, hasDate) + const position = positionChartTooltip({ + anchorX: 400, + anchorY, + width: WIDTH, + height: HEIGHT, + tooltipMaxWidth: estimateTooltipWidth(12), + tooltipHeight, + }) + return { ...position, tooltipHeight } +} + +describe('positionChartTooltip', () => { + /** Guards the height-aware vertical clamp — see `positionChartTooltip`. */ + it('keeps the whole box inside the chart when the cursor is at the very bottom', () => { + const { top, tooltipHeight } = place(HEIGHT) + expect(top + tooltipHeight).toBeLessThanOrEqual(HEIGHT) + }) + + it('holds for a taller multi-row tooltip, which overflows soonest', () => { + const { top, tooltipHeight } = place(HEIGHT, 5) + expect(top + tooltipHeight).toBeLessThanOrEqual(HEIGHT) + expect(top).toBeGreaterThanOrEqual(0) + }) + + it('never places the box above the chart when the cursor is at the top', () => { + expect(place(0).top).toBeGreaterThanOrEqual(0) + }) + + it('prefers the right of the cursor and flips left near the right edge', () => { + const boxWidth = estimateTooltipWidth(12) + const right = positionChartTooltip({ + anchorX: 100, + anchorY: 80, + width: WIDTH, + height: HEIGHT, + tooltipMaxWidth: boxWidth, + tooltipHeight: estimateTooltipHeight(1, true), + }) + expect(right.left).toBeGreaterThan(100) + + const flipped = positionChartTooltip({ + anchorX: WIDTH - CHART_PADDING.right, + anchorY: 80, + width: WIDTH, + height: HEIGHT, + tooltipMaxWidth: boxWidth, + tooltipHeight: estimateTooltipHeight(1, true), + }) + expect(flipped.left + boxWidth).toBeLessThanOrEqual(WIDTH - CHART_PADDING.right) + }) + + /** A chart with wide axis labels has a wider gutter, and the clamp must follow it. */ + it('clamps the left edge to the resolved gutter, not the shared constant', () => { + const padding = resolveChartPadding(['123456.7m']) + const { left } = positionChartTooltip({ + anchorX: 0, + anchorY: 80, + width: WIDTH, + height: HEIGHT, + tooltipMaxWidth: estimateTooltipWidth(12), + tooltipHeight: estimateTooltipHeight(1, true), + padding, + }) + expect(left).toBeGreaterThanOrEqual(padding.left) + expect(padding.left).toBeGreaterThan(CHART_PADDING.left) + }) +}) + +describe('estimateTooltipHeight', () => { + it('grows with each row and with the date header', () => { + expect(estimateTooltipHeight(2, true)).toBeGreaterThan(estimateTooltipHeight(1, true)) + expect(estimateTooltipHeight(1, true)).toBeGreaterThan(estimateTooltipHeight(1, false)) + }) + + it('reserves a row even when told there are none', () => { + expect(estimateTooltipHeight(0, false)).toBe(estimateTooltipHeight(1, false)) + }) + + /** + * The estimate is what the clamp measures against, and the chart clips its overflow, + * so it must never come in under the real box — an underestimate cuts the bottom off + * rather than moving the box up. Measured here against the box model the tooltip's + * own class string implies: `border` + `py-1.5`, a `text-micro` date with `mb-1`, + * and one `text-xs` row per value, every line at the ambient 1.5 line-height. + */ + it('never comes in under the box the tooltip actually renders', () => { + const chrome = 2 + 6 + 6 + const dateLine = 10 * 1.5 + 4 + const rowLine = 11 * 1.5 + + for (const rows of [1, 2, 5]) { + expect(estimateTooltipHeight(rows, true)).toBeGreaterThanOrEqual( + chrome + dateLine + rows * rowLine + ) + expect(estimateTooltipHeight(rows, false)).toBeGreaterThanOrEqual(chrome + rows * rowLine) + } + }) +}) diff --git a/apps/sim/components/charts/chart-tooltip.tsx b/apps/sim/components/charts/chart-tooltip.tsx new file mode 100644 index 00000000000..518a470479a --- /dev/null +++ b/apps/sim/components/charts/chart-tooltip.tsx @@ -0,0 +1,127 @@ +'use client' + +import type { ReactNode } from 'react' +import { CHART_PADDING, type ChartPadding } from '@/components/charts/chart-geometry' + +/** + * The chart family's hover surface. Defined once so a sibling chart cannot ship a + * tooltip that looks almost the same — this class string was previously duplicated + * between the line chart and the status bar. + */ +export const CHART_TOOLTIP_CLASSES = + 'pointer-events-none absolute rounded-lg border border-[var(--border)] bg-[var(--surface-1)] px-2 py-1.5 text-xs shadow-overlay' + +interface PositionChartTooltipArgs { + anchorX: number + anchorY: number + width: number + height: number + tooltipMaxWidth: number + tooltipHeight: number + /** The chart's resolved padding, whose left gutter varies with its axis labels. */ + padding?: ChartPadding +} + +/** + * Places the tooltip beside the cursor, preferring the right and flipping left when + * it would overflow, then clamping it wholly inside the chart box. + * + * The vertical clamp is against the tooltip's own height rather than a fixed inset. + * A fixed one let the box hang a pixel or two past the bottom near the foot of the + * plot, and because the scroll container's `overflow-x` forces `overflow-y` to `auto`, + * those pixels raised a vertical scrollbar the moment the cursor approached the axis. + */ +export function positionChartTooltip({ + anchorX, + anchorY, + width, + height, + tooltipMaxWidth, + tooltipHeight, + padding = CHART_PADDING, +}: PositionChartTooltipArgs): { left: number; top: number } { + const margin = 10 + const rightEdge = width - padding.right + const preferRight = anchorX + margin + tooltipMaxWidth <= rightEdge + const left = preferRight + ? Math.max(padding.left, Math.min(anchorX + margin, rightEdge - tooltipMaxWidth)) + : Math.max( + padding.left, + Math.min(anchorX - margin - tooltipMaxWidth, rightEdge - tooltipMaxWidth) + ) + const top = Math.max(0, Math.min(anchorY - 26, height - tooltipHeight)) + return { left, top } +} + +/** Width estimate for the longest `label value` row, used by {@link positionChartTooltip}. */ +export function estimateTooltipWidth(longestRowLength: number): number { + return Math.min(220, Math.max(80, 7 * longestRowLength + 24)) +} + +/** Border plus the `py-1.5` the tooltip's own class string sets. */ +const TOOLTIP_CHROME_HEIGHT = 2 + 12 + +/** + * The `text-micro` date's line box plus its `mb-1`. + * + * The type scale pairs no line-height with a font size, so a line occupies the + * ambient 1.5 rather than the font size itself — 15px for 10px `text-micro`, not 10. + */ +const TOOLTIP_DATE_HEIGHT = 15 + 4 + +/** One `text-xs` row's line box: 11px at the ambient 1.5, rounded up from 16.5. */ +const TOOLTIP_ROW_HEIGHT = 17 + +/** + * Height of the box {@link ChartTooltip} renders, from its own box model. + * + * Estimated rather than measured because the position is computed in the same render + * that mounts the tooltip — reading a real height would need a second paint, which + * shows up as the tooltip visibly jumping under the cursor. Every part rounds up: + * this is what {@link positionChartTooltip} clamps against and the chart clips its + * overflow, so an underestimate cuts the bottom off the box rather than moving it. + */ +export function estimateTooltipHeight(rowCount: number, hasDate: boolean): number { + return ( + TOOLTIP_CHROME_HEIGHT + + (hasDate ? TOOLTIP_DATE_HEIGHT : 0) + + Math.max(1, rowCount) * TOOLTIP_ROW_HEIGHT + ) +} + +interface ChartTooltipProps { + left: number + top: number + /** Header line; omitted when the timestamp could not be formatted. */ + date?: string + children: ReactNode +} + +export function ChartTooltip({ left, top, date, children }: ChartTooltipProps) { + return ( +
+ {date &&
{date}
} + {children} +
+ ) +} + +interface ChartTooltipRowProps { + color: string + label?: string + value: string +} + +export function ChartTooltipRow({ color, label, value }: ChartTooltipRowProps) { + return ( +
+
+ ) +} diff --git a/apps/sim/components/charts/index.ts b/apps/sim/components/charts/index.ts new file mode 100644 index 00000000000..1be948503bf --- /dev/null +++ b/apps/sim/components/charts/index.ts @@ -0,0 +1,13 @@ +export { BarChart, type BarChartPoint } from '@/components/charts/bar-chart' +export { + formatChartCompactNumber, + formatChartLatency, + formatChartTimestamp, +} from '@/components/charts/chart-format' +export { CHART_DEFAULT_HEIGHT } from '@/components/charts/chart-geometry' +export { + LineChart, + type LineChartMultiSeries, + type LineChartPoint, +} from '@/components/charts/line-chart' +export { RadarChart, type RadarChartAxis } from '@/components/charts/radar-chart' diff --git a/apps/sim/components/charts/line-chart.tsx b/apps/sim/components/charts/line-chart.tsx new file mode 100644 index 00000000000..0896f1df4ee --- /dev/null +++ b/apps/sim/components/charts/line-chart.tsx @@ -0,0 +1,660 @@ +'use client' + +import { memo, useId, useMemo, useState } from 'react' +import { Button, cn } from '@sim/emcn' +import { + formatChartCompactNumber, + formatChartLatency, + formatChartTimestamp, +} from '@/components/charts/chart-format' +import { + CHART_AXIS_LABEL_GAP, + CHART_DEFAULT_HEIGHT, + CHART_GRID_FRACTIONS, + CHART_TICK_FILL, + CHART_TICK_FONT_SIZE, + chartPlotBand, + formatTimeTick, + resolveChartPadding, + resolveSpanMs, + resolveTimeTickIndices, +} from '@/components/charts/chart-geometry' +import { + ChartTooltip, + ChartTooltipRow, + estimateTooltipHeight, + estimateTooltipWidth, + positionChartTooltip, +} from '@/components/charts/chart-tooltip' +import { + useChartWidth, + useIsDarkTheme, + useResolvedChartColors, +} from '@/components/charts/use-chart-theme' + +export interface LineChartPoint { + timestamp: string + value: number +} + +export interface LineChartMultiSeries { + id?: string + label: string + color: string + data: LineChartPoint[] + dashed?: boolean +} + +interface LineChartProps { + data: LineChartPoint[] + /** Pass `''` for the caller-owned-wrapper form: no card chrome, title, or legend. */ + label: string + color: string + unit?: string + series?: LineChartMultiSeries[] + height?: number +} + +/** + * Smoothed path through `points`, with every control point clamped into the plot + * band so a curve between two near-axis samples cannot bow over an axis rule. + * + * At module scope because the base line and each extra series need the identical + * curve: the two copies had drifted apart before, and a clamp fixed in one drew a + * different shape from the other. + */ +function buildSmoothPath( + points: ReadonlyArray<{ x: number; y: number }>, + yMin: number, + yMax: number +): string { + if (points.length <= 1) return '' + const tension = 0.2 + let d = `M ${points[0].x} ${points[0].y}` + for (let i = 0; i < points.length - 1; i++) { + const p0 = points[i - 1] || points[i] + const p1 = points[i] + const p2 = points[i + 1] + const p3 = points[i + 2] || points[i + 1] + const cp1x = p1.x + ((p2.x - p0.x) / 6) * tension + let cp1y = p1.y + ((p2.y - p0.y) / 6) * tension + const cp2x = p2.x - ((p3.x - p1.x) / 6) * tension + let cp2y = p2.y - ((p3.y - p1.y) / 6) * tension + cp1y = Math.max(yMin, Math.min(yMax, cp1y)) + cp2y = Math.max(yMin, Math.min(yMax, cp2y)) + d += ` C ${cp1x} ${cp1y}, ${cp2x} ${cp2y}, ${p2.x} ${p2.y}` + } + return d +} + +function LineChartComponent({ + data, + label, + color, + unit, + series, + height = CHART_DEFAULT_HEIGHT, +}: LineChartProps) { + /* + `useId`, not `useRef(generateShortId())`: a ref initializer is evaluated on + every render and all but the first result thrown away, and React already has + a hook whose whole job is a stable unique id. + */ + const uniqueId = useId().replace(/:/g, '') + const [containerRef, containerWidth] = useChartWidth() + const width = containerWidth ?? 0 + const isDark = useIsDarkTheme() + const [hoverSeriesId, setHoverSeriesId] = useState(null) + const [activeSeriesId, setActiveSeriesId] = useState(null) + const [hoverPos, setHoverPos] = useState<{ x: number; y: number } | null>(null) + + const colorTokens: Record = { base: color } + for (const s of series ?? []) { + const id = s.id || s.label || '' + if (id) colorTokens[id] = s.color + } + const resolvedColors = useResolvedChartColors(colorTokens) + + const hasExternalWrapper = !label || label === '' + + const allSeries = useMemo( + () => + (Array.isArray(series) && series.length > 0 + ? [{ id: 'base', label, color, data }, ...series] + : [{ id: 'base', label, color, data }] + ).map((s, idx) => ({ ...s, id: s.id || s.label || String(idx) })), + [series, label, color, data] + ) + + const { maxValue, minValue, valueRange } = useMemo(() => { + const flatValues = allSeries.flatMap((s) => s.data.map((d) => d.value)) + const rawMax = Math.max(...flatValues, 1) + const rawMin = Math.min(...flatValues, 0) + const paddedMax = rawMax === 0 ? 1 : rawMax * 1.1 + const paddedMin = Math.min(0, rawMin) + const unitSuffixPre = (unit || '').trim().toLowerCase() + let maxVal = Math.ceil(paddedMax) + let minVal = Math.floor(paddedMin) + if (unitSuffixPre === 'ms' || unitSuffixPre === 'latency') { + minVal = 0 + if (paddedMax < 10) { + maxVal = Math.ceil(paddedMax) + } else if (paddedMax < 100) { + maxVal = Math.ceil(paddedMax / 10) * 10 + } else if (paddedMax < 1000) { + maxVal = Math.ceil(paddedMax / 50) * 50 + } else if (paddedMax < 10000) { + maxVal = Math.ceil(paddedMax / 500) * 500 + } else { + maxVal = Math.ceil(paddedMax / 1000) * 1000 + } + } + return { + maxValue: maxVal, + minValue: minVal, + valueRange: maxVal - minVal || 1, + } + }, [allSeries, unit]) + + /** + * The two y-axis tick labels, resolved once so the gutter that has to hold them is + * measured from the same strings the axis draws. + */ + const yAxisLabels = useMemo(() => { + const unitSuffix = (unit || '').trim() + const isLatency = unitSuffix.toLowerCase() === 'latency' + const suffix = unitSuffix === '%' && !isLatency ? unitSuffix : '' + const compact = (value: number) => { + if (isLatency) return value === 0 ? '0' : formatChartLatency(value) + return `${formatChartCompactNumber(value)}${suffix}` + } + return [compact(maxValue), compact(minValue)] as const + }, [maxValue, minValue, unit]) + + const padding = resolveChartPadding(yAxisLabels) + const chartWidth = width - padding.left - padding.right + const chartHeight = height - padding.top - padding.bottom + + const { yMin, yMax } = chartPlotBand(height) + + const scaledPoints = useMemo( + () => + data.map((d, i) => { + const usableW = Math.max(1, chartWidth) + const x = padding.left + (i / (data.length - 1 || 1)) * usableW + const rawY = padding.top + chartHeight - ((d.value - minValue) / valueRange) * chartHeight + const y = Math.max(yMin, Math.min(yMax, rawY)) + return { x, y } + }), + [data, chartWidth, chartHeight, minValue, valueRange, yMin, yMax, padding.left, padding.top] + ) + + /** + * The hovered sample, derived from the stored cursor rather than stored beside it. + * + * Clamped here rather than relying on the stored x having been clamped at mousemove + * time: `padding.left` follows the axis labels and `chartWidth` follows the + * container, so either can move with no pointer event at all — a sidebar collapse + * mid-hover otherwise pushed the ratio past 1 and indexed off the end, and the dot, + * the rule and the tooltip all vanished until the cursor moved again. + */ + const hoverIndex = + hoverPos === null || scaledPoints.length === 0 + ? null + : Math.max( + 0, + Math.min( + scaledPoints.length - 1, + Math.round( + ((hoverPos.x - padding.left) / (chartWidth || 1)) * (scaledPoints.length - 1) + ) + ) + ) + + const scaledSeries = useMemo( + () => + allSeries.map((s) => { + const pts = s.data.map((d, i) => { + const usableW = Math.max(1, chartWidth) + const x = padding.left + (i / (s.data.length - 1 || 1)) * usableW + const rawY = padding.top + chartHeight - ((d.value - minValue) / valueRange) * chartHeight + const y = Math.max(yMin, Math.min(yMax, rawY)) + return { x, y } + }) + return { ...s, pts } + }), + [ + allSeries, + chartWidth, + chartHeight, + minValue, + valueRange, + yMin, + yMax, + padding.left, + padding.top, + ] + ) + + const getSeriesById = (id?: string | null) => scaledSeries.find((s) => s.id === id) + const visibleSeries = activeSeriesId + ? scaledSeries.filter((s) => s.id === activeSeriesId) + : scaledSeries + + const pathD = useMemo(() => buildSmoothPath(scaledPoints, yMin, yMax), [scaledPoints, yMin, yMax]) + + const currentHoverDate = + hoverIndex !== null && data[hoverIndex] ? formatChartTimestamp(data[hoverIndex].timestamp) : '' + + if (containerWidth === null) { + return ( +
+ ) + } + + if (data.length === 0) { + return ( +
+

No data

+
+ ) + } + + return ( +
+ {!hasExternalWrapper && ( +
+

{label}

+ {allSeries.length > 1 && ( +
+ {scaledSeries.slice(1).map((s) => { + const isActive = activeSeriesId ? activeSeriesId === s.id : true + const isHovered = hoverSeriesId === s.id + const dimmed = activeSeriesId ? !isActive : false + return ( + + ) + })} +
+ )} +
+ )} +
+ { + if (scaledPoints.length === 0) return + const rect = (e.currentTarget as SVGSVGElement).getBoundingClientRect() + const x = e.clientX - rect.left + const clamped = Math.max(padding.left, Math.min(width - padding.right, x)) + const ratio = (clamped - padding.left) / (chartWidth || 1) + const i = Math.round(ratio * (scaledPoints.length - 1)) + setHoverPos({ x: clamped, y: e.clientY - rect.top }) + const cursorY = e.clientY - rect.top + if (activeSeriesId) { + setHoverSeriesId(activeSeriesId) + } else { + let best: { id: string | null; dy: number } = { + id: null, + dy: Number.POSITIVE_INFINITY, + } + for (const s of scaledSeries.slice(1)) { + const pt = s.pts[i] + if (!pt) continue + const dy = Math.abs(pt.y - cursorY) + if (dy < best.dy) best = { id: s.id || null, dy } + } + setHoverSeriesId(best.dy <= 12 ? best.id : null) + } + }} + onMouseLeave={() => { + setHoverPos(null) + setHoverSeriesId(null) + }} + > + + + + + + + + + + + + + {CHART_GRID_FRACTIONS.map((p) => ( + + ))} + + {!activeSeriesId && scaledPoints.length > 1 && ( + + )} + + {!activeSeriesId && + scaledPoints.length === 1 && + (() => { + const strokeWidth = isDark ? 1.7 : 2.0 + const capExtension = strokeWidth / 2 + return ( + + ) + })()} + + {visibleSeries.map((s, idx) => { + const isActive = activeSeriesId ? activeSeriesId === s.id : true + const isHovered = hoverSeriesId ? hoverSeriesId === s.id : false + const baseOpacity = isActive ? 1 : 0.12 + const strokeOpacity = isHovered ? 1 : baseOpacity + const sw = (() => { + switch ((s.id || '').toLowerCase()) { + case 'p50': + return isDark ? 1.5 : 1.7 + case 'p90': + return isDark ? 1.9 : 2.1 + case 'p99': + return isDark ? 2.3 : 2.5 + default: + return isDark ? 1.7 : 2.0 + } + })() + if (s.pts.length <= 1) { + const y = s.pts[0]?.y + if (y === undefined) return null + return ( + + ) + } + const p = buildSmoothPath(s.pts, yMin, yMax) + return ( + setActiveSeriesId((prev) => (prev === s.id ? null : s.id || null))} + /> + ) + })} + + {hoverIndex !== null && + scaledPoints[hoverIndex] && + scaledPoints.length > 1 && + (() => { + const guideSeries = + getSeriesById(activeSeriesId) || getSeriesById(hoverSeriesId) || scaledSeries[0] + const active = guideSeries + const pt = active.pts[hoverIndex] || scaledPoints[hoverIndex] + return ( + + + {activeSeriesId && + (() => { + const s = getSeriesById(activeSeriesId) + const spt = s?.pts?.[hoverIndex] + if (!s || !spt) return null + return ( + + ) + })()} + + ) + })()} + + {(() => { + if (data.length < 2) return null + const usableW = Math.max(1, chartWidth) + const spanMs = resolveSpanMs(data) + const idx = resolveTimeTickIndices(data.length, usableW) + + return idx.map((i) => { + const x = padding.left + (i / (data.length - 1 || 1)) * usableW + const tsSource = data[i]?.timestamp + if (!tsSource) return null + const ts = new Date(tsSource) + const labelStr = Number.isNaN(ts.getTime()) ? '' : formatTimeTick(ts, spanMs) + return ( + + {labelStr} + + ) + }) + })()} + + + {yAxisLabels[0]} + + + {yAxisLabels[1]} + + + + + + {hoverIndex !== null && + scaledPoints[hoverIndex] && + (() => { + const active = + getSeriesById(activeSeriesId) || getSeriesById(hoverSeriesId) || scaledSeries[0] + const pt = active.pts[hoverIndex] || scaledPoints[hoverIndex] + const toDisplay = activeSeriesId + ? [getSeriesById(activeSeriesId)!] + : scaledSeries.length > 1 + ? scaledSeries.slice(1) + : [scaledSeries[0]] + + const fmt = (v?: number) => { + if (typeof v !== 'number' || !Number.isFinite(v)) return '—' + const u = unit || '' + if (u.includes('%')) return `${v.toFixed(1)}%` + if (u.toLowerCase() === 'latency') return formatChartLatency(v) + if (u.toLowerCase().includes('ms')) return `${Math.round(v)}ms` + if (u.toLowerCase().includes('exec')) return `${Math.round(v)}` + return `${Math.round(v)}${u}` + } + + const longest = toDisplay.reduce((m, s) => { + const seriesIndex = allSeries.findIndex((x) => x.id === s.id) + const v = allSeries[seriesIndex]?.data?.[hoverIndex]?.value + const valueStr = fmt(v) + const labelStr = s.label || String(s.id || '') + const len = `${labelStr} ${valueStr}`.length + return Math.max(m, len) + }, 0) + const { left, top } = positionChartTooltip({ + anchorX: hoverPos?.x ?? pt.x, + anchorY: hoverPos?.y ?? pt.y, + width, + height, + tooltipMaxWidth: estimateTooltipWidth(longest), + tooltipHeight: estimateTooltipHeight(toDisplay.length, Boolean(currentHoverDate)), + padding, + }) + return ( + + {toDisplay.map((s) => { + const seriesIndex = allSeries.findIndex((x) => x.id === s.id) + const val = allSeries[seriesIndex]?.data?.[hoverIndex]?.value + const seriesLabel = s.label || s.id + const showLabel = + seriesLabel && seriesLabel !== 'base' && seriesLabel.trim() !== '' + return ( + + ) + })} + + ) + })()} +
+
+ ) +} + +export const LineChart = memo(LineChartComponent) diff --git a/apps/sim/components/charts/radar-chart.tsx b/apps/sim/components/charts/radar-chart.tsx new file mode 100644 index 00000000000..0f33a02b783 --- /dev/null +++ b/apps/sim/components/charts/radar-chart.tsx @@ -0,0 +1,327 @@ +'use client' + +import { memo, useId, useMemo, useState } from 'react' +import { truncate } from '@sim/utils/string' +import { + CHART_GRID_FRACTIONS, + CHART_TICK_FILL, + CHART_TICK_FONT_SIZE, + estimateAxisLabelWidth, +} from '@/components/charts/chart-geometry' +import { + ChartTooltip, + ChartTooltipRow, + estimateTooltipHeight, + estimateTooltipWidth, + positionChartTooltip, +} from '@/components/charts/chart-tooltip' +import { + useChartWidth, + useIsDarkTheme, + useResolvedChartColors, +} from '@/components/charts/use-chart-theme' + +export interface RadarChartAxis { + label: string + value: number + /** Text shown for `value` in the hover row. Defaults to the raw number. */ + display?: string +} + +interface RadarChartProps { + axes: RadarChartAxis[] + color: string + height?: number +} + +/** Room above and below the web for the captions on the vertical centreline. */ +const LABEL_GUTTER = 52 + +/** Gap between the outer ring and a caption anchored beyond it. */ +const LABEL_GAP = 12 + +/** + * The web's rings: the family's gridline fractions plus the outer ring, which is this + * chart's axis rule. Read from the constant rather than divided into `RING_COUNT` + * even steps — the arithmetic agreed with the siblings only while the fractions + * happened to be uniform, which is exactly the drift `chart-geometry` exists to stop. + */ +const RING_FRACTIONS = [...CHART_GRID_FRACTIONS, 1] as const + +/** + * Caption budget. A long source name would otherwise run past the container, and the + * svg paints outside its box so it would not even clip — it would overlap the section + * beside it. The hover row carries the full name. + */ +const MAX_LABEL_LENGTH = 16 + +/** + * Polar coordinates for an axis. `-90°` puts the first axis at twelve o'clock, so a + * list read top-down and the web read clockwise start in the same place. + */ +function axisPoint(index: number, count: number, radius: number, cx: number, cy: number) { + const angle = (index / count) * Math.PI * 2 - Math.PI / 2 + return { x: cx + Math.cos(angle) * radius, y: cy + Math.sin(angle) * radius } +} + +function polygon(points: ReadonlyArray<{ x: number; y: number }>): string { + return points.map((p) => `${p.x.toFixed(2)},${p.y.toFixed(2)}`).join(' ') +} + +/** + * Shape of a distribution across a handful of named categories. + * + * The third member of the chart family, and built from the same tokens, tooltip, and + * theme hooks as {@link BarChart} and {@link LineChart}. It answers a question the + * other two cannot: a bar list ranks categories but says nothing about balance, and + * "one source dominates" versus "spend is spread evenly" is legible here at a glance + * and nowhere else on the panel. + * + * Every axis is scaled against the largest value rather than against its own range, + * so the polygon's area is proportional to the real distribution — normalising each + * axis independently would draw a balanced pentagon for any input at all. + */ +function RadarChartComponent({ axes, color, height = 200 }: RadarChartProps) { + const uniqueId = useId().replace(/:/g, '') + const [containerRef, containerWidth] = useChartWidth() + const isDark = useIsDarkTheme() + const [hoverIndex, setHoverIndex] = useState(null) + + const resolvedColors = useResolvedChartColors({ base: color }) + const resolvedColor = resolvedColors.base || color + + const width = containerWidth ?? 0 + const cx = width / 2 + const cy = height / 2 + /* + One memo over the whole web: hovering re-renders this component on every wedge + enter and leave, and none of this geometry can move under a hover. Guarding only + the point projection left the costlier half — a per-glyph estimate of every + caption — running on each of those renders. + + The horizontal budget is the caption's own estimated width, the same + `estimateAxisLabelWidth` the sibling charts use to size a gutter around SVG text + they cannot measure. A 16-glyph caption runs to ~84px, so a fixed inset let every + side caption run past the plot; budgeting the radius against the real caption + width is what keeps them inside the box the svg clips to. + */ + const { maxValue, radius, points } = useMemo(() => { + const labelWidth = axes.reduce( + (max, axis) => Math.max(max, estimateAxisLabelWidth(truncate(axis.label, MAX_LABEL_LENGTH))), + 0 + ) + const webRadius = Math.max( + 0, + Math.min(width / 2 - labelWidth - LABEL_GAP, height / 2 - LABEL_GUTTER / 2) + ) + const peak = Math.max(...axes.map((axis) => axis.value), 0) + return { + maxValue: peak, + radius: webRadius, + points: axes.map((axis, index) => { + const fraction = peak > 0 ? axis.value / peak : 0 + return { + axis, + outer: axisPoint(index, axes.length, webRadius, cx, cy), + value: axisPoint(index, axes.length, webRadius * fraction, cx, cy), + label: axisPoint(index, axes.length, webRadius + LABEL_GAP, cx, cy), + } + }), + } + }, [axes, width, height, cx, cy]) + + if (containerWidth === null) { + return
+ } + + /* + Three axes are the fewest that enclose an area; below that the "polygon" is a + line or a point and reads as a rendering fault rather than as a distribution. + */ + if (axes.length < 3 || maxValue <= 0) { + return ( +
+

No data

+
+ ) + } + + const hovered = hoverIndex !== null ? points[hoverIndex] : null + + return ( + /* + Two boxes, like the siblings: the outer one scrolls, the inner one is the + positioning context. `relative` on the scroll container itself left the + absolutely-positioned tooltip anchored to the viewport of the scroll rather than + to the plot — below CHART_MIN_WIDTH it stayed nailed while the web slid under it. + + Captions are inside the plot by construction, since `radius` is budgeted against + `labelWidth`, so the horizontal scroll never cuts one off. + */ +
+
+ + + {/* + Radial rather than the siblings' vertical linear gradient — a shape with + radial symmetry lit from the top reads as a rendering error. The stop + opacities stay in the family's range, and light is the more opaque theme + because dark composites through `screen` below. + */} + + + + + + + {RING_FRACTIONS.map((fraction) => ( + axisPoint(index, axes.length, radius * fraction, cx, cy)) + )} + fill='none' + stroke='var(--border)' + strokeOpacity={fraction === 1 ? 1 : 0.35} + strokeWidth='1' + /> + ))} + {points.map((point, index) => ( + + ))} + + + point.value))} + fill={`url(#radar-${uniqueId})`} + stroke={resolvedColor} + strokeWidth={isDark ? 1.7 : 2} + strokeLinejoin='round' + /> + {points.map((point, index) => ( + + ))} + + + {points.map((point, index) => ( + cx ? 'start' : 'end' + } + dominantBaseline={ + Math.abs(point.label.x - cx) >= 1 + ? 'middle' + : point.label.y > cy + ? 'hanging' + : 'auto' + } + fontSize={CHART_TICK_FONT_SIZE} + fill={CHART_TICK_FILL} + > + {truncate(point.axis.label, MAX_LABEL_LENGTH)} + + ))} + + {/* + Hit targets last so they sit above the painted web, and wedge-sized — a + vertex-sized target is far too small to hover on a 200px chart. + + An arc sector, not a triangle. A triangle's far edge is the chord, which + along its own spoke reaches only `reach·cos(π/n)` — at three axes that is + 50px against a 74px radius, so the largest value's vertex, the one a reader + aims at, sat outside its own target and outside every other. Sectors tile + identically and reach `reach` in every direction. The sweep flag is 1 + because SVG's y grows downward, and the arc is never a major one: 2π/n ≤ + 2π/3 < π for the three-or-more axes this chart requires. + */} + {points.map((point, index) => { + const half = Math.PI / axes.length + const angle = (index / axes.length) * Math.PI * 2 - Math.PI / 2 + const reach = radius + LABEL_GUTTER / 2 + const a = { + x: cx + Math.cos(angle - half) * reach, + y: cy + Math.sin(angle - half) * reach, + } + const b = { + x: cx + Math.cos(angle + half) * reach, + y: cy + Math.sin(angle + half) * reach, + } + return ( + setHoverIndex(index)} + onMouseLeave={() => setHoverIndex(null)} + /> + ) + })} + + + {hovered && + (() => { + const value = hovered.axis.display ?? String(hovered.axis.value) + /* + Beside the hovered vertex, through the same placer the siblings use, so + the box flips and clamps identically. Centring it on the web instead put + a filled panel over the densest part of the gradient — the concentration + this chart exists to show. The padding passed is the caption gap rather + than the axis-bearing charts' gutters: a radar has no axis rules to keep + clear of. + */ + const { left, top } = positionChartTooltip({ + anchorX: hovered.value.x, + anchorY: hovered.value.y, + width, + height, + tooltipMaxWidth: estimateTooltipWidth( + Math.max(hovered.axis.label.length, value.length) + ), + tooltipHeight: estimateTooltipHeight(1, true), + padding: { top: 0, right: LABEL_GAP, bottom: 0, left: LABEL_GAP }, + }) + return ( + + + + ) + })()} +
+
+ ) +} + +export const RadarChart = memo(RadarChartComponent) diff --git a/apps/sim/components/charts/use-chart-theme.ts b/apps/sim/components/charts/use-chart-theme.ts new file mode 100644 index 00000000000..b6b864b0644 --- /dev/null +++ b/apps/sim/components/charts/use-chart-theme.ts @@ -0,0 +1,102 @@ +'use client' + +import { type RefObject, useEffect, useRef, useState, useSyncExternalStore } from 'react' +import { CHART_MIN_WIDTH } from '@/components/charts/chart-geometry' + +function subscribeToDarkTheme(onStoreChange: () => void): () => void { + const observer = new MutationObserver(onStoreChange) + observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class'] }) + return () => observer.disconnect() +} + +function getDarkThemeSnapshot(): boolean { + return document.documentElement.classList.contains('dark') +} + +/** Dark is the assumed default before the class is readable, matching first paint. */ +function getServerDarkThemeSnapshot(): boolean { + return true +} + +/** + * Whether the document is in dark mode, read from the class the theme toggle writes. + * Charts need this as a *value* rather than a CSS class because SVG stroke opacity + * and blend mode are set per element, not by a selector. + * + * The class is an external store, so it is read through `useSyncExternalStore`: the + * first client render already sees the real value instead of painting the default and + * correcting it in an effect. + */ +export function useIsDarkTheme(): boolean { + return useSyncExternalStore( + subscribeToDarkTheme, + getDarkThemeSnapshot, + getServerDarkThemeSnapshot + ) +} + +/** Materializes one `var(--token)` into a concrete `rgb()` via a throwaway probe node. */ +function resolveColor(value: string): string { + if (!value.startsWith('var(')) return value + const probe = document.createElement('div') + probe.style.color = value + document.body.appendChild(probe) + const computed = window.getComputedStyle(probe).color + probe.remove() + return computed +} + +/** + * Resolves `var(--token)` colors to concrete `rgb()` strings. + * + * SVG `stroke` and gradient `stopColor` do not accept a CSS variable that is defined + * on an ancestor, so the value has to be computed once and passed literally. Callers + * still declare colors as tokens; this is the one place that materializes them. + */ +export function useResolvedChartColors(colors: Record): Record { + const [resolved, setResolved] = useState>({}) + const serialized = JSON.stringify(colors) + /* + A token resolves to a different `rgb()` per theme, and the probe runs once per + token set — so without this the colours resolved on the theme the chart mounted + under survived a toggle, and the series kept its dark-mode fill on a light page. + */ + const isDark = useIsDarkTheme() + + useEffect(() => { + if (typeof window === 'undefined') return + + const next: Record = {} + for (const [key, value] of Object.entries(JSON.parse(serialized) as Record)) { + next[key] = resolveColor(value) + } + setResolved(next) + }, [serialized, isDark]) + + return resolved +} + +/** + * Observed container width, floored at {@link CHART_MIN_WIDTH}. `null` until the + * first measurement, which callers render as an empty box of the right height so the + * chart does not reflow the page when it appears. + */ +export function useChartWidth(): [RefObject, number | null] { + const containerRef = useRef(null) + const [width, setWidth] = useState(null) + + useEffect(() => { + const element = containerRef.current + if (!element) return + const observer = new ResizeObserver((entries) => { + const measured = entries[0]?.contentRect?.width + if (measured && measured > 0) setWidth(Math.max(CHART_MIN_WIDTH, Math.floor(measured))) + }) + observer.observe(element) + const rect = element.getBoundingClientRect() + if (rect?.width > 0) setWidth(Math.max(CHART_MIN_WIDTH, Math.floor(rect.width))) + return () => observer.disconnect() + }, []) + + return [containerRef, width] +} diff --git a/apps/sim/components/emails/render-notifications.test.ts b/apps/sim/components/emails/render-notifications.test.ts index fb5286cfc08..49e6f08a881 100644 --- a/apps/sim/components/emails/render-notifications.test.ts +++ b/apps/sim/components/emails/render-notifications.test.ts @@ -73,7 +73,7 @@ describe('renderUsageLimitReachedEmail', () => { scope: 'organization', currentUsage: 500, limit: 500, - ctaLink: 'https://sim.ai/organization/org_1/settings/billing', + ctaLink: 'https://sim.ai/workspace/ws_1/settings/billing', }) expect(html).toContain('Raise Organization Limit') diff --git a/apps/sim/components/settings/navigation.test.ts b/apps/sim/components/settings/navigation.test.ts index 8420aba40bb..df3a5f0d5cd 100644 --- a/apps/sim/components/settings/navigation.test.ts +++ b/apps/sim/components/settings/navigation.test.ts @@ -11,17 +11,15 @@ import { buildUnifiedSettingsNavigation, canMutateWorkspaceSettingsSection, getAccountSettingsHref, - getOrganizationSettingsHref, getWorkspaceSettingsHref, isOrganizationSettingsSectionAvailable, ORGANIZATION_PLANE_UNIFIED_SECTIONS, - ORGANIZATION_SETTINGS_ITEMS, - ORGANIZATION_SETTINGS_PATH_ALIASES, parseSettingsPathSection, resolveOrganizationSectionAccess, resolveWorkspaceNavigation, SELFHOST_SETTINGS_ITEMS, SETTINGS_SECTION_REGISTRY, + UNIFIED_TO_ORGANIZATION_SECTION, WORKSPACE_SETTINGS_ITEMS, WORKSPACE_SETTINGS_PATH_ALIASES, } from '@/components/settings/navigation' @@ -55,6 +53,7 @@ describe('settings navigation boundaries', () => { 'billing', 'teammates', 'organization', + 'usage', 'secrets', 'credential-groups', 'custom-tools', @@ -83,17 +82,6 @@ describe('settings navigation boundaries', () => { 'mothership', ]) expect(SELFHOST_SETTINGS_ITEMS.map(({ id }) => id)).toEqual(['general', 'billing', 'chat-keys']) - expect(ORGANIZATION_SETTINGS_ITEMS.map(({ id }) => id)).toEqual([ - 'members', - 'billing', - 'access-control', - 'audit-logs', - 'sso', - 'sessions', - 'data-retention', - 'data-drains', - 'whitelabeling', - ]) expect(WORKSPACE_SETTINGS_ITEMS.map(({ id }) => id)).toEqual([ 'teammates', 'secrets', @@ -199,9 +187,6 @@ describe('settings navigation boundaries', () => { const accountIds = SETTINGS_SECTION_REGISTRY.flatMap(({ planes }) => planes?.account ? [planes.account.id] : [] ) - const organizationIds = SETTINGS_SECTION_REGISTRY.flatMap(({ planes }) => - planes?.organization ? [planes.organization.id] : [] - ) const selfHostIds = SETTINGS_SECTION_REGISTRY.flatMap(({ planes }) => planes?.selfhost ? [planes.selfhost.id] : [] ) @@ -211,7 +196,6 @@ describe('settings navigation boundaries', () => { expect(new Set(unifiedIds).size).toBe(unifiedIds.length) expect(new Set(accountIds).size).toBe(accountIds.length) - expect(new Set(organizationIds).size).toBe(organizationIds.length) expect(new Set(selfHostIds).size).toBe(selfHostIds.length) expect(new Set(workspaceIds).size).toBe(workspaceIds.length) expect([...unifiedIds].sort()).toEqual( @@ -220,9 +204,6 @@ describe('settings navigation boundaries', () => { .sort() ) expect([...accountIds].sort()).toEqual(ACCOUNT_SETTINGS_ITEMS.map(({ id }) => id).sort()) - expect([...organizationIds].sort()).toEqual( - ORGANIZATION_SETTINGS_ITEMS.map(({ id }) => id).sort() - ) expect([...selfHostIds].sort()).toEqual(SELFHOST_SETTINGS_ITEMS.map(({ id }) => id).sort()) expect([...workspaceIds].sort()).toEqual(WORKSPACE_SETTINGS_ITEMS.map(({ id }) => id).sort()) }) @@ -237,27 +218,37 @@ describe('settings navigation boundaries', () => { 'organization', 'sessions', 'sso', + 'usage', 'whitelabeling', ]) }) - it('shares labels, icons, and docs links across projections', () => { - const unifiedSso = buildUnifiedSettingsNavigation().find(({ id }) => id === 'sso') - const organizationSso = ORGANIZATION_SETTINGS_ITEMS.find(({ id }) => id === 'sso') - - expect(organizationSso?.label).toBe(unifiedSso?.label) - expect(organizationSso?.icon).toBe(unifiedSso?.icon) - expect(organizationSso?.docsLink).toBe(unifiedSso?.docsLink) + it('maps every organization-scoped unified section to its organization counterpart', () => { + // The section page reads this map to decide whether to apply the organization + // gate at all, so a section missing from it is not "ungated by omission" — it + // is a section any workspace member could open. + expect(UNIFIED_TO_ORGANIZATION_SECTION).toEqual({ + organization: 'members', + billing: 'billing', + 'access-control': 'access-control', + 'audit-logs': 'audit-logs', + sso: 'sso', + sessions: 'sessions', + 'data-retention': 'data-retention', + 'data-drains': 'data-drains', + whitelabeling: 'whitelabeling', + usage: 'usage', + }) + expect(Object.keys(UNIFIED_TO_ORGANIZATION_SECTION).sort()).toEqual( + [...ORGANIZATION_PLANE_UNIFIED_SECTIONS].sort() + ) }) - it('uses scope-specific labels consistently across settings surfaces', () => { - const organizationMembers = ORGANIZATION_SETTINGS_ITEMS.find(({ id }) => id === 'members') + it('labels the members section consistently', () => { const unifiedOrganization = buildUnifiedSettingsNavigation().find( ({ id }) => id === 'organization' ) - expect(organizationMembers?.label).toBe('Members') - expect(organizationMembers?.description).toBe('Manage organization members, roles, and seats.') expect(unifiedOrganization?.label).toBe('Members') }) @@ -293,9 +284,6 @@ describe('settings navigation boundaries', () => { it('builds canonical settings hrefs across all three planes', () => { expect(getAccountSettingsHref('general')).toBe('/account/settings/general') - expect(getOrganizationSettingsHref('organization-a', 'members')).toBe( - '/organization/organization-a/settings/members' - ) expect(getWorkspaceSettingsHref('workspace-a', 'teammates')).toBe( '/workspace/workspace-a/settings/teammates' ) @@ -328,20 +316,6 @@ describe('settings navigation boundaries', () => { expect(parseAccountPath('/account/settings', 'general')).toBe('general') }) - it('parses canonical, aliased, and invalid organization settings paths', () => { - const parseOrganizationPath = (path: string) => - parseSettingsPathSection({ - path, - items: ORGANIZATION_SETTINGS_ITEMS, - defaultSection: null, - aliases: ORGANIZATION_SETTINGS_PATH_ALIASES, - }) - - expect(parseOrganizationPath('sso')).toBe('sso') - expect(parseOrganizationPath('/organization/org-a/settings/organization')).toBe('members') - expect(parseOrganizationPath('/organization/org-a/settings/not-a-section')).toBeNull() - }) - it('parses canonical, aliased, and invalid workspace settings paths', () => { const parseWorkspacePath = (path: string) => parseSettingsPathSection({ @@ -359,7 +333,6 @@ describe('settings navigation boundaries', () => { it('keeps API keys split between account and workspace settings', () => { expect(ACCOUNT_SETTINGS_ITEMS.some(({ id }) => id === 'api-keys')).toBe(true) expect(WORKSPACE_SETTINGS_ITEMS.some(({ id }) => id === 'api-keys')).toBe(true) - expect(ORGANIZATION_SETTINGS_ITEMS.some(({ id }) => String(id) === 'api-keys')).toBe(false) }) it('requires target-organization membership and admin authority', () => { diff --git a/apps/sim/components/settings/navigation.ts b/apps/sim/components/settings/navigation.ts index 64704f531d5..dcb92ef1beb 100644 --- a/apps/sim/components/settings/navigation.ts +++ b/apps/sim/components/settings/navigation.ts @@ -1,5 +1,6 @@ import type { ComponentType } from 'react' import { + ChartColumn, ClipboardList, Clock, Credit, @@ -40,10 +41,11 @@ import { isSandboxesEnabled, isSessionPoliciesEnabled, isSsoEnabled, + isUsageMonitoringEnabled, isWhitelabelingEnabled, } from '@/lib/core/config/env-flags' -export type SettingsPlane = 'account' | 'organization' | 'selfhost' | 'workspace' +export type SettingsPlane = 'account' | 'selfhost' | 'workspace' export type AccountSettingsSection = 'general' | 'billing' | 'api-keys' | 'admin' | 'mothership' @@ -56,6 +58,7 @@ export type SelfHostSettingsSection = 'general' | 'billing' | 'chat-keys' export type OrganizationSettingsSection = | 'members' | 'billing' + | 'usage' | 'access-control' | 'audit-logs' | 'sso' @@ -86,8 +89,6 @@ export type SettingsSection = | SelfHostSettingsSection | WorkspaceSettingsSection -export type OrganizationSettingsRouteSection = OrganizationSettingsSection | 'unavailable' - export interface SettingsNavigationItem
{ id: Section label: string @@ -112,6 +113,7 @@ export type UnifiedSettingsSection = | 'billing' | 'teammates' | 'organization' + | 'usage' | 'sso' | 'whitelabeling' | 'forks' @@ -164,6 +166,16 @@ export interface UnifiedSettingsNavigationItem { hideForEnterprise?: boolean externalUrl?: string docsLink?: string + /** + * The organization-scoped counterpart of this section. Declaring it marks the + * section as acting on the host organization rather than the workspace, which + * routes it through the organization gate (host organization present, org-admin + * viewer, plan entitlement) in both the sidebar and the section page. + * + * This is the single source for {@link ORGANIZATION_PLANE_UNIFIED_SECTIONS} and + * {@link UNIFIED_TO_ORGANIZATION_SECTION}, so the two cannot drift apart. + */ + organizationSection?: OrganizationSettingsSection } interface UnifiedSettingsProjection @@ -173,7 +185,6 @@ interface UnifiedSettingsProjection interface SettingsPlaneSectionMap { account: AccountSettingsSection - organization: OrganizationSettingsSection selfhost: SelfHostSettingsSection workspace: WorkspaceSettingsSection } @@ -220,6 +231,7 @@ const SETTINGS_SELF_HOSTED_OVERRIDES = { sandboxes: isSandboxesEnabled, sessionPolicies: isSessionPoliciesEnabled, sso: isSsoEnabled, + usageMonitoring: isUsageMonitoringEnabled, whitelabeling: isWhitelabelingEnabled, } as const @@ -249,17 +261,6 @@ export function getSelfHostSettingsHref( return withSettingsSearchParams(`/selfhost/settings/${section}`, searchParams) } -export function getOrganizationSettingsHref( - organizationId: string, - section: OrganizationSettingsRouteSection, - searchParams?: SettingsHrefSearchParams -): string { - return withSettingsSearchParams( - `/organization/${organizationId}/settings/${section}`, - searchParams - ) -} - export function getWorkspaceSettingsHref( workspaceId: string, section: WorkspaceSettingsSection, @@ -272,12 +273,6 @@ export const ACCOUNT_SETTINGS_PATH_ALIASES = { apikeys: 'api-keys', } as const satisfies Readonly> -export const ORGANIZATION_SETTINGS_PATH_ALIASES = { - organization: 'members', - // Verified domains moved into the SSO page; keep old links working. - domains: 'sso', -} as const satisfies Readonly> - export const WORKSPACE_SETTINGS_PATH_ALIASES = { apikeys: 'api-keys', } as const satisfies Readonly> @@ -341,7 +336,6 @@ export const SETTINGS_PLANE_CHROME: Record< { label: string; showWordmark: boolean } > = { account: { label: 'Account', showWordmark: false }, - organization: { label: 'Organization', showWordmark: false }, selfhost: { label: 'Self-host', showWordmark: true }, } @@ -350,12 +344,6 @@ export const SELFHOST_SETTINGS_GROUPS = [ { key: 'developer', title: 'Developer' }, ] as const -export const ORGANIZATION_SETTINGS_GROUPS = [ - { key: 'organization', title: 'Organization' }, - { key: 'security', title: 'Security' }, - { key: 'enterprise', title: 'Enterprise' }, -] as const - export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] = [ { label: 'General', @@ -412,13 +400,11 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] id: 'access-control', description: 'Manage permission groups across your organization.', group: 'organization', - order: 3, + order: 4, requiresHosted: true, requiresEnterprise: true, selfHostedOverride: SETTINGS_SELF_HOSTED_OVERRIDES.accessControl, - }, - planes: { - organization: { id: 'access-control', group: 'security', order: 2 }, + organizationSection: 'access-control', }, }, { @@ -429,13 +415,11 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] id: 'audit-logs', description: 'Review activity and changes across your organization.', group: 'organization', - order: 4, + order: 5, requiresHosted: true, requiresEnterprise: true, selfHostedOverride: SETTINGS_SELF_HOSTED_OVERRIDES.auditLogs, - }, - planes: { - organization: { id: 'audit-logs', group: 'security', order: 3 }, + organizationSection: 'audit-logs', }, }, { @@ -446,7 +430,7 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] id: 'forks', description: 'Fork this workspace and sync changes with its parent.', group: 'organization', - order: 2, + order: 3, }, planes: { workspace: { id: 'forks', group: 'enterprise', order: 10 }, @@ -461,6 +445,7 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] group: 'account', order: 1, hideWhenBillingDisabled: true, + organizationSection: 'billing', }, planes: { account: { @@ -475,12 +460,6 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] group: 'account', order: 1, }, - organization: { - id: 'billing', - description: 'Manage the organization plan, usage, and invoices.', - group: 'organization', - order: 1, - }, }, }, { @@ -507,14 +486,38 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] hideWhenBillingDisabled: true, requiresHosted: true, requiresTeam: true, + /** + * A plain member sees the roster read-only — `resolveOrganizationSectionAccess` + * grants them `'view'` on this one section, and `TeamManagement` renders + * without management controls. Every other organization section stays + * admin-only. + */ + allowNonOrgAdmin: true, + organizationSection: 'members', }, - planes: { - organization: { - id: 'members', - description: 'Manage organization members, roles, and seats.', - group: 'organization', - order: 0, - }, + }, + { + label: 'Usage tracking', + icon: ChartColumn, + unified: { + id: 'usage', + description: 'Monitor credit usage across your organization.', + group: 'organization', + order: 1, + /** + * Deliberately no `hideWhenBillingDisabled`, unlike Members above. + * + * The sidebar applies that filter *before* it consults `selfHostedOverride`, + * so pairing the two hid this section from exactly the deployment the + * override exists to serve: self-hosted, billing off, `USAGE_MONITORING_ENABLED` + * on. Members can carry the flag because it has no override to reach. Here the + * two gates below already answer both cases — hosted needs the plan, and + * self-hosted needs the flag. + */ + requiresHosted: true, + requiresEnterprise: true, + selfHostedOverride: SETTINGS_SELF_HOSTED_OVERRIDES.usageMonitoring, + organizationSection: 'usage', }, }, { @@ -704,13 +707,11 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] id: 'sso', description: 'Configure single sign-on for your organization.', group: 'organization', - order: 6, + order: 7, requiresHosted: true, requiresEnterprise: true, selfHostedOverride: SETTINGS_SELF_HOSTED_OVERRIDES.sso, - }, - planes: { - organization: { id: 'sso', group: 'security', order: 4 }, + organizationSection: 'sso', }, }, { @@ -721,13 +722,11 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] id: 'sessions', description: 'Limit session lifetimes and sign out members org-wide.', group: 'organization', - order: 7, + order: 8, requiresHosted: true, requiresEnterprise: true, selfHostedOverride: SETTINGS_SELF_HOSTED_OVERRIDES.sessionPolicies, - }, - planes: { - organization: { id: 'sessions', group: 'security', order: 5 }, + organizationSection: 'sessions', }, }, { @@ -739,13 +738,11 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] description: 'Control data retention windows and PII redaction. Workspaces without an override inherit the organization defaults.', group: 'organization', - order: 8, + order: 9, requiresHosted: true, requiresEnterprise: true, selfHostedOverride: SETTINGS_SELF_HOSTED_OVERRIDES.dataRetention, - }, - planes: { - organization: { id: 'data-retention', group: 'enterprise', order: 6 }, + organizationSection: 'data-retention', }, }, { @@ -756,13 +753,11 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] id: 'data-drains', description: 'Stream your logs and events to external destinations.', group: 'organization', - order: 9, + order: 10, requiresHosted: true, requiresEnterprise: true, selfHostedOverride: SETTINGS_SELF_HOSTED_OVERRIDES.dataDrains, - }, - planes: { - organization: { id: 'data-drains', group: 'enterprise', order: 7 }, + organizationSection: 'data-drains', }, }, { @@ -773,13 +768,11 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] id: 'whitelabeling', description: 'Customize your workspace branding and appearance.', group: 'organization', - order: 5, + order: 6, requiresHosted: true, requiresEnterprise: true, selfHostedOverride: SETTINGS_SELF_HOSTED_OVERRIDES.whitelabeling, - }, - planes: { - organization: { id: 'whitelabeling', group: 'enterprise', order: 8 }, + organizationSection: 'whitelabeling', }, }, { @@ -790,7 +783,7 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] id: 'custom-blocks', description: 'Publish workflows as reusable blocks for your organization.', group: 'organization', - order: 1, + order: 2, requiresHosted: true, requiresEnterprise: true, allowNonOrgAdmin: true, @@ -878,9 +871,6 @@ function buildPlaneSettingsItems( export const ACCOUNT_SETTINGS_ITEMS: SettingsNavigationItem[] = buildPlaneSettingsItems('account') -export const ORGANIZATION_SETTINGS_ITEMS: SettingsNavigationItem[] = - buildPlaneSettingsItems('organization') - export const SELFHOST_SETTINGS_ITEMS: SettingsNavigationItem[] = buildPlaneSettingsItems('selfhost') @@ -895,7 +885,26 @@ export const WORKSPACE_SETTINGS_ITEMS: SettingsNavigationItem = new Set( SETTINGS_SECTION_REGISTRY.flatMap((entry) => - entry.planes?.organization && entry.unified ? [entry.unified.id] : [] + entry.unified?.organizationSection ? [entry.unified.id] : [] + ) +) + +/** + * Unified section id to the organization-scoped section it acts on, for the gates + * that take an {@link OrganizationSettingsSection} (`canOpenOrganizationSettingsSection`, + * `isOrganizationSettingsSectionAvailable`). + * + * Derived from the registry rather than hand-listed: a section added to one and + * forgotten in the other used to mean the page applied *no* organization gate at + * all, since an unmapped section reads as "not organization-scoped". + */ +export const UNIFIED_TO_ORGANIZATION_SECTION: Readonly< + Partial> +> = Object.fromEntries( + SETTINGS_SECTION_REGISTRY.flatMap((entry) => + entry.unified?.organizationSection + ? [[entry.unified.id, entry.unified.organizationSection] as const] + : [] ) ) @@ -938,6 +947,7 @@ export function getOrganizationSettingsFeatures( sessions: SETTINGS_SELF_HOSTED_OVERRIDES.sessionPolicies, 'data-retention': SETTINGS_SELF_HOSTED_OVERRIDES.dataRetention, 'data-drains': SETTINGS_SELF_HOSTED_OVERRIDES.dataDrains, + usage: SETTINGS_SELF_HOSTED_OVERRIDES.usageMonitoring, whitelabeling: SETTINGS_SELF_HOSTED_OVERRIDES.whitelabeling, }, } @@ -1097,11 +1107,9 @@ export function getSettingsSectionMeta( const catalog = plane === 'account' ? ACCOUNT_SETTINGS_ITEMS - : plane === 'organization' - ? ORGANIZATION_SETTINGS_ITEMS - : plane === 'selfhost' - ? SELFHOST_SETTINGS_ITEMS - : WORKSPACE_SETTINGS_ITEMS + : plane === 'selfhost' + ? SELFHOST_SETTINGS_ITEMS + : WORKSPACE_SETTINGS_ITEMS const item = catalog.find((candidate) => candidate.id === section) return item ? { label: item.label, description: item.description, docsLink: item.docsLink } : null } diff --git a/apps/sim/components/settings/organization-settings-renderer.tsx b/apps/sim/components/settings/organization-settings-renderer.tsx deleted file mode 100644 index 66ae7a1efc3..00000000000 --- a/apps/sim/components/settings/organization-settings-renderer.tsx +++ /dev/null @@ -1,79 +0,0 @@ -'use client' - -import { useEffect } from 'react' -import dynamic from 'next/dynamic' -import { usePostHog } from 'posthog-js/react' -import type { OrganizationSettingsSection } from '@/components/settings/navigation' -import { captureEvent } from '@/lib/posthog/client' - -const TeamManagement = dynamic(() => - import('@/app/workspace/[workspaceId]/settings/components/team-management/team-management').then( - (module) => module.TeamManagement - ) -) -const Billing = dynamic(() => - import('@/app/workspace/[workspaceId]/settings/components/billing/billing').then( - (module) => module.Billing - ) -) -const AccessControl = dynamic(() => - import('@/ee/access-control/components/access-control').then((module) => module.AccessControl) -) -const AuditLogs = dynamic(() => - import('@/ee/audit-logs/components/audit-logs').then((module) => module.AuditLogs) -) -const SSO = dynamic(() => import('@/ee/sso/components/sso-settings').then((module) => module.SSO)) -const SessionPolicySettings = dynamic(() => - import('@/ee/session-policy/components/session-policy-settings').then( - (module) => module.SessionPolicySettings - ) -) -const DataRetentionSettings = dynamic(() => - import('@/ee/data-retention/components/data-retention-settings').then( - (module) => module.DataRetentionSettings - ) -) -const DataDrainsSettings = dynamic(() => - import('@/ee/data-drains/components/data-drains-settings').then( - (module) => module.DataDrainsSettings - ) -) -const WhitelabelingSettings = dynamic( - () => - import('@/ee/whitelabeling/components/whitelabeling-settings').then( - (module) => module.WhitelabelingSettings - ), - { ssr: false } -) - -interface OrganizationSettingsRendererProps { - organizationId: string - section: OrganizationSettingsSection -} - -export function OrganizationSettingsRenderer({ - organizationId, - section, -}: OrganizationSettingsRendererProps) { - const posthog = usePostHog() - - useEffect(() => { - captureEvent(posthog, 'settings_tab_viewed', { plane: 'organization', section }) - }, [posthog, section]) - - if (section === 'members') return - if (section === 'billing') return - if (section === 'access-control') { - return - } - if (section === 'audit-logs') return - if (section === 'sso') return - if (section === 'sessions') { - return - } - if (section === 'data-retention') { - return - } - if (section === 'data-drains') return - return -} diff --git a/apps/sim/components/settings/settings-header-shell.test.tsx b/apps/sim/components/settings/settings-header-shell.test.tsx index e2d588159ff..ecf53c2706a 100644 --- a/apps/sim/components/settings/settings-header-shell.test.tsx +++ b/apps/sim/components/settings/settings-header-shell.test.tsx @@ -181,8 +181,8 @@ describe('SettingsHeaderShell static meta', () => { }) it('lets a body claim the header with nothing in it, suppressing the meta entirely', () => { - // How SettingsUnavailable opts out: it renders its own centred heading, so the routed - // section's catalog title must not caption it. + // How a surface that draws its own centred heading opts out: the routed section's + // catalog title must not also caption it. function OwnHeadingBody() { useSettingsHeader(EMPTY_HEADER) return
diff --git a/apps/sim/components/settings/settings-header.tsx b/apps/sim/components/settings/settings-header.tsx index 7c1b3ae4c69..27788ef7293 100644 --- a/apps/sim/components/settings/settings-header.tsx +++ b/apps/sim/components/settings/settings-header.tsx @@ -274,7 +274,7 @@ export function SettingsHeaderShell({ meta, children }: SettingsHeaderShellProps * field-by-field: a body that registers an explicit `title` and no `description` is * deliberately suppressing the meta description, so the two must never be merged — and a * body that registers an empty config is deliberately asking for no heading at all, which - * is how a surface that renders its own (`SettingsUnavailable`) opts out. + * is how a surface that renders its own heading opts out. */ const config: SettingsHeaderConfig = registered ?? meta ?? EMPTY_CONFIG const { title, description, docsLink, back, actions, search, scrollContainerRef } = config diff --git a/apps/sim/components/settings/settings-unavailable.tsx b/apps/sim/components/settings/settings-unavailable.tsx deleted file mode 100644 index 806ab078c3f..00000000000 --- a/apps/sim/components/settings/settings-unavailable.tsx +++ /dev/null @@ -1,28 +0,0 @@ -import { ChipLink, cn } from '@sim/emcn' - -interface SettingsUnavailableProps { - title?: string - description?: string - embedded?: boolean -} - -export function SettingsUnavailable({ - title = 'Settings unavailable', - description = 'You do not have access to manage this organization. Contact an organization owner or admin for help.', - embedded = false, -}: SettingsUnavailableProps) { - return ( -
-
-

{title}

-

{description}

- Back to workspaces -
-
- ) -} diff --git a/apps/sim/components/settings/standalone-settings-shell.test.ts b/apps/sim/components/settings/standalone-settings-shell.test.ts index 720bcdfff56..49d462e8f76 100644 --- a/apps/sim/components/settings/standalone-settings-shell.test.ts +++ b/apps/sim/components/settings/standalone-settings-shell.test.ts @@ -5,8 +5,6 @@ import { describe, expect, it } from 'vitest' import { ACCOUNT_SETTINGS_ITEMS, ACCOUNT_SETTINGS_PATH_ALIASES, - ORGANIZATION_SETTINGS_ITEMS, - ORGANIZATION_SETTINGS_PATH_ALIASES, parseSettingsPathSection, SELFHOST_SETTINGS_ITEMS, } from '@/components/settings/navigation' @@ -23,17 +21,6 @@ describe('standalone settings section resolution', () => { ).toBe('billing') }) - it('resolves the organization section from its pathname', () => { - expect( - parseSettingsPathSection({ - path: '/organization/org-1/settings/audit-logs', - items: ORGANIZATION_SETTINGS_ITEMS, - defaultSection: 'members', - aliases: ORGANIZATION_SETTINGS_PATH_ALIASES, - }) - ).toBe('audit-logs') - }) - it('keeps Subscription active for the self-host billing route', () => { expect( parseSettingsPathSection({ diff --git a/apps/sim/components/settings/standalone-settings-shell.tsx b/apps/sim/components/settings/standalone-settings-shell.tsx index 2991fe68cfe..ac31197e7b6 100644 --- a/apps/sim/components/settings/standalone-settings-shell.tsx +++ b/apps/sim/components/settings/standalone-settings-shell.tsx @@ -7,15 +7,8 @@ import { ACCOUNT_SETTINGS_ITEMS, ACCOUNT_SETTINGS_PATH_ALIASES, getAccountSettingsHref, - getOrganizationSettingsFeatures, - getOrganizationSettingsHref, getSelfHostSettingsHref, - isOrganizationSettingsSectionAvailable, - ORGANIZATION_SETTINGS_GROUPS, - ORGANIZATION_SETTINGS_ITEMS, - ORGANIZATION_SETTINGS_PATH_ALIASES, parseSettingsPathSection, - resolveOrganizationSectionAccess, SELFHOST_SETTINGS_GROUPS, SELFHOST_SETTINGS_ITEMS, SETTINGS_PLANE_CHROME, @@ -40,40 +33,19 @@ interface SelfHostSettingsShellProps extends StandaloneSettingsShellBaseProps { plane: 'selfhost' } -interface OrganizationSettingsShellProps extends StandaloneSettingsShellBaseProps { - plane: 'organization' - organizationId: string - hasEnterprisePlan: boolean - isOrganizationAdmin: boolean -} - -type StandaloneSettingsShellProps = - | AccountSettingsShellProps - | OrganizationSettingsShellProps - | SelfHostSettingsShellProps +type StandaloneSettingsShellProps = AccountSettingsShellProps | SelfHostSettingsShellProps export function StandaloneSettingsShell(props: StandaloneSettingsShellProps) { const { children, plane } = props useSettingsBeforeUnload() const pathname = usePathname() - const hasEnterprisePlan = plane === 'organization' ? props.hasEnterprisePlan : false - const isOrganizationAdmin = plane === 'organization' ? props.isOrganizationAdmin : false const isSuperUser = plane === 'account' ? (props.isSuperUser ?? false) : false - const organizationFeatures = getOrganizationSettingsFeatures(hasEnterprisePlan) const accountItems = ACCOUNT_SETTINGS_ITEMS.filter((item) => { if (item.id === 'billing' && !isBillingEnabled) return false if ((item.id === 'admin' || item.id === 'mothership') && !isSuperUser) return false return true }) - const organizationItems = ORGANIZATION_SETTINGS_ITEMS.filter( - (item) => - resolveOrganizationSectionAccess({ - section: item.id, - isTargetOrganizationMember: true, - isTargetOrganizationAdmin: isOrganizationAdmin, - }) !== 'unavailable' && isOrganizationSettingsSectionAvailable(item.id, organizationFeatures) - ) const selfHostItems = SELFHOST_SETTINGS_ITEMS.filter((item) => { if (item.id === 'billing' && !isBillingEnabled) return false // Chat keys are issued by the managed service, so there are none to list on @@ -93,28 +65,9 @@ export function StandaloneSettingsShell(props: StandaloneSettingsShellProps) { defaultSection: 'general', aliases: ACCOUNT_SETTINGS_PATH_ALIASES, }) - const organizationSection = parseSettingsPathSection({ - path: pathname, - items: ORGANIZATION_SETTINGS_ITEMS, - defaultSection: 'members', - aliases: ORGANIZATION_SETTINGS_PATH_ALIASES, - }) - const activeSection = - plane === 'account' - ? accountSection - : plane === 'selfhost' - ? selfHostSection - : organizationSection + const activeSection = plane === 'account' ? accountSection : selfHostSection const sidebar = - plane === 'selfhost' ? ( - - ) : plane === 'account' ? ( + plane === 'account' ? ( ) : ( getOrganizationSettingsHref(props.organizationId, section)} - items={organizationItems} + groups={SELFHOST_SETTINGS_GROUPS} + hrefForSection={getSelfHostSettingsHref} + items={selfHostItems} /> ) diff --git a/apps/sim/connectors/sftp/sftp.test.ts b/apps/sim/connectors/sftp/sftp.test.ts index a8026582bce..5bcf4286933 100644 --- a/apps/sim/connectors/sftp/sftp.test.ts +++ b/apps/sim/connectors/sftp/sftp.test.ts @@ -53,6 +53,21 @@ vi.mock('ssh2', () => { return this } + once(event: string, handler: (arg?: unknown) => void) { + const wrapped = (arg?: unknown) => { + this.off(event, wrapped) + handler(arg) + } + return this.on(event, wrapped) + } + + off(event: string, handler: (arg?: unknown) => void) { + this.handlers[event] = (this.handlers[event] ?? []).filter( + (registered) => registered !== handler + ) + return this + } + private emit(event: string, arg?: unknown) { for (const handler of this.handlers[event] ?? []) handler(arg) } @@ -74,8 +89,13 @@ vi.mock('ssh2', () => { cb(null, fakeSftp) } - end() {} - destroy() {} + end() { + this.emit('close') + } + + destroy() { + this.emit('close') + } } return { diff --git a/apps/sim/connectors/sftp/sftp.ts b/apps/sim/connectors/sftp/sftp.ts index f7da687f66b..6d03efc677f 100644 --- a/apps/sim/connectors/sftp/sftp.ts +++ b/apps/sim/connectors/sftp/sftp.ts @@ -8,7 +8,7 @@ import { getSftp, isPathSafe, readSftpFileCapped, -} from '@/app/api/tools/sftp/utils' +} from '@/lib/internal/sftp/client' import { sftpConnectorMeta } from '@/connectors/sftp/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { @@ -332,7 +332,7 @@ async function withSftpSession( let client: Client | undefined let timer: NodeJS.Timeout | undefined try { - client = await createSftpConnection({ + const connection = await createSftpConnection({ host: ctx.host, port: ctx.port, username: ctx.username, @@ -342,8 +342,8 @@ async function withSftpSession( readyTimeout: READY_TIMEOUT_MS, keepaliveInterval: KEEPALIVE_INTERVAL_MS, }) - const sftp = await getSftp(client) - const connection = client + client = connection + const sftp = await getSftp(connection) const deadline = new Promise((_, reject) => { timer = setTimeout(() => { /** diff --git a/apps/sim/content/library/ai-agent-vs-chatbot/index.mdx b/apps/sim/content/library/ai-agent-vs-chatbot/index.mdx index 4311e723e7b..a842e875518 100644 --- a/apps/sim/content/library/ai-agent-vs-chatbot/index.mdx +++ b/apps/sim/content/library/ai-agent-vs-chatbot/index.mdx @@ -3,214 +3,94 @@ slug: ai-agent-vs-chatbot title: 'AI Agent vs Chatbot: Understanding the Differences' description: Understand the key differences between AI agents vs chatbots, from architecture to real-world use cases. Learn when to use each and how to choose the right approach for your workflows. date: 2026-06-28 -updated: 2026-07-23 +updated: 2026-08-27 authors: - - emir -readingTime: 13 + - andrew +readingTime: 6 tags: [AI Agents, Chatbots, AI Workspace, Sim] ogImage: /library/ai-agent-vs-chatbot/cover.jpg canonical: https://www.sim.ai/library/ai-agent-vs-chatbot draft: false faq: - - q: "What is the difference between an AI agent and a chatbot?" - a: "A chatbot is a reactive system that matches user input to predefined responses using rules or basic NLP; it can only answer within its scripted boundaries. An AI agent is an autonomous system that reasons toward a goal, accesses external tools and data, retains memory across interactions, and executes multi-step actions with minimal human direction." - - q: "Can a chatbot become an AI agent?" - a: "Adding an LLM like GPT to a chatbot doesn't make it an agent — it makes it a chatbot with better language generation. True AI agents require a fundamentally different architecture: persistent memory (short-term and long-term), tool access for taking real actions across systems, and goal-directed planning that lets the system break down objectives into steps and execute them." - - q: "Are AI agents more expensive than chatbots?" - a: "AI agents can require more setup: integration with APIs and databases, memory configuration, LLM costs, and orchestration logic. But new tools like Sim make it easy to deploy a first agent in minutes. Once live, agents reduce manual overhead at scale — less time spent on repetitive multi-step processes, and fewer errors in cross-system workflows. For teams drowning in manual tasks or coordination, the ROI on an agent proves itself within months." - - q: "What are the best use cases for AI agents in 2026?" - a: "Common enterprise use cases include end-to-end support automation (resolving tickets without human escalation), data enrichment (pulling and combining data across CRMs, third-party sources, and internal databases), compliance workflows (monitoring regulatory requirements and flagging issues across documents), multi-system reporting (aggregating data from analytics, billing, and project platforms into actionable summaries), and employee or customer onboarding (coordinating account provisioning, training schedules, and documentation across multiple tools)." - - q: "How do I know if my team is ready to build AI agents?" - a: "Check for four things: clear workflows you want to automate with defined steps rather than a vague AI goal, the right access credentials to the systems involved (CRM, payment gateway, databases, communication tools), a chosen platform or framework such as a code-first tool like LangChain or a visual builder like Sim, and someone who will own the agent rollout and keep monitoring performance and expanding scope over time." + - q: "Do AI agents always produce more accurate answers than chatbots?" + a: "Answer accuracy is the degree to which a system's response is correct and supported by the available evidence. In [Sim](https://sim.ai), you can inspect each workflow step and control which instructions, data, and tools the model uses. This visibility helps you find the source of an error and improve the workflow without assuming that an agent is inherently more accurate than a chatbot." + - q: "Can a chatbot and an AI agent work together?" + a: "A hybrid application uses a chatbot for conversation and an AI agent for actions that require tools or multiple steps. Sim connects our [Chat interface](https://docs.sim.ai/execution/chat) to agent workflows you build in a visual workspace with connected integrations. You can give users one conversational interface while the agent handles work across connected applications." + - q: "Do I need to know how to code to build an AI agent?" + a: "A visual agent builder represents workflow logic as configurable blocks that connect models to tools, so you do not have to program every step. Sim provides a visual workspace for creating and testing these workflows. You can add code when needed without building the orchestration layer from scratch." + - q: "Can AI agents run with local models?" + a: "Local-model execution runs a model on infrastructure you control rather than through a hosted model provider. Sim supports Enterprise workflows that use local-model tools such as Ollama and also offers hosted access. This choice lets teams match deployment to their security, control, and infrastructure requirements." + - q: "How should an AI agent handle failed actions?" + a: "A fallback is a predefined response to an invalid, unavailable, or unsuccessful tool call. Sim lets you route a failed action to a retry, an approved alternative, human review, or a safe stop. These paths prevent the agent from continuing with unsupported data or taking an unapproved action." --- -You've deployed a chatbot. It answers the easy stuff fine: store hours, password resets, order status checks. Then a customer shows up with something slightly more complex: a refund tied to a promotional code, a shipping update across two orders, a billing question that requires pulling data from your CRM. The chatbot stalls. It loops. It punts to a human agent. Nearly one in five consumers who have used AI for customer service saw no benefit from the experience, according to the Qualtrics 2026 Customer Experience Trends Report. +## TL;DR -The market isn't helping with clarity: vendors slap "AI agent" on glorified decision trees and call rule-based bots "intelligent assistants." The term "AI agent" gets thrown around loosely, and the resulting confusion costs companies money. +- A chatbot responds to user messages within a defined conversation. +- An AI agent pursues a goal by deciding what steps to take and completing actions. +- AI agents can use retained context and external tools to manage multi-step work with less user direction. +- Use a chatbot for predictable conversations and an agent for tasks that require decisions or actions. +- [Sim](https://sim.ai) combines [Chat](https://docs.sim.ai/execution/chat) with more than [1,000 integrations](https://sim.ai/integrations) in a visual workspace for building agent-chatbot hybrids. -Chatbots and AI agents are fundamentally different architectures. They're built differently, they reason differently, they act differently. One follows a script. The other pursues a goal. Understanding the difference shapes whether you waste six months on the wrong tool or deploy something that moves the needle from day one. +We last verified the article on August 27, 2026. -We'll walk through what each one is, how they work under the hood, a direct side-by-side comparison, a decision framework for choosing the right tool, and a look at hybrid approaches where both work together. By the end, you'll know exactly which architecture fits your problem. +## AI agents vs. chatbots: key differences -## The Short Version +A chatbot is a conversational interface that responds to user messages, usually within a predefined flow or a single exchange. An AI agent is a goal-directed system that uses external tools to complete tasks with limited user guidance. [Anthropic defines an agent](https://www.anthropic.com/research/measuring-agent-autonomy) as an AI system equipped with tools that let it take actions, like running code, calling APIs, or messaging other agents. -- **Chatbots follow scripts:** They match user input to predefined responses using rules or basic NLP, handling FAQs and simple lookups well but breaking down when conversations go off-script. -- **AI agents pursue goals:** They combine LLMs, memory, tool access, and planning to reason through multi-step tasks, make decisions, and take actions across systems with minimal human direction. -- **The dividing lines are autonomy, memory, and tool use:** If a system can't remember context, access external tools, or adjust its approach mid-task, it's a chatbot, regardless of what it's marketed as. -- **Chatbots aren't inferior; they're scoped:** For high-volume FAQ handling, appointment booking, and lead qualification, a well-built chatbot is the right call. Don't over-engineer. -- **You don't have to build from scratch:** Visual agent builders like Sim let teams design, deploy, and iterate on agent workflows without writing orchestration code from the ground up. +AI agents can direct more of the work than chatbots. A chatbot waits for each prompt and returns a response, while an AI agent can plan a sequence of steps and adjust the plan after evaluating each result. An agent can also retain relevant context across steps or sessions and act through connected software, such as by searching a database or updating a CRM. -## What Is a Chatbot? +A hybrid product combines a chatbot interface with an AI agent. A conversational interface may collect a request, while an agent works behind it to complete a multi-step task. Treat a hybrid as an AI agent when it can choose and execute steps outside the conversation while preserving relevant context. -A chatbot is software designed to simulate conversation. At its core, it takes user input, matches that input against a set of predefined rules, decision trees, or basic natural language processing (NLP) patterns, and returns a scripted response. The interaction model is reactive: the user asks, the chatbot answers within the bounds of what it's been programmed to say. +## When to use a chatbot vs. an AI agent -### Two flavors, same ceiling +Choose based on how much work your product must complete beyond answering a user. A chatbot fits predictable conversations, while an AI agent fits tasks that require independent decisions and actions across connected systems. -Rule-based chatbots are the simplest form. They follow rigid decision trees: if the user says X, respond with Y. Think of the IVR menus of the chat world. "Press 1 for billing, press 2 for shipping." They're predictable, easy to build, and cheap to run. But the moment a user asks something outside the tree, the bot can hit a wall. - -NLP-enhanced chatbots add a layer of language understanding. They can parse intent ("I want to return something") and extract entities ("order #12345") to route the conversation more flexibly. They feel smarter because they handle variations in phrasing. But they're still reactive and script-bound; they don't reason, they don't plan, and they don't take actions outside their predefined flows. - -### What chatbots do well - -Chatbots are excellent at: - -- **High-volume FAQ handling:** Store hours, return policies, pricing questions, password resets. Consistent answers, zero wait time. -- **Simple transactional lookups:** Order status, account balance, appointment availability. If the answer lives in one system and requires one query, chatbots handle it fast. -- **Lead qualification with basic routing:** "What's your company size?" "What product are you interested in?" Route to the right sales rep. -- **Fast deployment, low cost:** A rule-based chatbot can go live in days. NLP-enhanced versions can take a few weeks. Neither requires a dedicated engineering team to maintain. - -### Where chatbots break - -The ceiling shows up fast when complexity increases. Consider two scenarios: - -**Scenario A: "Where is my order?"** - -The chatbot asks for the order number, queries the tracking system, and returns a status. Clean, fast, done. This is a chatbot's sweet spot. - -**Scenario B: "I want to return this item, get a refund to my original payment method, and update my shipping address for future orders."** - -Now the chatbot needs to initiate a return flow, process a refund through the payment system, and update the customer's profile in a separate database. That's three systems, conditional logic (is the item eligible for return? was it purchased within the return window?), and an action sequence that depends on real-time data. Most chatbots will either punt this to a human, ask the customer to complete each step separately, or get stuck looping through a flow that wasn't designed for multi-step requests. - -## What Is an AI Agent? - -An AI agent is an autonomous system that perceives context, reasons toward a goal, makes decisions, and executes actions across tools and systems, with minimal human direction. Where a chatbot waits for input and responds from a script, an agent receives an objective and figures out how to accomplish it. - -### The components that make agents different - -Four architectural elements separate AI agents from chatbots: - -- **LLMs for reasoning:** The agent uses a large language model as its "brain" to interpret context, break down complex requests into sub-tasks, and decide what to do next. -- **Memory (short-term and long-term):** Short-term memory tracks the current conversation and task state. Long-term memory retains information across sessions: past interactions, user preferences, and resolved tickets. -- **Tool access:** Agents connect to APIs, databases, CRMs, payment systems, email services, and other external tools. They can update a record, trigger a workflow, send a notification, or query multiple systems in sequence. -- **A feedback and learning loop:** Agents can evaluate the results of their actions, adjust their approach when something doesn't work, and improve over time based on outcomes. - -### What agents do that chatbots can't - -AI agents handle multi-step task completion, cross-system coordination, dynamic decision-making mid-task, and proactive action. They don't wait for you to tell them each step; they plan the sequence, execute it, and verify the result. - -Let's revisit the refund example from the chatbot section. Same customer, same request: "I want to return this item, get a refund, and update my shipping address." - -Here's how an AI agent handles it: - -- Verifies the customer's identity by pulling account data from the CRM -- Checks return eligibility by querying the order management system for purchase date, item condition policy, and return window status -- Initiates the return by creating a return authorization in the fulfillment system -- Processes the refund by triggering the payment gateway to reverse the charge to the original payment method -- Updates the shipping address in the customer profile database -- Notifies the finance team by logging the refund in the accounting system -- Sends a confirmation email to the customer with return instructions and the updated address - -### A common misconception worth clearing up - -Bolting GPT onto a chatbot doesn't make it an AI agent. An LLM-powered chatbot can generate more natural-sounding responses and handle a wider range of questions, but without memory, tool access, and goal-directed planning, it's still fundamentally reactive. An AI agent is not a chatbot. Nor is it a smarter search engine. True agents reason, remember, act, and adapt. - -## AI Agent vs Chatbot: Side-by-Side Comparison - -The table below breaks down how each architecture works across the dimensions that matter most when you're deciding what to build or buy. - -| Dimension | Chatbot | AI Agent | +| Criterion | Use a chatbot | Use an AI agent | | --- | --- | --- | -| Primary function | Answers questions from predefined scripts or NLP-matched patterns | Pursues goals by reasoning, planning, and executing multi-step tasks | -| Decision-making | Rule-based or intent-matching; follows fixed logic | Dynamic; evaluates context, weighs options, and adjusts approach mid-task | -| Memory | Session-only (forgets after conversation ends) | Short-term (task state) and long-term (cross-session context, user history) | -| Tool/system access | Limited or none; may query one data source | Connects to APIs, databases, CRMs, payment systems, and external services | -| Handles multi-step tasks | Poorly; breaks down when requests span multiple systems or require conditional logic | Core strength: plans and executes task sequences across systems | -| Learning over time | No, responses are static unless manually updated | Yes, adjusts based on outcomes, feedback, and accumulated context | -| Setup complexity | Low; can deploy in days to weeks | Higher; requires integration with tools, memory configuration, and orchestration | -| Best for | FAQs, simple lookups, lead qualification, and appointment booking | End-to-end support resolution, cross-system workflows, automated reporting, and complex onboarding | +| Task complexity | Simple questions and guided interactions | Complex tasks with changing requirements | +| Tool access | No external tools or limited lookups | Actions across connected applications and databases | +| Workflow length | One response or a short exchange | Multiple dependent steps | +| Context needs | Current conversation is sufficient | Past activity must inform later decisions | +| Predictability | Responses should follow fixed paths | The AI agent must choose the next step | +| Human oversight | A person handles exceptions | The AI agent handles routine exceptions within set limits | +| Risk level | Errors have limited consequences | Sensitive actions need approval, and every action needs controlled permissions plus an audit record | -### The three rows that matter most +Ask whether your product can finish the job in one bounded exchange. If it must review an intermediate result before choosing the next action, use an agent with task-specific permissions and approval controls. -If you take one thing from this comparison, focus on autonomy, memory, and tool use. These are the clearest dividing lines. +## Chatbots and AI agents side by side -Autonomy determines whether the system can figure out how to accomplish a goal or whether you have to pre-script every possible path. Chatbots need you to anticipate every conversation branch. Agents interpret the goal and plan their own route. +An agent's orchestration loop can complete more work than a chatbot's bounded response loop, but it also adds tool permissions, variable costs, latency, and additional failure paths. These operational tradeoffs affect how each system handles data, testing, monitoring, and safeguards. -Memory determines whether the system treats every interaction as brand new or builds on past context. A customer who called last week about a billing issue shouldn't have to re-explain the problem this week. - -Tool use determines whether the system can take action or only provide information. Chatbots tell you what to do. Agents do it for you: updating records, triggering workflows, and coordinating across systems. - -## When to Use a Chatbot vs. an AI Agent - -This isn't a "chatbots are bad, agents are good" conversation. Chatbots are the right tool for specific jobs, and deploying an AI agent where a chatbot would suffice is like hiring a senior engineer to update a spreadsheet. The goal is to match the tool to the task. - -### Where chatbots shine - -- **FAQ and knowledge base deflection:** If the majority of your support tickets are "what's your return policy?" or "how do I reset my password?", a chatbot handles this all day without breaking a sweat. -- **Appointment booking with fixed rules:** Dental office, salon, repair service. The variables are simple: available time slots, service type, and customer name. No conditional logic required. -- **Order status lookups:** Single-system queries with a predictable response. The customer provides an order number, and the chatbot returns tracking info. -- **Lead qualification with simple routing:** Collecting firmographic data (company size, industry, budget range) and routing to the right sales rep based on predefined criteria. -- **Website navigation assistance:** "Where do I find your pricing page?" "How do I contact support?" Directional, low-complexity. - -### Where AI agents earn their keep - -- **Automated reporting pipelines:** Aggregating data from multiple platforms (analytics, billing, project management), generating a summary, and distributing it to stakeholders on a schedule. -- **Multi-system data enrichment:** Pulling data from your CRM, enriching it with third-party sources, scoring it, and updating the record; all triggered by a single event. -- **Code review and release automation:** Scanning pull requests, checking for issues, running tests, and coordinating the release pipeline across tools like GitHub, Jira, and Slack. -- **Complex onboarding workflows:** New employee onboarding that involves provisioning accounts, sending welcome sequences, scheduling training, and updating multiple internal systems. -- **End-to-end customer support resolution:** The customer's issue spans returns, refunds, account updates, and follow-up scheduling. The agent resolves it in one pass. - -### Decision criteria at a glance - -| Scenario | Recommended Tool | Why | +| Comparison dimension | Chatbot | AI agent | | --- | --- | --- | -| Answering common product questions | Chatbot | Responses are static and predictable; no cross-system action needed | -| Processing a return that involves a refund, inventory update, and customer notification | AI Agent | Requires multi-step actions across payment, inventory, and CRM systems | -| Booking a demo meeting with a prospect | Chatbot | Fixed logic: check calendar availability, collect contact info, confirm | -| Enriching a lead record with firmographic and intent data from multiple sources | AI Agent | Requires querying and writing to multiple APIs in sequence | -| Routing a support ticket to the right department | Chatbot | Simple classification based on keywords or category selection | -| Generating a weekly performance report from three data platforms | AI Agent | Requires aggregation, analysis, and distribution across tools | - -### Signals that you need an agent - -If any of these describe your situation, a chatbot probably won't cut it: - -- The task requires data from more than one system -- The task has branching logic that changes based on real-time context (not just predefined rules) -- The task requires taking an action, not just providing information -- The task needs to remember context across sessions; the user shouldn't have to repeat themselves every time - -## The Hybrid Approach: When You Need Both - -In production, many effective deployments don't choose between chatbots and AI agents. They use both in a layered architecture where each handles the work it's built for. - -### The pattern - -The chatbot sits on the front line. It handles tier-1 volume: FAQs, routing, simple confirmations, and basic lookups. It's fast, cheap, and consistent. The moment a request crosses a complexity threshold, the chatbot hands off to an AI agent that can reason, access tools, and resolve the issue end-to-end. - -Think of it like a support team. The chatbot is your first-response rep who handles the quick wins. The AI agent is your senior specialist who steps in when the problem requires investigation, cross-system access, and judgment. - -### The build consideration - -This hybrid model doesn't work with duct tape. It requires an orchestration layer: something that manages the handoff between the chatbot and agent, passes full conversation context, and routes based on defined complexity triggers. Without it, you get the worst of both worlds: a chatbot that can't escalate gracefully and an agent that receives incomplete context. - -This is where an agent workspace or workflow builder becomes critical. You need a system that can define escalation rules, pass structured data between layers, and give your team visibility into what's happening at each stage. - -## How to Build an AI Agent (Without Starting From Scratch) - -Most content about AI agents stops at the "what" and never gets to the "how." You're left understanding the concept but with no clear path to building one. That gap is where teams stall: they know they need an agent, but the engineering lift feels enormous. - -### Two build paths - -Code-first frameworks like LangChain and CrewAI give you full control. You define the agent's reasoning chain, tool connections, memory management, and orchestration logic in code. The upside is flexibility and power. The downside is that you need engineers who understand both LLM application architecture and your business logic. Building, testing, and iterating takes time, and maintenance is ongoing. - -Visual/no-code agent builders take a different approach. They let teams design agent workflows through drag-and-drop interfaces, connecting LLMs, memory, APIs, and business tools without writing orchestration code from scratch. The tradeoff is less granular control for significantly faster deployment and lower maintenance overhead. For most business teams, this is the faster path to production. +| Software architecture | A chatbot sends each user message through a predefined response flow or language model call. | An AI agent combines a model with an orchestration loop that uses the result of each selected action to determine the next step. | +| Data handling | A chatbot usually reads the current conversation and approved reference material to produce a response. | An AI agent may read and write data across connected applications, so each tool needs scoped permissions and validation rules. | +| Cost and latency profile | A chatbot often completes one model request per exchange, which makes response time and usage easier to estimate. | An AI agent may make several model and tool calls for one request, which increases variable cost and response time. | +| Common failure modes | A chatbot can misread intent and produce an unsupported answer. | An AI agent can select the wrong tool or continue acting on incorrect data. | +| Testing and monitoring | Chatbot testing checks response quality and whether retrieval and routing work correctly. | Agent testing must verify each tool choice against its permissions and expected outcome. | +| Operational safeguards | A chatbot can restrict responses with filtered retrieval and clear escalation rules. | An AI agent needs strict action limits and logs of every tool call. Require approval for sensitive operations. | -### What a visual agent workspace looks like +## Why combine a chatbot with an AI agent -A visual agent builder gives you a canvas where each node represents a step in the workflow: an LLM call for reasoning, an API call to pull data, a conditional branch, a tool action, and an output. You wire these together, configure each node's parameters, and deploy the workflow as a live agent that can be triggered via chat, API, webhook, or on a schedule. +When users need one interface for both answers and actions, combining a chatbot with an AI agent separates the work between them. The chatbot clarifies the request and presents the result, while the agent retains relevant context, chooses workflow steps, and calls approved systems. This division lets each layer use controls suited to its role. -Sim is one example of this approach, an open-source AI agent workspace where teams build agent workflows visually, connecting over 1,000 integrations and multiple LLMs through a drag-and-drop canvas. It supports processing blocks (AI agents, API calls, custom functions), logic blocks (conditional branching, loops, routers), and output blocks (responses, evaluators). Workflows can run synchronously via API for real-time interactions or asynchronously via webhooks and scheduled triggers for background processes. +Production conversations and actions require different controls. A user may submit an ambiguous request, so the conversational layer can confirm the account or desired outcome before an agent proceeds. After confirmation, the agent can operate within permissions you define and pause for approval before a sensitive action. -The point isn't that visual builders replace code-first approaches. It's that they remove the orchestration overhead so your team can focus on the business logic: what the agent should do, not how to wire the plumbing. +With limited agent autonomy, one interface can handle both simple and complex requests. A chatbot can answer a policy question directly, but a refund request may require an agent to retrieve an order and assess eligibility. An employee can approve the refund before the agent issues it. -## The Bottom Line +[Sim's visual workspace](https://sim.ai) lets you build this hybrid. You can connect a conversational entry point to workflow branches that call tools after gathering the required information. Separate branches handle approval requests. Our [1,000+ integrations](https://sim.ai/integrations) connect those branches to the business applications that store the relevant records and execute the actions. -The AI agent vs chatbot distinction comes down to architecture. Chatbots react to inputs within predefined boundaries. AI agents reason toward goals, access tools, remember context, and take action across systems. They're different tools for different jobs, and teams often use both. +For example, Sim Chat can collect an order number before a workflow retrieves the matching purchase through an integration. The workflow can then apply refund rules and request approval when those rules require it. Chat returns the final status to the user after the workflow completes. For more patterns in this area, see the [best AI agents for customer support automation](https://www.sim.ai/library/best-ai-agents-for-customer-support-automation). -If your workflow requires multi-step reasoning, cross-system coordination, or actions that go beyond surfacing information, you need an agent, or a hybrid setup where the chatbot handles the front door and the agent handles the resolution. +## Building your own agent-chatbot hybrid with Sim -AI agents are moving from experimental to expected across enterprise teams, and the adoption curve is steep. For your team, the question isn't whether to bring agents in; it's where to start. +Sim's conversational layer gathers intent, the visual workflow routes the request, and connected tools perform approved actions. Configure those layers in five steps: -Pick one workflow that's currently breaking down: reports that take hours to compile manually, and employee onboarding sequences that require five people to coordinate. Build an agent for that, prove ROI, then expand from there. +1. Start with [Sim's Chat feature](https://docs.sim.ai/execution/chat). Chat gives users one place to submit a request and review the agent's response without exposing the workflow behind it. +2. Use Sim's visual workspace to [build the workflow](https://www.sim.ai/library/how-to-create-an-ai-agent) and configure instructions and routes for each request type. Specify when the agent should ask for clarification instead of acting. +3. Connect the services the agent needs through [Sim's integration library](https://sim.ai/integrations). For example, the workflow can retrieve a customer record and update a support ticket after Chat confirms the user's intent. +4. Choose between using [Sim's hosted access](https://sim.ai/pricing) and bringing your own API key (BYOK). Enterprise access also supports local-model workflows such as Ollama. +5. Before publishing, test how the workflow responds when required information is missing or a tool is unavailable. Confirm that the workflow seeks human approval before restricted actions. -For the adjacent comparison, [AI agents vs RPA](/library/ai-agents-vs-rpa) covers rule-based automation rather than conversational tools. If you've decided an agent is what you need, [10 AI agent ideas](/library/ai-agent-ideas) has starting points, [how to build AI agents](/library/how-to-create-an-ai-agent) walks through the first one, and [the best AI agents for customer support automation](/library/best-ai-agents-for-customer-support-automation) goes deep on the support use case specifically. +[Explore Sim's visual workspace](https://sim.ai) to build, test, and publish an agent-chatbot hybrid with the integrations and approval steps your workflow requires. If you need a starting point, these [AI agent ideas](https://www.sim.ai/library/ai-agent-ideas) cover a range of workflow patterns. diff --git a/apps/sim/content/library/how-to-turn-a-workflow-into-a-reusable-mcp-tool/index.mdx b/apps/sim/content/library/how-to-turn-a-workflow-into-a-reusable-mcp-tool/index.mdx index 30f805b8687..4d28edb704e 100644 --- a/apps/sim/content/library/how-to-turn-a-workflow-into-a-reusable-mcp-tool/index.mdx +++ b/apps/sim/content/library/how-to-turn-a-workflow-into-a-reusable-mcp-tool/index.mdx @@ -19,7 +19,7 @@ faq: - q: "Does Sim support MCP as both a client and server?" a: "Yes. Sim workflows can call tools on external MCP servers, and deployed Sim workflows can be published as MCP tools for other compatible clients." - q: "Can Claude or another assistant call a Sim workflow?" - a: "Yes, when the assistant supports MCP connections. Sim provides connection configurations for Claude Desktop, Cursor, VS Code, and Claude Code, while other compatible clients can use the same server details." + a: "Yes, when the assistant supports MCP connections. Sim provides connection configurations for Codex, Claude Desktop, Cursor, VS Code, and Claude Code, while other compatible clients can use the same server details." - q: "How is a published Sim MCP server authenticated?" a: "A Sim MCP server can use API Key access, where clients send an X-API-Key header containing a Sim API key, or Public access without authentication. Tool calls run the live deployment and consume workspace credits like other executions." - q: "Is MCP different from a REST API deployment?" @@ -34,7 +34,7 @@ To turn a workflow into a reusable MCP tool, you build the workflow with clearly - Build the workflow and define its Start-block inputs and outputs. - Deploy a versioned snapshot of the workflow. - Create an MCP server and add the deployed workflow to it as a tool. -- Connect the server to Claude Desktop, Cursor, VS Code, Claude Code, or another MCP-compatible client. +- Connect the server to Codex, Claude Desktop, Cursor, VS Code, Claude Code, or another MCP-compatible client. - As of August 2026, [n8n](https://docs.n8n.io/advanced-ai/accessing-n8n-mcp-server/), [Gumloop](https://docs.gumloop.com/nodes/mcp), and [Zapier](https://zapier.com/mcp) all support MCP on both sides in some form. What differs is the publishing model: node or instance configuration, a platform control plane, and an action catalog respectively. ## What does it mean to turn a workflow into a reusable MCP tool? @@ -54,7 +54,7 @@ Sim turns a deployed workflow into an MCP tool on an MCP server that external as 1. **Build the workflow.** Create the capability in Sim. Wire the Start trigger, Agent blocks, integrations, data, code, and control logic needed to complete the task. Define clear Start-block inputs and outputs, because those become the tool's parameters and shape how an external assistant calls it. 2. **Deploy a versioned snapshot.** Deploy when the behavior is ready for external callers. Sim freezes an immutable snapshot as a numbered version and marks one version live. Canvas edits stay in the draft until you publish an update, and promoting an earlier version rolls the live tool back. Every surface—API, chat, and MCP—runs that same live snapshot. 3. **Create an MCP server and add the workflow as a tool.** In Settings, add an MCP server with a name and an access mode. Then open the deployed workflow, go to the MCP tab in the Deploy view, set the tool name and description, review the parameter descriptions derived from the Start inputs, select one or more MCP servers, and save the tool. One server can host many workflow tools, and a workflow must already be deployed before it can be added. -4. **Connect an external MCP client.** From the server's details view, copy the ready-made configuration for Cursor, Claude Desktop, VS Code, Claude Code, or another host. Private servers expect an `X-API-Key` header carrying a Sim API key. When the assistant invokes the tool, Sim runs the live snapshot and returns the output over MCP. +4. **Connect an external MCP client.** From the server's details view, copy the ready-made configuration for Codex, Cursor, Claude Desktop, VS Code, Claude Code, or another host. Private servers expect an `X-API-Key` header carrying a Sim API key. When the assistant invokes the tool, Sim runs the live snapshot and returns the output over MCP. Sim also works in the opposite direction. A Sim workflow can [connect to external MCP servers](https://docs.sim.ai/mcp) and call their tools. Sim therefore acts as both an MCP client and an MCP server, which lets one workflow consume outside capabilities and publish its own capability for reuse. @@ -109,6 +109,6 @@ Current as of August 2026. ## Get started -Build a new workflow or open an existing one in [Sim](https://sim.ai). Once the workflow behaves as expected, deploy a versioned snapshot, then create an MCP server and add the workflow to it as a tool by following the [MCP deployment documentation](https://docs.sim.ai/workflows/deployment/mcp). You can then connect the server to Claude Desktop, Cursor, VS Code, Claude Code, or another MCP-compatible client. +Build a new workflow or open an existing one in [Sim](https://sim.ai). Once the workflow behaves as expected, deploy a versioned snapshot, then create an MCP server and add the workflow to it as a tool by following the [MCP deployment documentation](https://docs.sim.ai/workflows/deployment/mcp). You can then connect the server to Codex, Claude Desktop, Cursor, VS Code, Claude Code, or another MCP-compatible client. Sim suits developers who want one maintained workflow to provide the same callable capability across every MCP-compatible assistant they use. diff --git a/apps/sim/content/library/openai-vs-n8n-vs-sim/index.mdx b/apps/sim/content/library/openai-vs-n8n-vs-sim/index.mdx index fc2f84d49f5..9d7a9452770 100644 --- a/apps/sim/content/library/openai-vs-n8n-vs-sim/index.mdx +++ b/apps/sim/content/library/openai-vs-n8n-vs-sim/index.mdx @@ -1,9 +1,9 @@ --- slug: openai-vs-n8n-vs-sim -title: 'OpenAI AgentKit vs n8n vs Sim: AI Agent Workflow Builder Comparison' -description: OpenAI just released AgentKit for building AI agents. How does it compare to workflow automation platforms like n8n and purpose-built AI agent builders like Sim? +title: 'Sim vs n8n vs OpenAI AgentKit: AI Agent Builder Comparison (2026)' +description: 'Compare Sim with n8n and OpenAI AgentKit on integrations and deployment. See how Sim''s open-source platform works with multiple model providers.' date: 2025-10-06 -updated: 2026-07-23 +updated: 2026-08-28 authors: - emir readingTime: 9 diff --git a/apps/sim/content/library/what-is-an-mcp-server/index.mdx b/apps/sim/content/library/what-is-an-mcp-server/index.mdx index 37102908960..29ce13d5c3c 100644 --- a/apps/sim/content/library/what-is-an-mcp-server/index.mdx +++ b/apps/sim/content/library/what-is-an-mcp-server/index.mdx @@ -1,124 +1,112 @@ --- slug: what-is-an-mcp-server title: 'What Is an MCP Server?' -description: 'Learn what an MCP server is, how Model Context Protocol discovery, tools, and transports work, and how Sim consumes and publishes MCP tools for AI agents.' +description: 'Learn what an MCP server is, how Model Context Protocol tools, resources, and prompts work, and how Sim acts as both an MCP client and server.' date: 2026-07-24 -updated: 2026-07-24 +updated: 2026-08-27 authors: - andrew -readingTime: 10 +readingTime: 7 tags: [MCP, AI Agents, Model Context Protocol, Sim] ogImage: /library/what-is-an-mcp-server/cover.jpg canonical: https://www.sim.ai/library/what-is-an-mcp-server draft: false faq: - - q: "What does an MCP server do?" - a: "An MCP server exposes tools, resources, and prompts through the Model Context Protocol so a compatible AI client can discover and use those capabilities through a standard interface." - - q: "How is an MCP server different from a REST API?" - a: "A REST integration usually requires client-specific endpoint, schema, and authentication code. MCP adds a standard discovery and invocation layer, allowing compatible clients to list a server's capabilities and call its tools through the same protocol." - - q: "Which MCP transport should I use?" - a: "Use stdio when the client and server run on the same machine. Use Streamable HTTP for remote or multi-client deployments. HTTP+SSE is the deprecated legacy transport and is mainly relevant for backwards compatibility." - - q: "Can Sim use and publish MCP tools?" - a: "Yes. Sim can connect to external MCP servers and execute their tools inside workflows. It can also expose deployed workflows as tools on a workflow MCP server for compatible clients to discover and call." + - q: "What does MCP server mean?" + a: "An MCP server is a program that gives AI applications standardized access to tools, data, and reusable prompts. With Sim, you can connect to external MCP servers and publish Sim workflows through our MCP server. This lets you add external capabilities to workflows or publish a complete workflow as one reusable tool." + - q: "Where can I find the official Model Context Protocol documentation?" + a: "The official Model Context Protocol documentation is the primary source for the specification and implementation guides. At Sim, we follow this protocol when connecting to external MCP servers and publishing workflows as MCP tools. Use the documentation to verify compatibility and technical requirements before you configure a connection." + - q: "Is an MCP server a real server?" + a: "An MCP server is a software component that can run as a local process or a remote network service. At Sim, we act as an MCP client when connecting to these servers and as an MCP server when publishing workflows. Running an MCP server locally or remotely lets you choose a setup that can reach the required tools and data." + - q: "How does an MCP server differ from a microservice?" + a: "An MCP server exposes capabilities through a standard interface for AI clients, while a microservice usually exposes application functions through an API. In Sim, a workflow can call services through their APIs, and you can publish the workflow as a single MCP tool. This lets AI clients use multi-step service logic without integrating separately with every underlying API." + - q: "Does MCP replace APIs?" + a: "MCP is a standard interface for discovering and invoking capabilities that may rely on APIs, not a replacement for those APIs. A Sim workflow can combine multiple API calls and expose the result as an MCP tool. This approach lets you reuse existing APIs while giving compatible AI clients one callable interface." + - q: "Can Sim act as both an MCP client and an MCP server?" + a: "An MCP client consumes server capabilities, while an MCP server exposes capabilities to compatible clients. At Sim, we support both roles by connecting workflows to external MCP servers and publishing workflows as MCP tools. Supporting both roles lets you consume external capabilities and distribute reusable workflows through the same protocol." --- ## TL;DR -An MCP server is a program that exposes tools, data, and prompts to AI models through the [Model Context Protocol](https://modelcontextprotocol.io/specification/2025-06-18/architecture), a standard interface any MCP-compatible client can call. - -- An MCP server lets an LLM or agent [discover and call external capabilities](https://modelcontextprotocol.io/specification/2025-06-18/server/tools) without a developer hardcoding each one. -- [Anthropic introduced and open-sourced MCP in November 2024](https://www.anthropic.com/news/model-context-protocol). It uses [JSON-RPC 2.0 messages](https://modelcontextprotocol.io/specification/2025-06-18/basic) and is supported by clients including [Claude Desktop](https://modelcontextprotocol.io/quickstart/user), [Cursor](https://docs.cursor.com/context/model-context-protocol), and [VS Code](https://code.visualstudio.com/docs/copilot/chat/mcp-servers). -- MCP replaces dozens of bespoke API integrations with a [single client-server architecture](https://modelcontextprotocol.io/specification/2025-06-18/architecture), so compatible clients can use the same server. -- Sim works in both directions. It calls external MCP servers as tools, and it publishes deployed Sim workflows as tools that other AI systems can call through MCP. -- The [spec defines two current transports](https://modelcontextprotocol.io/specification/2025-06-18/basic/transports), stdio for local processes and Streamable HTTP for remote calls. The older HTTP+SSE transport is deprecated and should not be used for new builds. -- The comparison table below shows how MCP servers, traditional API integrations, and Sim workflows-as-MCP-tools differ on setup effort, discovery, and reusability. +- An MCP server gives AI applications access to external tools and data through the Model Context Protocol. An MCP server can also provide reusable prompts. +- An MCP host is the AI application. The host creates an MCP client for each server connection. Each client handles capability discovery and requests. +- MCP gives AI applications a consistent interface for discovering and using capabilities, while each MCP server handles the service-specific connection to a remote service or local file system. For example, an MCP server can connect an application to GitHub or a database. +- [Sim is open source under Apache 2.0](https://github.com/simstudioai/sim). We support both MCP roles. You can use Sim as an MCP client to connect workflows to external servers or expose Sim workflows as MCP tools. ## What is an MCP server? -An MCP server is a program that exposes tools, data sources, and prompts to an AI model through the [Model Context Protocol](https://modelcontextprotocol.io/specification/2025-06-18/architecture), a shared standard for connecting language models to external capabilities. - -The point of an MCP server is to give an LLM or agent a consistent way to reach the outside world. Instead of writing custom glue for every API a model needs, you run an MCP server that advertises its capabilities in a format the model understands. An [MCP-compatible client](https://modelcontextprotocol.io/specification/2025-06-18/architecture) can then connect, list what the server offers, and call it. If you are new to the software on the other side of that connection, this guide to [AI agents and chatbots](https://www.sim.ai/library/ai-agent-vs-chatbot) explains why tool use matters. - -[Model Context Protocol](https://modelcontextprotocol.io) was [introduced and open-sourced by Anthropic in November 2024](https://www.anthropic.com/news/model-context-protocol) to solve a coordination problem. Each new tool, database, or service used to require its own integration written against a specific model or framework, and none of that work carried over. MCP defines one interface for all of them, built on [JSON-RPC 2.0](https://modelcontextprotocol.io/specification/2025-06-18/basic), so a server built once works with clients that speak the protocol. Adoption moved fast: [Claude Desktop](https://modelcontextprotocol.io/quickstart/user), [Cursor](https://docs.cursor.com/context/model-context-protocol), and [VS Code](https://code.visualstudio.com/docs/copilot/chat/mcp-servers) all document MCP client support. - -A single MCP server can expose three kinds of capability. [**Tools**](https://modelcontextprotocol.io/specification/2025-06-18/server/tools) are actions the model can invoke, like sending an email or querying a database. [**Resources**](https://modelcontextprotocol.io/specification/2025-06-18/server/resources) are data the model can read, like files or records. [**Prompts**](https://modelcontextprotocol.io/specification/2025-06-18/server/prompts) are reusable templates the server offers to guide a task. The client asks the server what it supports, and the model or user decides what to call. - -Because the interface is standardized, the same MCP server can work across [compatible clients](https://modelcontextprotocol.io/specification/2025-06-18/architecture) without modification. A server you build for one agent can answer calls from another when it connects and negotiates a supported protocol version and capability set. - -## How does an MCP server differ from a traditional API integration? +An MCP server is a program that gives AI applications standardized access to tools and data through the Model Context Protocol. The server can also provide reusable prompts. -A traditional API integration forces a developer to hardcode each endpoint by hand, while an MCP server lets a client [discover the available tools at runtime](https://modelcontextprotocol.io/specification/2025-06-18/server/tools). When you connect to a REST API, you read the docs, map the request and response shapes, and write code specific to that one service. Connect to a second service, and you repeat the process with a different auth scheme and schema. An MCP server flips this. Through the protocol's [tool discovery flow](https://modelcontextprotocol.io/specification/2025-06-18/server/tools), a client queries the server and receives tool names, descriptions, and input schemas. It can then call those tools without the developer wiring each underlying endpoint into that client ahead of time. The server owns its tool definitions rather than forcing every client to maintain a field-by-field mapping. +MCP connects an AI host to each server through a client. The host is an AI application, such as a chatbot that supports coding or workflow building. Inside the host, the MCP client establishes the server connection and discovers its capabilities. The client sends requests to the server and relays the server's responses to the host. To understand how these hosts differ, see this comparison of [AI agents and chatbots](https://www.sim.ai/library/ai-agent-vs-chatbot). -Authentication becomes a per-server concern from the client's perspective. Every conventional API may use a different scheme, which can mean new token handling, headers, and refresh logic for each integration. For HTTP transports, MCP defines an [authorization framework based on OAuth 2.1](https://modelcontextprotocol.io/specification/2025-06-18/basic/authorization), so a client authenticates to the MCP server rather than directly implementing auth for every downstream tool it exposes. The server still has to manage its downstream credentials. Adding another tool behind an existing server does not necessarily require new auth code in the client. REST APIs can publish machine-readable descriptions through formats such as the [OpenAPI Specification](https://spec.openapis.org/oas/latest.html), but that description is separate from the invocation layer and is not shared automatically by every client. MCP makes capability discovery part of the protocol, which is what lets exposed tools travel across compatible clients. +An MCP server can expose tools that perform actions and resources that provide context. The server can also supply reusable prompts. For example, a server might offer a tool for creating a GitHub issue or a resource for reading a repository file. The host controls which capabilities the model can access and when a request requires user approval. -## How is an MCP server built? +The word server describes the program's role rather than a specific type of computer. An MCP server may run as a local process on the same device as the host or as a remote service that the host reaches over a network. An MCP server can also translate MCP requests into calls to an existing API or data store, so the connected service does not need to support MCP directly. -Three things determine how a server behaves once an agent connects to it. Discovery answers "what can you do?", tool wrapping defines the individual actions, and the transport layer decides how bytes move between client and server. Build all three well and any compatible client can use your server without a custom adapter. +## What is the Model Context Protocol? -### Discovery +The Model Context Protocol, or MCP, is an open standard that defines how AI applications connect to external data sources and software tools. [Anthropic's November 2024 announcement](https://www.anthropic.com/news/model-context-protocol) introduced MCP as a shared protocol for connecting AI assistants to systems where information and actions already live. -[Discovery](https://modelcontextprotocol.io/specification/2025-06-18/architecture) lets a client query an MCP server for its available tools, resources, and prompts at connection time. The protocol provides [list operations and capability negotiation](https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle), and a [tool list](https://modelcontextprotocol.io/specification/2025-06-18/server/tools#listing-tools) includes each tool's name, description, and input schema. An agent reads that response and decides which tool to call, so endpoint shapes do not have to be hardcoded into the client. [Servers can also notify clients that a tool list changed](https://modelcontextprotocol.io/specification/2025-06-18/server/tools#list-changed-notification) when both sides declare support for that capability. +MCP replaces service-specific connection logic with a consistent interface for discovering and using capabilities. A compatible MCP server can therefore work with different AI applications without requiring a separate interface for each one. -### Tool wrapping +The [official MCP specification](https://modelcontextprotocol.io/specification/latest) defines an architecture in which a client connects a host to a server. For each server connection, the AI host creates a client that exchanges structured messages with the MCP server. MCP standardizes that exchange, but the host application still controls access to underlying systems and asks for user consent when needed. -Tool wrapping turns an existing function, database query, or REST API into a callable MCP tool. You write a thin adapter that declares the tool's schema and maps the model's arguments onto whatever the underlying code expects. A weather API becomes a `get_forecast` tool, a SQL query becomes a `search_orders` tool, and the agent calls both through the same [MCP tool invocation interface](https://modelcontextprotocol.io/specification/2025-06-18/server/tools). Wrapping is where most of your build effort goes, since it defines the contract the model reasons about. +MCP does not replace the APIs or databases behind an integration. An MCP implementation translates those existing capabilities into a common interface that an AI application can inspect and use. You can continue using the underlying service without building a separate AI-specific integration for every model or application. -### Transport options +## How an MCP server works -The [MCP transport specification](https://modelcontextprotocol.io/specification/2025-06-18/basic/transports) defines two current transports. A third, the original HTTP+SSE design, is deprecated and should not be used for new builds. +An MCP server exposes selected capabilities to an AI application through a structured connection. After opening a session and confirming the protocol version, the MCP client discovers the capabilities that the server makes available. Both sides exchange JSON-RPC messages over a supported local or remote transport. -- [**stdio**](https://modelcontextprotocol.io/specification/2025-06-18/basic/transports#stdio) runs the server as a local subprocess and pipes JSON-RPC messages over standard input and output. Use it for local tools, desktop clients, and processes on the same machine where you want no network setup. Messages are newline-delimited UTF-8, and the server can write logs to stderr. -- [**Streamable HTTP**](https://modelcontextprotocol.io/specification/2025-06-18/basic/transports#streamable-http) exposes a single MCP endpoint that handles both directions. The client POSTs JSON-RPC requests to it, and the server returns either a standard HTTP response or an SSE stream. This is the transport for remote, multi-client, or horizontally scaled deployments. -- [**HTTP+SSE (legacy, deprecated)**](https://modelcontextprotocol.io/specification/2025-06-18/basic/transports#backwards-compatibility) was the original network design. It used separate SSE and POST endpoints. Streamable HTTP replaced it, and the specification retains compatibility guidance for clients and servers that still need to interoperate with older implementations. +The [official MCP specification](https://modelcontextprotocol.io/specification/latest) groups server capabilities into tools, resources, and prompts. Each capability has a different purpose and uses its own discovery and request methods. A server can expose any subset of these capabilities. -A point worth clearing up, because it trips up a lot of teams: "SSE" and "HTTP" are not two peer transports sitting alongside stdio. [SSE is a response mechanism used by Streamable HTTP](https://modelcontextprotocol.io/specification/2025-06-18/basic/transports#streamable-http), while "the SSE transport" commonly refers to the deprecated two-endpoint HTTP+SSE design. If you are building something new, use Streamable HTTP. If you maintain a legacy server, plan migration around the clients you need to support. +Tools let a model perform an action. Each tool includes a name and description. Its input schema tells the client which arguments the tool accepts. The client discovers available actions with `tools/list` and invokes one with `tools/call`, such as `create_issue` with a repository name and the issue details. -You can wire the same set of wrapped tools to either current transport without rewriting the tools themselves, since the [protocol architecture separates server capabilities from transports](https://modelcontextprotocol.io/specification/2025-06-18/architecture). A common pattern is stdio for local development and Streamable HTTP for production. +Resources give the client readable context without asking the model to execute an action. A resource has a URI and metadata describing its contents. The client can discover resources with `resources/list` and retrieve one with `resources/read`, such as reading a configuration file or database schema. -The [Sim MCP docs](https://docs.sim.ai/mcp) show how discovery, wrapping, and transport selection appear when you connect a server in practice, including which transport to pick for local versus deployed setups. +Prompts provide reusable message templates for common tasks. The client discovers them with `prompts/list` and requests a selected template with `prompts/get`. A prompt can accept arguments, such as a repository name for a code review template, before the host sends the completed messages to the model. -## Can Sim act as both an MCP client and an MCP server? +The MCP client controls the connection and routes each request to the server. After validating a request, the server runs the relevant operation and returns structured content or an error. The host application decides whether to add the result to the model's context or present it to the user. The host can require user approval before allowing a protected action to run. -Yes. Sim works in both directions, which means you can consume external tools and expose your own from the same platform. +## MCP server examples: GitHub, filesystems, and databases -As an MCP client, Sim calls external MCP servers as tools inside your workflows. You configure a server in a Sim workspace, select its tools for an agent or MCP block, and Sim can invoke those published tools. If you run an MCP server for a database, search index, or internal service, Sim discovers its tools and calls them alongside native integrations. You get access to the MCP ecosystem without writing a custom Sim integration for each server. +MCP servers can wrap familiar systems, including GitHub and local data stores. Each server describes its available capabilities so an MCP client can discover and use them without service-specific client code. In Sim, you can connect these servers to workflows and use their tools and data in later steps. -As an MCP server, Sim can publish deployed workflows as tools on a workflow MCP server that other AI systems call. Build and deploy a workflow in Sim, add it to a workflow MCP server, and a compatible client can discover and invoke it. Whatever logic you assembled inside Sim, whether it chains models, calls APIs, or runs branching decisions, appears to the caller as a tool with a defined schema. For the broader build-and-deploy flow, see [how to create an AI agent with Sim](https://www.sim.ai/library/how-to-create-an-ai-agent). +### GitHub MCP server -That two-way capability changes how you compose systems. You can connect Sim to a third-party MCP server, build a workflow on top of it, and then publish that workflow as an MCP tool for something else to consume. One Sim deployment can sit in the middle of a chain, acting as a client to the servers below it and a server to the clients above it. +[GitHub's official MCP server](https://github.com/github/github-mcp-server) gives an AI assistant controlled access to repository content and lets it manage issues or pull requests within granted permissions. The server can also provide repository data as resources and offer prompts for tasks such as summarizing a pull request. -The practical payoff is reuse. You build a workflow once, deploy it, and compatible clients can call it without knowing how it works internally. You skip the usual step of writing and maintaining a separate API wrapper for each consumer. +For example, you ask an AI assistant to find open authentication bugs in a repository. The MCP client calls the issue search tool with the repository name and relevant filters. The server queries GitHub using your authorized account and returns structured issue data for the model to summarize. -Setup for both roles lives in the [Sim MCP docs](https://docs.sim.ai/mcp), which walk through connecting external servers as tools and publishing deployed workflows through a workflow MCP server. +### Filesystem MCP server -## Why teams run MCP servers on Sim +The [filesystem MCP server reference implementation](https://github.com/modelcontextprotocol/servers) limits an assistant's access to configured directories while supporting file search and reading. The server can also allow approved writes and provide file metadata as resources. Reusable prompts can guide tasks such as comparing documents within the permitted directories. -Standing up an MCP server yourself means writing the wrapper, hosting it, handling auth, and maintaining it as the specification evolves. Sim provides a workflow layer around that work, and three qualities matter for teams that need to control their deployment. +A filesystem MCP server lets an assistant compare two reports stored in a project folder. The client calls the file-reading tool for each approved path, and the server returns the contents. The model can then identify differences without receiving access to files outside the configured folder. -**It is Apache 2.0 licensed.** Sim's repository ships under Apache 2.0, a permissive open-source license that allows use, modification, commercial distribution, and sublicensing and includes an express patent grant. The practical implications are covered in [Apache 2.0 vs fair-code](https://www.sim.ai/library/apache-2-0-vs-fair-code). +### Database query MCP server -**You can run it on your own infrastructure, including isolated environments.** A self-hosted Sim deployment keeps the workflow and MCP execution layer within infrastructure you control. That matters when exposed tools touch regulated or private data. Teams comparing deployment and governance models can also review these [open-source AI agent platforms](https://www.sim.ai/library/open-source-ai-agent-platforms). +The MCP project maintains [database reference implementations](https://github.com/modelcontextprotocol/servers) that give an AI assistant controlled access to database structure and permitted queries. The server can provide schema details and saved query definitions as resources. Reusable prompts can instruct the model to write read-only queries that follow the database's conventions. -**Admins control access to MCP capabilities.** Sim's workspace permissions govern MCP server management, and permission-group settings can hide MCP deployment or disable MCP tools for selected users. Workflow execution traces record activity block by block, giving operators a detailed view of what the workflow did when a tool was called. +A database MCP server can help an assistant identify which products generated the most revenue last month. The client first retrieves the relevant schema, then asks the server to execute a read-only query. The server applies its credentials and access rules before returning structured rows that the model can explain. -You can build the workflow behind the server in natural language with Mothership, assemble it on the visual canvas, or define it through the API. The exposed MCP tool presents the same deployed workflow logic either way. +## MCP server vs. traditional API integration vs. a Sim workflow as an MCP tool -## MCP server vs traditional API integration vs Sim workflow-as-MCP-tool +A direct API integration connects an application to one service-specific interface, while MCP gives compatible clients a shared way to discover and call server capabilities. When you expose a workflow through Sim as an MCP tool, we package its API calls and model-processing steps behind one callable interface. All three approaches can still rely on conventional APIs underneath. -Three approaches let an AI system reach an external tool, and they differ in how much work you do up front and how far the result travels. The table below compares them on setup effort, discovery support, reusability across clients, and who can call the result. +| Approach | Connection model | What the model can access | Maintenance | +| --- | --- | --- | --- | +| Traditional API integration | You write service-specific authentication and request-handling logic. | The model can use only the functions your integration defines. | You update custom code when an API or application changes. | +| MCP server | An MCP client discovers and calls capabilities through a shared protocol. | The model can access the capabilities the server exposes. | The server owner maintains the implementation behind the MCP interface. | +| Sim workflow as MCP tool | Sim publishes a workflow through an MCP server. | The model can call the complete workflow as a reusable tool. | You update the workflow in Sim without rebuilding the client integration. | -| | Setup effort | Discovery support | Reusability across clients | Who can call it | -| --- | --- | --- | --- | --- | -| **MCP server** | Moderate. You wrap tools once behind the protocol. | [Built in](https://modelcontextprotocol.io/specification/2025-06-18/server/tools). Clients query the server for its available tools. | High. [Compatible clients](https://modelcontextprotocol.io/specification/2025-06-18/architecture) connect without a bespoke integration. | Any MCP client that supports the server's transport and protocol version. | -| **Traditional API integration** | High. You implement endpoints, auth flows, and schemas for each client integration. | Not required by REST itself. Discovery depends on documentation or an additional description format. | Low by default. Each new client needs integration code. | Applications with code written for that API. | -| **Sim workflow-as-MCP-tool** | Low. You build and deploy a workflow, then add it to a workflow MCP server. | Built in. Sim exposes the deployed workflow as a discoverable tool. | High. Compatible AI systems can connect to its MCP server. | MCP clients that can reach and authenticate to the Sim endpoint. | +## Using Sim as an MCP client and MCP server -A traditional API integration can lock a tool to one application and force the next team to repeat the client work. An MCP server breaks that pattern by exposing tools through a discovery layer compatible clients can read, so one server serves many callers. A Sim workflow-as-MCP-tool pushes the effort down further: you build and deploy the logic in Sim, then publish it through a workflow MCP server. See [docs.sim.ai/mcp](https://docs.sim.ai/mcp) for setup details. +In Sim, you can connect a workflow to an external MCP server and call its tools. For example, your workflow can use a GitHub MCP server to read an issue or create a pull request, then use the returned data in later steps. -## Conclusion +You can also publish a Sim workflow as a callable MCP tool. External MCP clients can discover and run the tool with the expected inputs, then receive the workflow's output. One MCP tool can contain several workflow steps behind a single interface. For a broader workflow tutorial, learn [how to create an AI agent with Sim](https://www.sim.ai/library/how-to-create-an-ai-agent). -MCP servers solve a real problem for anyone building agentic systems. Instead of writing a custom integration for every tool an agent needs, you expose those tools through one standard interface that compatible clients can discover and call. That standardization is why the protocol spread quickly after Anthropic released it. +You can [explore Sim's MCP integration](https://sim.ai) by connecting a workflow to an existing MCP server or publishing a workflow for other MCP clients to call. -Sim gives you both directions in one place. You can wire external MCP servers into a workflow as tools, and you can expose deployed Sim workflows through an MCP server for other AI systems to call, on infrastructure you control under the Apache 2.0 license. +## Getting started with MCP and Sim -Start by reading [docs.sim.ai/mcp](https://docs.sim.ai/mcp) and deploy your first MCP server today. +MCP gives AI applications a shared way to connect to external tools. At Sim, we support both MCP client and server roles. You can use external MCP servers inside Sim workflows, then expose those workflows as MCP tools for other compatible clients. Supporting both roles lets you build and publish integrations through the same platform rather than maintaining separate connection methods. Teams evaluating deployment options can also compare [open-source AI agent platforms](https://www.sim.ai/library/open-source-ai-agent-platforms). You can explore the [Sim workflow builder](https://sim.ai) to connect an MCP server or prepare a workflow for MCP access. diff --git a/apps/sim/ee/audit-logs/components/audit-logs.test.ts b/apps/sim/ee/audit-logs/components/audit-logs.test.ts new file mode 100644 index 00000000000..bef651a2835 --- /dev/null +++ b/apps/sim/ee/audit-logs/components/audit-logs.test.ts @@ -0,0 +1,51 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import type { AuditLogPage } from '@/lib/api/contracts/audit-logs' +import { presentableAuditEntries } from '@/ee/audit-logs/components/audit-logs' + +function page(...ids: string[]): AuditLogPage { + return { + success: true, + data: ids.map((id) => ({ + id, + workspaceId: null, + actorId: null, + actorName: null, + actorEmail: null, + action: 'organization.updated', + resourceType: 'organization', + resourceId: null, + resourceName: null, + description: null, + metadata: null, + createdAt: '2026-01-01T00:00:00.000Z', + })), + } +} + +describe('presentableAuditEntries', () => { + it('flattens every loaded page while the scope is answerable', () => { + expect(presentableAuditEntries([page('a', 'b'), page('c')], true).map((e) => e.id)).toEqual([ + 'a', + 'b', + 'c', + ]) + }) + + /** + * The case this exists for: an unresolved workspace scope drops the filter, so its + * query key equals the unscoped feed's. Disabling the query does not clear that + * cache entry, so an admin who had just been reading the organization-wide feed + * would have kept its rows on screen under a scoped URL — and Export, which gates + * on this list being non-empty, stayed armed against them. + */ + it('presents nothing when the scope cannot be answered, even with pages cached', () => { + expect(presentableAuditEntries([page('a', 'b')], false)).toEqual([]) + }) + + it('presents nothing before any page has loaded', () => { + expect(presentableAuditEntries(undefined, true)).toEqual([]) + }) +}) diff --git a/apps/sim/ee/audit-logs/components/audit-logs.tsx b/apps/sim/ee/audit-logs/components/audit-logs.tsx index 62f41b4c16d..2c8f1c8993b 100644 --- a/apps/sim/ee/audit-logs/components/audit-logs.tsx +++ b/apps/sim/ee/audit-logs/components/audit-logs.tsx @@ -1,27 +1,27 @@ 'use client' -import { type ReactNode, useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { type ReactNode, useEffect, useMemo, useRef, useState } from 'react' import { Badge, Button, Calendar, + Chip, ChipCombobox, ChipInput, ChipSelect, type ComboboxOption, - Download, OverflowText, Popover, PopoverAnchor, PopoverContent, - RefreshCw, - Search, toast, } from '@sim/emcn' +import { Download, RefreshCw, Search, X } from '@sim/emcn/icons' import { createLogger } from '@sim/logger' import { formatDateTime } from '@sim/utils/formatting' import { isRecordLike } from '@sim/utils/object' import { useQueryStates } from 'nuqs' +import type { AuditLogPage } from '@/lib/api/contracts/audit-logs' import { formatDateShort } from '@/lib/core/utils/date-display' import { getEndDateFromTimeRange, getStartDateFromTimeRange } from '@/lib/logs/filters' import { SEARCH_DEBOUNCE_MS } from '@/lib/url-state' @@ -33,6 +33,7 @@ import { import { SettingsEmptyState } from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state' import { SettingsPanel } from '@/app/workspace/[workspaceId]/settings/components/settings-panel' import { useSettingsSearch } from '@/app/workspace/[workspaceId]/settings/components/use-settings-search' +import { useOrganizationWorkspaces } from '@/ee/access-control/hooks/permission-groups' import { RESOURCE_TYPE_OPTIONS } from '@/ee/audit-logs/constants' import { type AuditLogFilters, useAuditLogs } from '@/ee/audit-logs/hooks/audit-logs' import { @@ -150,12 +151,15 @@ function renderMetadataValue(value: unknown) { ) } +/** Already rendered as their own labelled rows, so the metadata block would repeat them. */ +const HIDDEN_METADATA_KEYS = new Set(['name', 'description']) + function getMetadataEntries(metadata: unknown) { if (!isRecordLike(metadata)) return [] return Object.entries(metadata).filter(([key, value]) => { if (value === undefined) return false - return !['name', 'description'].includes(key) + return !HIDDEN_METADATA_KEYS.has(key) }) } @@ -237,6 +241,24 @@ interface AuditLogsProps { organizationId: string } +/** + * Entries the feed is allowed to present. + * + * A disabled query still serves whatever is cached under its key, and an unresolved + * workspace scope resolves to the same key as the unscoped feed — so an admin looking + * at the organization-wide feed who then followed a stale scoped link kept those rows + * on screen, with Export still armed against them. The scope a link asks for is a + * ceiling, so when it cannot be honoured the feed presents nothing rather than + * whatever it happens to be holding. + */ +export function presentableAuditEntries( + pages: AuditLogPage[] | undefined, + isScopeAnswerable: boolean +): EnterpriseAuditLogEntry[] { + if (!isScopeAnswerable || !pages) return [] + return pages.flatMap((page) => page.data) +} + export function AuditLogs({ organizationId }: AuditLogsProps) { const [urlFilters, setUrlFilters] = useQueryStates(auditLogFilterParsers, auditLogFilterUrlKeys) const { types: selectedTypes } = urlFilters @@ -251,30 +273,83 @@ export function AuditLogs({ organizationId }: AuditLogsProps) { urlFilters.timeRange === 'Custom range' && (!customStartDate || !customEndDate) ? DEFAULT_AUDIT_TIME_RANGE : urlFilters.timeRange + /** + * Resolved, not merely present. Only the id lives in the URL, and the filter is + * applied once it matches a workspace the organization actually owns — a stale id + * from an old link would otherwise be shown under a chip labelled with a bare uuid. + */ + const workspaceScope = urlFilters.workspace + const orgWorkspaces = useOrganizationWorkspaces(organizationId, Boolean(workspaceScope)) + const scopedWorkspace = workspaceScope + ? orgWorkspaces.data?.find((entry) => entry.id === workspaceScope) + : undefined + const [datePickerOpen, setDatePickerOpen] = useState(false) const dateRangeAppliedRef = useRef(false) const [searchTerm, setSearchTerm] = useSettingsSearch() const debouncedSearch = useDebounce(searchTerm, SEARCH_DEBOUNCE_MS).trim() const [isVisuallyRefreshing, setIsVisuallyRefreshing] = useState(false) - const refreshTimersRef = useRef(new Set()) + const refreshTimersRef = useRef | null>(null) + refreshTimersRef.current ??= new Set() + const refreshTimers = refreshTimersRef.current const [isExporting, setIsExporting] = useState(false) useEffect(() => { - const timers = refreshTimersRef.current return () => { - for (const timerId of timers) window.clearTimeout(timerId) + for (const timerId of refreshTimers) window.clearTimeout(timerId) } - }, []) - - const filters = useMemo(() => { - return { - search: debouncedSearch || undefined, - resourceType: selectedTypes.length > 0 ? selectedTypes.join(',') : undefined, - startDate: getStartDateFromTimeRange(timeRange, customStartDate)?.toISOString(), - endDate: getEndDateFromTimeRange(timeRange, customEndDate)?.toISOString(), - } - }, [debouncedSearch, selectedTypes, timeRange, customStartDate, customEndDate]) + }, [refreshTimers]) + + /* + Not memoized: this object is only ever hashed, never compared by identity — React + Query hashes a query key structurally, and the export handler reads its fields + directly. The same rule `useUsageWindow` applies to its window object. + */ + const filters: AuditLogFilters = { + search: debouncedSearch || undefined, + resourceType: selectedTypes.length > 0 ? selectedTypes.join(',') : undefined, + workspaceId: scopedWorkspace?.id, + startDate: getStartDateFromTimeRange(timeRange, customStartDate)?.toISOString(), + endDate: getEndDateFromTimeRange(timeRange, customEndDate)?.toISOString(), + } + + /** + * A deep-linked workspace scope is only resolvable once the organization's workspace + * list has loaded. Querying before then fetches the whole organization's feed and + * immediately refetches it narrowed — two requests, with a flash of rows the link + * did not ask for in between. + */ + const isWorkspaceScopePending = Boolean(workspaceScope) && orgWorkspaces.isPending + /** + * The lookup itself failed, so whether the workspace exists is simply unknown. + * + * Kept apart from {@link isWorkspaceScopeUnresolved}: telling an admin their + * workspace is not part of the organization because a request timed out is a wrong + * answer, not a cautious one, and it offers nothing to do about it. Refresh retries + * this lookup alongside the feed. + */ + const isWorkspaceScopeUnavailable = Boolean(workspaceScope) && orgWorkspaces.isError + + /** + * The link named a workspace this organization does not have — deleted since, or + * never one of ours. + * + * The feed stays closed rather than falling back to the organization. Every other + * deep-linked id in the app degrades to the unfiltered view, but an audit feed is + * the one place where widening is the dangerous direction: dropping the filter + * would answer a request for one workspace's history with everybody's, under a URL + * that still claims to be scoped, and the CSV export would follow. + */ + const isWorkspaceScopeUnresolved = + Boolean(workspaceScope) && + !isWorkspaceScopePending && + !isWorkspaceScopeUnavailable && + !scopedWorkspace + + /** The feed can answer the scope the URL asks for — the gate on reading or exporting. */ + const isScopeAnswerable = + !isWorkspaceScopePending && !isWorkspaceScopeUnresolved && !isWorkspaceScopeUnavailable const { data, isLoading, @@ -283,12 +358,12 @@ export function AuditLogs({ organizationId }: AuditLogsProps) { hasNextPage, fetchNextPage, refetch, - } = useAuditLogs(organizationId, filters) + } = useAuditLogs(organizationId, filters, !isWorkspaceScopePending && !isWorkspaceScopeUnresolved) - const allEntries = useMemo(() => { - if (!data?.pages) return [] - return data.pages.flatMap((page) => page.data) - }, [data]) + const allEntries = useMemo( + () => presentableAuditEntries(data?.pages, isScopeAnswerable), + [data, isScopeAnswerable] + ) const typeDisplayLabel = selectedTypes.length === 0 @@ -324,25 +399,38 @@ export function AuditLogs({ organizationId }: AuditLogsProps) { setDatePickerOpen(false) } - const handleRefresh = useCallback(() => { + const handleRefresh = () => { setIsVisuallyRefreshing(true) const timerId = window.setTimeout(() => { setIsVisuallyRefreshing(false) - refreshTimersRef.current.delete(timerId) + refreshTimers.delete(timerId) }, REFRESH_SPINNER_DURATION_MS) - refreshTimersRef.current.add(timerId) - refetch().catch((error: unknown) => { + refreshTimers.add(timerId) + const pending: Promise[] = [] + /* + `refetch` ignores `enabled`, so this has to repeat the gate. While the scope is + unanswerable the feed's filter carries no workspace, and refreshing it would + issue exactly the organization-wide read the gate exists to prevent. + */ + if (isScopeAnswerable) pending.push(refetch()) + /* + The lookup is what has to succeed for a closed feed to reopen, so it is retried + whenever a scope asked for it — and skipped entirely when none did, where it is + a disabled query with nothing to say. + */ + if (workspaceScope) pending.push(orgWorkspaces.refetch()) + Promise.all(pending).catch((error: unknown) => { logger.error('Failed to refresh audit logs', { error }) }) - }, [refetch]) + } - const handleLoadMore = useCallback(() => { + const handleLoadMore = () => { if (hasNextPage && !isFetchingNextPage) { fetchNextPage().catch((error: unknown) => { logger.error('Failed to load more audit logs', { error }) }) } - }, [hasNextPage, isFetchingNextPage, fetchNextPage]) + } const handleExportCsv = async () => { setIsExporting(true) @@ -351,6 +439,7 @@ export function AuditLogs({ organizationId }: AuditLogsProps) { params.set('organizationId', organizationId) if (filters.search) params.set('search', filters.search) if (filters.resourceType) params.set('resourceType', filters.resourceType) + if (filters.workspaceId) params.set('workspaceId', filters.workspaceId) if (filters.startDate) params.set('startDate', filters.startDate) if (filters.endDate) params.set('endDate', filters.endDate) @@ -385,7 +474,13 @@ export function AuditLogs({ organizationId }: AuditLogsProps) { text: 'Export', icon: Download, onSelect: () => void handleExportCsv(), - disabled: allEntries.length === 0 || isExporting || isPlaceholderData, + /* + `isScopeAnswerable` explicitly, not just via the empty `allEntries` it + implies: the export is the action that leaves the building, so the + condition that makes it safe belongs where it is read. + */ + disabled: + !isScopeAnswerable || allEntries.length === 0 || isExporting || isPlaceholderData, }, ]} > @@ -410,6 +505,28 @@ export function AuditLogs({ organizationId }: AuditLogsProps) { allOptionLabel='All types' align='start' /> + {workspaceScope && ( + /* + A deep-linked scope, not a picker: the organization can hold hundreds of + workspaces, so this narrows the feed only when a link asks it to and + offers exactly one action — take it back off. Trailing `X` and a bounded + width, matching the app's other removable filter chips; the label names + the dimension because a bare workspace name gives no clue what it scopes. + */ + void setUrlFilters({ workspace: null })} + aria-label='Clear the workspace filter' + className='max-w-[280px] shrink-0' + > + {/* Rendered for an unresolved scope too, or a bad link would leave the + feed closed with no control to reopen it. */} + + + )}
{/* ChipCombobox (Radix Popover, non-modal), not ChipSelect (Radix DropdownMenu, modal by default) — a modal trigger closing in the @@ -469,7 +586,15 @@ export function AuditLogs({ organizationId }: AuditLogsProps) { + Couldn't check that workspace. Refresh to try again. + + ) : isWorkspaceScopeUnresolved ? ( + + That workspace is not part of this organization. + + ) : debouncedSearch ? ( No results for "{debouncedSearch}" diff --git a/apps/sim/ee/audit-logs/hooks/audit-logs.test.tsx b/apps/sim/ee/audit-logs/hooks/audit-logs.test.tsx index 9fd71da3ada..70d1a2fea8f 100644 --- a/apps/sim/ee/audit-logs/hooks/audit-logs.test.tsx +++ b/apps/sim/ee/audit-logs/hooks/audit-logs.test.tsx @@ -50,8 +50,16 @@ let container: HTMLDivElement let root: Root let queryClient: QueryClient -function AuditProbe({ organizationId }: { organizationId: string }) { - const auditLogs = useAuditLogs(organizationId, {}) +function AuditProbe({ + organizationId, + workspaceId, + search, +}: { + organizationId: string + workspaceId?: string + search?: string +}) { + const auditLogs = useAuditLogs(organizationId, { workspaceId, search }) const entries = auditLogs.data?.pages.flatMap((page) => page.data) ?? [] return ( @@ -62,11 +70,20 @@ function AuditProbe({ organizationId }: { organizationId: string }) { ) } -function renderAuditLogs(organizationId: string) { +interface RenderOptions { + workspaceId?: string + search?: string +} + +function renderAuditLogs(organizationId: string, options: RenderOptions = {}) { act(() => { root.render( - + ) }) @@ -130,4 +147,50 @@ describe('useAuditLogs identity transitions', () => { }) ) }) + + /** Blanking the feed on each keystroke is what the placeholder exists to stop. */ + it('holds the current entries while a filter change loads, within one scope', async () => { + const filteredPage = createDeferred() + mockRequestJson.mockImplementation( + (contract: unknown, input: { query?: { search?: string } }) => { + if (contract !== listAuditLogsContract) throw new Error('Unexpected contract') + return input.query?.search ? filteredPage.promise : Promise.resolve(AUDIT_PAGE_A) + } + ) + + renderAuditLogs('org-a') + await flushQueries() + expect(container).toHaveTextContent('Updated Organization A') + + renderAuditLogs('org-a', { search: 'canary' }) + await flushQueries() + + expect(container).toHaveTextContent('Updated Organization A') + }) + + /** + * The other side of that rule. A workspace is a scope, not a filter: holding the + * organization-wide rows while the scoped page loads would show, under a + * workspace-scoped URL, entries that scope does not cover — with Export armed + * against them, since it gates on this list being non-empty. + */ + it('clears the entries when the workspace scope changes, within one organization', async () => { + const scopedPage = createDeferred() + mockRequestJson.mockImplementation( + (contract: unknown, input: { query?: { workspaceId?: string } }) => { + if (contract !== listAuditLogsContract) throw new Error('Unexpected contract') + return input.query?.workspaceId ? scopedPage.promise : Promise.resolve(AUDIT_PAGE_A) + } + ) + + renderAuditLogs('org-a') + await flushQueries() + expect(container).toHaveTextContent('Updated Organization A') + + renderAuditLogs('org-a', { workspaceId: 'workspace-a' }) + await flushQueries() + + expect(container).not.toHaveTextContent('Updated Organization A') + expect(container.querySelector('button')).toBeNull() + }) }) diff --git a/apps/sim/ee/audit-logs/hooks/audit-logs.ts b/apps/sim/ee/audit-logs/hooks/audit-logs.ts index 7685f83b3ff..0880b13c064 100644 --- a/apps/sim/ee/audit-logs/hooks/audit-logs.ts +++ b/apps/sim/ee/audit-logs/hooks/audit-logs.ts @@ -1,4 +1,4 @@ -import { useInfiniteQuery } from '@tanstack/react-query' +import { hashKey, useInfiniteQuery } from '@tanstack/react-query' import { requestJson } from '@/lib/api/client/request' import { type AuditLogPage, listAuditLogsContract } from '@/lib/api/contracts/audit-logs' @@ -7,8 +7,22 @@ export const AUDIT_LOG_LIST_STALE_TIME = 30 * 1000 export const auditLogKeys = { all: ['audit-logs'] as const, lists: () => [...auditLogKeys.all, 'list'] as const, + /** + * What a key is allowed to see: the organization, and the workspace within it. + * + * It leads the key, ahead of the filters, because previous data may be held across + * a filter change but never across a scope change — and a leading scope makes that + * a prefix comparison rather than a reach inside the filter object. + */ + scope: (organizationId: string, workspaceId?: string) => + [...auditLogKeys.lists(), organizationId, workspaceId ?? ''] as const, list: (organizationId: string, filters: AuditLogFilters) => - [...auditLogKeys.lists(), organizationId, filters] as const, + [...auditLogKeys.scope(organizationId, filters.workspaceId), filters] as const, +} + +/** The scope a key reads from, which is everything but its trailing filter object. */ +function auditListScopeIdentity(key: readonly unknown[]): string { + return hashKey(key.slice(0, -1)) } export interface AuditLogFilters { @@ -16,6 +30,8 @@ export interface AuditLogFilters { action?: string resourceType?: string actorId?: string + /** Narrows the feed to one workspace in the organization. */ + workspaceId?: string startDate?: string endDate?: string } @@ -34,6 +50,7 @@ async function fetchAuditLogs( action: filters.action, resourceType: filters.resourceType, actorId: filters.actorId, + workspaceId: filters.workspaceId, startDate: filters.startDate, endDate: filters.endDate, cursor, @@ -43,12 +60,29 @@ async function fetchAuditLogs( } export function useAuditLogs(organizationId: string, filters: AuditLogFilters, enabled = true) { + const queryKey = auditLogKeys.list(organizationId, filters) return useInfiniteQuery({ - queryKey: auditLogKeys.list(organizationId, filters), + queryKey, queryFn: ({ pageParam, signal }) => fetchAuditLogs(organizationId, filters, pageParam, signal), initialPageParam: undefined as string | undefined, getNextPageParam: (lastPage) => lastPage.nextCursor, enabled: Boolean(organizationId) && enabled, staleTime: AUDIT_LOG_LIST_STALE_TIME, + /** + * Held across a filter change, never across a scope change. + * + * Search, types and the window are all part of the key, so without a placeholder + * the feed blanks to its empty state on each keystroke and the Export action's + * `isPlaceholderData` guard is dead. But the organization and the workspace are in + * the key too, and holding across either shows rows the current scope does not + * cover — one tenant's entries under another's heading, or the organization's + * under a workspace-scoped URL — with Export armed against them. + */ + placeholderData: (previous, previousQuery) => + previous && + previousQuery && + auditListScopeIdentity(previousQuery.queryKey) === auditListScopeIdentity(queryKey) + ? previous + : undefined, }) } diff --git a/apps/sim/ee/audit-logs/search-params.ts b/apps/sim/ee/audit-logs/search-params.ts index 1c4a5284a9e..b28f253b942 100644 --- a/apps/sim/ee/audit-logs/search-params.ts +++ b/apps/sim/ee/audit-logs/search-params.ts @@ -1,4 +1,4 @@ -import { parseAsArrayOf, parseAsString } from 'nuqs/server' +import { createSerializer, parseAsArrayOf, parseAsString } from 'nuqs/server' import { parseAsDateString, parseAsTimeRange, @@ -20,6 +20,13 @@ export const DEFAULT_AUDIT_TIME_RANGE: TimeRange = 'Past 30 days' */ export const auditLogFilterParsers = { types: parseAsArrayOf(parseAsString).withDefault([]), + /** + * Nullable by design: the feed is organization-wide unless a link narrows it, and + * the usage panel's workspace drill-down is what does. Only the id is stored — the + * name is resolved from the loaded workspace list, so a stale id from an old link + * clears the filter rather than labelling it with nothing. + */ + workspace: parseAsString, timeRange: parseAsTimeRange.withDefault(DEFAULT_AUDIT_TIME_RANGE), startDate: parseAsDateString, endDate: parseAsDateString, @@ -36,3 +43,13 @@ export const auditLogFilterUrlKeys = { endDate: 'end-date', }, } as const + +/** + * Outbound links into the audit feed — the usage panel's workspace drill-down builds + * one — serialized from the map the feed itself parses rather than by concatenation, + * which emitted a bare `?workspace=` for a null id and left the value unencoded. + */ +export const serializeAuditLogFilters = createSerializer(auditLogFilterParsers, { + clearOnDefault: true, + urlKeys: auditLogFilterUrlKeys.urlKeys, +}) diff --git a/apps/sim/ee/organization-usage/components/usage-consumers.test.ts b/apps/sim/ee/organization-usage/components/usage-consumers.test.ts new file mode 100644 index 00000000000..510d065a2a2 --- /dev/null +++ b/apps/sim/ee/organization-usage/components/usage-consumers.test.ts @@ -0,0 +1,15 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { USAGE_PROVIDER_ICON_IDS } from '@/ee/organization-usage/components/usage-consumers' +import { PROVIDER_DEFINITIONS } from '@/providers/models' + +describe('PROVIDER_ICONS', () => { + /** A gap is silent: the row simply renders with no mark. */ + it('covers every provider the model registry defines', () => { + const covered = new Set(USAGE_PROVIDER_ICON_IDS) + const missing = Object.keys(PROVIDER_DEFINITIONS).filter((id) => !covered.has(id)) + expect(missing).toEqual([]) + }) +}) diff --git a/apps/sim/ee/organization-usage/components/usage-consumers.tsx b/apps/sim/ee/organization-usage/components/usage-consumers.tsx new file mode 100644 index 00000000000..610056f46cd --- /dev/null +++ b/apps/sim/ee/organization-usage/components/usage-consumers.tsx @@ -0,0 +1,300 @@ +'use client' + +import type { ComponentType } from 'react' +import { cn, disclosureChevronClass } from '@sim/emcn' +import { ArrowRight, ChevronDown } from '@sim/emcn/icons' +import { formatChartCompactNumber } from '@/components/charts' +import { + AnthropicIcon, + AzureIcon, + BasetenIcon, + BedrockIcon, + CerebrasIcon, + DeepseekIcon, + FireworksIcon, + GeminiIcon, + GroqIcon, + KimiIcon, + LitellmIcon, + MetaIcon, + MistralIcon, + NvidiaIcon, + OllamaIcon, + OpenAIIcon, + OpenRouterIcon, + SakanaIcon, + TogetherIcon, + VertexIcon, + VllmIcon, + xAIIcon, + ZaiIcon, +} from '@/components/icons' +import type { + OrganizationUsageBreakdown, + OrganizationUsageBreakdownRow, + UsageBreakdownDimension, +} from '@/lib/api/contracts/organization-usage' +import { + type RowAction, + RowActionsMenu, +} from '@/app/workspace/[workspaceId]/settings/components/row-actions-menu' +import { SettingsEmptyState } from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state' +import { RESOURCE_ROW_ARROW_CLASSES } from '@/app/workspace/[workspaceId]/settings/components/settings-resource-row' +import { USAGE_TAB_EMPTY_COPY } from '@/ee/organization-usage/constants' + +/** + * Provider brand marks, keyed by the `providerId` the server resolves. + * + * Kept here rather than read from `PROVIDER_DEFINITIONS`: that module carries the + * whole model registry and would land in this settings chunk for two dozen glyphs. + * The icons themselves come from the same `@/components/icons` module the registry + * imports, so this is a re-keying, never a second set of artwork. + * + * It must list every provider the registry defines, or a model resolving to a + * missing one renders an unexplained blank where every neighbouring row has a mark + * — which is how `zai` (GLM) shipped iconless. `usage-consumers.test.ts` fails when + * the two drift, so the coverage is checked rather than remembered. + */ +const PROVIDER_ICONS: Readonly>> = { + anthropic: AnthropicIcon, + baseten: BasetenIcon, + bedrock: BedrockIcon, + cerebras: CerebrasIcon, + deepseek: DeepseekIcon, + fireworks: FireworksIcon, + google: GeminiIcon, + groq: GroqIcon, + kimi: KimiIcon, + litellm: LitellmIcon, + meta: MetaIcon, + mistral: MistralIcon, + nvidia: NvidiaIcon, + ollama: OllamaIcon, + 'ollama-cloud': OllamaIcon, + openai: OpenAIIcon, + openrouter: OpenRouterIcon, + sakana: SakanaIcon, + together: TogetherIcon, + vertex: VertexIcon, + vllm: VllmIcon, + xai: xAIIcon, + zai: ZaiIcon, + 'azure-anthropic': AzureIcon, + /** Not a registry provider — a BYOK credential kind the breakdown can also emit. */ + 'azure-openai': AzureIcon, +} + +export const USAGE_PROVIDER_ICON_IDS = Object.keys(PROVIDER_ICONS) + +interface UsageConsumerRowProps { + row: OrganizationUsageBreakdownRow + /** BYOK rows carry no cost, so tokens are the only usage they can show. */ + showTokensOnly: boolean + onSelect?: (row: OrganizationUsageBreakdownRow) => void + actions?: RowAction[] + /** + * Width of the affordance some other row in this list carries, reserved here so + * every figure stays in one column — including when the only row that carries one + * is `Other`. + */ + reservedTrailing?: string +} + +/** + * Width of each trailing affordance, so a list that carries one can reserve the + * same slot on its `Other` row and keep every figure in one column. + */ +const TRAILING_SLOT_CLASSES = { + arrow: 'size-4', + /** `RowActionsMenu`'s trigger: a 14px glyph in a `chipVariants()` pill. */ + menu: 'size-[30px]', + /** The disclosure chevron on an expandable `Other` row, at the default icon size. */ + disclosure: 'size-[14px]', +} as const + +/** + * Geometry of the bespoke tabular usage row — the sanctioned exception to + * `SettingsResourceRow` in `sim-settings-pages.md`. One definition, so the breakdown + * rows, the `Other` row, and the events ledger cannot drift apart. + */ +export const USAGE_ROW_CLASSES = 'flex w-full items-center gap-2.5 rounded-lg p-2 text-left' + +/** + * A tabular row, not `SettingsResourceRow` — tabular columns are the sanctioned + * exception in `sim-settings-pages.md`, alongside billing invoices and credit usage. + */ +function UsageConsumerRow({ + row, + showTokensOnly, + onSelect, + actions, + reservedTrailing, +}: UsageConsumerRowProps) { + const ProviderIcon = row.providerId ? PROVIDER_ICONS[row.providerId] : undefined + const Row = onSelect ? 'button' : 'div' + + return ( + onSelect(row), + 'aria-label': `Open ${row.label}`, + } + : {})} + className={cn( + USAGE_ROW_CLASSES, + onSelect && 'transition-colors hover-hover:bg-[var(--surface-active)]' + )} + > + {ProviderIcon && ( + + )} + {row.label} + diff --git a/apps/sim/enrichments/types.ts b/apps/sim/enrichments/types.ts index 845d3369d71..64ad738b7eb 100644 --- a/apps/sim/enrichments/types.ts +++ b/apps/sim/enrichments/types.ts @@ -23,9 +23,8 @@ export interface EnrichmentOutputField { } /** - * Execution context for an enrichment run (runs server-side). `tableId`/`rowId` - * are present for the table per-row path but optional — the workflow block path - * (`/api/tools/enrichment/run`) has no table/row and passes only `workspaceId`. + * Execution context for an enrichment run. `tableId` and `rowId` are present + * for the table per-row path but optional for workflow tool execution. */ export interface EnrichmentRunContext { tableId?: string diff --git a/apps/sim/executor/handlers/agent/agent-handler.test.ts b/apps/sim/executor/handlers/agent/agent-handler.test.ts index 60da25f4d0f..5d7591cc673 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.test.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.test.ts @@ -38,10 +38,18 @@ import { ToolSchemaEnrichmentError } from '@/tools/params' process.env.NEXT_PUBLIC_APP_URL = 'http://localhost:3000' -const { mockImportWorkspaceFileSecretProvenanceForModelView } = vi.hoisted(() => ({ +const { + mockDiscoverMcpServerToolsAsExecutor, + mockImportWorkspaceFileSecretProvenanceForModelView, +} = vi.hoisted(() => ({ + mockDiscoverMcpServerToolsAsExecutor: vi.fn().mockResolvedValue([]), mockImportWorkspaceFileSecretProvenanceForModelView: vi.fn().mockResolvedValue(true), })) +vi.mock('@/lib/internal/mcp/discover-tools', () => ({ + discoverMcpServerToolsAsExecutor: mockDiscoverMcpServerToolsAsExecutor, +})) + vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({ importWorkspaceFileSecretProvenanceForModelView: mockImportWorkspaceFileSecretProvenanceForModelView, @@ -123,10 +131,11 @@ const MCP_SERVER_ROWS = [ { id: 'mcp-legacy-server', connectionStatus: 'connected' }, ] -const mockGetCustomToolById = vi.fn() +const mockReadAvailableCustomToolByIdOrTitleAsExecutor = vi.fn() -vi.mock('@/lib/workflows/custom-tools/operations', () => ({ - getCustomToolById: (...args: unknown[]) => mockGetCustomToolById(...args), +vi.mock('@/lib/internal/custom-tools/read-available-by-id-or-title', () => ({ + readAvailableCustomToolByIdOrTitleAsExecutor: (...args: unknown[]) => + mockReadAvailableCustomToolByIdOrTitleAsExecutor(...args), })) const mockGetAllBlocks = getAllBlocks as Mock @@ -153,6 +162,7 @@ describe('AgentBlockHandler', () => { beforeEach(() => { handler = new AgentBlockHandler() vi.clearAllMocks() + mockDiscoverMcpServerToolsAsExecutor.mockResolvedValue([]) mockImportWorkspaceFileSecretProvenanceForModelView.mockResolvedValue(true) resetDbChainMock() // The MCP server lookup awaits select().from(mcpServers).where(...) directly; @@ -3296,40 +3306,12 @@ describe('AgentBlockHandler', () => { }) it('should use cached schema for MCP tools (no discovery needed)', async () => { - const fetchCalls: any[] = [] - - mockFetch.mockImplementation((url: string, options: any) => { - fetchCalls.push({ url, options }) - - if (url.includes('/api/providers')) { - return Promise.resolve({ - ok: true, - headers: { - get: (name: string) => (name === 'Content-Type' ? 'application/json' : null), - }, - json: () => - Promise.resolve({ - content: 'Used MCP tool successfully', - model: 'gpt-4o', - tokens: { input: 10, output: 10, total: 20 }, - toolCalls: [], - timing: { total: 50 }, - }), - }) - } - - if (url.includes('/api/mcp/tools/execute')) { - return Promise.resolve({ - ok: true, - json: () => - Promise.resolve({ - success: true, - data: { output: { content: [{ type: 'text', text: 'Tool executed' }] } }, - }), - }) - } - - return Promise.resolve({ ok: true, json: () => Promise.resolve({}) }) + mockExecuteProviderRequest.mockResolvedValueOnce({ + content: 'Used MCP tool successfully', + model: 'gpt-4o', + tokens: { input: 10, output: 10, total: 20 }, + toolCalls: [], + timing: { total: 50 }, }) const inputs = { @@ -3367,15 +3349,11 @@ describe('AgentBlockHandler', () => { await handler.execute(contextWithWorkspace, mockBlock, inputs) - const discoveryCalls = fetchCalls.filter((c) => c.url.includes('/api/mcp/tools/discover')) - expect(discoveryCalls.length).toBe(0) - + expect(mockDiscoverMcpServerToolsAsExecutor).not.toHaveBeenCalled() expect(mockExecuteProviderRequest).toHaveBeenCalled() }) - it('should pass toolSchema to execution endpoint when using cached schema', async () => { - let executionCall: any = null - + it('should pass the cached tool schema to the provider', async () => { mockExecuteProviderRequest.mockResolvedValueOnce({ content: 'Tool executed', model: 'gpt-4o', @@ -3389,22 +3367,6 @@ describe('AgentBlockHandler', () => { timing: { total: 50 }, }) - mockFetch.mockImplementation((url: string, options: any) => { - if (url.includes('/api/mcp/tools/execute')) { - executionCall = { url, body: JSON.parse(options.body) } - return Promise.resolve({ - ok: true, - json: () => - Promise.resolve({ - success: true, - data: { output: { content: [{ type: 'text', text: 'Search results' }] } }, - }), - }) - } - - return Promise.resolve({ ok: true, json: () => Promise.resolve({}) }) - }) - const cachedSchema = { type: 'object', properties: { @@ -3613,29 +3575,12 @@ describe('AgentBlockHandler', () => { }) it('should handle multiple MCP tools from the same server efficiently', async () => { - const fetchCalls: any[] = [] - - mockFetch.mockImplementation((url: string, options: any) => { - fetchCalls.push({ url, options }) - - if (url.includes('/api/providers')) { - return Promise.resolve({ - ok: true, - headers: { - get: (name: string) => (name === 'Content-Type' ? 'application/json' : null), - }, - json: () => - Promise.resolve({ - content: 'Used tools', - model: 'gpt-4o', - tokens: { input: 10, output: 10, total: 20 }, - toolCalls: [], - timing: { total: 50 }, - }), - }) - } - - return Promise.resolve({ ok: true, json: () => Promise.resolve({}) }) + mockExecuteProviderRequest.mockResolvedValueOnce({ + content: 'Used tools', + model: 'gpt-4o', + tokens: { input: 10, output: 10, total: 20 }, + toolCalls: [], + timing: { total: 50 }, }) const inputs = { @@ -3689,58 +3634,27 @@ describe('AgentBlockHandler', () => { await handler.execute(contextWithWorkspace, mockBlock, inputs) - const discoveryCalls = fetchCalls.filter((c) => c.url.includes('/api/mcp/tools/discover')) - expect(discoveryCalls.length).toBe(0) - + expect(mockDiscoverMcpServerToolsAsExecutor).not.toHaveBeenCalled() expect(mockExecuteProviderRequest).toHaveBeenCalled() const providerCallArgs = mockExecuteProviderRequest.mock.calls[0] expect(providerCallArgs[1].tools.length).toBe(3) }) - it('should fallback to discovery for MCP tools without cached schema', async () => { - const fetchCalls: any[] = [] - - mockFetch.mockImplementation((url: string, options: any) => { - fetchCalls.push({ url, options }) - - if (url.includes('/api/mcp/tools/discover')) { - return Promise.resolve({ - ok: true, - json: () => - Promise.resolve({ - success: true, - data: { - tools: [ - { - name: 'legacy_tool', - description: 'A legacy tool without cached schema', - inputSchema: { type: 'object', properties: {} }, - serverName: 'legacy-server', - }, - ], - }, - }), - }) - } - - if (url.includes('/api/providers')) { - return Promise.resolve({ - ok: true, - headers: { - get: (name: string) => (name === 'Content-Type' ? 'application/json' : null), - }, - json: () => - Promise.resolve({ - content: 'Used legacy tool', - model: 'gpt-4o', - tokens: { input: 10, output: 10, total: 20 }, - toolCalls: [], - timing: { total: 50 }, - }), - }) - } - - return Promise.resolve({ ok: true, json: () => Promise.resolve({}) }) + it('should discover MCP tools without cached schema through the application operation', async () => { + mockDiscoverMcpServerToolsAsExecutor.mockResolvedValue([ + { + name: 'legacy_tool', + description: 'A legacy tool without cached schema', + inputSchema: { type: 'object', properties: {} }, + serverName: 'legacy-server', + }, + ]) + mockExecuteProviderRequest.mockResolvedValueOnce({ + content: 'Used legacy tool', + model: 'gpt-4o', + tokens: { input: 10, output: 10, total: 20 }, + toolCalls: [], + timing: { total: 50 }, }) const inputs = { @@ -3763,6 +3677,7 @@ describe('AgentBlockHandler', () => { const contextWithWorkspace = { ...mockContext, + userId: 'user-1', workspaceId: 'test-workspace-123', workflowId: 'test-workflow-456', } @@ -3771,10 +3686,20 @@ describe('AgentBlockHandler', () => { await handler.execute(contextWithWorkspace, mockBlock, inputs) - const discoveryCalls = fetchCalls.filter((c) => c.url.includes('/api/mcp/tools/discover')) - expect(discoveryCalls.length).toBe(1) - - expect(discoveryCalls[0].url).toContain('serverId=mcp-legacy-server') + expect(mockDiscoverMcpServerToolsAsExecutor).toHaveBeenCalledWith( + expect.objectContaining({ + workspaceId: 'test-workspace-123', + context: expect.objectContaining({ + userId: contextWithWorkspace.userId, + workflowId: 'test-workflow-456', + }), + serverId: 'mcp-legacy-server', + }) + ) + expect(mockFetch).not.toHaveBeenCalledWith( + expect.stringContaining('/api/mcp/tools/discover'), + expect.anything() + ) }) describe('customToolId resolution - DB as source of truth', () => { @@ -3813,8 +3738,9 @@ describe('AgentBlockHandler', () => { const dbCode = 'return { title, content, format };' function mockDBForCustomTool(toolId: string) { - mockGetCustomToolById.mockImplementation(({ toolId: id }: { toolId: string }) => { - if (id === toolId) { + mockReadAvailableCustomToolByIdOrTitleAsExecutor.mockImplementation( + ({ identifier }: { identifier: string }) => { + if (identifier !== toolId) return Promise.resolve(null) return Promise.resolve({ id: toolId, title: 'formatReport', @@ -3822,12 +3748,13 @@ describe('AgentBlockHandler', () => { code: dbCode, }) } - return Promise.resolve(null) - }) + ) } function mockDBFailure() { - mockGetCustomToolById.mockRejectedValue(new Error('DB connection failed')) + mockReadAvailableCustomToolByIdOrTitleAsExecutor.mockRejectedValue( + new Error('DB connection failed') + ) } beforeEach(() => { @@ -3836,7 +3763,7 @@ describe('AgentBlockHandler', () => { writable: true, configurable: true, }) - mockGetCustomToolById.mockReset() + mockReadAvailableCustomToolByIdOrTitleAsExecutor.mockReset() mockContext.userId = 'test-user' }) @@ -3933,7 +3860,9 @@ describe('AgentBlockHandler', () => { await handler.execute(mockContext, mockBlock, inputs) - expect(mockGetCustomToolById).toHaveBeenCalledWith(expect.objectContaining({ toolId })) + expect(mockReadAvailableCustomToolByIdOrTitleAsExecutor).toHaveBeenCalledWith( + expect.objectContaining({ context: mockContext, identifier: toolId, lookup: 'id' }) + ) const providerRequest = mockExecuteProviderRequest.mock.calls[0][1] expect(providerRequest.tools).toHaveLength(1) expect(providerRequest.tools[0].id).toBe('custom_formatReport') @@ -4040,7 +3969,9 @@ describe('AgentBlockHandler', () => { await expect(handler.execute(mockContext, mockBlock, inputs)).rejects.toBe(failure) - expect(mockGetCustomToolById).toHaveBeenCalledWith(expect.objectContaining({ toolId })) + expect(mockReadAvailableCustomToolByIdOrTitleAsExecutor).toHaveBeenCalledWith( + expect.objectContaining({ context: mockContext, identifier: toolId, lookup: 'id' }) + ) expect(inputs.tools[0].customToolId).toBe('{{CANARY_CUSTOM_TOOL_ID}}') expect(mockContext.resolvedSecretTraceRegistry?.getActiveMatches()).toEqual([]) expect(mockExecuteProviderRequest).not.toHaveBeenCalled() @@ -4189,7 +4120,7 @@ describe('AgentBlockHandler', () => { await handler.execute(mockContext, mockBlock, inputs) - expect(mockGetCustomToolById).not.toHaveBeenCalled() + expect(mockReadAvailableCustomToolByIdOrTitleAsExecutor).not.toHaveBeenCalled() expect(mockExecuteProviderRequest).toHaveBeenCalled() const providerCall = mockExecuteProviderRequest.mock.calls[0] diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index e6076426b24..3664fb82227 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -2,7 +2,6 @@ import { db } from '@sim/db' import { mcpServers } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' -import { sleep } from '@sim/utils/helpers' import { isPlainRecord } from '@sim/utils/object' import { truncate } from '@sim/utils/string' import { and, eq, inArray, isNull } from 'drizzle-orm' @@ -12,6 +11,12 @@ import { projectResolvedModelInput, selectModelSchemaInputPaths, } from '@/lib/execution/model-input-provenance' +import { readAvailableCustomToolByIdOrTitleAsExecutor } from '@/lib/internal/custom-tools/read-available-by-id-or-title' +import { discoverMcpServerToolsAsExecutor } from '@/lib/internal/mcp/discover-tools' +import { + readWorkflowInputFieldsForTool, + readWorkflowMetadataForTool, +} from '@/lib/internal/workflows/read-tool-enrichment' import type { McpToolSchema } from '@/lib/mcp/types' import { createMcpToolId } from '@/lib/mcp/utils' import { @@ -31,7 +36,6 @@ import { import { selectModelBoundFileInputPaths } from '@/lib/uploads/utils/model-input' import { hydrateUserFilesWithBase64 } from '@/lib/uploads/utils/user-file-base64.server' import { resolveCustomBlockToolBinding } from '@/lib/workflows/custom-blocks/operations' -import { getCustomToolById } from '@/lib/workflows/custom-tools/operations' import { getAllBlocks, getBlock } from '@/blocks' import { assembleCustomBlockInputMapping, isCustomBlockType } from '@/blocks/custom/build-config' import type { BlockOutput } from '@/blocks/types' @@ -59,7 +63,6 @@ import type { import { parseResponseFormat } from '@/executor/handlers/shared/response-format' import type { BlockHandler, ExecutionContext, StreamingExecution } from '@/executor/types' import { collectBlockData } from '@/executor/utils/block-data' -import { buildAPIUrl, buildAuthHeaders } from '@/executor/utils/http' import { stringifyJSON } from '@/executor/utils/json' import { projectResolvedSecretDiagnosticContent } from '@/executor/utils/resolved-secret-content-projection' import { prepareResolvedSecretProjectedInputs } from '@/executor/utils/resolved-secret-input-projection' @@ -984,7 +987,7 @@ export class AgentBlockHandler implements BlockHandler { ctx: ExecutionContext, customToolId: string ): Promise<{ schema: any; title: string } | null> { - if (!ctx.userId) { + if (!ctx.userId && !ctx.executorDelegationOrigin?.subjectUserId) { logger.error( 'Cannot fetch custom tool without userId', projectAgentDiagnosticMetadata( @@ -997,10 +1000,10 @@ export class AgentBlockHandler implements BlockHandler { } try { - const tool = await getCustomToolById({ - toolId: customToolId, - userId: ctx.userId, - workspaceId: ctx.workspaceId, + const tool = await readAvailableCustomToolByIdOrTitleAsExecutor({ + context: ctx, + identifier: customToolId, + lookup: 'id', }) if (!tool) { @@ -1267,83 +1270,30 @@ export class AgentBlockHandler implements BlockHandler { return results } - /** - * Discover tools from a single MCP server with retry logic. - */ + /** Discovers one server's tools through the authorized MCP operation. */ private async discoverMcpToolsForServer(ctx: ExecutionContext, serverId: string): Promise { + if (!ctx.userId) { + throw new Error('userId is required for MCP tool discovery') + } if (!ctx.workspaceId) { throw new Error('workspaceId is required for MCP tool discovery') } if (!ctx.workflowId) { - throw new Error('workflowId is required for internal JWT authentication') + throw new Error('workflowId is required for MCP tool discovery') } - const headers = await buildAuthHeaders(ctx.userId) - const url = buildAPIUrl('/api/mcp/tools/discover', { - serverId, + return discoverMcpServerToolsAsExecutor({ workspaceId: ctx.workspaceId, - workflowId: ctx.workflowId, - ...(ctx.userId ? { userId: ctx.userId } : {}), + context: { + workflowId: ctx.workflowId, + workspaceId: ctx.workspaceId, + executionId: ctx.executionId, + userId: ctx.userId, + executorDelegationOrigin: ctx.executorDelegationOrigin, + }, + serverId, + signal: ctx.abortSignal, }) - - const maxAttempts = 2 - for (let attempt = 0; attempt < maxAttempts; attempt++) { - try { - const response = await fetch(url.toString(), { method: 'GET', headers }) - - if (!response.ok) { - const errorText = await response.text() - if (this.isRetryableError(errorText) && attempt < maxAttempts - 1) { - logger.warn( - '[AgentHandler] Session error discovering tools, retrying', - projectAgentDiagnosticMetadata( - ctx, - { serverId, attempt: attempt + 1 }, - { hasServerId: serverId.length > 0, attempt: attempt + 1 } - ) - ) - await sleep(100) - continue - } - throw new Error(`Failed to discover tools: ${response.status} ${errorText}`) - } - - const data = await response.json() - if (!data.success) { - throw new Error(data.error || 'Failed to discover MCP tools') - } - - return data.data.tools - } catch (error) { - const errorMsg = toError(error).message - if (this.isRetryableError(errorMsg) && attempt < maxAttempts - 1) { - logger.warn( - '[AgentHandler] Retryable error discovering tools', - projectAgentDiagnosticMetadata( - ctx, - { serverId, attempt: attempt + 1, ...getErrorDiagnosticMetadata(error) }, - { - hasServerId: serverId.length > 0, - attempt: attempt + 1, - ...getErrorDiagnosticFallback(error), - } - ) - ) - await sleep(100) - continue - } - throw error - } - } - - throw new Error( - `Failed to discover tools from server ${serverId} after ${maxAttempts} attempts` - ) - } - - private isRetryableError(errorMsg: string): boolean { - const lowerMsg = errorMsg.toLowerCase() - return lowerMsg.includes('session') || lowerMsg.includes('400') || lowerMsg.includes('404') } private async createMcpToolFromDiscoveredData( @@ -1394,9 +1344,7 @@ export class AgentBlockHandler implements BlockHandler { getAllBlocks, getToolAsync: (toolId: string) => getToolAsync(toolId, { - workflowId: ctx.workflowId, - userId: ctx.userId, - workspaceId: ctx.workspaceId, + executionContext: ctx, }), getTool, canonicalModes, @@ -1405,10 +1353,13 @@ export class AgentBlockHandler implements BlockHandler { workspaceId: ctx.workspaceId, executionId: ctx.executionId, userId: ctx.userId, + executorDelegationOrigin: ctx.executorDelegationOrigin, }, toolIndex, resolveCustomBlockBinding: (blockType: string) => resolveCustomBlockToolBinding(blockType, ctx.workspaceId), + readWorkflowInputFields: readWorkflowInputFieldsForTool, + readWorkflowMetadata: readWorkflowMetadataForTool, }) if (transformedTool) { diff --git a/apps/sim/executor/handlers/generic/generic-handler.ts b/apps/sim/executor/handlers/generic/generic-handler.ts index f8dcbb42aaa..dec5e6787e5 100644 --- a/apps/sim/executor/handlers/generic/generic-handler.ts +++ b/apps/sim/executor/handlers/generic/generic-handler.ts @@ -10,7 +10,7 @@ import { prepareResolvedSecretProjectedInputs } from '@/executor/utils/resolved- import type { ResolvedSecretInputPath } from '@/executor/utils/resolved-secret-trace-registry' import type { SerializedBlock } from '@/serializer/types' import { executeTool } from '@/tools' -import type { ToolConfig } from '@/tools/types' +import { isInternalToolConfig, type ToolConfig } from '@/tools/types' import { getTool } from '@/tools/utils' const logger = createLogger('GenericBlockHandler') @@ -217,7 +217,8 @@ export class GenericBlockHandler implements BlockHandler { } } - const boundary = tool ? selectBlockBoundaryPaths(tool, finalInputs) : undefined + const requestTool = tool && !isInternalToolConfig(tool) ? tool : undefined + const boundary = requestTool ? selectBlockBoundaryPaths(requestTool, finalInputs) : undefined const projectedInputs = boundary && boundary.paths.length > 0 && registry?.hasResolvedInputProjections() ? registry.projectResolvedInputSelections(inputs) @@ -228,11 +229,11 @@ export class GenericBlockHandler implements BlockHandler { }) } - if (projectedInputs?.complete && boundary && tool && registry) { + if (projectedInputs?.complete && boundary && requestTool && registry) { for (const projection of projectedInputs.values) { const preserveFileDescriptorGrammar = - isFileBoundaryPath(tool, projection.path) || - boundary.paths.some((path) => isFileBoundaryPath(tool, path)) + isFileBoundaryPath(requestTool, projection.path) || + boundary.paths.some((path) => isFileBoundaryPath(requestTool, path)) let projectedFinalInputs = prepareResolvedSecretProjectedInputs( projection.value, blockConfig?.inputs, @@ -248,7 +249,7 @@ export class GenericBlockHandler implements BlockHandler { } } catch (error) { const structuredProjection = createStructuredModelProjection( - tool, + requestTool, finalInputs, projection.path, projection.projectedValue @@ -270,7 +271,7 @@ export class GenericBlockHandler implements BlockHandler { registry.markIncomplete('structural-input-root-unprojected', { detail: { blockType, - tool: tool.id, + tool: requestTool.id, inputPath: projection.path.join('.'), failure: toError(error).name, }, diff --git a/apps/sim/executor/handlers/pi/core/backend.ts b/apps/sim/executor/handlers/pi/core/backend.ts index 3a044615af3..4fb3b3ee3d2 100644 --- a/apps/sim/executor/handlers/pi/core/backend.ts +++ b/apps/sim/executor/handlers/pi/core/backend.ts @@ -9,7 +9,7 @@ */ import type { TSchema } from 'typebox' -import type { SSHConnectionConfig } from '@/app/api/tools/ssh/utils' +import type { SSHConnectionConfig } from '@/lib/internal/ssh/client' import type { Message } from '@/executor/handlers/agent/types' import type { PiEvent, PiRunTotals } from '@/executor/handlers/pi/core/events' import type { PiSearchProvider } from '@/executor/handlers/pi/core/keys' diff --git a/apps/sim/executor/handlers/pi/local/sim-tools.test.ts b/apps/sim/executor/handlers/pi/local/sim-tools.test.ts index 649d3c9b27e..60b99bb95ac 100644 --- a/apps/sim/executor/handlers/pi/local/sim-tools.test.ts +++ b/apps/sim/executor/handlers/pi/local/sim-tools.test.ts @@ -162,6 +162,69 @@ describe('buildSimToolSpecs', () => { expect(callParams._context.workflowId).toBe('wf-1') }) + it('executes Function tools with resolved inputs and the complete trusted execution context', async () => { + mockTransformBlockTool.mockResolvedValue({ + id: 'function_execute', + name: 'Function Execute', + description: 'Execute code', + params: { + code: 'return [{{API_KEY}}, __blockRef_0.field, workflowVariables.customer]', + envVars: { API_KEY: 'resolved-secret' }, + workflowVariables: { customer: 'Ada' }, + contextVariables: { __blockRef_0: { field: 'resolved-output' } }, + }, + parameters: { type: 'object', properties: {} }, + }) + const abortController = new AbortController() + const trustedCtx = { + workspaceId: 'ws-1', + workflowId: 'wf-1', + userId: 'user-1', + executionId: 'execution-1', + largeValueExecutionIds: ['execution-1'], + largeValueKeys: ['lv_ABCDEFGHIJKL'], + fileKeys: ['file-1'], + allowLargeValueWorkflowScope: true, + abortSignal: abortController.signal, + resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry(), + } as ExecutionContext + mockExecuteTool.mockResolvedValue({ + success: true, + output: { result: ['resolved-secret', 'resolved-output', 'Ada'] }, + }) + + const [spec] = await buildSimToolSpecs(trustedCtx, [ + { type: 'function', operation: 'execute', usageControl: 'auto' }, + ]) + const result = await spec.execute({ + _context: { userId: 'attacker', workspaceId: 'evil-workspace' }, + }) + + expect(mockExecuteTool).toHaveBeenCalledWith( + 'function_execute', + expect.objectContaining({ + code: 'return [{{API_KEY}}, __blockRef_0.field, workflowVariables.customer]', + envVars: { API_KEY: 'resolved-secret' }, + workflowVariables: { customer: 'Ada' }, + contextVariables: { __blockRef_0: { field: 'resolved-output' } }, + _context: expect.objectContaining({ + userId: 'user-1', + workspaceId: 'ws-1', + workflowId: 'wf-1', + executionId: 'execution-1', + }), + }), + expect.objectContaining({ + executionContext: trustedCtx, + resolvedSecretTraceRegistry: expect.any(ResolvedSecretTraceRegistry), + }) + ) + expect(result).toEqual({ + text: JSON.stringify({ result: ['resolved-secret', 'resolved-output', 'Ada'] }), + isError: false, + }) + }) + it('projects named provenance in successful Sim tool output', async () => { mockToolAdapter({ apiKey: 'secret-value' }) encryptionMockFns.mockDecryptSecret.mockResolvedValue({ decrypted: 'secret-value' }) diff --git a/apps/sim/executor/handlers/pi/local/sim-tools.ts b/apps/sim/executor/handlers/pi/local/sim-tools.ts index fa55a7faa3f..483eb87f0a6 100644 --- a/apps/sim/executor/handlers/pi/local/sim-tools.ts +++ b/apps/sim/executor/handlers/pi/local/sim-tools.ts @@ -9,6 +9,10 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { + readWorkflowInputFieldsForTool, + readWorkflowMetadataForTool, +} from '@/lib/internal/workflows/read-tool-enrichment' import { resolveCustomBlockToolBinding } from '@/lib/workflows/custom-blocks/operations' import { getAllBlocks } from '@/blocks/registry' import type { ToolInput } from '@/executor/handlers/agent/types' @@ -216,9 +220,12 @@ export async function buildSimToolSpecs( workspaceId: ctx.workspaceId, executionId: ctx.executionId, userId: ctx.userId, + executorDelegationOrigin: ctx.executorDelegationOrigin, }, resolveCustomBlockBinding: (blockType: string) => resolveCustomBlockToolBinding(blockType, ctx.workspaceId), + readWorkflowInputFields: readWorkflowInputFieldsForTool, + readWorkflowMetadata: readWorkflowMetadataForTool, }) if (!provider?.id) continue diff --git a/apps/sim/executor/handlers/pi/local/ssh-tools.test.ts b/apps/sim/executor/handlers/pi/local/ssh-tools.test.ts index 05a4b21e9be..b2323b31960 100644 --- a/apps/sim/executor/handlers/pi/local/ssh-tools.test.ts +++ b/apps/sim/executor/handlers/pi/local/ssh-tools.test.ts @@ -7,7 +7,7 @@ const { mockExecuteSSHCommand } = vi.hoisted(() => ({ mockExecuteSSHCommand: vi.fn(), })) -vi.mock('@/app/api/tools/ssh/utils', () => ({ +vi.mock('@/lib/internal/ssh/client', () => ({ createSSHConnection: vi.fn(), executeSSHCommand: mockExecuteSSHCommand, escapeShellArg: (value: string) => value.replace(/'/g, "'\\''"), diff --git a/apps/sim/executor/handlers/pi/local/ssh-tools.ts b/apps/sim/executor/handlers/pi/local/ssh-tools.ts index 182d8caf6fb..0a7c0f8b78a 100644 --- a/apps/sim/executor/handlers/pi/local/ssh-tools.ts +++ b/apps/sim/executor/handlers/pi/local/ssh-tools.ts @@ -14,7 +14,7 @@ import { executeSSHCommand, sanitizeCommand, sanitizePath, -} from '@/app/api/tools/ssh/utils' +} from '@/lib/internal/ssh/client' import type { PiSshConnection, PiToolResult, PiToolSpec } from '@/executor/handlers/pi/core/backend' const logger = createLogger('PiSshTools') diff --git a/apps/sim/executor/handlers/workflow/workflow-handler.test.ts b/apps/sim/executor/handlers/workflow/workflow-handler.test.ts index 010ae1250fc..b1dc669b4bf 100644 --- a/apps/sim/executor/handlers/workflow/workflow-handler.test.ts +++ b/apps/sim/executor/handlers/workflow/workflow-handler.test.ts @@ -7,6 +7,7 @@ import { } from '@sim/testing' import { afterAll, beforeAll, beforeEach, describe, expect, it, type Mock, vi } from 'vitest' import { createTimeoutAbortController, getExecutionDeadlineAt } from '@/lib/core/execution-limits' +import { OrchestrationError } from '@/lib/core/orchestration/types' import { getBlock } from '@/blocks/registry' import { BlockType } from '@/executor/constants' import { BoundarySafeError } from '@/executor/errors/boundary' @@ -44,7 +45,7 @@ const { mockSetExecutionDeadlineAt, mockSetTraceLargeValueAccess, mockDispose, - mockBuildExecutorDelegationHeaders, + mockReadWorkflowDefinitionAsExecutor, mockCheckWorkspaceAccess, mockProjectTraceSpansForLiveDisplay, executorOptions, @@ -68,7 +69,7 @@ const { mockSetExecutionDeadlineAt: vi.fn(), mockSetTraceLargeValueAccess: vi.fn(), mockDispose: vi.fn(), - mockBuildExecutorDelegationHeaders: vi.fn(), + mockReadWorkflowDefinitionAsExecutor: vi.fn(), executorOptions: [] as Array>, loggingSessionArgs: [] as Array, })) @@ -140,6 +141,10 @@ vi.mock('@/lib/users/queries', () => ({ getUserEmailById: mockGetUserEmailById, })) +vi.mock('@/lib/internal/workflows/read-definition', () => ({ + readWorkflowDefinitionAsExecutor: mockReadWorkflowDefinitionAsExecutor, +})) + /** * Overrides the global registry mock's getBlock so the Serializer can carry the * start block's runMetadata param through child deployed-state serialization. @@ -193,20 +198,6 @@ vi.mock('@/lib/auth/internal', () => ({ generateInternalToken: vi.fn().mockResolvedValue('test-token'), })) -vi.mock('@/executor/utils/http', () => ({ - buildExecutorDelegationHeaders: mockBuildExecutorDelegationHeaders, - buildAPIUrl: vi.fn((path: string) => new URL(path, 'http://localhost:3000')), - extractAPIErrorMessage: vi.fn(async (response: Response) => { - const defaultMessage = `API request failed with status ${response.status}` - try { - const errorData = await response.json() - return errorData.error || defaultMessage - } catch { - return defaultMessage - } - }), -})) - describe('WorkflowBlockHandler', () => { let handler: WorkflowBlockHandler let mockBlock: SerializedBlock @@ -238,8 +229,16 @@ describe('WorkflowBlockHandler', () => { mockContext = { workflowId: 'parent-workflow-id', + executionId: 'parent-execution-id', userId: 'user-1', principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + executorDelegationOrigin: { + subjectUserId: 'user-1', + workflowId: 'parent-workflow-id', + executionId: 'parent-execution-id', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + currentWorkflow: { workflowId: 'parent-workflow-id', mode: 'draft' }, + }, blockStates: new Map(), blockLogs: [], metadata: { @@ -267,11 +266,6 @@ describe('WorkflowBlockHandler', () => { mockSafeStart.mockResolvedValue(true) mockAdmitCustomBlockChildExecution.mockResolvedValue(undefined) mockBuildTraceSpans.mockReturnValue({ traceSpans: [], totalDuration: 0 }) - mockBuildExecutorDelegationHeaders.mockResolvedValue({ - 'Content-Type': 'application/json', - Authorization: 'Bearer executor-token', - }) - // Setup default fetch mock mockFetch.mockResolvedValue({ ok: true, @@ -298,6 +292,65 @@ describe('WorkflowBlockHandler', () => { }, }), }) + mockReadWorkflowDefinitionAsExecutor.mockImplementation( + async ({ workflowId, state }: { workflowId: string; state: 'draft' | 'deployed' }) => { + const response = await mockFetch( + state === 'deployed' + ? `http://localhost:3000/api/workflows/${workflowId}/deployed` + : `http://localhost:3000/api/workflows/${workflowId}` + ) + if (!response.ok) { + if (response.status === 404) { + throw new OrchestrationError('not_found', 'Workflow not found') + } + throw new Error(`Failed to read workflow: ${response.status} ${response.statusText}`) + } + + const json = await response.json() + if (state === 'draft') { + const data = json.data + return { + workflow: { + id: workflowId, + name: data?.name, + workspaceId: data?.workspaceId, + variables: data?.variables ?? {}, + }, + workspaceId: data?.workspaceId, + state: data?.state, + } + } + + const deployedState = json?.data?.deployedState ?? json?.deployedState ?? null + if (!deployedState) { + return { + workflow: { id: workflowId, name: workflowId, variables: {} }, + workspaceId: undefined, + state: null, + } + } + + const metadataResponse = await mockFetch( + `http://localhost:3000/api/workflows/${workflowId}` + ) + if (!metadataResponse.ok) { + throw new Error( + `Failed to read workflow metadata: ${metadataResponse.status} ${metadataResponse.statusText}` + ) + } + const metadata = (await metadataResponse.json())?.data + return { + workflow: { + id: workflowId, + name: metadata?.name, + workspaceId: metadata?.workspaceId, + variables: metadata?.variables ?? {}, + }, + workspaceId: metadata?.workspaceId, + state: deployedState, + } + } + ) }) describe('canHandle', () => { @@ -386,12 +439,17 @@ describe('WorkflowBlockHandler', () => { ) expect(mockCreateSnapshot).not.toHaveBeenCalled() expect(mockExecutorExecute).not.toHaveBeenCalled() - expect(mockBuildExecutorDelegationHeaders).toHaveBeenCalledWith({ - subjectUserId: 'user-1', - workflowId: 'parent-workflow-id', - executionId: 'parent-execution-id', - principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, - }) + expect(mockReadWorkflowDefinitionAsExecutor).toHaveBeenCalledWith( + expect.objectContaining({ + origin: { + subjectUserId: 'user-1', + workflowId: 'parent-workflow-id', + executionId: 'parent-execution-id', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + currentWorkflow: { workflowId: 'parent-workflow-id', mode: 'draft' }, + }, + }) + ) }) it('should fail a cross-workspace child in the deployed loader path', async () => { @@ -634,10 +692,14 @@ describe('WorkflowBlockHandler', () => { await handler.execute(ctx, customBlock, {}) - expect(mockBuildExecutorDelegationHeaders).toHaveBeenCalledWith({ - subjectUserId: 'owner-9', - workflowId: 'source-workflow-id', - }) + expect(mockReadWorkflowDefinitionAsExecutor).toHaveBeenCalledWith( + expect.objectContaining({ + origin: { + subjectUserId: 'owner-9', + workflowId: 'source-workflow-id', + }, + }) + ) expect(mockResolveBillingAttribution).toHaveBeenCalledWith({ actorUserId: 'owner-9', workspaceId: 'workspace-source', @@ -2036,12 +2098,17 @@ describe('WorkflowBlockHandler', () => { currentWorkflow: { workflowId: 'child-workflow-id', mode: 'draft' }, principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, }) - expect(mockBuildExecutorDelegationHeaders).toHaveBeenCalledWith({ - subjectUserId: 'user-1', - workflowId: 'parent-workflow-id', - executionId: 'parent-execution-id', - principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, - }) + expect(mockReadWorkflowDefinitionAsExecutor).toHaveBeenCalledWith( + expect.objectContaining({ + origin: { + subjectUserId: 'user-1', + workflowId: 'parent-workflow-id', + executionId: 'parent-execution-id', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + currentWorkflow: { workflowId: 'parent-workflow-id', mode: 'draft' }, + }, + }) + ) expect(extensions.onStream).toBe(ctx.onStream) expect(extensions.childWorkflowContext).toBeDefined() }) @@ -2072,7 +2139,9 @@ describe('WorkflowBlockHandler', () => { await handler.execute(ctx, mockBlock, { workflowId: 'grandchild-workflow-id' }) - expect(mockBuildExecutorDelegationHeaders).toHaveBeenCalledWith(ctx.executorDelegationOrigin) + expect(mockReadWorkflowDefinitionAsExecutor).toHaveBeenCalledWith( + expect.objectContaining({ origin: ctx.executorDelegationOrigin }) + ) expect(executorOptions[0].contextExtensions.executorDelegationOrigin).toEqual({ ...ctx.executorDelegationOrigin, currentWorkflow: { workflowId: 'grandchild-workflow-id', mode: 'draft' }, diff --git a/apps/sim/executor/handlers/workflow/workflow-handler.ts b/apps/sim/executor/handlers/workflow/workflow-handler.ts index cacfdc06aba..c8a29c2fe8d 100644 --- a/apps/sim/executor/handlers/workflow/workflow-handler.ts +++ b/apps/sim/executor/handlers/workflow/workflow-handler.ts @@ -1,12 +1,14 @@ -import { resolvePrincipalSubject } from '@sim/auth/principal' import { createLogger } from '@sim/logger' import { findCause, getErrorMessage, toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { isRecordLike } from '@sim/utils/object' +import type { Variable, WorkflowState } from '@sim/workflow-types/workflow' import { resolveBillingAttribution } from '@/lib/billing/core/billing-attribution' import { getExecutionDeadlineAt } from '@/lib/core/execution-limits' +import { asOrchestrationError } from '@/lib/core/orchestration/types' import { getPersonalAndWorkspaceEnv } from '@/lib/environment/utils' import { buildNextCallChain, validateCallChain } from '@/lib/execution/call-chain' +import { readWorkflowDefinitionAsExecutor } from '@/lib/internal/workflows/read-definition' import { LoggingSession } from '@/lib/logs/execution/logging-session' import { snapshotService } from '@/lib/logs/execution/snapshot/service' import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans' @@ -20,6 +22,7 @@ import { } from '@/lib/workflows/custom-blocks/child-execution' import { getCustomBlockAuthority } from '@/lib/workflows/custom-blocks/operations' import { extractInputFieldsFromBlocks } from '@/lib/workflows/input-format' +import { parseWorkflowVariables } from '@/lib/workflows/variables/parse' import { type CustomBlockOutput, isCustomBlockType } from '@/blocks/custom/build-config' import type { BlockOutput } from '@/blocks/types' import { Executor } from '@/executor' @@ -28,7 +31,6 @@ import { CHILD_EXECUTION_ID_OUTPUT_KEY, CHILD_TRACE_DISABLED_OUTPUT_KEY, DEFAULTS, - HTTP, } from '@/executor/constants' import { BoundarySafeError, @@ -54,7 +56,6 @@ import { type StreamingExecution, } from '@/executor/types' import { hasExecutionResult } from '@/executor/utils/errors' -import { buildAPIUrl, buildExecutorDelegationHeaders } from '@/executor/utils/http' import { getIterationContext } from '@/executor/utils/iteration-context' import { parseJSON } from '@/executor/utils/json' import { lazyCleanupInputMapping } from '@/executor/utils/lazy-cleanup' @@ -328,29 +329,25 @@ export class WorkflowBlockHandler implements BlockHandler { if (!ctx.principal) { throw new Error('Workflow child loading requires an execution principal') } - const principalSubject = resolvePrincipalSubject(ctx.principal) - const workflowReadDelegationOrigin: ExecutorDelegationOrigin = isCustomBlock - ? { - ...(loadUserId ? { subjectUserId: loadUserId } : {}), - workflowId, - } - : (ctx.executorDelegationOrigin ?? { - ...(principalSubject?.kind === 'sim_user' - ? { subjectUserId: principalSubject.userId } - : {}), - workflowId: ctx.workflowId, - ...(ctx.executionId ? { executionId: ctx.executionId } : {}), - principal: ctx.principal, - }) + let workflowReadDelegationOrigin: ExecutorDelegationOrigin + if (isCustomBlock) { + workflowReadDelegationOrigin = { + ...(loadUserId ? { subjectUserId: loadUserId } : {}), + workflowId, + } + } else { + if (!ctx.executorDelegationOrigin) { + throw new Error('Child workflow loading requires executor delegation authority') + } + workflowReadDelegationOrigin = ctx.executorDelegationOrigin + } if (!isCustomBlock) childExecutorDelegationOrigin = workflowReadDelegationOrigin - const workflowReadHeaders = await buildExecutorDelegationHeaders(workflowReadDelegationOrigin) - // A custom block runs the source's latest deployment; if the source has been // undeployed there's nothing to run. `BoundarySafeError` marks the message as // safe to cross the invocation boundary verbatim (it names no source // internals), so the catch forwards it instead of the generic failure. if (isCustomBlock) { - const deployed = await this.checkChildDeployment(workflowId, workflowReadHeaders) + const deployed = await this.checkChildDeployment(workflowId, workflowReadDelegationOrigin) if (!deployed) { throw new BoundarySafeError({ errorType: 'not_deployed', @@ -360,7 +357,10 @@ export class WorkflowBlockHandler implements BlockHandler { } if (useDeployed && !isCustomBlock) { - const hasActiveDeployment = await this.checkChildDeployment(workflowId, workflowReadHeaders) + const hasActiveDeployment = await this.checkChildDeployment( + workflowId, + workflowReadDelegationOrigin + ) if (!hasActiveDeployment) { throw new Error( `Child workflow is not deployed. Please deploy the workflow before invoking it.` @@ -369,8 +369,8 @@ export class WorkflowBlockHandler implements BlockHandler { } const childWorkflow = useDeployed - ? await this.loadChildWorkflowDeployed(workflowId, workflowReadHeaders) - : await this.loadChildWorkflow(workflowId, workflowReadHeaders) + ? await this.loadChildWorkflowDeployed(workflowId, workflowReadDelegationOrigin) + : await this.loadChildWorkflow(workflowId, workflowReadDelegationOrigin) if (!childWorkflow) { throw new Error(`Child workflow ${workflowId} not found`) @@ -1165,28 +1165,51 @@ export class WorkflowBlockHandler implements BlockHandler { } } - private async loadChildWorkflow(workflowId: string, headers: Record) { - const url = buildAPIUrl(`/api/workflows/${workflowId}`) + private getWorkflowVariables( + workflowId: string, + persistedVariables: unknown + ): Record { + const persisted = parseWorkflowVariables(persistedVariables) + const variables: Record = {} + for (const [variableId, variable] of Object.entries(persisted ?? {})) { + variables[variableId] = { ...variable, workflowId } + } + return variables + } - const response = await fetch(url.toString(), { headers }) + private getWorkflowStateMetadata(state: unknown): NonNullable { + if (!isRecordLike(state) || !isRecordLike(state.metadata)) return {} - if (!response.ok) { - await response.text().catch(() => {}) - if (response.status === HTTP.STATUS.NOT_FOUND) { + const metadata: NonNullable = {} + if (typeof state.metadata.name === 'string') metadata.name = state.metadata.name + if (typeof state.metadata.description === 'string') { + metadata.description = state.metadata.description + } + if (typeof state.metadata.exportedAt === 'string') { + metadata.exportedAt = state.metadata.exportedAt + } + return metadata + } + + private async loadChildWorkflow(workflowId: string, origin: ExecutorDelegationOrigin) { + let definition + try { + definition = await readWorkflowDefinitionAsExecutor({ + origin, + workflowId, + state: 'draft', + }) + } catch (error) { + if (asOrchestrationError(error)?.code === 'not_found') { logger.warn(`Child workflow ${workflowId} not found`) return null } - throw new Error(`Failed to fetch workflow: ${response.status} ${response.statusText}`) - } - - const { data: workflowData } = await response.json() - - if (!workflowData) { - throw new Error(`Child workflow ${workflowId} returned empty data`) + throw error } + const workflowData = definition.workflow + const workflowState = definition.state logger.info(`Loaded child workflow: ${workflowData.name} (${workflowId})`) - const workflowState = workflowData.state if (!workflowState || !workflowState.blocks) { throw new Error(`Child workflow ${workflowId} has invalid state`) @@ -1200,12 +1223,12 @@ export class WorkflowBlockHandler implements BlockHandler { true ) - const workflowVariables = (workflowData.variables as Record) || {} - const workflowStateWithVariables = { + const workflowVariables = this.getWorkflowVariables(workflowId, workflowData.variables) + const workflowStateWithVariables: WorkflowState = { ...workflowState, variables: workflowVariables, metadata: { - ...(workflowState.metadata || {}), + ...this.getWorkflowStateMetadata(workflowState), name: workflowData.name || DEFAULTS.WORKFLOW_NAME, }, } @@ -1218,7 +1241,7 @@ export class WorkflowBlockHandler implements BlockHandler { return { name: workflowData.name, - workspaceId: (workflowData.workspaceId ?? null) as string | null, + workspaceId: definition.workspaceId, deploymentVersionId: undefined, serializedState: serializedWorkflow, variables: workflowVariables, @@ -1229,20 +1252,15 @@ export class WorkflowBlockHandler implements BlockHandler { private async checkChildDeployment( workflowId: string, - headers: Record + origin: ExecutorDelegationOrigin ): Promise { try { - const url = buildAPIUrl(`/api/workflows/${workflowId}/deployed`) - - const response = await fetch(url.toString(), { - headers, - cache: 'no-store', + const definition = await readWorkflowDefinitionAsExecutor({ + origin, + workflowId, + state: 'deployed', }) - - if (!response.ok) return false - - const json = await response.json() - return !!json?.data?.deployedState || !!json?.deployedState + return definition.state !== null } catch (error) { logger.error('Failed to check child deployment', { errorName: toError(error).name, @@ -1252,39 +1270,30 @@ export class WorkflowBlockHandler implements BlockHandler { } } - private async loadChildWorkflowDeployed(workflowId: string, headers: Record) { - const deployedUrl = buildAPIUrl(`/api/workflows/${workflowId}/deployed`) - - const deployedRes = await fetch(deployedUrl.toString(), { - headers, - cache: 'no-store', - }) - - if (!deployedRes.ok) { - if (deployedRes.status === HTTP.STATUS.NOT_FOUND) { + private async loadChildWorkflowDeployed(workflowId: string, origin: ExecutorDelegationOrigin) { + let definition + try { + definition = await readWorkflowDefinitionAsExecutor({ + origin, + workflowId, + state: 'deployed', + }) + } catch (error) { + if (asOrchestrationError(error)?.code === 'not_found') { return null } - throw new Error( - `Failed to fetch deployed workflow: ${deployedRes.status} ${deployedRes.statusText}` - ) - } - const deployedJson = await deployedRes.json() - const deployedState = deployedJson?.data?.deployedState || deployedJson?.deployedState - if (!deployedState || !deployedState.blocks) { - throw new Error(`Deployed state missing or invalid for child workflow ${workflowId}`) + throw error } - const metaUrl = buildAPIUrl(`/api/workflows/${workflowId}`) - const metaRes = await fetch(metaUrl.toString(), { - headers, - cache: 'no-store', - }) - - if (!metaRes.ok) { - throw new Error(`Failed to fetch workflow metadata: ${metaRes.status} ${metaRes.statusText}`) + const deployedState = definition.state + if ( + !deployedState || + !deployedState.blocks || + !('deploymentVersionId' in deployedState) || + typeof deployedState.deploymentVersionId !== 'string' + ) { + throw new Error(`Deployed state missing or invalid for child workflow ${workflowId}`) } - const metaJson = await metaRes.json() - const wfData = metaJson?.data const serializedWorkflow = this.serializer.serializeWorkflow( deployedState.blocks, @@ -1294,21 +1303,21 @@ export class WorkflowBlockHandler implements BlockHandler { true ) - const workflowVariables = (wfData?.variables as Record) || {} - const childName = wfData?.name || DEFAULTS.WORKFLOW_NAME - const workflowStateWithVariables = { + const workflowVariables = this.getWorkflowVariables(workflowId, definition.workflow.variables) + const childName = definition.workflow.name || DEFAULTS.WORKFLOW_NAME + const workflowStateWithVariables: WorkflowState = { ...deployedState, variables: workflowVariables, metadata: { - ...(deployedState.metadata || {}), + ...this.getWorkflowStateMetadata(deployedState), name: childName, }, } return { name: childName, - workspaceId: (wfData?.workspaceId ?? null) as string | null, - deploymentVersionId: deployedState.deploymentVersionId as string | undefined, + workspaceId: definition.workspaceId, + deploymentVersionId: deployedState.deploymentVersionId, serializedState: serializedWorkflow, variables: workflowVariables, workflowState: workflowStateWithVariables, diff --git a/apps/sim/executor/utils/errors.ts b/apps/sim/executor/utils/errors.ts index ba0d35d0ff1..deb1306e6ac 100644 --- a/apps/sim/executor/utils/errors.ts +++ b/apps/sim/executor/utils/errors.ts @@ -36,6 +36,55 @@ export function attachExecutionResult(error: Error, executionResult: ExecutionRe Object.assign(error, { executionResult }) } +/** + * Dispatched-run ids, keyed by the thrown value itself. + * + * A side table rather than a property on the error, for the same reason + * {@link markExecutionFinalizedByCore} keeps one: a thrown value is not reliably writable. + * `Object.assign` throws on a frozen or sealed failure, and guarding that throw would drop + * the marker instead — silently converting "this run exists" into "nothing started", which + * is the one direction that duplicates work. Identity keying also means no id can arrive + * through a prototype chain, and nothing is added to the error's own surface, so a + * serialized error carries no stray field. + */ +const attemptedExecutionIds = new WeakMap() + +/** + * Names the run a failure belongs to once dispatch has been attempted. + * + * A caller that only sees the thrown error cannot tell an authorization refusal — which + * created nothing — from a crash after the run was already dispatched, and those need + * opposite retry decisions. Recording the id at the point of no return makes its absence + * mean "nothing was started" rather than "we do not know", and its presence a key that + * resolves to zero or one executions. + * + * Distinct from {@link attachExecutionResult}: that says the workflow ran and produced a + * result, this says only that it was dispatched. + */ +export function attachAttemptedExecutionId(error: unknown, executionId: string): void { + if (!isRecordedThrown(error) || !executionId) return + if (attemptedExecutionIds.has(error)) return + attemptedExecutionIds.set(error, executionId) +} + +/** Reads the dispatched-run id a thrown value carries, if dispatch was reached at all. */ +export function readAttemptedExecutionId(error: unknown): string | undefined { + return isRecordedThrown(error) ? attemptedExecutionIds.get(error) : undefined +} + +/** + * Any non-null object, not only an `Error`. + * + * Restricting this to `Error` would silently invert the invariant for a thrown plain object: + * no id would be recorded, its absence would read as "nothing was started", and the caller + * would retry a run that already exists. A thrown primitive cannot be keyed at all, which + * costs nothing today because every throw site past the dispatch boundary raises an `Error`. + */ +function isRecordedThrown(value: unknown): value is object { + /** Functions key a WeakMap as well as objects do, so excluding them would drop the record. */ + return (typeof value === 'object' || typeof value === 'function') && value !== null +} + export interface BlockExecutionErrorDetails { block: SerializedBlock error: Error | string diff --git a/apps/sim/executor/utils/file-tool-processor.ts b/apps/sim/executor/utils/file-tool-processor.ts index 72bdaa3bad3..eb641728695 100644 --- a/apps/sim/executor/utils/file-tool-processor.ts +++ b/apps/sim/executor/utils/file-tool-processor.ts @@ -5,7 +5,7 @@ import { uploadExecutionFile, uploadFileFromRawData } from '@/lib/uploads/contex import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server' import { MAX_FILE_SIZE, sniffImageContentType } from '@/lib/uploads/utils/validation' import type { ExecutionContext, UserFile } from '@/executor/types' -import type { ToolConfig, ToolFileData } from '@/tools/types' +import type { ToolDefinition, ToolFileData } from '@/tools/types' const logger = createLogger('FileToolProcessor') @@ -57,7 +57,7 @@ export class FileToolProcessor { */ static async processToolOutputs( toolOutput: any, - toolConfig: ToolConfig, + toolConfig: ToolDefinition, executionContext: ExecutionContext ): Promise { if (!toolConfig.outputs) { @@ -236,7 +236,7 @@ export class FileToolProcessor { /** * Check if a tool has any file-typed outputs */ - static hasFileOutputs(toolConfig: ToolConfig): boolean { + static hasFileOutputs(toolConfig: ToolDefinition): boolean { if (!toolConfig.outputs) { return false } diff --git a/apps/sim/hooks/queries/organization-usage.ts b/apps/sim/hooks/queries/organization-usage.ts new file mode 100644 index 00000000000..6eb65523ae5 --- /dev/null +++ b/apps/sim/hooks/queries/organization-usage.ts @@ -0,0 +1,134 @@ +'use client' + +import { hashKey, keepPreviousData, useInfiniteQuery, useQuery } from '@tanstack/react-query' +import { requestJson } from '@/lib/api/client/request' +import { + getOrganizationUsageBreakdownContract, + getOrganizationUsageSummaryContract, + listOrganizationUsageEventsContract, + type OrganizationUsageBreakdown, + type OrganizationUsageEventPage, + type OrganizationUsageSummary, + type UsageBreakdownDimension, +} from '@/lib/api/contracts/organization-usage' +import { + type OrganizationUsageWindowKey, + organizationUsageKeys, +} from '@/hooks/queries/utils/organization-usage-keys' + +export const ORGANIZATION_USAGE_SUMMARY_STALE_TIME = 60 * 1000 +/** + * Longer than the summary: a ranking does not move meaningfully within a minute, and + * three of the five dimensions heap-scan the ledger. + */ +export const ORGANIZATION_USAGE_BREAKDOWN_STALE_TIME = 5 * 60 * 1000 +export const ORGANIZATION_USAGE_EVENTS_STALE_TIME = 30 * 1000 + +const EVENTS_PAGE_SIZE = 50 + +export function useOrganizationUsageSummary( + organizationId: string | undefined, + window: OrganizationUsageWindowKey +) { + return useQuery({ + queryKey: organizationUsageKeys.summary(organizationId ?? '', window), + queryFn: ({ signal }): Promise => + requestJson(getOrganizationUsageSummaryContract, { + params: { id: organizationId as string }, + query: { ...window }, + signal, + }), + enabled: Boolean(organizationId), + staleTime: ORGANIZATION_USAGE_SUMMARY_STALE_TIME, + // Changing the period should dim the current figures rather than blank them. + placeholderData: keepPreviousData, + }) +} + +interface UseBreakdownOptions { + limit?: number + /** The panel passes the selected tab, so only the visible list is ever fetched. */ + enabled?: boolean + /** Narrows to one workspace, for the Workspaces drill-down. */ + workspaceId?: string +} + +/** + * A breakdown key with its trailing row limit removed — the identity of the list, + * which is what "the same list, more rows" has to compare on. + */ +function breakdownListIdentity(key: readonly unknown[]): string { + return hashKey(key.slice(0, -1)) +} + +export function useOrganizationUsageBreakdown( + organizationId: string | undefined, + window: OrganizationUsageWindowKey, + dimension: UsageBreakdownDimension, + options: UseBreakdownOptions = {} +) { + const limit = options.limit ?? 10 + const { workspaceId } = options + const queryKey = organizationUsageKeys.breakdown( + organizationId ?? '', + window, + dimension, + limit, + workspaceId + ) + return useQuery({ + queryKey, + queryFn: ({ signal }): Promise => + requestJson(getOrganizationUsageBreakdownContract, { + params: { id: organizationId as string }, + query: { + ...window, + dimension, + limit, + ...(workspaceId ? { workspaceId } : {}), + }, + signal, + }), + enabled: Boolean(organizationId) && (options.enabled ?? true), + staleTime: ORGANIZATION_USAGE_BREAKDOWN_STALE_TIME, + /** + * Kept only across a row-limit change — opening the `Other` row asks the same + * question of the same list, and the visible rows are a prefix of the answer, so + * dimming beats blanking. Any other key change (dimension, window, workspace) + * would put a stale ranking under a new label, which reads as wrong data and is + * worse than a brief skeleton. + */ + placeholderData: (previous, previousQuery) => + previous && + previousQuery && + breakdownListIdentity(previousQuery.queryKey) === breakdownListIdentity(queryKey) + ? previous + : undefined, + }) +} + +export function useOrganizationUsageEvents( + organizationId: string | undefined, + window: OrganizationUsageWindowKey, + sources: string[] = [] +) { + return useInfiniteQuery({ + queryKey: organizationUsageKeys.events(organizationId ?? '', window, sources), + queryFn: ({ pageParam, signal }): Promise => + requestJson(listOrganizationUsageEventsContract, { + params: { id: organizationId as string }, + query: { + ...window, + ...(sources.length ? { source: sources } : {}), + limit: EVENTS_PAGE_SIZE, + cursor: pageParam, + }, + signal, + }), + initialPageParam: undefined as string | undefined, + getNextPageParam: (lastPage) => lastPage.nextCursor, + enabled: Boolean(organizationId), + staleTime: ORGANIZATION_USAGE_EVENTS_STALE_TIME, + placeholderData: keepPreviousData, + }) +} diff --git a/apps/sim/hooks/queries/utils/organization-usage-keys.ts b/apps/sim/hooks/queries/utils/organization-usage-keys.ts new file mode 100644 index 00000000000..3c147137bc0 --- /dev/null +++ b/apps/sim/hooks/queries/utils/organization-usage-keys.ts @@ -0,0 +1,52 @@ +import type { + UsageBreakdownDimension, + UsageWindowPreset, +} from '@/lib/api/contracts/organization-usage' + +/** + * Query keys for organization usage. + * + * Not a `'use client'` module: a key factory imported by a server-evaluated + * prefetch would become a client reference and throw when called. + * + * The keys nest under `['organizations', 'detail', orgId]` deliberately, so the + * existing `invalidateQueries({ queryKey: organizationKeys.detail(orgId) })` calls — + * after a member's credit limit changes, for instance — sweep usage too. + */ + +export interface OrganizationUsageWindowKey { + preset: UsageWindowPreset + startDate?: string + endDate?: string + timezone: string +} + +export const organizationUsageKeys = { + all: (organizationId: string) => ['organizations', 'detail', organizationId, 'usage'] as const, + summary: (organizationId: string, window: OrganizationUsageWindowKey) => + [...organizationUsageKeys.all(organizationId), 'summary', window] as const, + breakdowns: (organizationId: string, window: OrganizationUsageWindowKey) => + [...organizationUsageKeys.all(organizationId), 'breakdown', window] as const, + breakdown: ( + organizationId: string, + window: OrganizationUsageWindowKey, + dimension: UsageBreakdownDimension, + limit: number, + workspaceId?: string + ) => + [ + ...organizationUsageKeys.breakdowns(organizationId, window), + dimension, + workspaceId ?? '', + /* + Last deliberately: it is the one segment the breakdown's `placeholderData` + ignores, so the list's identity is a plain prefix rather than an index-based + filter that would silently drop `workspaceId` if a segment were ever appended. + It also lets an invalidation target one dimension and workspace across every + row limit. + */ + limit, + ] as const, + events: (organizationId: string, window: OrganizationUsageWindowKey, sources: string[]) => + [...organizationUsageKeys.all(organizationId), 'events', window, sources] as const, +} diff --git a/apps/sim/lib/api/contracts/audit-logs.ts b/apps/sim/lib/api/contracts/audit-logs.ts index ebf09243741..df7a7dbeca4 100644 --- a/apps/sim/lib/api/contracts/audit-logs.ts +++ b/apps/sim/lib/api/contracts/audit-logs.ts @@ -1,5 +1,5 @@ import { z } from 'zod' -import { organizationIdSchema } from '@/lib/api/contracts/primitives' +import { organizationIdSchema, workspaceIdSchema } from '@/lib/api/contracts/primitives' import { defineRouteContract } from '@/lib/api/contracts/types' export const auditLogsQuerySchema = z.object({ @@ -11,6 +11,13 @@ export const auditLogsQuerySchema = z.object({ action: z.string().optional(), resourceType: z.string().optional(), actorId: z.string().optional(), + /** + * Narrows the org-scoped feed to one workspace. The use case already refuses an + * id outside the caller's organization; this only opens the door the internal + * surface had left shut while `buildFilterConditions` and the v1 contract both + * supported it. + */ + workspaceId: workspaceIdSchema.optional(), startDate: z .string() .optional() diff --git a/apps/sim/lib/api/contracts/hotspots.test.ts b/apps/sim/lib/api/contracts/hotspots.test.ts index 9a984ea169b..4533aa2433b 100644 --- a/apps/sim/lib/api/contracts/hotspots.test.ts +++ b/apps/sim/lib/api/contracts/hotspots.test.ts @@ -2,10 +2,10 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' -import { functionExecuteContract } from '@/lib/api/contracts/hotspots' +import { functionExecuteBodySchema } from '@/lib/api/contracts/hotspots' -describe('function execute contract', () => { - const bodySchema = functionExecuteContract.body! +describe('function execute input', () => { + const bodySchema = functionExecuteBodySchema it.each(['javascript', 'python', 'shell'] as const)('accepts the %s language', (language) => { const result = bodySchema.safeParse({ code: 'echo ok', language }) @@ -19,7 +19,7 @@ describe('function execute contract', () => { expect(result.language).toBe('javascript') }) - it('accepts unknown legacy languages for the route fallback', () => { + it('accepts unknown legacy languages for executor compatibility', () => { const result = bodySchema.safeParse({ code: 'puts :ok', language: 'ruby' }) expect(result.success).toBe(true) diff --git a/apps/sim/lib/api/contracts/hotspots.ts b/apps/sim/lib/api/contracts/hotspots.ts index 272d42a1220..b2ec180bb40 100644 --- a/apps/sim/lib/api/contracts/hotspots.ts +++ b/apps/sim/lib/api/contracts/hotspots.ts @@ -2,60 +2,12 @@ import { z } from 'zod' import { customPatternSchema, privateSecretProvenanceBundleSchema, - resolvedSecretTraceProvenanceSchema, stringRecordSchema, unknownRecordSchema, } from '@/lib/api/contracts/primitives' import { defineRouteContract } from '@/lib/api/contracts/types' import { DEFAULT_CODE_LANGUAGE } from '@/lib/execution/languages' -import { - PRIVATE_SECRET_PROVENANCE_FIELD, - RESOLVED_SECRET_PROVENANCE_FIELD, -} from '@/lib/execution/private-tool-metadata' -export const guardrailsValidateContract = defineRouteContract({ - method: 'POST', - path: '/api/guardrails/validate', - body: z.object({ - validationType: z.string().optional(), - input: z.unknown().optional(), - regex: z.string().optional(), - knowledgeBaseId: z.string().optional(), - threshold: z.string().optional(), - topK: z.string().optional(), - model: z.string().optional(), - apiKey: z.string().optional(), - azureEndpoint: z.string().optional(), - azureApiVersion: z.string().optional(), - vertexProject: z.string().optional(), - vertexLocation: z.string().optional(), - vertexCredential: z.string().optional(), - bedrockAccessKeyId: z.string().optional(), - bedrockSecretKey: z.string().optional(), - bedrockRegion: z.string().optional(), - workflowId: z.string().optional(), - piiEntityTypes: z.array(z.string()).optional(), - piiMode: z.string().optional(), - piiLanguage: z.string().optional(), - piiCustomPatterns: z.array(customPatternSchema).max(20).optional(), - [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), - }), - response: { - mode: 'json', - schema: z.object({ - success: z.boolean(), - output: z.object({ - passed: z.boolean(), - validationType: z.string(), - input: z.unknown().optional(), - error: z.string().optional(), - score: z.number().optional(), - reasoning: z.string().optional(), - detectedEntities: z.array(z.unknown()).optional(), - maskedText: z.string().optional(), - }), - }), - }, -}) +import { PRIVATE_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' const guardrailsMaskBatchBodySchema = z.object({ texts: z.array(z.string()).max(100_000), @@ -244,13 +196,3 @@ export const functionExecuteBodySchema = z export type FunctionExecuteBody = z.input export type ParsedFunctionExecuteBody = z.output - -export const functionExecuteContract = defineRouteContract({ - method: 'POST', - path: '/api/function/execute', - body: functionExecuteBodySchema, - response: { - mode: 'json', - schema: unknownRecordSchema, - }, -}) diff --git a/apps/sim/lib/api/contracts/mcp.ts b/apps/sim/lib/api/contracts/mcp.ts index 4c071657695..d720802b55c 100644 --- a/apps/sim/lib/api/contracts/mcp.ts +++ b/apps/sim/lib/api/contracts/mcp.ts @@ -219,16 +219,6 @@ export const mcpToolDiscoveryQuerySchema = z.object({ refresh: z.string().optional(), }) -export const mcpToolExecutionBodySchema = z - .object({ - serverId: z.string().min(1), - toolName: z.string().min(1), - arguments: z.record(z.string(), z.unknown()).optional(), - workflowId: z.string().optional(), - }) - .passthrough() -export type McpToolExecutionBody = z.input - export const mcpToolResultSchema = z .object({ content: z.array(z.unknown()).optional(), @@ -237,13 +227,6 @@ export const mcpToolResultSchema = z }) .passthrough() -export const mcpToolExecutionResultSchema = z.object({ - success: z.boolean(), - output: mcpToolResultSchema.optional(), - error: z.string().optional(), -}) -export type McpToolExecutionResult = z.output - export const mcpJsonRpcRequestSchema = z .object({ jsonrpc: z.literal('2.0'), @@ -434,17 +417,6 @@ export const testMcpServerConnectionContract = defineRouteContract({ }, }) -export const executeMcpToolContract = defineRouteContract({ - method: 'POST', - path: '/api/mcp/tools/execute', - body: mcpToolExecutionBodySchema, - response: { - mode: 'json', - schema: mcpSuccessResponseSchema(mcpToolExecutionResultSchema), - }, -}) -export type ExecuteMcpToolResponse = ContractJsonResponse - export const startMcpOauthQuerySchema = z.object({ serverId: z.string().min(1, 'serverId is required'), workspaceId: z.string().min(1, 'workspaceId is required'), diff --git a/apps/sim/lib/api/contracts/memory.ts b/apps/sim/lib/api/contracts/memory.ts index 7064bf47934..c0f1259cb06 100644 --- a/apps/sim/lib/api/contracts/memory.ts +++ b/apps/sim/lib/api/contracts/memory.ts @@ -11,26 +11,6 @@ export const memoryWorkspaceQuerySchema = z.object({ workspaceId: z.string().uuid('Invalid workspace ID format'), }) -const agentMemoryDataSchema = z.object({ - role: z.enum(['user', 'assistant', 'system'], { - error: 'Role must be user, assistant, or system', - }), - content: z.string().min(1, 'Content is required'), -}) - -const genericMemoryDataSchema = z.record(z.string(), z.unknown()) - -export const memoryPutBodySchema = z.object({ - data: z.union([agentMemoryDataSchema, genericMemoryDataSchema], { - error: 'Invalid memory data structure', - }), - workspaceId: z.string().uuid('Invalid workspace ID format'), - [PRIVATE_SECRET_PROVENANCE_FIELD]: privateSecretProvenanceBundleSchema.optional(), -}) -export type MemoryPutBody = z.input - -export const agentMemoryDataSchemaContract = agentMemoryDataSchema - export const memoryListQuerySchema = z.object({ workspaceId: z.string().optional(), query: z.string().nullable().optional(), @@ -125,25 +105,3 @@ export const getMemoryByIdContract = defineRouteContract({ schema: memorySuccessResponseSchema(memoryRecordSchema.nullable()), }, }) - -export const deleteMemoryByIdContract = defineRouteContract({ - method: 'DELETE', - path: '/api/memory/[id]', - params: memoryIdParamsSchema, - query: memoryWorkspaceQuerySchema, - response: { - mode: 'json', - schema: memorySuccessResponseSchema(z.object({ message: z.string() })), - }, -}) - -export const updateMemoryByIdContract = defineRouteContract({ - method: 'PUT', - path: '/api/memory/[id]', - params: memoryIdParamsSchema, - body: memoryPutBodySchema, - response: { - mode: 'json', - schema: memorySuccessResponseSchema(memoryRecordSchema), - }, -}) diff --git a/apps/sim/lib/api/contracts/organization-usage.test.ts b/apps/sim/lib/api/contracts/organization-usage.test.ts new file mode 100644 index 00000000000..91b25dbeac1 --- /dev/null +++ b/apps/sim/lib/api/contracts/organization-usage.test.ts @@ -0,0 +1,70 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { organizationUsageEventsQuerySchema } from '@/lib/api/contracts/organization-usage' + +/** The shared window fields every usage contract extends, exercised through one of them. */ +function parseWindow(input: Record) { + return organizationUsageEventsQuerySchema.safeParse({ preset: 'custom', ...input }) +} + +describe('organization usage window contract', () => { + it('accepts a real calendar date', () => { + expect(parseWindow({ startDate: '2026-08-01', endDate: '2026-08-31' }).success).toBe(true) + }) + + it('refuses a date that does not exist', () => { + // `Date.parse` accepts this and rolls it forward to March 2, so a request for + // February would otherwise be answered about March without saying so. + expect(parseWindow({ startDate: '2026-02-30' }).success).toBe(false) + }) + + it('refuses a parseable non-date such as a bare month', () => { + // `new Date('2026-08')` is August 1. Accepting it returned a window the caller + // never asked for, with nothing to indicate the value had been reinterpreted. + expect(parseWindow({ startDate: '2026-08' }).success).toBe(false) + }) + + it('refuses anything after the date, including a well-formed datetime', () => { + // The picker sends bare dates only, and every looser rule broke differently: + // `…Tgarbage` parsed to an Invalid Date that made the resolver throw from + // `toISOString` (a 500 for a bad query string), and an offset datetime validated + // on its date part while the resolver read a different UTC day off the whole + // value — so the range shown and the range queried disagreed. + expect(parseWindow({ startDate: '2026-08-01Tgarbage' }).success).toBe(false) + expect(parseWindow({ startDate: '2026-08-01T00:00:00+05:00' }).success).toBe(false) + expect(parseWindow({ startDate: '2026-02-30T00:00:00' }).success).toBe(false) + }) + + it('refuses an empty date but allows an absent one', () => { + // The picker clears the param rather than blanking it, so `?start-date=` is a + // malformed request — and treating it as absent silently answered about the + // current period instead of the range the caller named. + expect(parseWindow({ startDate: '' }).success).toBe(false) + expect(parseWindow({}).success).toBe(true) + }) + + it('treats an empty limit as omitted rather than as zero', () => { + // `z.coerce.number()` turns `''` into `0`, which then fails `.min(1)` — so a + // client serializing an unset filter got a 400 instead of the declared default. + const parsed = parseWindow({ limit: '' }) + expect(parsed.success).toBe(true) + if (parsed.success) expect(parsed.data.limit).toBe(50) + }) + + it('normalizes a single source to a one-item array', () => { + // One selected filter arrives as a scalar, which a bare `z.array` rejected. + const parsed = parseWindow({ source: 'workflow' }) + expect(parsed.success).toBe(true) + if (parsed.success) expect(parsed.data.source).toEqual(['workflow']) + }) + + it('refuses an unknown source instead of matching nothing', () => { + expect(parseWindow({ source: 'not-a-source' }).success).toBe(false) + }) + + it('refuses a timezone the runtime does not recognize', () => { + expect(parseWindow({ timezone: 'Mars/Olympus_Mons' }).success).toBe(false) + }) +}) diff --git a/apps/sim/lib/api/contracts/organization-usage.ts b/apps/sim/lib/api/contracts/organization-usage.ts new file mode 100644 index 00000000000..658a9f040c1 --- /dev/null +++ b/apps/sim/lib/api/contracts/organization-usage.ts @@ -0,0 +1,271 @@ +import { z } from 'zod' +import { organizationIdSchema, workspaceIdSchema } from '@/lib/api/contracts/primitives' +import { defineRouteContract } from '@/lib/api/contracts/types' +import { INTERNAL_USAGE_LOG_SOURCES } from '@/lib/billing/usage-sources' +import { isValidTimezone } from '@/lib/core/utils/timezone' + +/** + * Organization usage monitoring (enterprise). + * + * Everything on the wire is denominated in **credits**, never dollars: the ledger + * stores dollars and the use cases convert at this boundary, matching every other + * usage surface in the product. + */ + +export const USAGE_WINDOW_PRESETS = [ + 'current-period', + 'previous-period', + '7d', + '30d', + 'custom', +] as const +export const usageWindowPresetSchema = z.enum(USAGE_WINDOW_PRESETS).default('current-period') +export type UsageWindowPreset = z.output + +export const USAGE_BREAKDOWN_DIMENSIONS = [ + 'member', + 'workspace', + 'workflow', + 'model', + 'byok', + 'source', +] as const +export const usageBreakdownDimensionSchema = z.enum(USAGE_BREAKDOWN_DIMENSIONS) +export type UsageBreakdownDimension = z.output + +/** + * The longest custom range the ledger will scan. Declared on the contract so the + * picker states the same limit the window resolver enforces, rather than the client + * discovering it from a rejected request. + */ +export const MAX_CUSTOM_RANGE_DAYS = 92 + +/** + * A bare `YYYY-MM-DD` calendar date, and nothing else. + * + * Strict on purpose. The picker sends only bare dates — it has no time component — + * and every looser rule tried here has been wrong in a different way: + * + * - `Date.parse` alone accepts `2026-02-30` and rolls it forward, so February was + * answered about March. The round-trip below is what makes this a *calendar* + * check: a day that does not survive re-serialization never existed. + * - Validating only a `YYYY-MM-DD` prefix let `2026-08` through as August 1, and + * `2026-08-01Tgarbage` through as an `Invalid Date` that made the window resolver + * throw from `toISOString` — a 500 for a malformed query string. + * - A datetime with an offset would validate on its date part while the resolver + * read a different UTC day off the full value, so the range shown and the range + * queried could disagree. + * + * Accepting only the one form the client actually sends removes all three at once. + */ +const isoDateSchema = z + .string() + .optional() + .refine( + (value) => { + /* + Absent is allowed; empty is not. A missing bound is a real state — the picker + clears the param rather than blanking it — and the resolver falls back to the + current period for it. An explicit `?start-date=` is a malformed request, and + treating it as absent silently answered about a different window than the one + asked for. + + Deliberately unlike `usageLimitSchema`, which does coerce `''` to its default: + that field declares a default, so omission has a documented meaning. These + bounds have none — omitting one changes which period you get. + */ + if (value === undefined) return true + if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) return false + return new Date(`${value}T00:00:00.000Z`).toISOString().slice(0, 10) === value + }, + { message: 'Expected a calendar date in YYYY-MM-DD form, such as 2026-08-01' } + ) + +/** + * A page size that treats an empty or absent parameter as omitted. + * + * `z.coerce.number()` turns `''` into `0`, which then fails `.min(1)` — so a client + * that serializes an unset filter as `?limit=` got a 400 instead of the default the + * schema declares. Explicit numeric values still validate normally. + */ +function usageLimitSchema(max: number, fallback: number) { + return z.preprocess( + (value) => (value === '' || value === null ? undefined : value), + z.coerce.number().int().min(1).max(max).default(fallback) + ) +} + +/** + * Shared by all four contracts so the four surfaces cannot describe different + * windows — a mismatch here is how the tiles and the event log would disagree. + */ +const organizationUsageWindowQuerySchema = z.object({ + /* + No `organizationId` here. The organization is the path parameter, and that is the + one every handler authorizes and reads. Accepting a second copy in the query meant + a request could name one organization and be answered about another — not an + authorization hole, since `params.id` is what gets checked, but an API that reads + as though the query mattered. + */ + preset: usageWindowPresetSchema, + startDate: isoDateSchema, + endDate: isoDateSchema, + /** + * IANA name; bucket boundaries are the viewer's calendar days. + * + * Validated here rather than only at the SQL boundary. `assertValidTimezone` is a + * hard gate — the value reaches `AT TIME ZONE`, which takes an identifier and not + * a bound parameter — but it throws a plain `Error`, which surfaced as a 500 for + * what is an ordinary bad query param. Rejecting it as a contract violation makes + * it a 400 with a usable message and leaves that gate as the backstop it is. + */ + timezone: z + .string() + .min(1, 'timezone cannot be empty') + .refine(isValidTimezone, 'Expected an IANA timezone such as America/Los_Angeles') + .default('UTC'), +}) + +export const organizationUsageSummaryQuerySchema = organizationUsageWindowQuerySchema +export type OrganizationUsageSummaryQuery = z.input + +export const organizationUsageBreakdownQuerySchema = organizationUsageWindowQuerySchema.extend({ + dimension: usageBreakdownDimensionSchema, + /** Narrows the breakdown to one workspace, for the Workspaces drill-down. */ + workspaceId: workspaceIdSchema.optional(), + limit: usageLimitSchema(50, 10), +}) +export type OrganizationUsageBreakdownQuery = z.input + +/** + * Ledger sources, as an enum rather than free strings. + * + * Two problems this closes. A single selected source arrives on the wire as one + * scalar, not a one-item array, so an `z.array(...)` alone rejected the commonest + * filter outright — hence the union and normalization. And an unrecognized value + * used to survive validation and reach the query as an unchecked cast, where it + * matched nothing and returned an empty page that looked like "no usage" rather + * than a bad request. + */ +const usageLogSourceFilterSchema = z + .union([z.string(), z.array(z.string())]) + .transform((value) => (Array.isArray(value) ? value : [value])) + .pipe(z.array(z.enum(INTERNAL_USAGE_LOG_SOURCES)).max(20)) + +export const organizationUsageEventsQuerySchema = organizationUsageWindowQuerySchema.extend({ + source: usageLogSourceFilterSchema.optional(), + limit: usageLimitSchema(100, 50), + cursor: z.string().min(1).optional(), +}) +export type OrganizationUsageEventsQuery = z.input + +export const organizationUsageExportQuerySchema = organizationUsageEventsQuerySchema.omit({ + limit: true, + cursor: true, +}) +export type OrganizationUsageExportQuery = z.input + +/** Only the headline figure — see `readUsageTotals` for why nothing else lives here. */ +const usageTotalsSchema = z.object({ + credits: z.number(), +}) + +const usageSeriesPointSchema = z.object({ + timestamp: z.string(), + credits: z.number(), + events: z.number().int(), +}) + +export const organizationUsageSummaryResponseSchema = z.object({ + window: z.object({ + start: z.string(), + end: z.string(), + source: z.enum(['reporting', 'stripe', 'default', 'range']), + }), + bucket: z.enum(['day', 'week', 'month']), + totals: usageTotalsSchema, + /** `null` when the prior window is not exactly derivable — no delta beats a wrong one. */ + previousTotals: usageTotalsSchema.nullable(), + series: z.array(usageSeriesPointSchema), +}) +export type OrganizationUsageSummary = z.output + +export const organizationUsageBreakdownRowSchema = z.object({ + id: z.string(), + label: z.string(), + credits: z.number(), + events: z.number().int(), + /** 0..1 of the window total, not of the visible rows. */ + share: z.number().min(0).max(1), + /** Model dimensions only — resolved server-side so the client needs no model registry. */ + providerId: z.string().optional(), + /** Model dimensions only; BYOK rows carry no cost, so this is their only usage figure. */ + tokens: z.number().int().optional(), +}) +export type OrganizationUsageBreakdownRow = z.output + +export const organizationUsageBreakdownResponseSchema = z.object({ + dimension: usageBreakdownDimensionSchema, + rows: z.array(organizationUsageBreakdownRowSchema), + /** The truncated tail, so the visible rows plus this reconcile to `totalCredits`. */ + other: z.object({ + credits: z.number(), + events: z.number().int(), + rowCount: z.number().int(), + /** Tokens for the omitted rows, so the token-denominated BYOK tab still adds up. */ + tokens: z.number().int().nonnegative(), + }), + totalCredits: z.number(), +}) +export type OrganizationUsageBreakdown = z.output + +export const organizationUsageEventSchema = z.object({ + id: z.string(), + createdAt: z.string(), + source: z.string(), + description: z.string(), + workflowName: z.string().nullable(), + credits: z.number(), + hasCost: z.boolean(), +}) +export type OrganizationUsageEvent = z.output + +export const organizationUsageEventsResponseSchema = z.object({ + events: z.array(organizationUsageEventSchema), + nextCursor: z.string().optional(), + hasMore: z.boolean(), +}) +export type OrganizationUsageEventPage = z.output + +export const getOrganizationUsageSummaryContract = defineRouteContract({ + method: 'GET', + path: '/api/organizations/[id]/usage/summary', + params: z.object({ id: organizationIdSchema }), + query: organizationUsageSummaryQuerySchema, + response: { mode: 'json', schema: organizationUsageSummaryResponseSchema }, +}) + +export const getOrganizationUsageBreakdownContract = defineRouteContract({ + method: 'GET', + path: '/api/organizations/[id]/usage/breakdown', + params: z.object({ id: organizationIdSchema }), + query: organizationUsageBreakdownQuerySchema, + response: { mode: 'json', schema: organizationUsageBreakdownResponseSchema }, +}) + +export const listOrganizationUsageEventsContract = defineRouteContract({ + method: 'GET', + path: '/api/organizations/[id]/usage/events', + params: z.object({ id: organizationIdSchema }), + query: organizationUsageEventsQuerySchema, + response: { mode: 'json', schema: organizationUsageEventsResponseSchema }, +}) + +/** `mode: 'text'` — a CSV body has no JSON schema to validate. */ +export const exportOrganizationUsageContract = defineRouteContract({ + method: 'GET', + path: '/api/organizations/[id]/usage/export', + params: z.object({ id: organizationIdSchema }), + query: organizationUsageExportQuerySchema, + response: { mode: 'text' }, +}) diff --git a/apps/sim/lib/api/contracts/providers.ts b/apps/sim/lib/api/contracts/providers.ts index 115794a10ab..3b7f1d5eb28 100644 --- a/apps/sim/lib/api/contracts/providers.ts +++ b/apps/sim/lib/api/contracts/providers.ts @@ -1,7 +1,5 @@ import { z } from 'zod' -import { resolvedSecretTraceProvenanceSchema } from '@/lib/api/contracts/primitives' import { defineRouteContract } from '@/lib/api/contracts/types' -import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' export const openRouterModelInfoSchema = z.object({ id: z.string(), @@ -145,93 +143,6 @@ export const ollamaUpstreamResponseSchema = z.object({ .default([]), }) -const providerToolSchema = z - .object({ - id: z.string(), - name: z.string(), - description: z.string(), - params: z.record(z.string(), z.unknown()), - parameters: z - .object({ - type: z.string(), - properties: z.record(z.string(), z.unknown()), - required: z.array(z.string()), - }) - .passthrough(), - usageControl: z.enum(['auto', 'force', 'none']).optional(), - }) - .passthrough() - -const providerMessageSchema = z - .object({ - role: z.enum(['system', 'user', 'assistant', 'function', 'tool']), - content: z.string().nullable(), - name: z.string().optional(), - function_call: z - .object({ - name: z.string(), - arguments: z.string(), - }) - .optional(), - tool_calls: z - .array( - z.object({ - id: z.string(), - type: z.literal('function'), - function: z.object({ - name: z.string(), - arguments: z.string(), - }), - }) - ) - .optional(), - tool_call_id: z.string().optional(), - }) - .passthrough() - -const providerResponseFormatSchema = z - .object({ - name: z.string(), - // untyped-response: caller-supplied JSON Schema (request body field, not a route response) - schema: z.unknown(), - strict: z.boolean().optional(), - }) - .passthrough() - -export const providerApiRequestBodySchema = z - .object({ - provider: z.string().min(1), - model: z.string().min(1), - systemPrompt: z.string().optional(), - context: z.string().optional(), - tools: z.array(providerToolSchema).optional(), - temperature: z.number().optional(), - maxTokens: z.number().optional(), - apiKey: z.string().optional(), - azureEndpoint: z.string().optional(), - azureApiVersion: z.string().optional(), - vertexProject: z.string().optional(), - vertexLocation: z.string().optional(), - vertexCredential: z.string().optional(), - bedrockAccessKeyId: z.string().optional(), - bedrockSecretKey: z.string().optional(), - bedrockRegion: z.string().optional(), - responseFormat: providerResponseFormatSchema.optional(), - workflowId: z.string().optional(), - workspaceId: z.string().optional(), - stream: z.boolean().optional(), - messages: z.array(providerMessageSchema).optional(), - environmentVariables: z.record(z.string(), z.string()).optional(), - workflowVariables: z.record(z.string(), z.unknown()).optional(), - blockData: z.record(z.string(), z.unknown()).optional(), - blockNameMapping: z.record(z.string(), z.string()).optional(), - reasoningEffort: z.string().optional(), - verbosity: z.string().optional(), - [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), - }) - .passthrough() -export type ProviderApiRequestBody = z.input - export const getBaseProviderModelsContract = defineRouteContract({ method: 'GET', path: '/api/providers/base/models', @@ -325,42 +236,3 @@ export const getBasetenProviderModelsContract = defineRouteContract({ schema: providerModelsResponseSchema, }, }) - -/** - * `POST /api/providers` returns either a streamed response (handled at the - * runtime level — this contract models only the JSON case) or a JSON provider - * payload. The JSON case mirrors the canonical `ProviderResponse` shape from - * `@/providers/types`, but provider-specific fields are tolerated via - * passthrough so raw provider output flows through without contract drift. - */ -const executeProviderResponseSchema = z - .object({ - content: z.string(), - model: z.string(), - tokens: z - .object({ - input: z.number().optional(), - output: z.number().optional(), - total: z.number().optional(), - /** Prompt-cache buckets, reported separately from base input tokens. */ - cacheRead: z.number().optional(), - cacheWrite: z.number().optional(), - }) - .optional(), - toolCalls: z.array(z.record(z.string(), z.unknown())).optional(), - toolResults: z.array(z.record(z.string(), z.unknown())).optional(), - timing: z.record(z.string(), z.unknown()).optional(), - cost: z.record(z.string(), z.unknown()).optional(), - interactionId: z.string().optional(), - }) - .passthrough() - -export const executeProviderContract = defineRouteContract({ - method: 'POST', - path: '/api/providers', - body: providerApiRequestBodySchema, - response: { - mode: 'json', - schema: executeProviderResponseSchema, - }, -}) diff --git a/apps/sim/lib/api/contracts/selectors/google.ts b/apps/sim/lib/api/contracts/selectors/google.ts index 7b464048c79..c002725e3aa 100644 --- a/apps/sim/lib/api/contracts/selectors/google.ts +++ b/apps/sim/lib/api/contracts/selectors/google.ts @@ -17,26 +17,11 @@ import type { } from '@/lib/api/contracts/types' const googleCalendarSchema = z.object({ id: z.string(), summary: z.string() }).passthrough() -const gmailLabelSchema = z - .object({ - id: z.string(), - name: z.string(), - type: z.string().optional(), - messagesTotal: z.number().optional(), - messagesUnread: z.number().optional(), - }) - .passthrough() - export const labelsQuerySchema = credentialIdQuerySchema.extend({ query: optionalString, impersonateEmail: optionalString, }) -export const gmailLabelQuerySchema = credentialIdQuerySchema.extend({ - labelId: z.string().min(1), - impersonateEmail: optionalString, -}) - export const googleCalendarQuerySchema = credentialIdQuerySchema.extend({ workflowId: optionalString, impersonateEmail: optionalString, @@ -69,12 +54,6 @@ export const gmailLabelsSelectorContract = defineGetSelector( z.object({ labels: z.array(folderOptionSchema) }) ) -export const gmailLabelSelectorContract = defineGetSelector( - '/api/tools/gmail/label', - gmailLabelQuerySchema, - z.object({ label: gmailLabelSchema }) -) - export const googleCalendarSelectorContract = defineGetSelector( '/api/tools/google_calendar/calendars', googleCalendarQuerySchema, @@ -106,7 +85,6 @@ export const googleSheetsSelectorContract = defineGetSelector( ) export type GmailLabelsSelectorQuery = ContractQueryInput -export type GmailLabelSelectorQuery = ContractQueryInput export type GoogleCalendarSelectorQuery = ContractQueryInput export type GoogleTasksTaskListsSelectorBody = ContractBodyInput< typeof googleTasksTaskListsSelectorContract @@ -120,7 +98,6 @@ export type GoogleDriveFileSelectorQuery = ContractQueryInput< export type GoogleSheetsSelectorQuery = ContractQueryInput export type GmailLabelsSelectorResponse = ContractJsonResponse -export type GmailLabelSelectorResponse = ContractJsonResponse export type GoogleCalendarSelectorResponse = ContractJsonResponse< typeof googleCalendarSelectorContract > diff --git a/apps/sim/lib/api/contracts/selectors/index.ts b/apps/sim/lib/api/contracts/selectors/index.ts index c8d7e3c411a..bf69e28d6cc 100644 --- a/apps/sim/lib/api/contracts/selectors/index.ts +++ b/apps/sim/lib/api/contracts/selectors/index.ts @@ -35,7 +35,6 @@ import { confluenceSpacesSelectorContract, } from '@/lib/api/contracts/selectors/confluence' import { - gmailLabelSelectorContract, gmailLabelsSelectorContract, googleCalendarSelectorContract, googleDriveFileSelectorContract, @@ -76,7 +75,6 @@ import { microsoftPlannerTasksSelectorContract, microsoftTeamsSelectorContract, onedriveFilesSelectorContract, - onedriveFolderSelectorContract, onedriveFoldersSelectorContract, outlookCalendarsSelectorContract, outlookFoldersSelectorContract, @@ -93,7 +91,6 @@ import { import { pipedrivePipelinesSelectorContract } from '@/lib/api/contracts/selectors/pipedrive' import { sharepointListsSelectorContract, - sharepointSiteSelectorContract, sharepointSitesSelectorContract, } from '@/lib/api/contracts/selectors/sharepoint' import { @@ -104,7 +101,6 @@ import { import { snowflakeObjectsSelectorContract } from '@/lib/api/contracts/selectors/snowflake' import { trelloBoardsSelectorContract } from '@/lib/api/contracts/selectors/trello' import { - wealthboxItemContract, wealthboxItemsSelectorContract, wealthboxOAuthItemContract, wealthboxOAuthItemsContract, @@ -179,7 +175,6 @@ export const selectorContractsByPath = { '/api/tools/notion/pages': notionPagesSelectorContract, '/api/tools/pipedrive/pipelines': pipedrivePipelinesSelectorContract, '/api/tools/sharepoint/lists': sharepointListsSelectorContract, - '/api/tools/sharepoint/site': sharepointSiteSelectorContract, '/api/tools/sharepoint/sites': sharepointSitesSelectorContract, '/api/tools/trello/boards': trelloBoardsSelectorContract, '/api/tools/zoho_desk/organizations': zohoDeskOrganizationsSelectorContract, @@ -191,7 +186,6 @@ export const selectorContractsByPath = { '/api/tools/slack/users': slackUsersSelectorContract, '/api/tools/slack/users:detail': slackUserSelectorContract, '/api/tools/gmail/labels': gmailLabelsSelectorContract, - '/api/tools/gmail/label': gmailLabelSelectorContract, '/api/tools/hubspot/properties': hubspotPropertiesSelectorContract, '/api/tools/hubspot/lists': hubspotListsSelectorContract, '/api/tools/hubspot/pipelines': hubspotPipelinesSelectorContract, @@ -203,7 +197,6 @@ export const selectorContractsByPath = { '/api/tools/microsoft-teams/chats': microsoftChatsSelectorContract, '/api/tools/microsoft-teams/channels': microsoftChannelsSelectorContract, '/api/tools/wealthbox/items': wealthboxItemsSelectorContract, - '/api/tools/wealthbox/item': wealthboxItemContract, '/api/auth/oauth/wealthbox/items': wealthboxOAuthItemsContract, '/api/auth/oauth/wealthbox/item': wealthboxOAuthItemContract, '/api/tools/jira/projects': jiraProjectsSelectorContract, @@ -218,7 +211,6 @@ export const selectorContractsByPath = { '/api/tools/confluence/pages': confluencePagesSelectorContract, '/api/tools/confluence/page': confluencePageSelectorContract, '/api/tools/onedrive/files': onedriveFilesSelectorContract, - '/api/tools/onedrive/folder': onedriveFolderSelectorContract, '/api/tools/onedrive/folders': onedriveFoldersSelectorContract, '/api/tools/drive/files': googleDriveFilesSelectorContract, '/api/tools/drive/file': googleDriveFileSelectorContract, diff --git a/apps/sim/lib/api/contracts/selectors/jira.ts b/apps/sim/lib/api/contracts/selectors/jira.ts index c9cf30fd65b..bf4641667d6 100644 --- a/apps/sim/lib/api/contracts/selectors/jira.ts +++ b/apps/sim/lib/api/contracts/selectors/jira.ts @@ -2,7 +2,6 @@ import { z } from 'zod' import { idNameSchema, optionalString } from '@/lib/api/contracts/selectors/shared' import type { ContractBody, ContractJsonResponse, ContractQuery } from '@/lib/api/contracts/types' import { defineRouteContract } from '@/lib/api/contracts/types' -import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' const jiraIssueSectionSchema = z .object({ @@ -63,62 +62,6 @@ export const jiraIssuesBodySchema = z.object({ issueKeys: z.array(z.string().min(1)).default([]), }) -export const jiraParentReferenceSchema = z.union([ - z.string().min(1), - z.object({ key: z.string().min(1) }).passthrough(), - z.object({ id: z.string().min(1) }).passthrough(), -]) -export type JiraParentReference = z.input - -export const jiraWriteBodySchema = z.object({ - domain: z.string({ error: 'Domain is required' }).min(1, 'Domain is required'), - accessToken: z.string({ error: 'Access token is required' }).min(1, 'Access token is required'), - projectId: z.string({ error: 'Project ID is required' }).min(1, 'Project ID is required'), - summary: z.string({ error: 'Summary is required' }).min(1, 'Summary is required'), - description: z.union([z.string(), z.record(z.string(), z.unknown())]).optional(), - priority: z.string().optional(), - assignee: z.string().optional(), - cloudId: z.string().optional(), - issueType: z.string().optional(), - parent: jiraParentReferenceSchema.optional(), - labels: z.array(z.string()).optional(), - duedate: z.string().optional(), - reporter: z.string().optional(), - environment: z.union([z.string(), z.record(z.string(), z.unknown())]).optional(), - customFieldId: z.string().optional(), - customFieldValue: z.string().optional(), - components: z.array(z.string()).optional(), - fixVersions: z.array(z.string()).optional(), -}) - -export const jiraUpdateBodySchema = z.object({ - domain: z.string().min(1, 'Domain is required'), - accessToken: z.string().min(1, 'Access token is required'), - issueKey: z.string().min(1, 'Issue key is required'), - summary: z.string().optional(), - title: z.string().optional(), - description: z.union([z.string(), z.record(z.string(), z.unknown())]).optional(), - priority: z.string().optional(), - assignee: z.string().optional(), - labels: z.array(z.string()).optional(), - components: z.array(z.string()).optional(), - duedate: z.string().optional(), - fixVersions: z.array(z.string()).optional(), - environment: z.union([z.string(), z.record(z.string(), z.unknown())]).optional(), - customFieldId: z.string().optional(), - customFieldValue: z.string().optional(), - notifyUsers: z.boolean().optional(), - cloudId: z.string().optional(), -}) - -export const jiraAddAttachmentBodySchema = z.object({ - accessToken: z.string().min(1, 'Access token is required'), - domain: z.string().min(1, 'Domain is required'), - issueKey: z.string().min(1, 'Issue key is required'), - files: RawFileInputArraySchema, - cloudId: z.string().optional().nullable(), -}) - export const jiraProjectsSelectorContract = defineRouteContract({ method: 'GET', path: '/api/tools/jira/projects', @@ -170,88 +113,10 @@ export const jiraIssueSelectorContract = defineRouteContract({ }, }) -const jiraWriteResponseSchema = z.object({ - success: z.literal(true), - output: z.object({ - ts: z.string(), - id: z.string(), - issueKey: z.string(), - self: z.string(), - summary: z.string(), - success: z.literal(true), - url: z.string(), - assigneeId: z.string().optional(), - }), -}) - -const jiraUpdateResponseSchema = z.object({ - success: z.literal(true), - output: z.object({ - ts: z.string(), - issueKey: z.string(), - summary: z.string(), - success: z.literal(true), - }), -}) - -const jiraAttachmentSchema = z.object({ - id: z.string(), - filename: z.string(), - mimeType: z.string(), - size: z.number(), - content: z.string(), -}) - -const jiraAddAttachmentUserFileSchema = z - .object({ - id: z.string().optional(), - name: z.string(), - url: z.string().optional(), - size: z.number(), - type: z.string().optional(), - key: z.string(), - }) - .passthrough() - -const jiraAddAttachmentResponseSchema = z.object({ - success: z.literal(true), - output: z.object({ - ts: z.string(), - issueKey: z.string(), - attachments: z.array(jiraAttachmentSchema), - attachmentIds: z.array(z.string()), - files: z.array(jiraAddAttachmentUserFileSchema), - }), -}) - -export const jiraWriteContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/jira/write', - body: jiraWriteBodySchema, - response: { mode: 'json', schema: jiraWriteResponseSchema }, -}) - -export const jiraUpdateContract = defineRouteContract({ - method: 'PUT', - path: '/api/tools/jira/update', - body: jiraUpdateBodySchema, - response: { mode: 'json', schema: jiraUpdateResponseSchema }, -}) - -export const jiraAddAttachmentContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/jira/add-attachment', - body: jiraAddAttachmentBodySchema, - response: { mode: 'json', schema: jiraAddAttachmentResponseSchema }, -}) - export type JiraProjectsQuery = ContractQuery export type JiraProjectBody = ContractBody export type JiraIssuesQuery = ContractQuery export type JiraIssuesBody = ContractBody -export type JiraWriteBody = ContractBody -export type JiraUpdateBody = ContractBody -export type JiraAddAttachmentBody = ContractBody export type JiraProjectsSelectorResponse = ContractJsonResponse export type JiraProjectSelectorResponse = ContractJsonResponse export type JiraIssuesSelectorResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/selectors/microsoft.ts b/apps/sim/lib/api/contracts/selectors/microsoft.ts index 5876080d003..8123e6c6403 100644 --- a/apps/sim/lib/api/contracts/selectors/microsoft.ts +++ b/apps/sim/lib/api/contracts/selectors/microsoft.ts @@ -56,17 +56,6 @@ export const microsoftFileQuerySchema = credentialIdQuerySchema.extend({ workflowId: optionalString, }) -export const onedriveFolderQuerySchema = z.object({ - credentialId: z.preprocess( - (value) => value ?? '', - z.string().min(1, 'Credential ID and File ID are required') - ), - fileId: z.preprocess( - (value) => value ?? '', - z.string().min(1, 'Credential ID and File ID are required') - ), -}) - export const onedriveFilesQuerySchema = credentialIdQueryWithSearchSchema /** * Folder listing is drive-scoped like the file listing above: without `driveId` @@ -145,12 +134,6 @@ export const onedriveFoldersSelectorContract = defineGetSelector( z.object({ files: z.array(fileOptionSchema) }) ) -export const onedriveFolderSelectorContract = defineGetSelector( - '/api/tools/onedrive/folder', - onedriveFolderQuerySchema, - z.object({ file: fileOptionSchema.optional() }).passthrough() -) - export const microsoftExcelSheetsSelectorContract = defineGetSelector( '/api/tools/microsoft_excel/sheets', microsoftExcelSheetsQuerySchema, @@ -221,10 +204,6 @@ export type OnedriveFoldersSelectorResponse = ContractJsonResponse< typeof onedriveFoldersSelectorContract > export type OnedriveFoldersSelectorQuery = ContractQuery -export type OnedriveFolderSelectorResponse = ContractJsonResponse< - typeof onedriveFolderSelectorContract -> -export type OnedriveFolderSelectorQuery = ContractQuery export type MicrosoftExcelSheetsSelectorResponse = ContractJsonResponse< typeof microsoftExcelSheetsSelectorContract > diff --git a/apps/sim/lib/api/contracts/selectors/sharepoint.ts b/apps/sim/lib/api/contracts/selectors/sharepoint.ts index fed88488728..14a1ba341fb 100644 --- a/apps/sim/lib/api/contracts/selectors/sharepoint.ts +++ b/apps/sim/lib/api/contracts/selectors/sharepoint.ts @@ -1,13 +1,12 @@ import { z } from 'zod' import { credentialWorkflowBodySchema, - defineGetSelector, definePostSelector, fileOptionSchema, idDisplayNameSchema, optionalString, } from '@/lib/api/contracts/selectors/shared' -import type { ContractBody, ContractJsonResponse, ContractQuery } from '@/lib/api/contracts/types' +import type { ContractBody, ContractJsonResponse } from '@/lib/api/contracts/types' export const sharepointListsBodySchema = credentialWorkflowBodySchema.extend({ siteId: z.string().min(1), @@ -17,17 +16,6 @@ export const sharepointSitesBodySchema = credentialWorkflowBodySchema.extend({ query: optionalString, }) -export const sharepointSiteQuerySchema = z.object({ - credentialId: z.preprocess( - (value) => value ?? '', - z.string().min(1, 'Credential ID and Site ID are required') - ), - siteId: z.preprocess( - (value) => value ?? '', - z.string().min(1, 'Credential ID and Site ID are required') - ), -}) - export const sharepointListsSelectorContract = definePostSelector( '/api/tools/sharepoint/lists', sharepointListsBodySchema, @@ -40,12 +28,6 @@ export const sharepointSitesSelectorContract = definePostSelector( z.object({ files: z.array(fileOptionSchema) }) ) -export const sharepointSiteSelectorContract = defineGetSelector( - '/api/tools/sharepoint/site', - sharepointSiteQuerySchema, - z.object({ site: fileOptionSchema.optional() }).passthrough() -) - export type SharepointListsSelectorResponse = ContractJsonResponse< typeof sharepointListsSelectorContract > @@ -54,7 +36,3 @@ export type SharepointSitesSelectorResponse = ContractJsonResponse< typeof sharepointSitesSelectorContract > export type SharepointSitesSelectorBody = ContractBody -export type SharepointSiteSelectorResponse = ContractJsonResponse< - typeof sharepointSiteSelectorContract -> -export type SharepointSiteSelectorQuery = ContractQuery diff --git a/apps/sim/lib/api/contracts/selectors/wealthbox.ts b/apps/sim/lib/api/contracts/selectors/wealthbox.ts index c66e8b59147..7f2072e36ad 100644 --- a/apps/sim/lib/api/contracts/selectors/wealthbox.ts +++ b/apps/sim/lib/api/contracts/selectors/wealthbox.ts @@ -51,12 +51,6 @@ export const wealthboxItemsSelectorContract = defineGetSelector( wealthboxItemsResponseSchema ) -export const wealthboxItemContract = defineGetSelector( - '/api/tools/wealthbox/item', - wealthboxItemQuerySchema, - wealthboxItemResponseSchema -) - export const wealthboxOAuthItemsContract = defineGetSelector( '/api/auth/oauth/wealthbox/items', wealthboxItemsQuerySchema, @@ -72,6 +66,5 @@ export const wealthboxOAuthItemContract = defineGetSelector( export type WealthboxItemsSelectorResponse = ContractJsonResponse< typeof wealthboxItemsSelectorContract > -export type WealthboxItemResponse = ContractJsonResponse export type WealthboxOAuthItemsResponse = ContractJsonResponse export type WealthboxOAuthItemResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/storage-transfer.ts b/apps/sim/lib/api/contracts/storage-transfer.ts index 3bfe28e8794..9a18a22a9c5 100644 --- a/apps/sim/lib/api/contracts/storage-transfer.ts +++ b/apps/sim/lib/api/contracts/storage-transfer.ts @@ -7,11 +7,7 @@ import { type ContractQueryInput, defineRouteContract, } from '@/lib/api/contracts/types' -import { - FileInputSchema, - RawFileInputArraySchema, - RawFileInputSchema, -} from '@/lib/uploads/utils/file-schemas' +import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' const jsonResponseSchema = z.unknown() @@ -34,25 +30,6 @@ export function requirePasswordOrPrivateKey(schema: S): S { ) as S } -export const boxUploadBodySchema = z.object({ - accessToken: z.string().min(1, 'Access token is required'), - parentFolderId: z.string().min(1, 'Parent folder ID is required'), - file: FileInputSchema.optional().nullable(), - fileContent: z.string().optional().nullable(), - fileName: z.string().optional().nullable(), -}) - -export const dropboxUploadBodySchema = z.object({ - accessToken: z.string().min(1, 'Access token is required'), - path: z.string().min(1, 'Destination path is required'), - file: FileInputSchema.optional().nullable(), - fileContent: z.string().optional().nullable(), - fileName: z.string().optional().nullable(), - mode: z.enum(['add', 'overwrite']).optional().nullable(), - autorename: z.boolean().optional().nullable(), - mute: z.boolean().optional().nullable(), -}) - export const jupyterUploadBodySchema = z.object({ serverUrl: z.string().min(1, 'Server URL is required'), token: z.string().min(1, 'Token is required'), @@ -62,61 +39,6 @@ export const jupyterUploadBodySchema = z.object({ fileName: z.string().optional().nullable(), }) -export const wordpressUploadBodySchema = z.object({ - accessToken: z.string().min(1, 'Access token is required'), - siteId: z.string().min(1, 'Site ID is required'), - file: RawFileInputSchema.optional().nullable(), - filename: z.string().optional().nullable(), - title: z.string().optional().nullable(), - caption: z.string().optional().nullable(), - altText: z.string().optional().nullable(), - description: z.string().optional().nullable(), -}) - -export const sftpListBodySchema = requirePasswordOrPrivateKey( - z.object({ - ...connectionFields, - remotePath: z.string().min(1, 'Remote path is required'), - detailed: z.boolean().default(false), - }) -) - -export const sftpDeleteBodySchema = requirePasswordOrPrivateKey( - z.object({ - ...connectionFields, - remotePath: z.string().min(1, 'Remote path is required'), - recursive: z.boolean().default(false), - }) -) - -export const sftpMkdirBodySchema = requirePasswordOrPrivateKey( - z.object({ - ...connectionFields, - remotePath: z.string().min(1, 'Remote path is required'), - recursive: z.boolean().default(false), - }) -) - -export const sftpDownloadBodySchema = requirePasswordOrPrivateKey( - z.object({ - ...connectionFields, - remotePath: z.string().min(1, 'Remote path is required'), - encoding: z.enum(['utf-8', 'base64']).default('utf-8'), - }) -) - -export const sftpUploadBodySchema = requirePasswordOrPrivateKey( - z.object({ - ...connectionFields, - remotePath: z.string().min(1, 'Remote path is required'), - files: RawFileInputArraySchema.optional().nullable(), - fileContent: z.string().nullish(), - fileName: z.string().nullish(), - overwrite: z.boolean().default(true), - permissions: z.string().nullish(), - }) -) - export const sshCheckCommandExistsBodySchema = requirePasswordOrPrivateKey( z.object({ ...connectionFields, @@ -291,20 +213,6 @@ export const fileExportParamsSchema = z.object({ id: workspaceFileIdSchema, }) -export const boxUploadContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/box/upload', - body: boxUploadBodySchema, - response: { mode: 'json', schema: jsonResponseSchema }, -}) - -export const dropboxUploadContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/dropbox/upload', - body: dropboxUploadBodySchema, - response: { mode: 'json', schema: jsonResponseSchema }, -}) - export const jupyterUploadContract = defineRouteContract({ method: 'POST', path: '/api/tools/jupyter/upload', @@ -312,41 +220,6 @@ export const jupyterUploadContract = defineRouteContract({ response: { mode: 'json', schema: jsonResponseSchema }, }) -export const wordpressUploadContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/wordpress/upload', - body: wordpressUploadBodySchema, - response: { mode: 'json', schema: jsonResponseSchema }, -}) - -export const sftpDeleteContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/sftp/delete', - body: sftpDeleteBodySchema, - response: { mode: 'json', schema: jsonResponseSchema }, -}) - -export const sftpMkdirContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/sftp/mkdir', - body: sftpMkdirBodySchema, - response: { mode: 'json', schema: jsonResponseSchema }, -}) - -export const sftpDownloadContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/sftp/download', - body: sftpDownloadBodySchema, - response: { mode: 'json', schema: jsonResponseSchema }, -}) - -export const sftpUploadContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/sftp/upload', - body: sftpUploadBodySchema, - response: { mode: 'json', schema: jsonResponseSchema }, -}) - export const sshCheckCommandExistsContract = defineRouteContract({ method: 'POST', path: '/api/tools/ssh/check-command-exists', @@ -493,18 +366,8 @@ export const fileStorageStatusContract = defineRouteContract({ export type FileStorageStatusResponse = ContractJsonResponse -export type BoxUploadBody = ContractBodyInput -export type BoxUploadResponse = ContractJsonResponse -export type DropboxUploadBody = ContractBodyInput -export type DropboxUploadResponse = ContractJsonResponse export type JupyterUploadBody = ContractBodyInput export type JupyterUploadResponse = ContractJsonResponse -export type WordPressUploadBody = ContractBodyInput -export type WordPressUploadResponse = ContractJsonResponse -export type SftpDownloadBody = ContractBodyInput -export type SftpUploadBody = ContractBodyInput -export type SftpDeleteBody = ContractBodyInput -export type SftpMkdirBody = ContractBodyInput export type SshCheckCommandExistsBody = ContractBodyInput export type SshCheckFileExistsBody = ContractBodyInput export type SshCreateDirectoryBody = ContractBodyInput diff --git a/apps/sim/lib/api/contracts/tables.ts b/apps/sim/lib/api/contracts/tables.ts index 0a85964f137..2b8d863ec00 100644 --- a/apps/sim/lib/api/contracts/tables.ts +++ b/apps/sim/lib/api/contracts/tables.ts @@ -1524,6 +1524,23 @@ export const batchUpdateTableRowsContract = defineRouteContract({ }, }) +export const updateTableRowsByFilterContract = defineRouteContract({ + method: 'PUT', + path: '/api/table/[tableId]/rows', + params: tableIdParamsSchema, + body: updateRowsByFilterBodySchema, + response: { + mode: 'json', + schema: successResponseSchema( + z.object({ + message: z.string(), + updatedCount: z.number(), + updatedRowIds: z.array(z.string()).optional(), + }) + ), + }, +}) + export const deleteTableRowContract = defineRouteContract({ method: 'DELETE', path: '/api/table/[tableId]/rows/[rowId]', diff --git a/apps/sim/lib/api/contracts/tiktok-tools.ts b/apps/sim/lib/api/contracts/tiktok-tools.ts deleted file mode 100644 index b14c5eda42c..00000000000 --- a/apps/sim/lib/api/contracts/tiktok-tools.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { z } from 'zod' -import { - type ContractBodyInput, - type ContractJsonResponse, - defineRouteContract, -} from '@/lib/api/contracts/types' -import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' - -export const tiktokUploadVideoDraftBodySchema = z.object({ - accessToken: z.string().min(1, 'Access token is required'), - file: RawFileInputSchema, -}) - -export const tiktokUploadVideoDraftResponseSchema = z.object({ - success: z.boolean(), - output: z.object({ publishId: z.string() }).optional(), - error: z.string().optional(), -}) - -export const tiktokUploadVideoDraftContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/tiktok/upload-video-draft', - body: tiktokUploadVideoDraftBodySchema, - response: { mode: 'json', schema: tiktokUploadVideoDraftResponseSchema }, -}) - -export type TikTokUploadVideoDraftBody = ContractBodyInput -export type TikTokUploadVideoDraftResponse = ContractJsonResponse< - typeof tiktokUploadVideoDraftContract -> diff --git a/apps/sim/lib/api/contracts/tools/a2a.ts b/apps/sim/lib/api/contracts/tools/a2a.ts deleted file mode 100644 index e8a51463d02..00000000000 --- a/apps/sim/lib/api/contracts/tools/a2a.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { z } from 'zod' -import { resolvedSecretTraceProvenanceSchema } from '@/lib/api/contracts/primitives' -import { genericToolResponseSchema } from '@/lib/api/contracts/tools/shared' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' -import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' - -const a2aBaseBodySchema = z.object({ - agentUrl: z.string().url('Agent URL must be a valid URL').max(2048), - apiKey: z.string().optional(), -}) - -export const a2aSendMessageBodySchema = a2aBaseBodySchema.extend({ - message: z.string().min(1, 'Message is required'), - data: z.union([z.string(), z.record(z.string(), z.unknown())]).optional(), - files: z.array(RawFileInputSchema).max(20).optional(), - taskId: z.string().optional(), - contextId: z.string().optional(), - [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), -}) - -export const a2aGetTaskBodySchema = a2aBaseBodySchema.extend({ - taskId: z.string().min(1, 'Task ID is required'), - historyLength: z - .number() - .int() - .positive() - .max(1000, 'History length cannot exceed 1000') - .optional(), -}) - -export const a2aCancelTaskBodySchema = a2aBaseBodySchema.extend({ - taskId: z.string().min(1, 'Task ID is required'), -}) - -export const a2aGetAgentCardContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/a2a/get-agent-card', - body: a2aBaseBodySchema, - response: { mode: 'json', schema: genericToolResponseSchema }, -}) - -export const a2aSendMessageContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/a2a/send-message', - body: a2aSendMessageBodySchema, - response: { mode: 'json', schema: genericToolResponseSchema }, -}) - -export const a2aGetTaskContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/a2a/get-task', - body: a2aGetTaskBodySchema, - response: { mode: 'json', schema: genericToolResponseSchema }, -}) - -export const a2aCancelTaskContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/a2a/cancel-task', - body: a2aCancelTaskBodySchema, - response: { mode: 'json', schema: genericToolResponseSchema }, -}) diff --git a/apps/sim/lib/api/contracts/tools/aws/sqs-send.ts b/apps/sim/lib/api/contracts/tools/aws/sqs-send.ts deleted file mode 100644 index f304370c46d..00000000000 --- a/apps/sim/lib/api/contracts/tools/aws/sqs-send.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { z } from 'zod' -import type { - ContractBody, - ContractBodyInput, - ContractJsonResponse, -} from '@/lib/api/contracts/types' -import { defineRouteContract } from '@/lib/api/contracts/types' - -const SendMessageSchema = z.object({ - region: z.string().min(1, 'AWS region is required'), - accessKeyId: z.string().min(1, 'AWS access key ID is required'), - secretAccessKey: z.string().min(1, 'AWS secret access key is required'), - queueUrl: z.string().min(1, 'Queue URL is required'), - messageGroupId: z.string().nullish(), - messageDeduplicationId: z.string().nullish(), - data: z.record(z.string(), z.unknown()).refine((obj) => Object.keys(obj).length > 0, { - message: 'Data object must have at least one field', - }), -}) - -const SendMessageResponseSchema = z.object({ - message: z.string(), - id: z.string().optional(), -}) - -export const awsSqsSendContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/sqs/send', - body: SendMessageSchema, - response: { mode: 'json', schema: SendMessageResponseSchema }, -}) -export type AwsSqsSendRequest = ContractBodyInput -export type AwsSqsSendBody = ContractBody -export type AwsSqsSendResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/tools/aws/sts-assume-role-with-saml.ts b/apps/sim/lib/api/contracts/tools/aws/sts-assume-role-with-saml.ts deleted file mode 100644 index fdf7d9ff984..00000000000 --- a/apps/sim/lib/api/contracts/tools/aws/sts-assume-role-with-saml.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { z } from 'zod' -import type { - ContractBody, - ContractBodyInput, - ContractJsonResponse, -} from '@/lib/api/contracts/types' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { validateAwsRegion } from '@/lib/core/security/input-validation' - -const AssumeRoleWithSAMLSchema = z.object({ - region: z - .string() - .min(1, 'AWS region is required') - .refine((v) => validateAwsRegion(v).isValid, { - message: 'Invalid AWS region format (e.g., us-east-1, eu-west-2)', - }), - roleArn: z.string().min(20, 'Role ARN is required').max(2048), - principalArn: z.string().min(20, 'SAML provider ARN is required').max(2048), - samlAssertion: z - .string() - .min(4, 'SAML assertion is required') - .max(100000, 'SAML assertion must not exceed 100000 characters'), - policy: z.string().max(2048).nullish(), - policyArns: z - .string() - .nullish() - .refine((v) => !v || v.split(',').filter((arn) => arn.trim().length > 0).length <= 10, { - message: 'A maximum of 10 policy ARNs can be provided', - }), - durationSeconds: z.number().int().min(900).max(43200).nullish(), -}) - -const AssumeRoleWithSAMLResponseSchema = z.object({ - accessKeyId: z.string(), - secretAccessKey: z.string(), - sessionToken: z.string(), - expiration: z.string().nullable(), - assumedRoleArn: z.string(), - assumedRoleId: z.string(), - subject: z.string().nullable(), - subjectType: z.string().nullable(), - issuer: z.string().nullable(), - audience: z.string().nullable(), - nameQualifier: z.string().nullable(), - packedPolicySize: z.number().nullable(), - sourceIdentity: z.string().nullable(), -}) - -export const awsStsAssumeRoleWithSAMLContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/sts/assume-role-with-saml', - body: AssumeRoleWithSAMLSchema, - response: { mode: 'json', schema: AssumeRoleWithSAMLResponseSchema }, -}) -export type AwsStsAssumeRoleWithSAMLRequest = ContractBodyInput< - typeof awsStsAssumeRoleWithSAMLContract -> -export type AwsStsAssumeRoleWithSAMLBody = ContractBody -export type AwsStsAssumeRoleWithSAMLResponse = ContractJsonResponse< - typeof awsStsAssumeRoleWithSAMLContract -> diff --git a/apps/sim/lib/api/contracts/tools/aws/sts-assume-role-with-web-identity.ts b/apps/sim/lib/api/contracts/tools/aws/sts-assume-role-with-web-identity.ts deleted file mode 100644 index 9a08299b9ce..00000000000 --- a/apps/sim/lib/api/contracts/tools/aws/sts-assume-role-with-web-identity.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { z } from 'zod' -import type { - ContractBody, - ContractBodyInput, - ContractJsonResponse, -} from '@/lib/api/contracts/types' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { validateAwsRegion } from '@/lib/core/security/input-validation' - -const AssumeRoleWithWebIdentitySchema = z.object({ - region: z - .string() - .min(1, 'AWS region is required') - .refine((v) => validateAwsRegion(v).isValid, { - message: 'Invalid AWS region format (e.g., us-east-1, eu-west-2)', - }), - roleArn: z.string().min(20, 'Role ARN is required').max(2048), - roleSessionName: z.string().min(2, 'Role session name is required').max(64), - webIdentityToken: z - .string() - .min(4, 'Web identity token is required') - .max(20000, 'Web identity token must not exceed 20000 characters'), - providerId: z.string().min(4).max(2048).nullish(), - policy: z.string().max(2048).nullish(), - policyArns: z - .string() - .nullish() - .refine((v) => !v || v.split(',').filter((arn) => arn.trim().length > 0).length <= 10, { - message: 'A maximum of 10 policy ARNs can be provided', - }), - durationSeconds: z.number().int().min(900).max(43200).nullish(), -}) - -const AssumeRoleWithWebIdentityResponseSchema = z.object({ - accessKeyId: z.string(), - secretAccessKey: z.string(), - sessionToken: z.string(), - expiration: z.string().nullable(), - assumedRoleArn: z.string(), - assumedRoleId: z.string(), - subjectFromWebIdentityToken: z.string(), - audience: z.string().nullable(), - provider: z.string().nullable(), - packedPolicySize: z.number().nullable(), - sourceIdentity: z.string().nullable(), -}) - -export const awsStsAssumeRoleWithWebIdentityContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/sts/assume-role-with-web-identity', - body: AssumeRoleWithWebIdentitySchema, - response: { mode: 'json', schema: AssumeRoleWithWebIdentityResponseSchema }, -}) -export type AwsStsAssumeRoleWithWebIdentityRequest = ContractBodyInput< - typeof awsStsAssumeRoleWithWebIdentityContract -> -export type AwsStsAssumeRoleWithWebIdentityBody = ContractBody< - typeof awsStsAssumeRoleWithWebIdentityContract -> -export type AwsStsAssumeRoleWithWebIdentityResponse = ContractJsonResponse< - typeof awsStsAssumeRoleWithWebIdentityContract -> diff --git a/apps/sim/lib/api/contracts/tools/aws/sts-assume-role.ts b/apps/sim/lib/api/contracts/tools/aws/sts-assume-role.ts deleted file mode 100644 index f02dfe6e566..00000000000 --- a/apps/sim/lib/api/contracts/tools/aws/sts-assume-role.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { isRecordLike } from '@sim/utils/object' -import { z } from 'zod' -import type { - ContractBody, - ContractBodyInput, - ContractJsonResponse, -} from '@/lib/api/contracts/types' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { validateAwsRegion } from '@/lib/core/security/input-validation' - -const AssumeRoleSchema = z.object({ - region: z - .string() - .min(1, 'AWS region is required') - .refine((v) => validateAwsRegion(v).isValid, { - message: 'Invalid AWS region format (e.g., us-east-1, eu-west-2)', - }), - accessKeyId: z.string().min(1, 'AWS access key ID is required'), - secretAccessKey: z.string().min(1, 'AWS secret access key is required'), - roleArn: z.string().min(1, 'Role ARN is required'), - roleSessionName: z.string().min(1, 'Role session name is required'), - durationSeconds: z.number().int().min(900).max(43200).nullish(), - policy: z.string().max(2048).nullish(), - externalId: z.string().min(2).max(1224).nullish(), - serialNumber: z.string().nullish(), - tokenCode: z.string().nullish(), - policyArns: z - .string() - .nullish() - .refine((v) => !v || v.split(',').filter((arn) => arn.trim().length > 0).length <= 10, { - message: 'A maximum of 10 policy ARNs can be provided', - }), - tags: z - .string() - .nullish() - .refine( - (v) => { - if (!v) return true - try { - const parsed = JSON.parse(v) - return isRecordLike(parsed) - } catch { - return false - } - }, - { message: 'tags must be a valid JSON object string' } - ), - transitiveTagKeys: z - .string() - .nullish() - .refine((v) => !v || v.split(',').filter((key) => key.trim().length > 0).length <= 50, { - message: 'A maximum of 50 transitive tag keys can be provided', - }), -}) - -const AssumeRoleResponseSchema = z.object({ - accessKeyId: z.string(), - secretAccessKey: z.string(), - sessionToken: z.string(), - expiration: z.string().nullable(), - assumedRoleArn: z.string(), - assumedRoleId: z.string(), - packedPolicySize: z.number().nullable(), - sourceIdentity: z.string().nullable(), -}) - -export const awsStsAssumeRoleContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/sts/assume-role', - body: AssumeRoleSchema, - response: { mode: 'json', schema: AssumeRoleResponseSchema }, -}) -export type AwsStsAssumeRoleRequest = ContractBodyInput -export type AwsStsAssumeRoleBody = ContractBody -export type AwsStsAssumeRoleResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/tools/aws/sts-get-access-key-info.ts b/apps/sim/lib/api/contracts/tools/aws/sts-get-access-key-info.ts deleted file mode 100644 index 541dbbf53d3..00000000000 --- a/apps/sim/lib/api/contracts/tools/aws/sts-get-access-key-info.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { z } from 'zod' -import type { - ContractBody, - ContractBodyInput, - ContractJsonResponse, -} from '@/lib/api/contracts/types' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { validateAwsRegion } from '@/lib/core/security/input-validation' - -const GetAccessKeyInfoSchema = z.object({ - region: z - .string() - .min(1, 'AWS region is required') - .refine((v) => validateAwsRegion(v).isValid, { - message: 'Invalid AWS region format (e.g., us-east-1, eu-west-2)', - }), - accessKeyId: z.string().min(1, 'AWS access key ID is required'), - secretAccessKey: z.string().min(1, 'AWS secret access key is required'), - targetAccessKeyId: z.string().min(1, 'Target access key ID is required'), -}) - -const GetAccessKeyInfoResponseSchema = z.object({ - account: z.string(), -}) - -export const awsStsGetAccessKeyInfoContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/sts/get-access-key-info', - body: GetAccessKeyInfoSchema, - response: { mode: 'json', schema: GetAccessKeyInfoResponseSchema }, -}) -export type AwsStsGetAccessKeyInfoRequest = ContractBodyInput -export type AwsStsGetAccessKeyInfoBody = ContractBody -export type AwsStsGetAccessKeyInfoResponse = ContractJsonResponse< - typeof awsStsGetAccessKeyInfoContract -> diff --git a/apps/sim/lib/api/contracts/tools/aws/sts-get-caller-identity.ts b/apps/sim/lib/api/contracts/tools/aws/sts-get-caller-identity.ts deleted file mode 100644 index b65cb6be979..00000000000 --- a/apps/sim/lib/api/contracts/tools/aws/sts-get-caller-identity.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { z } from 'zod' -import type { - ContractBody, - ContractBodyInput, - ContractJsonResponse, -} from '@/lib/api/contracts/types' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { validateAwsRegion } from '@/lib/core/security/input-validation' - -const GetCallerIdentitySchema = z.object({ - region: z - .string() - .min(1, 'AWS region is required') - .refine((v) => validateAwsRegion(v).isValid, { - message: 'Invalid AWS region format (e.g., us-east-1, eu-west-2)', - }), - accessKeyId: z.string().min(1, 'AWS access key ID is required'), - secretAccessKey: z.string().min(1, 'AWS secret access key is required'), -}) - -const GetCallerIdentityResponseSchema = z.object({ - account: z.string(), - arn: z.string(), - userId: z.string(), -}) - -export const awsStsGetCallerIdentityContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/sts/get-caller-identity', - body: GetCallerIdentitySchema, - response: { mode: 'json', schema: GetCallerIdentityResponseSchema }, -}) -export type AwsStsGetCallerIdentityRequest = ContractBodyInput< - typeof awsStsGetCallerIdentityContract -> -export type AwsStsGetCallerIdentityBody = ContractBody -export type AwsStsGetCallerIdentityResponse = ContractJsonResponse< - typeof awsStsGetCallerIdentityContract -> diff --git a/apps/sim/lib/api/contracts/tools/aws/sts-get-session-token.ts b/apps/sim/lib/api/contracts/tools/aws/sts-get-session-token.ts deleted file mode 100644 index ce33e96ed6b..00000000000 --- a/apps/sim/lib/api/contracts/tools/aws/sts-get-session-token.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { z } from 'zod' -import type { - ContractBody, - ContractBodyInput, - ContractJsonResponse, -} from '@/lib/api/contracts/types' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { validateAwsRegion } from '@/lib/core/security/input-validation' - -const GetSessionTokenSchema = z.object({ - region: z - .string() - .min(1, 'AWS region is required') - .refine((v) => validateAwsRegion(v).isValid, { - message: 'Invalid AWS region format (e.g., us-east-1, eu-west-2)', - }), - accessKeyId: z.string().min(1, 'AWS access key ID is required'), - secretAccessKey: z.string().min(1, 'AWS secret access key is required'), - durationSeconds: z.number().int().min(900).max(129600).nullish(), - serialNumber: z.string().nullish(), - tokenCode: z.string().nullish(), -}) - -const GetSessionTokenResponseSchema = z.object({ - accessKeyId: z.string(), - secretAccessKey: z.string(), - sessionToken: z.string(), - expiration: z.string().nullable(), -}) - -export const awsStsGetSessionTokenContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/sts/get-session-token', - body: GetSessionTokenSchema, - response: { mode: 'json', schema: GetSessionTokenResponseSchema }, -}) -export type AwsStsGetSessionTokenRequest = ContractBodyInput -export type AwsStsGetSessionTokenBody = ContractBody -export type AwsStsGetSessionTokenResponse = ContractJsonResponse< - typeof awsStsGetSessionTokenContract -> diff --git a/apps/sim/lib/api/contracts/tools/azure_data_explorer.ts b/apps/sim/lib/api/contracts/tools/azure_data_explorer.ts deleted file mode 100644 index 2abfa8c32fe..00000000000 --- a/apps/sim/lib/api/contracts/tools/azure_data_explorer.ts +++ /dev/null @@ -1,168 +0,0 @@ -import { isPrivateIpHost } from '@sim/security/ssrf' -import { z } from 'zod' -import { genericToolResponseSchema } from '@/lib/api/contracts/tools/shared' -import { defineRouteContract } from '@/lib/api/contracts/types' - -/** - * Kusto service domains Sim will talk to, each paired with the Microsoft Entra - * authority that issues tokens for it. - * - * A cluster URI is user-supplied, so the proxy is pinned to the documented Azure - * Data Explorer and Fabric Eventhouse domains rather than trusting any HTTPS - * host. Host and authority are declared together on purpose: a sovereign cloud - * authenticates against its own isolated Entra instance, so accepting a cluster - * host without its authority would pass validation and then fail to get a token. - * - * Apex hosts match as well as subdomains, because the documented token - * audiences (`https://api.kusto.windows.net`, - * `https://kusto.fabric.microsoft.com`) sit at the apex. - * - * Every entry is a domain Microsoft documents: the Kusto connection-string - * reference for `kusto.windows.net`, the national-cloud endpoint tables for the - * two sovereign domains, and the Fabric KQL-database REST reference for - * `kusto.fabric.microsoft.com` (both its `queryServiceUri` and - * `ingestionServiceUri` sit under it). Do not add a host without one. - */ -const KUSTO_CLOUDS = [ - { hostSuffix: 'kusto.windows.net', authority: 'https://login.microsoftonline.com' }, - { hostSuffix: 'kusto.fabric.microsoft.com', authority: 'https://login.microsoftonline.com' }, - { hostSuffix: 'kusto.usgovcloudapi.net', authority: 'https://login.microsoftonline.us' }, - { hostSuffix: 'kusto.chinacloudapi.cn', authority: 'https://login.partner.microsoftonline.cn' }, -] as const - -const ALLOWED_CLUSTER_HOSTS = KUSTO_CLOUDS.map((cloud) => cloud.hostSuffix).join(', ') - -function matchKustoCloud(host: string): (typeof KUSTO_CLOUDS)[number] | null { - return ( - KUSTO_CLOUDS.find( - (cloud) => host === cloud.hostSuffix || host.endsWith(`.${cloud.hostSuffix}`) - ) ?? null - ) -} - -/** - * Resolves the Entra authority that issues tokens for a cluster host. Callers - * pass a host already accepted by {@link checkAzureDataExplorerClusterUri}, so - * an unmatched host here means the two fell out of sync and is a bug, not input. - */ -export function resolveEntraAuthority(clusterHost: string): string { - const cloud = matchKustoCloud(clusterHost.toLowerCase()) - if (!cloud) { - throw new Error(`No Microsoft Entra authority is configured for cluster host ${clusterHost}`) - } - return cloud.authority -} - -export function checkAzureDataExplorerClusterUri( - rawUrl: string, - label = 'clusterUri' -): { ok: true; url: URL } | { ok: false; message: string } { - let parsed: URL - try { - parsed = new URL(rawUrl) - } catch { - return { - ok: false, - message: `${label} must be a full URL (e.g., https://mycluster.eastus.kusto.windows.net)`, - } - } - if (parsed.protocol !== 'https:') { - return { ok: false, message: `${label} must use https://` } - } - const host = parsed.hostname.toLowerCase() - if (isPrivateIpHost(host)) { - return { ok: false, message: `${label} host is not allowed (private/loopback range)` } - } - if (!matchKustoCloud(host)) { - return { - ok: false, - message: `${label} host must be an Azure Data Explorer or Fabric Eventhouse endpoint (${ALLOWED_CLUSTER_HOSTS})`, - } - } - return { ok: true, url: parsed } -} - -export function assertSafeAzureDataExplorerClusterUri(rawUrl: string, label?: string): URL { - const result = checkAzureDataExplorerClusterUri(rawUrl, label) - if (!result.ok) throw new Error(result.message) - return result.url -} - -/** - * The exact character set Kusto documents for an identifier: letters, digits, - * underscores, spaces, dots, and dashes, 1-1024 characters. An allowlist rather - * than a denylist, so nothing that could terminate `["..."]` name quoting — or - * that Kusto would reject anyway — reaches a command string. - */ -const entityNameSchema = z - .string() - .trim() - .min(1, 'name is required') - .max(1024, 'name must be at most 1024 characters') - .regex( - /^[\p{L}\p{N}_ .-]+$/u, - 'name may contain only letters, digits, underscores, spaces, dots, and dashes' - ) - -/** A Microsoft Entra tenant is addressed by GUID or by verified domain name. */ -const tenantIdSchema = z - .string() - .trim() - .min(1, 'tenantId is required') - .max(253, 'tenantId is too long') - .regex( - /^[A-Za-z0-9][A-Za-z0-9.-]*$/, - 'tenantId must be a GUID or a domain name (e.g., contoso.onmicrosoft.com)' - ) - -export const azureDataExplorerEndpointSchema = z.enum(['query', 'mgmt']) - -export const azureDataExplorerProxyBodySchema = z - .object({ - clusterUri: z.string().min(1, 'clusterUri is required'), - tenantId: tenantIdSchema, - clientId: z.string().min(1, 'clientId is required'), - clientSecret: z.string().min(1, 'clientSecret is required'), - /** - * Microsoft Entra token audience. Defaults to the cluster's own origin, which - * is the form the Kusto REST reference uses for client-credential tokens. - */ - resource: z.string().optional(), - endpoint: azureDataExplorerEndpointSchema, - database: entityNameSchema.optional(), - csl: z.string().min(1, 'csl is required').max(1_000_000, 'csl is too long'), - properties: z.record(z.string(), z.unknown()).optional(), - /** Sends `x-ms-readonly`, which makes the cluster reject data-changing requests. */ - readOnly: z.boolean().optional(), - }) - .superRefine((req, ctx) => { - const clusterCheck = checkAzureDataExplorerClusterUri(req.clusterUri) - if (!clusterCheck.ok) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ['clusterUri'], - message: clusterCheck.message, - }) - } - if (req.resource === undefined) return - const resourceCheck = checkAzureDataExplorerClusterUri(req.resource, 'resource') - if (!resourceCheck.ok) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ['resource'], - message: resourceCheck.message, - }) - } - }) - -export type AzureDataExplorerProxyRequest = z.infer - -export const azureDataExplorerProxyContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/azure_data_explorer/proxy', - body: azureDataExplorerProxyBodySchema, - response: { - mode: 'json', - schema: genericToolResponseSchema, - }, -}) diff --git a/apps/sim/lib/api/contracts/tools/brex.ts b/apps/sim/lib/api/contracts/tools/brex.ts deleted file mode 100644 index 80ae1fccc31..00000000000 --- a/apps/sim/lib/api/contracts/tools/brex.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { z } from 'zod' -import type { ContractBodyInput, ContractJsonResponse } from '@/lib/api/contracts/types' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' - -export const brexUploadReceiptBodySchema = z.object({ - apiKey: z - .string() - .min(1, 'API key is required') - .max(512, 'API key is too long') - .regex(/^[\x21-\x7e]+$/, 'API key contains invalid characters'), - expenseId: z - .string() - .trim() - .min(1, 'Expense ID cannot be empty') - .max(255, 'Expense ID must be at most 255 characters') - .optional(), - file: RawFileInputSchema, - receiptName: z - .string() - .trim() - .min(1, 'Receipt name cannot be empty') - .max(255, 'Receipt name must be at most 255 characters') - .optional(), -}) - -export const brexUploadReceiptResponseSchema = z.object({ - success: z.literal(true), - output: z.object({ - receiptId: z.string(), - receiptName: z.string(), - expenseId: z.string().nullable(), - }), -}) - -export const brexUploadReceiptContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/brex/upload-receipt', - body: brexUploadReceiptBodySchema, - response: { mode: 'json', schema: brexUploadReceiptResponseSchema }, -}) - -export type BrexUploadReceiptBody = ContractBodyInput -export type BrexUploadReceiptRouteResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/tools/buffer.ts b/apps/sim/lib/api/contracts/tools/buffer.ts deleted file mode 100644 index 948b9d708d7..00000000000 --- a/apps/sim/lib/api/contracts/tools/buffer.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { z } from 'zod' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' - -/** - * Internal contracts for the Buffer create-post / edit-post routes. The Buffer - * GraphQL API attaches media by publicly accessible URL only, so the tools post - * a JSON envelope to these internal routes, which verify access to any - * referenced file, mint a short-lived presigned URL for it, and forward the - * mutation to Buffer. - */ - -const postErrorSchema = z.object({ - message: z.string(), - supportUrl: z.string().nullable(), - rawError: z.string().nullable(), -}) - -const postAssetSchema = z.object({ - id: z.string().nullable(), - type: z.string(), - mimeType: z.string(), - source: z.string(), - thumbnail: z.string(), -}) - -const postSchema = z.object({ - id: z.string(), - text: z.string(), - status: z.string(), - via: z.string(), - channelId: z.string(), - channelService: z.string(), - schedulingType: z.string().nullable(), - shareMode: z.string(), - isCustomScheduled: z.boolean(), - sharedNow: z.boolean(), - createdAt: z.string(), - updatedAt: z.string(), - dueAt: z.string().nullable(), - sentAt: z.string().nullable(), - externalLink: z.string().nullable(), - error: postErrorSchema.nullable(), - assets: z.array(postAssetSchema), -}) - -const postRouteResponseSchema = z.object({ - success: z.boolean(), - output: z.object({ post: postSchema }).optional(), - error: z.string().optional(), -}) - -const postSharedFields = { - apiKey: z.string().min(1, 'API key is required'), - text: z.string().max(50000, 'text is too long').optional().nullable(), - mode: z.enum(['addToQueue', 'shareNext', 'shareNow', 'customScheduled']), - schedulingType: z.enum(['automatic', 'notification']).default('automatic'), - dueAt: z - .string() - .datetime({ offset: true, message: 'dueAt must be an ISO 8601 timestamp' }) - .optional() - .nullable(), - saveToDraft: z.boolean().optional().nullable(), - media: FileInputSchema.optional().nullable(), - mediaType: z.enum(['auto', 'image', 'video']).default('auto'), - mediaAltText: z.string().max(1000, 'mediaAltText is too long').optional().nullable(), -} - -/** - * Cross-field rule shared by create and edit: a custom-scheduled post needs a - * publish time. - */ -function validateDueAt(body: { mode: string; dueAt?: string | null }, ctx: z.RefinementCtx): void { - if (body.mode === 'customScheduled' && !body.dueAt) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ['dueAt'], - message: 'dueAt is required when mode is customScheduled', - }) - } -} - -export const bufferCreatePostBodySchema = z - .object({ - ...postSharedFields, - channelId: z.string().min(1, 'channelId is required'), - }) - .superRefine((body, ctx) => { - validateDueAt(body, ctx) - if (!body.text?.trim() && !body.media) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ['text'], - message: 'Either text or media is required', - }) - } - }) - -export type BufferCreatePostBody = z.input - -export const bufferCreatePostContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/buffer/create-post', - body: bufferCreatePostBodySchema, - response: { mode: 'json', schema: postRouteResponseSchema }, -}) - -export const bufferEditPostBodySchema = z - .object({ - ...postSharedFields, - postId: z.string().min(1, 'postId is required'), - }) - .superRefine(validateDueAt) - -export type BufferEditPostBody = z.input - -export const bufferEditPostContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/buffer/edit-post', - body: bufferEditPostBodySchema, - response: { mode: 'json', schema: postRouteResponseSchema }, -}) diff --git a/apps/sim/lib/api/contracts/tools/clickup.ts b/apps/sim/lib/api/contracts/tools/clickup.ts deleted file mode 100644 index cebc498265f..00000000000 --- a/apps/sim/lib/api/contracts/tools/clickup.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { z } from 'zod' -import type { ContractBody, ContractJsonResponse } from '@/lib/api/contracts/types' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' - -export const clickupUploadAttachmentBodySchema = z.object({ - accessToken: z.string().min(1, 'Access token is required'), - taskId: z.string().min(1, 'Task ID is required'), - file: FileInputSchema, -}) - -const clickupAttachmentSchema = z.object({ - id: z.string(), - version: z.string().nullable(), - title: z.string().nullable(), - extension: z.string().nullable(), - url: z.string().nullable(), - date: z.number().nullable(), - thumbnailSmall: z.string().nullable(), - thumbnailLarge: z.string().nullable(), -}) - -const clickupUserFileSchema = z - .object({ - id: z.string(), - name: z.string(), - url: z.string(), - size: z.number(), - type: z.string(), - key: z.string(), - }) - .passthrough() - -const clickupUploadAttachmentResponseSchema = z.object({ - success: z.literal(true), - output: z.object({ - attachment: clickupAttachmentSchema, - files: z.array(clickupUserFileSchema), - }), -}) - -export const clickupUploadAttachmentContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickup/upload-attachment', - body: clickupUploadAttachmentBodySchema, - response: { mode: 'json', schema: clickupUploadAttachmentResponseSchema }, -}) - -export type ClickUpUploadAttachmentBody = ContractBody -export type ClickUpUploadAttachmentApiResponse = ContractJsonResponse< - typeof clickupUploadAttachmentContract -> diff --git a/apps/sim/lib/api/contracts/tools/communication/discord.ts b/apps/sim/lib/api/contracts/tools/communication/discord.ts deleted file mode 100644 index 2f798f389c0..00000000000 --- a/apps/sim/lib/api/contracts/tools/communication/discord.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { z } from 'zod' -import { - defineCommunicationToolContract, - discordBotTokenSelectorSchema, - discordIdSchema, - discordRequiredIdSchema, -} from '@/lib/api/contracts/tools/communication/shared' -import type { ContractBodyInput, ContractJsonResponse } from '@/lib/api/contracts/types' -import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' - -export const discordSendMessageBodySchema = z.object({ - botToken: z.string().min(1, 'Bot token is required'), - channelId: z.string().min(1, 'Channel ID is required'), - content: z.string().optional().nullable(), - files: RawFileInputArraySchema.optional().nullable(), -}) - -export const discordChannelsBodySchema = z.object({ - botToken: discordBotTokenSelectorSchema, - serverId: discordRequiredIdSchema('Server ID is required'), - channelId: discordIdSchema.optional().nullable(), -}) - -export const discordServersBodySchema = z.object({ - botToken: discordBotTokenSelectorSchema, - serverId: discordIdSchema.optional().nullable(), -}) - -export const discordSendMessageContract = defineCommunicationToolContract( - '/api/tools/discord/send-message', - discordSendMessageBodySchema -) -export const discordChannelsContract = defineCommunicationToolContract( - '/api/tools/discord/channels', - discordChannelsBodySchema -) -export const discordServersContract = defineCommunicationToolContract( - '/api/tools/discord/servers', - discordServersBodySchema -) - -export type DiscordSendMessageBody = ContractBodyInput -export type DiscordChannelsBody = ContractBodyInput -export type DiscordServersBody = ContractBodyInput - -export type DiscordSendMessageResponse = ContractJsonResponse -export type DiscordChannelsResponse = ContractJsonResponse -export type DiscordServersResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/tools/communication/email.ts b/apps/sim/lib/api/contracts/tools/communication/email.ts deleted file mode 100644 index 6bdfbf6d289..00000000000 --- a/apps/sim/lib/api/contracts/tools/communication/email.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { z } from 'zod' -import { defineCommunicationToolContract } from '@/lib/api/contracts/tools/communication/shared' -import type { ContractBodyInput, ContractJsonResponse } from '@/lib/api/contracts/types' -import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' - -export const smtpSendBodySchema = z.object({ - smtpHost: z.string().min(1, 'SMTP host is required'), - smtpPort: z.number().min(1).max(65535, 'Port must be between 1 and 65535'), - smtpUsername: z.string().min(1, 'SMTP username is required'), - smtpPassword: z.string().min(1, 'SMTP password is required'), - smtpSecure: z.enum(['TLS', 'SSL', 'None']), - from: z.string().email('Invalid from email address').min(1, 'From address is required'), - to: z.string().min(1, 'To email is required'), - subject: z.string().min(1, 'Subject is required'), - body: z.string().min(1, 'Email body is required'), - contentType: z.enum(['text', 'html']).optional().nullable(), - fromName: z.string().optional().nullable(), - cc: z.string().optional().nullable(), - bcc: z.string().optional().nullable(), - replyTo: z.string().optional().nullable(), - attachments: RawFileInputArraySchema.optional().nullable(), -}) - -export const sendGridSendMailBodySchema = z.object({ - apiKey: z.string().min(1, 'API key is required'), - from: z.string().min(1, 'From email is required'), - fromName: z.string().optional().nullable(), - to: z.string().min(1, 'To email is required'), - toName: z.string().optional().nullable(), - subject: z.string().optional().nullable(), - content: z.string().optional().nullable(), - contentType: z.string().optional().nullable(), - cc: z.string().optional().nullable(), - bcc: z.string().optional().nullable(), - replyTo: z.string().optional().nullable(), - replyToName: z.string().optional().nullable(), - templateId: z.string().optional().nullable(), - dynamicTemplateData: z.unknown().optional().nullable(), - attachments: RawFileInputArraySchema.optional().nullable(), -}) - -export const smtpSendContract = defineCommunicationToolContract( - '/api/tools/smtp/send', - smtpSendBodySchema -) -export const sendGridSendMailContract = defineCommunicationToolContract( - '/api/tools/sendgrid/send-mail', - sendGridSendMailBodySchema -) - -export type SmtpSendBody = ContractBodyInput -export type SendGridSendMailBody = ContractBodyInput - -export type SmtpSendResponse = ContractJsonResponse -export type SendGridSendMailResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/tools/communication/index.ts b/apps/sim/lib/api/contracts/tools/communication/index.ts index b8e0c3034f7..a0ee32b46ab 100644 --- a/apps/sim/lib/api/contracts/tools/communication/index.ts +++ b/apps/sim/lib/api/contracts/tools/communication/index.ts @@ -1,5 +1,3 @@ -export * from '@/lib/api/contracts/tools/communication/discord' -export * from '@/lib/api/contracts/tools/communication/email' export * from '@/lib/api/contracts/tools/communication/messaging' export * from '@/lib/api/contracts/tools/communication/shared' export * from '@/lib/api/contracts/tools/communication/slack' diff --git a/apps/sim/lib/api/contracts/tools/communication/messaging.ts b/apps/sim/lib/api/contracts/tools/communication/messaging.ts index 207afb82826..0a107455e3a 100644 --- a/apps/sim/lib/api/contracts/tools/communication/messaging.ts +++ b/apps/sim/lib/api/contracts/tools/communication/messaging.ts @@ -1,57 +1,6 @@ import { z } from 'zod' -import { defineCommunicationToolContract } from '@/lib/api/contracts/tools/communication/shared' -import type { ContractBodyInput, ContractJsonResponse } from '@/lib/api/contracts/types' -import { FileInputSchema, RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' export const smsSendBodySchema = z.object({ to: z.string().min(1, 'To phone number is required'), body: z.string().min(1, 'SMS body is required'), }) - -export const telegramSendDocumentBodySchema = z.object({ - botToken: z.string().min(1, 'Bot token is required'), - chatId: z.string().min(1, 'Chat ID is required'), - files: RawFileInputArraySchema.optional().nullable(), - caption: z.string().optional().nullable(), -}) - -export const twilioGetRecordingBodySchema = z.object({ - accountSid: z.string().min(1, 'Account SID is required'), - authToken: z.string().min(1, 'Auth token is required'), - recordingSid: z.string().min(1, 'Recording SID is required'), -}) - -export const linqUploadAttachmentBodySchema = z.object({ - apiKey: z.string().min(1, 'API key is required'), - file: FileInputSchema.optional().nullable(), - fileContent: z.string().optional().nullable(), - filename: z.string().min(1).max(1024).optional().nullable(), - contentType: z.string().min(1).max(255).optional().nullable(), -}) - -export const smsSendContract = defineCommunicationToolContract( - '/api/tools/sms/send', - smsSendBodySchema -) -export const telegramSendDocumentContract = defineCommunicationToolContract( - '/api/tools/telegram/send-document', - telegramSendDocumentBodySchema -) -export const twilioGetRecordingContract = defineCommunicationToolContract( - '/api/tools/twilio/get-recording', - twilioGetRecordingBodySchema -) -export const linqUploadAttachmentContract = defineCommunicationToolContract( - '/api/tools/linq/upload', - linqUploadAttachmentBodySchema -) - -export type SmsSendBody = ContractBodyInput -export type TelegramSendDocumentBody = ContractBodyInput -export type TwilioGetRecordingBody = ContractBodyInput -export type LinqUploadAttachmentBody = ContractBodyInput - -export type SmsSendResponse = ContractJsonResponse -export type TelegramSendDocumentResponse = ContractJsonResponse -export type TwilioGetRecordingResponse = ContractJsonResponse -export type LinqUploadAttachmentResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/tools/communication/shared.ts b/apps/sim/lib/api/contracts/tools/communication/shared.ts index 216610e42c7..71498eaa7d4 100644 --- a/apps/sim/lib/api/contracts/tools/communication/shared.ts +++ b/apps/sim/lib/api/contracts/tools/communication/shared.ts @@ -3,18 +3,6 @@ import { defineRouteContract } from '@/lib/api/contracts/types' export const communicationToolResponseSchema = z.unknown() export const slackBlocksSchema = z.array(z.record(z.string(), z.unknown())) -export const discordIdSchema = z.union([z.string(), z.number()]) - -export const discordRequiredIdSchema = (message: string) => - z.preprocess( - (value) => (value === null || value === undefined ? '' : value), - discordIdSchema.refine((value) => value !== '', { message }) - ) - -export const discordBotTokenSelectorSchema = z.preprocess( - (value) => (value === null || value === undefined ? '' : value), - z.string().min(1, 'Bot token is required') -) export const defineCommunicationToolContract = ( path: string, diff --git a/apps/sim/lib/api/contracts/tools/cursor.ts b/apps/sim/lib/api/contracts/tools/cursor.ts deleted file mode 100644 index c8e04fb4296..00000000000 --- a/apps/sim/lib/api/contracts/tools/cursor.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { z } from 'zod' -import { genericToolResponseSchema } from '@/lib/api/contracts/tools/shared' -import { defineRouteContract } from '@/lib/api/contracts/types' - -export const cursorDownloadArtifactBodySchema = z.object({ - apiKey: z.string().min(1, 'API key is required'), - agentId: z.string().min(1, 'Agent ID is required'), - path: z.string().min(1, 'Artifact path is required'), -}) - -export const cursorDownloadArtifactContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/cursor/download-artifact', - body: cursorDownloadArtifactBodySchema, - response: { - mode: 'json', - schema: genericToolResponseSchema, - }, -}) diff --git a/apps/sim/lib/api/contracts/tools/databases/clickhouse.ts b/apps/sim/lib/api/contracts/tools/databases/clickhouse.ts deleted file mode 100644 index 57158d3de8e..00000000000 --- a/apps/sim/lib/api/contracts/tools/databases/clickhouse.ts +++ /dev/null @@ -1,319 +0,0 @@ -import { z } from 'zod' -import { - introspectionResponseSchema, - nonEmptyRecordSchema, - sqlRowsResponseSchema, -} from '@/lib/api/contracts/tools/databases/shared' -import { - type ContractBodyInput, - type ContractJsonResponse, - defineRouteContract, -} from '@/lib/api/contracts/types' - -const secureFlagSchema = z - .union([z.boolean(), z.string()]) - .transform((value) => (typeof value === 'string' ? value.toLowerCase() === 'true' : value)) - .default(true) - -export const clickhouseConnectionBodySchema = z.object({ - host: z.string().min(1, 'Host is required'), - port: z.coerce.number().int().positive('Port must be a positive integer'), - database: z.string().min(1, 'Database name is required'), - username: z.string().min(1, 'Username is required'), - password: z.string().default(''), - secure: secureFlagSchema, -}) - -export const clickhouseQueryBodySchema = clickhouseConnectionBodySchema.extend({ - query: z.string().min(1, 'Query is required'), -}) - -export const clickhouseExecuteBodySchema = clickhouseQueryBodySchema - -export const clickhouseInsertBodySchema = clickhouseConnectionBodySchema.extend({ - table: z.string().min(1, 'Table name is required'), - data: nonEmptyRecordSchema('Data object cannot be empty'), -}) - -export const clickhouseUpdateBodySchema = clickhouseConnectionBodySchema.extend({ - table: z.string().min(1, 'Table name is required'), - data: nonEmptyRecordSchema('Data object cannot be empty'), - where: z.string().min(1, 'WHERE clause is required'), -}) - -export const clickhouseDeleteBodySchema = clickhouseConnectionBodySchema.extend({ - table: z.string().min(1, 'Table name is required'), - where: z.string().min(1, 'WHERE clause is required'), -}) - -export const clickhouseIntrospectBodySchema = clickhouseConnectionBodySchema - -export const clickhouseQueryContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/query', - body: clickhouseQueryBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const clickhouseExecuteContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/execute', - body: clickhouseExecuteBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const clickhouseInsertContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/insert', - body: clickhouseInsertBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const clickhouseUpdateContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/update', - body: clickhouseUpdateBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const clickhouseDeleteContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/delete', - body: clickhouseDeleteBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const clickhouseIntrospectContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/introspect', - body: clickhouseIntrospectBodySchema, - response: { mode: 'json', schema: introspectionResponseSchema }, -}) - -const clickhouseTableBodySchema = clickhouseConnectionBodySchema.extend({ - table: z.string().min(1, 'Table name is required'), -}) - -const clickhouseCountResponseSchema = z.object({ - message: z.string(), - count: z.number(), -}) - -const clickhouseDdlResponseSchema = z.object({ - message: z.string(), - ddl: z.string(), -}) - -export const clickhouseListDatabasesBodySchema = clickhouseConnectionBodySchema -export const clickhouseListTablesBodySchema = clickhouseConnectionBodySchema -export const clickhouseDescribeTableBodySchema = clickhouseTableBodySchema -export const clickhouseShowCreateTableBodySchema = clickhouseTableBodySchema -export const clickhouseCountRowsBodySchema = clickhouseTableBodySchema.extend({ - where: z.string().optional(), -}) -export const clickhouseListPartitionsBodySchema = clickhouseTableBodySchema -export const clickhouseListMutationsBodySchema = clickhouseConnectionBodySchema.extend({ - table: z.string().optional(), - onlyRunning: z - .union([z.boolean(), z.string()]) - .transform((value) => (typeof value === 'string' ? value.toLowerCase() === 'true' : value)) - .default(false), -}) -export const clickhouseListRunningQueriesBodySchema = clickhouseConnectionBodySchema -export const clickhouseTableStatsBodySchema = clickhouseConnectionBodySchema.extend({ - table: z.string().optional(), -}) -export const clickhouseListClustersBodySchema = clickhouseConnectionBodySchema -export const clickhouseCreateDatabaseBodySchema = clickhouseConnectionBodySchema.extend({ - name: z.string().min(1, 'Database name is required'), -}) -export const clickhouseDropDatabaseBodySchema = clickhouseConnectionBodySchema.extend({ - name: z.string().min(1, 'Database name is required'), -}) -export const clickhouseCreateTableBodySchema = clickhouseConnectionBodySchema.extend({ - table: z.string().min(1, 'Table name is required'), - columns: z - .array( - z.object({ - name: z.string().min(1, 'Column name is required'), - type: z.string().min(1, 'Column type is required'), - }) - ) - .min(1, 'At least one column is required'), - engine: z.string().min(1).default('MergeTree'), - orderBy: z.string().min(1, 'ORDER BY expression is required'), - partitionBy: z.string().optional(), -}) -export const clickhouseDropTableBodySchema = clickhouseTableBodySchema -export const clickhouseTruncateTableBodySchema = clickhouseTableBodySchema -export const clickhouseRenameTableBodySchema = clickhouseTableBodySchema.extend({ - newTable: z.string().min(1, 'New table name is required'), -}) -export const clickhouseOptimizeTableBodySchema = clickhouseTableBodySchema.extend({ - final: z - .union([z.boolean(), z.string()]) - .transform((value) => (typeof value === 'string' ? value.toLowerCase() === 'true' : value)) - .default(false), -}) -export const clickhouseDropPartitionBodySchema = clickhouseTableBodySchema.extend({ - partition: z.string().min(1, 'Partition expression is required'), -}) -export const clickhouseKillQueryBodySchema = clickhouseConnectionBodySchema.extend({ - queryId: z.string().min(1, 'Query ID is required'), -}) -export const clickhouseInsertRowsBodySchema = clickhouseTableBodySchema.extend({ - rows: z.array(z.record(z.string(), z.unknown())).min(1, 'At least one row is required'), -}) - -export const clickhouseListDatabasesContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/list-databases', - body: clickhouseListDatabasesBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const clickhouseListTablesContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/list-tables', - body: clickhouseListTablesBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const clickhouseDescribeTableContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/describe-table', - body: clickhouseDescribeTableBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const clickhouseShowCreateTableContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/show-create-table', - body: clickhouseShowCreateTableBodySchema, - response: { mode: 'json', schema: clickhouseDdlResponseSchema }, -}) - -export const clickhouseCountRowsContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/count-rows', - body: clickhouseCountRowsBodySchema, - response: { mode: 'json', schema: clickhouseCountResponseSchema }, -}) - -export const clickhouseListPartitionsContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/list-partitions', - body: clickhouseListPartitionsBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const clickhouseListMutationsContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/list-mutations', - body: clickhouseListMutationsBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const clickhouseListRunningQueriesContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/list-running-queries', - body: clickhouseListRunningQueriesBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const clickhouseTableStatsContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/table-stats', - body: clickhouseTableStatsBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const clickhouseListClustersContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/list-clusters', - body: clickhouseListClustersBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const clickhouseCreateDatabaseContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/create-database', - body: clickhouseCreateDatabaseBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const clickhouseDropDatabaseContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/drop-database', - body: clickhouseDropDatabaseBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const clickhouseCreateTableContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/create-table', - body: clickhouseCreateTableBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const clickhouseDropTableContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/drop-table', - body: clickhouseDropTableBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const clickhouseTruncateTableContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/truncate-table', - body: clickhouseTruncateTableBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const clickhouseRenameTableContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/rename-table', - body: clickhouseRenameTableBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const clickhouseOptimizeTableContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/optimize-table', - body: clickhouseOptimizeTableBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const clickhouseDropPartitionContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/drop-partition', - body: clickhouseDropPartitionBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const clickhouseKillQueryContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/kill-query', - body: clickhouseKillQueryBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const clickhouseInsertRowsContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/clickhouse/insert-rows', - body: clickhouseInsertRowsBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export type ClickHouseQueryRequest = ContractBodyInput -export type ClickHouseQueryResponse = ContractJsonResponse -export type ClickHouseExecuteRequest = ContractBodyInput -export type ClickHouseExecuteResponse = ContractJsonResponse -export type ClickHouseInsertRequest = ContractBodyInput -export type ClickHouseInsertResponse = ContractJsonResponse -export type ClickHouseUpdateRequest = ContractBodyInput -export type ClickHouseUpdateResponse = ContractJsonResponse -export type ClickHouseDeleteRequest = ContractBodyInput -export type ClickHouseDeleteResponse = ContractJsonResponse -export type ClickHouseIntrospectRequest = ContractBodyInput -export type ClickHouseIntrospectResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/tools/databases/index.ts b/apps/sim/lib/api/contracts/tools/databases/index.ts index 4d5eb0e40ac..c71189701f2 100644 --- a/apps/sim/lib/api/contracts/tools/databases/index.ts +++ b/apps/sim/lib/api/contracts/tools/databases/index.ts @@ -1,9 +1,2 @@ -export * from '@/lib/api/contracts/tools/databases/mongodb' -export * from '@/lib/api/contracts/tools/databases/mssql' -export * from '@/lib/api/contracts/tools/databases/mysql' export * from '@/lib/api/contracts/tools/databases/neo4j' -export * from '@/lib/api/contracts/tools/databases/postgresql' -export * from '@/lib/api/contracts/tools/databases/rds' -export * from '@/lib/api/contracts/tools/databases/redis' export * from '@/lib/api/contracts/tools/databases/shared' -export * from '@/lib/api/contracts/tools/databases/supabase' diff --git a/apps/sim/lib/api/contracts/tools/databases/mongodb.ts b/apps/sim/lib/api/contracts/tools/databases/mongodb.ts deleted file mode 100644 index b12d5cbe577..00000000000 --- a/apps/sim/lib/api/contracts/tools/databases/mongodb.ts +++ /dev/null @@ -1,217 +0,0 @@ -import { z } from 'zod' -import { - introspectionResponseSchema, - mongoDocumentsResponseSchema, - sslModeSchema, -} from '@/lib/api/contracts/tools/databases/shared' -import { - type ContractBodyInput, - type ContractJsonResponse, - defineRouteContract, -} from '@/lib/api/contracts/types' - -// Un-refined base so the downstream operation schemas can .extend it; each -// reattaches mongoUsernamePasswordPaired after its own .extend. -const mongoConnectionBaseSchema = z.object({ - host: z.string().min(1, 'Host is required'), - port: z.coerce.number().int().positive('Port must be a positive integer'), - database: z.string().min(1, 'Database name is required'), - username: z.string().min(1, 'Username is required').optional(), - password: z.string().min(1, 'Password is required').optional(), - authSource: z.string().optional(), - ssl: sslModeSchema, -}) - -const mongoUsernamePasswordPaired = (data: { username?: string; password?: string }) => - Boolean(data.username) === Boolean(data.password) -const mongoUsernamePasswordPairedError = { - message: 'Username and password must be provided together', - path: ['password' as const], -} - -const mongoJsonStringOrObjectSchema = (message: string) => - z - .union([z.string(), z.object({}).passthrough()]) - .transform((val) => { - if (typeof val === 'object' && val !== null) { - return JSON.stringify(val) - } - return val - }) - .refine((val) => val && val.trim() !== '', { message }) - -const booleanStringSchema = z - .union([z.boolean(), z.string(), z.undefined()]) - .optional() - .transform((val) => { - if (val === 'true' || val === true) return true - if (val === 'false' || val === false) return false - return false - }) - -export const mongodbQueryBodySchema = mongoConnectionBaseSchema - .extend({ - collection: z.string().min(1, 'Collection name is required'), - query: z - .union([z.string(), z.object({}).passthrough()]) - .optional() - .default('{}') - .transform((val) => { - if (typeof val === 'object' && val !== null) { - return JSON.stringify(val) - } - return val || '{}' - }), - limit: z - .union([z.coerce.number().int().positive(), z.literal(''), z.undefined()]) - .optional() - .transform((val) => { - if (val === '' || val === undefined || val === null) { - return 100 - } - return val - }), - sort: z - .union([z.string(), z.object({}).passthrough(), z.null()]) - .optional() - .transform((val) => { - if (typeof val === 'object' && val !== null) { - return JSON.stringify(val) - } - return val - }), - }) - .refine(mongoUsernamePasswordPaired, mongoUsernamePasswordPairedError) - -export const mongodbExecuteBodySchema = mongoConnectionBaseSchema - .extend({ - collection: z.string().min(1, 'Collection name is required'), - pipeline: z - .union([z.string(), z.array(z.object({}).passthrough())]) - .transform((val) => { - if (Array.isArray(val)) { - return JSON.stringify(val) - } - return val - }) - .refine((val) => val && val.trim() !== '', { - message: 'Pipeline is required', - }), - }) - .refine(mongoUsernamePasswordPaired, mongoUsernamePasswordPairedError) - -export const mongodbInsertBodySchema = mongoConnectionBaseSchema - .extend({ - collection: z.string().min(1, 'Collection name is required'), - documents: z - .union([z.array(z.record(z.string(), z.unknown())), z.string()]) - .transform((val) => { - if (typeof val === 'string') { - try { - const parsed = JSON.parse(val) - return Array.isArray(parsed) ? parsed : [parsed] - } catch { - throw new Error('Invalid JSON in documents field') - } - } - return val - }) - .refine((val) => Array.isArray(val) && val.length > 0, { - message: 'At least one document is required', - }), - }) - .refine(mongoUsernamePasswordPaired, mongoUsernamePasswordPairedError) - -export const mongodbUpdateBodySchema = mongoConnectionBaseSchema - .extend({ - collection: z.string().min(1, 'Collection name is required'), - filter: mongoJsonStringOrObjectSchema('Filter is required for MongoDB Update').refine( - (val) => val !== '{}', - { message: 'Filter is required for MongoDB Update' } - ), - update: mongoJsonStringOrObjectSchema('Update is required'), - upsert: booleanStringSchema, - multi: booleanStringSchema, - }) - .refine(mongoUsernamePasswordPaired, mongoUsernamePasswordPairedError) - -export const mongodbDeleteBodySchema = mongoConnectionBaseSchema - .extend({ - collection: z.string().min(1, 'Collection name is required'), - filter: mongoJsonStringOrObjectSchema('Filter is required for MongoDB Delete').refine( - (val) => val !== '{}', - { message: 'Filter is required for MongoDB Delete' } - ), - multi: booleanStringSchema, - }) - .refine(mongoUsernamePasswordPaired, mongoUsernamePasswordPairedError) - -export const mongodbIntrospectBodySchema = z - .object({ - host: z.string().min(1, 'Host is required'), - port: z.coerce.number().int().positive('Port must be a positive integer'), - database: z.string().optional(), - username: z.string().optional(), - password: z.string().optional(), - authSource: z.string().optional(), - ssl: sslModeSchema, - }) - .refine((data) => Boolean(data.username) === Boolean(data.password), { - message: 'Username and password must be provided together', - path: ['password'], - }) - -export const mongodbQueryContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/mongodb/query', - body: mongodbQueryBodySchema, - response: { mode: 'json', schema: mongoDocumentsResponseSchema }, -}) - -export const mongodbExecuteContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/mongodb/execute', - body: mongodbExecuteBodySchema, - response: { mode: 'json', schema: mongoDocumentsResponseSchema }, -}) - -export const mongodbInsertContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/mongodb/insert', - body: mongodbInsertBodySchema, - response: { mode: 'json', schema: mongoDocumentsResponseSchema }, -}) - -export const mongodbUpdateContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/mongodb/update', - body: mongodbUpdateBodySchema, - response: { mode: 'json', schema: mongoDocumentsResponseSchema }, -}) - -export const mongodbDeleteContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/mongodb/delete', - body: mongodbDeleteBodySchema, - response: { mode: 'json', schema: mongoDocumentsResponseSchema }, -}) - -export const mongodbIntrospectContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/mongodb/introspect', - body: mongodbIntrospectBodySchema, - response: { mode: 'json', schema: introspectionResponseSchema }, -}) - -export type MongoDBQueryRequest = ContractBodyInput -export type MongoDBQueryResponse = ContractJsonResponse -export type MongoDBExecuteRequest = ContractBodyInput -export type MongoDBExecuteResponse = ContractJsonResponse -export type MongoDBInsertRequest = ContractBodyInput -export type MongoDBInsertResponse = ContractJsonResponse -export type MongoDBUpdateRequest = ContractBodyInput -export type MongoDBUpdateResponse = ContractJsonResponse -export type MongoDBDeleteRequest = ContractBodyInput -export type MongoDBDeleteResponse = ContractJsonResponse -export type MongoDBIntrospectRequest = ContractBodyInput -export type MongoDBIntrospectResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/tools/databases/mssql.ts b/apps/sim/lib/api/contracts/tools/databases/mssql.ts deleted file mode 100644 index a16af9f4a90..00000000000 --- a/apps/sim/lib/api/contracts/tools/databases/mssql.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { z } from 'zod' -import { - introspectionResponseSchema, - sqlInsertDataSchema, - sqlRowsResponseSchema, - sqlUpdateDataSchema, -} from '@/lib/api/contracts/tools/databases/shared' -import { - type ContractBodyInput, - type ContractJsonResponse, - defineRouteContract, -} from '@/lib/api/contracts/types' - -/** - * Toggle schema for the two TLS switches the `mssql` (Tedious) driver exposes. - * Modelled as an enum rather than a boolean because block dropdowns serialize - * strings, and `z.coerce.boolean()` would turn the string `'false'` into `true`. - * @see https://github.com/tediousjs/node-mssql#tedious - */ -export const mssqlToggleSchema = z.enum(['enabled', 'disabled']) - -/** - * Connection fields shared by every Microsoft SQL Server tool route. - * Field placement mirrors the driver: `connectionTimeout` and `requestTimeout` - * are top-level, while `encrypt` and `trustServerCertificate` live under - * `options`. Named instances are unsupported because the driver resolves them - * with an unpinnable SQL Server Browser lookup; use a static TCP port instead. - * @see https://github.com/tediousjs/node-mssql#general-same-for-all-drivers - */ -export const mssqlConnectionBodySchema = z.object({ - host: z.string().min(1, 'Host is required'), - port: z.coerce - .number() - .int() - .min(1, 'Port must be between 1 and 65535') - .max(65535, 'Port must be between 1 and 65535') - .default(1433), - database: z.string().min(1, 'Database name is required'), - username: z.string().min(1, 'Username is required'), - password: z.string().min(1, 'Password is required'), - encrypt: mssqlToggleSchema.default('enabled'), - trustServerCertificate: mssqlToggleSchema.default('disabled'), - connectionTimeout: z.coerce - .number() - .int() - .min(1000, 'connectionTimeout must be at least 1000 ms') - .max(120000, 'connectionTimeout must be at most 120000 ms') - .default(15000), -}) - -export const mssqlQueryBodySchema = mssqlConnectionBodySchema.extend({ - query: z.string().min(1, 'Query is required'), -}) - -export const mssqlExecuteBodySchema = mssqlQueryBodySchema - -export const mssqlInsertBodySchema = mssqlConnectionBodySchema.extend({ - table: z.string().min(1, 'Table name is required'), - data: sqlInsertDataSchema, -}) - -export const mssqlUpdateBodySchema = mssqlConnectionBodySchema.extend({ - table: z.string().min(1, 'Table name is required'), - data: sqlUpdateDataSchema, - where: z.string().min(1, 'WHERE clause is required'), -}) - -export const mssqlDeleteBodySchema = mssqlConnectionBodySchema.extend({ - table: z.string().min(1, 'Table name is required'), - where: z.string().min(1, 'WHERE clause is required'), -}) - -export const mssqlIntrospectBodySchema = mssqlConnectionBodySchema.extend({ - schema: z.string().min(1, 'Schema name cannot be empty').default('dbo'), -}) - -export const mssqlQueryContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/mssql/query', - body: mssqlQueryBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const mssqlExecuteContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/mssql/execute', - body: mssqlExecuteBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const mssqlInsertContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/mssql/insert', - body: mssqlInsertBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const mssqlUpdateContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/mssql/update', - body: mssqlUpdateBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const mssqlDeleteContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/mssql/delete', - body: mssqlDeleteBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const mssqlIntrospectContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/mssql/introspect', - body: mssqlIntrospectBodySchema, - response: { mode: 'json', schema: introspectionResponseSchema }, -}) - -export type MSSQLQueryRequest = ContractBodyInput -export type MSSQLQueryResponse = ContractJsonResponse -export type MSSQLExecuteRequest = ContractBodyInput -export type MSSQLExecuteResponse = ContractJsonResponse -export type MSSQLInsertRequest = ContractBodyInput -export type MSSQLInsertResponse = ContractJsonResponse -export type MSSQLUpdateRequest = ContractBodyInput -export type MSSQLUpdateResponse = ContractJsonResponse -export type MSSQLDeleteRequest = ContractBodyInput -export type MSSQLDeleteResponse = ContractJsonResponse -export type MSSQLIntrospectRequest = ContractBodyInput -export type MSSQLIntrospectResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/tools/databases/mysql.ts b/apps/sim/lib/api/contracts/tools/databases/mysql.ts deleted file mode 100644 index 8e46c1eb352..00000000000 --- a/apps/sim/lib/api/contracts/tools/databases/mysql.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { - introspectionResponseSchema, - sqlConnectionBodySchema, - sqlDeleteBodySchema, - sqlInsertBodySchema, - sqlQueryBodySchema, - sqlRowsResponseSchema, - sqlUpdateBodySchema, -} from '@/lib/api/contracts/tools/databases/shared' -import { - type ContractBodyInput, - type ContractJsonResponse, - defineRouteContract, -} from '@/lib/api/contracts/types' - -export const mysqlQueryBodySchema = sqlQueryBodySchema -export const mysqlExecuteBodySchema = sqlQueryBodySchema -export const mysqlInsertBodySchema = sqlInsertBodySchema -export const mysqlUpdateBodySchema = sqlUpdateBodySchema -export const mysqlDeleteBodySchema = sqlDeleteBodySchema -export const mysqlIntrospectBodySchema = sqlConnectionBodySchema - -export const mysqlQueryContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/mysql/query', - body: mysqlQueryBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const mysqlExecuteContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/mysql/execute', - body: mysqlExecuteBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const mysqlInsertContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/mysql/insert', - body: mysqlInsertBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const mysqlUpdateContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/mysql/update', - body: mysqlUpdateBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const mysqlDeleteContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/mysql/delete', - body: mysqlDeleteBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const mysqlIntrospectContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/mysql/introspect', - body: mysqlIntrospectBodySchema, - response: { mode: 'json', schema: introspectionResponseSchema }, -}) - -export type MySQLQueryRequest = ContractBodyInput -export type MySQLQueryResponse = ContractJsonResponse -export type MySQLExecuteRequest = ContractBodyInput -export type MySQLExecuteResponse = ContractJsonResponse -export type MySQLInsertRequest = ContractBodyInput -export type MySQLInsertResponse = ContractJsonResponse -export type MySQLUpdateRequest = ContractBodyInput -export type MySQLUpdateResponse = ContractJsonResponse -export type MySQLDeleteRequest = ContractBodyInput -export type MySQLDeleteResponse = ContractJsonResponse -export type MySQLIntrospectRequest = ContractBodyInput -export type MySQLIntrospectResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/tools/databases/postgresql.ts b/apps/sim/lib/api/contracts/tools/databases/postgresql.ts deleted file mode 100644 index 4358a17c835..00000000000 --- a/apps/sim/lib/api/contracts/tools/databases/postgresql.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { z } from 'zod' -import { - introspectionResponseSchema, - sqlConnectionBodySchema, - sqlDeleteBodySchema, - sqlInsertBodySchema, - sqlQueryBodySchema, - sqlRowsResponseSchema, - sqlUpdateBodySchema, -} from '@/lib/api/contracts/tools/databases/shared' -import { - type ContractBodyInput, - type ContractJsonResponse, - defineRouteContract, -} from '@/lib/api/contracts/types' - -export const postgresqlQueryBodySchema = sqlQueryBodySchema -export const postgresqlExecuteBodySchema = sqlQueryBodySchema -export const postgresqlInsertBodySchema = sqlInsertBodySchema -export const postgresqlUpdateBodySchema = sqlUpdateBodySchema -export const postgresqlDeleteBodySchema = sqlDeleteBodySchema -export const postgresqlIntrospectBodySchema = sqlConnectionBodySchema.extend({ - schema: z.string().default('public'), -}) - -export const postgresqlQueryContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/postgresql/query', - body: postgresqlQueryBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const postgresqlExecuteContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/postgresql/execute', - body: postgresqlExecuteBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const postgresqlInsertContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/postgresql/insert', - body: postgresqlInsertBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const postgresqlUpdateContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/postgresql/update', - body: postgresqlUpdateBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const postgresqlDeleteContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/postgresql/delete', - body: postgresqlDeleteBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const postgresqlIntrospectContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/postgresql/introspect', - body: postgresqlIntrospectBodySchema, - response: { mode: 'json', schema: introspectionResponseSchema }, -}) - -export type PostgreSQLQueryRequest = ContractBodyInput -export type PostgreSQLQueryResponse = ContractJsonResponse -export type PostgreSQLExecuteRequest = ContractBodyInput -export type PostgreSQLExecuteResponse = ContractJsonResponse -export type PostgreSQLInsertRequest = ContractBodyInput -export type PostgreSQLInsertResponse = ContractJsonResponse -export type PostgreSQLUpdateRequest = ContractBodyInput -export type PostgreSQLUpdateResponse = ContractJsonResponse -export type PostgreSQLDeleteRequest = ContractBodyInput -export type PostgreSQLDeleteResponse = ContractJsonResponse -export type PostgreSQLIntrospectRequest = ContractBodyInput -export type PostgreSQLIntrospectResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/tools/databases/rds.ts b/apps/sim/lib/api/contracts/tools/databases/rds.ts deleted file mode 100644 index af0f84aed39..00000000000 --- a/apps/sim/lib/api/contracts/tools/databases/rds.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { z } from 'zod' -import { - introspectionResponseSchema, - nonEmptyRecordSchema, - sqlRowsResponseSchema, -} from '@/lib/api/contracts/tools/databases/shared' -import { - type ContractBodyInput, - type ContractJsonResponse, - defineRouteContract, -} from '@/lib/api/contracts/types' - -const rdsConnectionBodySchema = z.object({ - region: z.string().min(1, 'AWS region is required'), - accessKeyId: z.string().min(1, 'AWS access key ID is required'), - secretAccessKey: z.string().min(1, 'AWS secret access key is required'), - resourceArn: z.string().min(1, 'Resource ARN is required'), - secretArn: z.string().min(1, 'Secret ARN is required'), - database: z.string().optional(), -}) - -export const rdsQueryBodySchema = rdsConnectionBodySchema.extend({ - query: z.string().min(1, 'Query is required'), -}) - -export const rdsInsertBodySchema = rdsConnectionBodySchema.extend({ - table: z.string().min(1, 'Table name is required'), - data: nonEmptyRecordSchema('Data object must have at least one field'), -}) -export const rdsUpdateBodySchema = rdsConnectionBodySchema.extend({ - table: z.string().min(1, 'Table name is required'), - data: nonEmptyRecordSchema('Data object must have at least one field'), - conditions: nonEmptyRecordSchema('At least one condition is required'), -}) -export const rdsDeleteBodySchema = rdsConnectionBodySchema.extend({ - table: z.string().min(1, 'Table name is required'), - conditions: nonEmptyRecordSchema('At least one condition is required'), -}) -export const rdsIntrospectBodySchema = rdsConnectionBodySchema.extend({ - schema: z.string().optional(), - engine: z.enum(['aurora-postgresql', 'aurora-mysql']).optional(), -}) - -export const rdsQueryContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/rds/query', - body: rdsQueryBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const rdsExecuteContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/rds/execute', - body: rdsQueryBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const rdsInsertContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/rds/insert', - body: rdsInsertBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const rdsUpdateContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/rds/update', - body: rdsUpdateBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const rdsDeleteContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/rds/delete', - body: rdsDeleteBodySchema, - response: { mode: 'json', schema: sqlRowsResponseSchema }, -}) - -export const rdsIntrospectContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/rds/introspect', - body: rdsIntrospectBodySchema, - response: { mode: 'json', schema: introspectionResponseSchema }, -}) - -export type RdsQueryRequest = ContractBodyInput -export type RdsQueryResponse = ContractJsonResponse -export type RdsExecuteRequest = ContractBodyInput -export type RdsExecuteResponse = ContractJsonResponse -export type RdsInsertRequest = ContractBodyInput -export type RdsInsertResponse = ContractJsonResponse -export type RdsUpdateRequest = ContractBodyInput -export type RdsUpdateResponse = ContractJsonResponse -export type RdsDeleteRequest = ContractBodyInput -export type RdsDeleteResponse = ContractJsonResponse -export type RdsIntrospectRequest = ContractBodyInput -export type RdsIntrospectResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/tools/databases/redis.ts b/apps/sim/lib/api/contracts/tools/databases/redis.ts deleted file mode 100644 index 5baff481928..00000000000 --- a/apps/sim/lib/api/contracts/tools/databases/redis.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { z } from 'zod' -import { redisExecuteResponseSchema } from '@/lib/api/contracts/tools/databases/shared' -import { - type ContractBodyInput, - type ContractJsonResponse, - defineRouteContract, -} from '@/lib/api/contracts/types' - -export const redisExecuteBodySchema = z.object({ - url: z.string().min(1, 'Redis connection URL is required'), - command: z.string().min(1, 'Redis command is required'), - args: z.array(z.union([z.string(), z.number()])).default([]), -}) - -export const redisExecuteContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/redis/execute', - body: redisExecuteBodySchema, - response: { mode: 'json', schema: redisExecuteResponseSchema }, -}) - -export type RedisExecuteRequest = ContractBodyInput -export type RedisExecuteResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/tools/databases/shared.ts b/apps/sim/lib/api/contracts/tools/databases/shared.ts index 092fdddeb55..f48d0bc91d0 100644 --- a/apps/sim/lib/api/contracts/tools/databases/shared.ts +++ b/apps/sim/lib/api/contracts/tools/databases/shared.ts @@ -1,93 +1,7 @@ -import { getErrorMessage } from '@sim/utils/errors' import { z } from 'zod' -export const sslModeSchema = z.enum(['disabled', 'required', 'preferred']).default('preferred') export const neo4jEncryptionSchema = z.enum(['enabled', 'disabled']).default('disabled') -export const nonEmptyRecordSchema = (message: string) => - z.record(z.string(), z.unknown()).refine((obj) => Object.keys(obj).length > 0, { message }) - -const jsonObjectStringSchema = (message: string, includeReceivedValue = false) => - z - .string() - .min(1) - .transform((str) => { - try { - const parsed = JSON.parse(str) - if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { - throw new Error('Data must be a JSON object') - } - return parsed - } catch (error) { - if (!includeReceivedValue) { - throw new Error(message) - } - - const errorMessage = getErrorMessage(error, 'Unknown error') - throw new Error(`${message}: ${errorMessage}. Received: ${str.substring(0, 100)}...`) - } - }) - -export const sqlConnectionBodySchema = z.object({ - host: z.string().min(1, 'Host is required'), - port: z.coerce.number().int().positive('Port must be a positive integer'), - database: z.string().min(1, 'Database name is required'), - username: z.string().min(1, 'Username is required'), - password: z.string().min(1, 'Password is required'), - ssl: sslModeSchema, -}) - -export const sqlQueryBodySchema = sqlConnectionBodySchema.extend({ - query: z.string().min(1, 'Query is required'), -}) - -export const sqlInsertDataSchema = z.union([ - nonEmptyRecordSchema('Data object cannot be empty'), - jsonObjectStringSchema('Invalid JSON format in data field', true), -]) - -export const sqlUpdateDataSchema = z.union([ - nonEmptyRecordSchema('Data object cannot be empty'), - jsonObjectStringSchema('Invalid JSON format in data field'), -]) - -export const sqlInsertBodySchema = sqlConnectionBodySchema.extend({ - table: z.string().min(1, 'Table name is required'), - data: sqlInsertDataSchema, -}) - -export const sqlUpdateBodySchema = sqlConnectionBodySchema.extend({ - table: z.string().min(1, 'Table name is required'), - data: sqlUpdateDataSchema, - where: z.string().min(1, 'WHERE clause is required'), -}) - -export const sqlDeleteBodySchema = sqlConnectionBodySchema.extend({ - table: z.string().min(1, 'Table name is required'), - where: z.string().min(1, 'WHERE clause is required'), -}) - -export const sqlRowsResponseSchema = z.object({ - message: z.string(), - rows: z.array(z.unknown()), - rowCount: z.number(), - /** - * Present only when the driver returned more than the route was willing to - * serialize. Absent means the recordset is complete, so a caller that ignores - * these two fields still reads a whole result correctly. - */ - truncated: z.boolean().optional(), - truncationReason: z.string().optional(), -}) - -export const mongoDocumentsResponseSchema = z - .object({ - message: z.string(), - documents: z.array(z.unknown()).optional(), - documentCount: z.number().optional(), - }) - .passthrough() - export const neo4jResponseSchema = z .object({ message: z.string(), @@ -100,10 +14,6 @@ export const introspectionResponseSchema = z }) .passthrough() -export const redisExecuteResponseSchema = z.object({ - result: z.unknown(), -}) - export const supabaseStorageUploadResponseSchema = z.object({ success: z.literal(true), output: z.object({ diff --git a/apps/sim/lib/api/contracts/tools/databases/supabase.ts b/apps/sim/lib/api/contracts/tools/databases/supabase.ts deleted file mode 100644 index acb24feca7a..00000000000 --- a/apps/sim/lib/api/contracts/tools/databases/supabase.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { z } from 'zod' -import { supabaseStorageUploadResponseSchema } from '@/lib/api/contracts/tools/databases/shared' -import { - type ContractBodyInput, - type ContractJsonResponse, - defineRouteContract, -} from '@/lib/api/contracts/types' -import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' - -export const supabaseStorageUploadBodySchema = z.object({ - projectId: z - .string() - .min(1, 'Project ID is required') - .regex(/^[a-z0-9]+$/, 'Project ID must contain only lowercase alphanumeric characters'), - apiKey: z.string().min(1, 'API key is required'), - bucket: z.string().min(1, 'Bucket name is required'), - fileName: z.string().min(1, 'File name is required'), - path: z.string().optional().nullable(), - fileData: FileInputSchema, - contentType: z.string().optional().nullable(), - cacheControl: z.string().optional().nullable(), - upsert: z.boolean().optional().default(false), -}) - -export const supabaseStorageUploadContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/supabase/storage-upload', - body: supabaseStorageUploadBodySchema, - response: { mode: 'json', schema: supabaseStorageUploadResponseSchema }, -}) - -export type SupabaseStorageUploadRequest = ContractBodyInput -export type SupabaseStorageUploadResponse = ContractJsonResponse< - typeof supabaseStorageUploadContract -> diff --git a/apps/sim/lib/api/contracts/tools/daytona.ts b/apps/sim/lib/api/contracts/tools/daytona.ts deleted file mode 100644 index 71ee1ff83c4..00000000000 --- a/apps/sim/lib/api/contracts/tools/daytona.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { z } from 'zod' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' - -export const daytonaUploadFileBodySchema = z.object({ - apiKey: z.string().min(1, 'API key is required'), - sandboxId: z.string().min(1, 'Sandbox ID is required'), - destinationPath: z.string().min(1, 'Destination path is required'), - file: FileInputSchema.optional().nullable(), - fileContent: z.string().nullish(), - fileName: z.string().nullish(), -}) - -export const daytonaUploadFileResponseSchema = z.object({ - success: z.boolean(), - uploadedPath: z.string().optional(), - name: z.string().optional(), - size: z.number().optional(), - error: z.string().optional(), -}) - -export const daytonaUploadFileContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/daytona/upload', - body: daytonaUploadFileBodySchema, - response: { mode: 'json', schema: daytonaUploadFileResponseSchema }, -}) diff --git a/apps/sim/lib/api/contracts/tools/deployments.ts b/apps/sim/lib/api/contracts/tools/deployments.ts deleted file mode 100644 index a27d4c5a989..00000000000 --- a/apps/sim/lib/api/contracts/tools/deployments.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { z } from 'zod' -import { deploymentVersionMetadataFieldsSchema } from '@/lib/api/contracts/deployments' -import { workflowIdSchema, workspaceIdSchema } from '@/lib/api/contracts/primitives' -import { genericToolResponseSchema } from '@/lib/api/contracts/tools/shared' -import { defineRouteContract } from '@/lib/api/contracts/types' - -/** Bounded to the Postgres `integer` range of `workflow_deployment_version.version`. */ -const versionSchema = z - .number() - .int('Version must be an integer') - .min(1, 'Version must be a positive integer') - .max(2147483647, 'Version is out of range') - -export const deploymentsDeployBodySchema = z.object({ - workflowId: workflowIdSchema, - workspaceId: workspaceIdSchema, - name: deploymentVersionMetadataFieldsSchema.shape.name, - description: deploymentVersionMetadataFieldsSchema.shape.description, -}) - -export type DeploymentsDeployBody = z.input - -export const deploymentsDeployContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/deployments/deploy', - body: deploymentsDeployBodySchema, - response: { - mode: 'json', - schema: genericToolResponseSchema, - }, -}) - -export const deploymentsUndeployBodySchema = z.object({ - workflowId: workflowIdSchema, - workspaceId: workspaceIdSchema, -}) - -export type DeploymentsUndeployBody = z.input - -export const deploymentsUndeployContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/deployments/undeploy', - body: deploymentsUndeployBodySchema, - response: { - mode: 'json', - schema: genericToolResponseSchema, - }, -}) - -export const deploymentsPromoteBodySchema = z.object({ - workflowId: workflowIdSchema, - workspaceId: workspaceIdSchema, - version: versionSchema, -}) - -export type DeploymentsPromoteBody = z.input - -export const deploymentsPromoteContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/deployments/promote', - body: deploymentsPromoteBodySchema, - response: { - mode: 'json', - schema: genericToolResponseSchema, - }, -}) - -export const deploymentsListVersionsQuerySchema = z.object({ - workflowId: workflowIdSchema, - workspaceId: workspaceIdSchema, -}) - -export type DeploymentsListVersionsQuery = z.input - -export const deploymentsListVersionsContract = defineRouteContract({ - method: 'GET', - path: '/api/tools/deployments/versions', - query: deploymentsListVersionsQuerySchema, - response: { - mode: 'json', - schema: genericToolResponseSchema, - }, -}) - -export const deploymentsGetVersionQuerySchema = z.object({ - workflowId: workflowIdSchema, - workspaceId: workspaceIdSchema, - version: z.coerce.number().pipe(versionSchema), -}) - -export type DeploymentsGetVersionQuery = z.input - -export const deploymentsGetVersionContract = defineRouteContract({ - method: 'GET', - path: '/api/tools/deployments/version', - query: deploymentsGetVersionQuerySchema, - response: { - mode: 'json', - schema: genericToolResponseSchema, - }, -}) diff --git a/apps/sim/lib/api/contracts/tools/embeddings.ts b/apps/sim/lib/api/contracts/tools/embeddings.ts deleted file mode 100644 index 7b3eb02abd3..00000000000 --- a/apps/sim/lib/api/contracts/tools/embeddings.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { z } from 'zod' -import { defineRouteContract } from '@/lib/api/contracts/types' -import type { EmbeddingCatalogProvider, EmbeddingTaskType } from '@/lib/embeddings/types' - -/** - * `satisfies` ties the wire enums to the catalog's own unions: renaming or - * removing a catalog member breaks the build here. It does NOT catch an - * addition — a new catalog provider or task type stays absent from the wire - * enum until it is added below. - */ -type EmbeddingToolProvider = EmbeddingCatalogProvider | 'openrouter' - -export const embeddingProviders = [ - 'openai', - 'openrouter', - 'gemini', - 'cohere', - 'mistral', -] as const satisfies readonly EmbeddingToolProvider[] - -export const embeddingTaskTypes = [ - 'document', - 'query', - 'similarity', - 'classification', - 'clustering', -] as const satisfies readonly EmbeddingTaskType[] - -/** Guards the route against unbounded fan-out into a paid provider. */ -export const MAX_EMBEDDING_INPUTS = 1000 -/** Caps total payload size independently of the input count. */ -export const MAX_EMBEDDING_TOTAL_CHARS = 1_000_000 - -const MISSING_EMBEDDING_INPUT_ERROR = 'Missing required field: input' -const embeddingCatalogProviders = [ - 'openai', - 'gemini', - 'cohere', - 'mistral', -] as const satisfies readonly EmbeddingCatalogProvider[] - -const embeddingToolCommonShape = { - model: z.string().min(1, 'model cannot be empty').optional(), - /** A single text, or an array of texts embedded in one call. */ - input: z.union( - [ - z.string().min(1, 'input cannot be empty'), - z - .array(z.string().min(1, 'input entries cannot be empty')) - .min(1, 'input must contain at least one text') - .max(MAX_EMBEDDING_INPUTS, `input cannot exceed ${MAX_EMBEDDING_INPUTS} texts`), - ], - { error: MISSING_EMBEDDING_INPUT_ERROR } - ), - taskType: z.enum(embeddingTaskTypes).optional(), - /** Matryoshka output size. Omitted means the model's native dimensionality. */ - dimensions: z.coerce - .number() - .int('dimensions must be an integer') - .min(1, 'dimensions must be at least 1') - .max(4096, 'dimensions cannot exceed 4096') - .optional(), -} - -export const embeddingsToolBodySchema = z.discriminatedUnion('provider', [ - z.object({ - ...embeddingToolCommonShape, - provider: z.enum(embeddingCatalogProviders), - apiKey: z.string({ error: 'apiKey is required' }).min(1, 'apiKey cannot be empty'), - }), - z.object({ - ...embeddingToolCommonShape, - provider: z.literal('openrouter'), - apiKey: z.string({ error: 'apiKey is required' }).min(1, 'apiKey cannot be empty'), - }), -]) - -const embeddingsUsageSchema = z.object({ - prompt_tokens: z.number(), - total_tokens: z.number(), -}) - -export const embeddingsToolResponseSchema = z.discriminatedUnion('success', [ - z.object({ - success: z.literal(true), - embeddings: z.array(z.array(z.number())), - model: z.string(), - provider: z.enum(embeddingProviders), - dimensions: z.number(), - usage: embeddingsUsageSchema, - /** Token count echoed back so the tool's hosted-key pricing hook can bill it. */ - __embeddingTokens: z.number(), - }), - z.object({ - success: z.literal(false), - error: z.string(), - }), -]) - -export type EmbeddingsToolBody = z.input -export type EmbeddingsToolResponse = z.output -export type EmbeddingProvider = (typeof embeddingProviders)[number] -export type EmbeddingTaskTypeName = (typeof embeddingTaskTypes)[number] - -export const embeddingsToolContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/embeddings', - body: embeddingsToolBodySchema, - response: { mode: 'json', schema: embeddingsToolResponseSchema }, -}) diff --git a/apps/sim/lib/api/contracts/tools/enrichment.ts b/apps/sim/lib/api/contracts/tools/enrichment.ts deleted file mode 100644 index c3279dba681..00000000000 --- a/apps/sim/lib/api/contracts/tools/enrichment.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { z } from 'zod' -import { defineRouteContract } from '@/lib/api/contracts/types' - -export const runEnrichmentBodySchema = z.object({ - enrichmentId: z.string().min(1, 'enrichmentId is required'), - /** Per-enrichment input map: enrichment input id → mapped value. */ - inputs: z.record(z.string(), z.unknown()).default({}), - workspaceId: z.string().min(1, 'workspaceId is required'), -}) - -const runEnrichmentResponseSchema = z.object({ - matched: z.boolean(), - // untyped-response: per-enrichment output map — keys and value types vary by enrichment - result: z.record(z.string(), z.unknown()), - cost: z.number(), - error: z.string().nullable(), - /** Label of the provider whose result was returned, null on no match. */ - provider: z.string().nullable(), -}) - -export const runEnrichmentContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/enrichment/run', - body: runEnrichmentBodySchema, - response: { mode: 'json', schema: runEnrichmentResponseSchema }, -}) - -export type RunEnrichmentBody = z.input -export type RunEnrichmentResponse = z.output diff --git a/apps/sim/lib/api/contracts/tools/firecrawl.ts b/apps/sim/lib/api/contracts/tools/firecrawl.ts deleted file mode 100644 index c89db4fc27f..00000000000 --- a/apps/sim/lib/api/contracts/tools/firecrawl.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { z } from 'zod' -import { resolvedSecretTraceProvenanceSchema } from '@/lib/api/contracts/primitives' -import type { - ContractBody, - ContractBodyInput, - ContractJsonResponse, -} from '@/lib/api/contracts/types' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' -import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' - -const firecrawlParseResponseSchema = z.object({ - success: z.literal(true), - // untyped-response: forwards firecrawl /v2/parse response unchanged for downstream tool consumers - output: z.unknown(), -}) - -export const firecrawlParseBodySchema = z.object({ - apiKey: z.string().min(1, 'API key is required'), - file: RawFileInputSchema, - options: z.record(z.string(), z.unknown()).optional(), - [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), -}) - -export const firecrawlParseContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/firecrawl/parse', - body: firecrawlParseBodySchema, - response: { mode: 'json', schema: firecrawlParseResponseSchema }, -}) - -export type FirecrawlParseBody = ContractBody -export type FirecrawlParseBodyInput = ContractBodyInput -export type FirecrawlParseResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/tools/fireflies.ts b/apps/sim/lib/api/contracts/tools/fireflies.ts deleted file mode 100644 index a59d402e4ad..00000000000 --- a/apps/sim/lib/api/contracts/tools/fireflies.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { z } from 'zod' -import { resolvedSecretTraceProvenanceSchema } from '@/lib/api/contracts/primitives' -import type { - ContractBody, - ContractBodyInput, - ContractJsonResponse, -} from '@/lib/api/contracts/types' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' - -const firefliesAudioFileSchema = z - .object({ - id: z.string().optional(), - key: z.string().optional(), - path: z.string().optional(), - url: z.string().optional(), - name: z.string().optional(), - size: z.number().nonnegative().optional(), - type: z.string().optional(), - context: z.string().optional(), - }) - .passthrough() - -const firefliesGraphqlErrorSchema = z - .object({ - message: z.string(), - }) - .passthrough() - -const firefliesUploadAudioResultSchema = z - .object({ - success: z.boolean(), - title: z.string().nullable().optional(), - message: z.string().nullable().optional(), - }) - .passthrough() - -export const firefliesUploadAudioBodySchema = z - .object({ - apiKey: z.string().min(1, 'Missing API key for Fireflies API request'), - audioFile: firefliesAudioFileSchema.optional(), - audioUrl: z.string().optional(), - title: z.string().optional(), - webhook: z.string().optional(), - language: z.string().optional(), - attendees: z.unknown().optional(), - clientReferenceId: z.string().optional(), - [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), - }) - .superRefine((data, context) => { - const file = data.audioFile - if (!file?.key && !file?.url && !file?.path && !data.audioUrl) { - context.addIssue({ - code: 'custom', - message: 'Either an audio file or audio URL is required', - path: ['audioUrl'], - }) - } - }) - -export const firefliesUploadAudioResponseSchema = z - .object({ - data: z - .object({ - uploadAudio: firefliesUploadAudioResultSchema.nullable().optional(), - }) - .nullable() - .optional(), - errors: z.array(firefliesGraphqlErrorSchema).optional(), - }) - .passthrough() - -export const firefliesUploadAudioContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/fireflies/upload-audio', - body: firefliesUploadAudioBodySchema, - response: { mode: 'json', schema: firefliesUploadAudioResponseSchema }, -}) - -export type FirefliesUploadAudioBody = ContractBody -export type FirefliesUploadAudioBodyInput = ContractBodyInput -export type FirefliesUploadAudioResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/tools/github.ts b/apps/sim/lib/api/contracts/tools/github.ts deleted file mode 100644 index c656ea15866..00000000000 --- a/apps/sim/lib/api/contracts/tools/github.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { z } from 'zod' -import type { - ContractBody, - ContractBodyInput, - ContractJsonResponse, -} from '@/lib/api/contracts/types' -import { defineRouteContract } from '@/lib/api/contracts/types' - -const githubUserSummarySchema = z.object({ - name: z.string(), - login: z.string(), - avatar_url: z.string(), - html_url: z.string(), -}) - -const githubCommitFileSchema = z.object({ - filename: z.string(), - additions: z.number(), - deletions: z.number(), - changes: z.number(), - status: z.string(), - raw_url: z.string().nullable().optional(), - blob_url: z.string().nullable().optional(), - patch: z.string().optional(), - content: z.string().optional(), -}) - -export const githubLatestCommitResponseSchema = z.object({ - success: z.literal(true), - output: z.object({ - content: z.string(), - metadata: z.object({ - sha: z.string(), - html_url: z.string(), - commit_message: z.string(), - author: githubUserSummarySchema, - committer: githubUserSummarySchema, - stats: z - .object({ - additions: z.number(), - deletions: z.number(), - total: z.number(), - }) - .optional(), - files: z.array(githubCommitFileSchema).optional(), - }), - }), -}) - -export const githubLatestCommitBodySchema = z.object({ - owner: z.string().min(1, 'Owner is required'), - repo: z.string().min(1, 'Repo is required'), - branch: z.string().optional().nullable(), - apiKey: z.string().min(1, 'API key is required'), -}) - -export const githubLatestCommitContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/github/latest-commit', - body: githubLatestCommitBodySchema, - response: { mode: 'json', schema: githubLatestCommitResponseSchema }, -}) - -export type GithubLatestCommitBody = ContractBody -export type GithubLatestCommitBodyInput = ContractBodyInput -export type GithubLatestCommitResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/tools/google.ts b/apps/sim/lib/api/contracts/tools/google.ts index 17d79fe6bf1..024c6870bc8 100644 --- a/apps/sim/lib/api/contracts/tools/google.ts +++ b/apps/sim/lib/api/contracts/tools/google.ts @@ -1,7 +1,7 @@ import { z } from 'zod' import type { ContractBodyInput, ContractJsonResponse } from '@/lib/api/contracts/types' import { defineRouteContract } from '@/lib/api/contracts/types' -import { RawFileInputArraySchema, RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' export const googleAccessTokenSchema = z.string().min(1, 'Access token is required') export const gmailMessageIdSchema = z.string().min(1, 'Message ID is required') @@ -37,58 +37,6 @@ export const gmailEditDraftBodySchema = gmailMailBodySchema.extend({ draftId: z.string().min(1, 'Draft ID is required'), }) -export const googleDriveUploadBodySchema = z.object({ - accessToken: googleAccessTokenSchema, - fileName: z.string().min(1, 'File name is required'), - file: RawFileInputSchema.optional().nullable(), - mimeType: z.string().optional().nullable(), - folderId: z.string().optional().nullable(), -}) - -export const googleDriveDownloadBodySchema = z.object({ - accessToken: googleAccessTokenSchema, - fileId: z.string().min(1, 'File ID is required'), - mimeType: z.string().optional().nullable(), - fileName: z.string().optional().nullable(), - includeRevisions: z.boolean().optional().default(true), -}) - -export const googleDriveExportBodySchema = z.object({ - accessToken: googleAccessTokenSchema, - fileId: z.string().min(1, 'File ID is required'), - mimeType: z.string().min(1, 'Target export MIME type is required'), - fileName: z.string().optional().nullable(), -}) - -export const googleVaultDownloadExportFileBodySchema = z.object({ - accessToken: googleAccessTokenSchema, - bucketName: z.string().min(1, 'Bucket name is required'), - objectName: z.string().min(1, 'Object name is required'), - fileName: z.string().optional().nullable(), -}) - -export const googleSlidesExportFormatSchema = z.preprocess((value) => { - if (typeof value !== 'string') return value - const normalized = value.trim().toUpperCase() - return normalized || undefined -}, z.enum(['PDF', 'PPTX', 'ODP', 'TXT', 'PNG', 'JPEG', 'SVG']).optional()) - -/** Google Drive / Slides file IDs are opaque base62-ish strings without URL metacharacters. */ -export const googlePresentationIdSchema = z - .string() - .trim() - .min(1, 'Presentation ID is required') - .regex(/^[a-zA-Z0-9_-]+$/, 'Presentation ID contains invalid characters') - -export const googleSlidesExportPresentationBodySchema = z.object({ - accessToken: googleAccessTokenSchema, - presentationId: googlePresentationIdSchema, - exportFormat: googleSlidesExportFormatSchema, - workspaceId: z.string().optional(), - workflowId: z.string().optional(), - executionId: z.string().optional(), -}) - const toolJsonResponseSchema = z.unknown() export const gmailAddLabelContract = defineRouteContract({ @@ -168,41 +116,6 @@ export const gmailUnarchiveContract = defineRouteContract({ response: { mode: 'json', schema: toolJsonResponseSchema }, }) -export const googleDriveUploadContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/google_drive/upload', - body: googleDriveUploadBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) - -export const googleDriveDownloadContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/google_drive/download', - body: googleDriveDownloadBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) - -export const googleDriveExportContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/google_drive/export', - body: googleDriveExportBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) - -export const googleVaultDownloadExportFileContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/google_vault/download-export-file', - body: googleVaultDownloadExportFileBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) - -export const googleSlidesExportPresentationContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/google_slides/export-presentation', - body: googleSlidesExportPresentationBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) - export type GmailAddLabelBody = ContractBodyInput export type GmailArchiveBody = ContractBodyInput export type GmailDeleteBody = ContractBodyInput @@ -214,15 +127,6 @@ export type GmailMoveBody = ContractBodyInput export type GmailRemoveLabelBody = ContractBodyInput export type GmailSendBody = ContractBodyInput export type GmailUnarchiveBody = ContractBodyInput -export type GoogleDriveUploadBody = ContractBodyInput -export type GoogleDriveDownloadBody = ContractBodyInput -export type GoogleDriveExportBody = ContractBodyInput -export type GoogleVaultDownloadExportFileBody = ContractBodyInput< - typeof googleVaultDownloadExportFileContract -> -export type GoogleSlidesExportPresentationBody = ContractBodyInput< - typeof googleSlidesExportPresentationContract -> export type GmailAddLabelResponse = ContractJsonResponse export type GmailArchiveResponse = ContractJsonResponse @@ -235,12 +139,3 @@ export type GmailMoveResponse = ContractJsonResponse export type GmailRemoveLabelResponse = ContractJsonResponse export type GmailSendResponse = ContractJsonResponse export type GmailUnarchiveResponse = ContractJsonResponse -export type GoogleDriveUploadResponse = ContractJsonResponse -export type GoogleDriveDownloadResponse = ContractJsonResponse -export type GoogleDriveExportResponse = ContractJsonResponse -export type GoogleVaultDownloadExportFileResponse = ContractJsonResponse< - typeof googleVaultDownloadExportFileContract -> -export type GoogleSlidesExportPresentationResponse = ContractJsonResponse< - typeof googleSlidesExportPresentationContract -> diff --git a/apps/sim/lib/api/contracts/tools/grafana.ts b/apps/sim/lib/api/contracts/tools/grafana.ts deleted file mode 100644 index 227353dabf5..00000000000 --- a/apps/sim/lib/api/contracts/tools/grafana.ts +++ /dev/null @@ -1,199 +0,0 @@ -import { z } from 'zod' -import type { ContractBody, ContractJsonResponse } from '@/lib/api/contracts/types' -import { defineRouteContract } from '@/lib/api/contracts/types' - -const grafanaUpdateDashboardBodySchema = z.object({ - apiKey: z.string().min(1, 'Grafana Service Account Token is required'), - baseUrl: z.string().min(1, 'Grafana instance URL is required'), - organizationId: z.string().optional(), - dashboardUid: z.string().min(1, 'Dashboard UID is required'), - title: z.string().optional(), - folderUid: z.string().optional(), - tags: z.string().optional(), - timezone: z.string().optional(), - refresh: z.string().optional(), - panels: z.string().optional(), - overwrite: z.boolean().optional(), - message: z.string().optional(), -}) - -const grafanaUpdateDashboardOutputSchema = z.object({ - id: z.number().optional(), - uid: z.string().optional(), - url: z.string().optional(), - status: z.string().optional(), - version: z.number().optional(), - slug: z.string().optional(), -}) - -export const grafanaUpdateDashboardResponseSchema = z.object({ - success: z.boolean(), - /** Absent on the auth short-circuit, `{}` on handled failures. */ - output: grafanaUpdateDashboardOutputSchema.partial().optional(), - error: z.string().optional(), - /** untyped-response: Zod issue objects, whose shape is Zod's, not ours to pin. */ - details: z.array(z.unknown()).optional(), -}) - -const grafanaUpdateAlertRuleBodySchema = z.object({ - apiKey: z.string().min(1, 'Grafana Service Account Token is required'), - baseUrl: z.string().min(1, 'Grafana instance URL is required'), - organizationId: z.string().optional(), - alertRuleUid: z.string().min(1, 'Alert rule UID is required'), - title: z.string().optional(), - folderUid: z.string().optional(), - ruleGroup: z.string().optional(), - condition: z.string().optional(), - data: z.string().optional(), - forDuration: z.string().optional(), - noDataState: z.string().optional(), - execErrState: z.string().optional(), - annotations: z.string().optional(), - labels: z.string().optional(), - isPaused: z.boolean().optional(), - keepFiringFor: z.string().optional(), - missingSeriesEvalsToResolve: z.number().optional(), - notificationSettings: z.string().optional(), - record: z.string().optional(), - disableProvenance: z.boolean().optional(), -}) - -const grafanaUpdateAlertRuleOutputSchema = z.object({ - id: z.number().nullable(), - uid: z.string().nullable(), - title: z.string().nullable(), - condition: z.string().nullable(), - /** untyped-response: alert query stages are opaque, data-source-specific payloads. */ - data: z.array(z.unknown()), - updated: z.string().nullable(), - noDataState: z.string().nullable(), - execErrState: z.string().nullable(), - for: z.string().nullable(), - keepFiringFor: z.string().nullable(), - missingSeriesEvalsToResolve: z.number().nullable(), - annotations: z.record(z.string(), z.string()), - labels: z.record(z.string(), z.string()), - isPaused: z.boolean(), - folderUID: z.string().nullable(), - ruleGroup: z.string().nullable(), - orgID: z.number().nullable(), - provenance: z.string(), - /** untyped-response: Grafana's notification settings shape is undocumented. */ - notification_settings: z.record(z.string(), z.unknown()).nullable(), - /** untyped-response: recording-rule config is passed through opaquely. */ - record: z.record(z.string(), z.unknown()).nullable(), -}) - -export const grafanaUpdateAlertRuleResponseSchema = z.object({ - success: z.boolean(), - /** Absent on the auth short-circuit, `{}` on handled failures. */ - output: z.union([grafanaUpdateAlertRuleOutputSchema, z.object({})]).optional(), - error: z.string().optional(), - /** untyped-response: Zod issue objects, whose shape is Zod's, not ours to pin. */ - details: z.array(z.unknown()).optional(), -}) - -const grafanaUpdateFolderBodySchema = z.object({ - apiKey: z.string().min(1, 'Grafana Service Account Token is required'), - baseUrl: z.string().min(1, 'Grafana instance URL is required'), - organizationId: z.string().optional(), - folderUid: z.string().min(1, 'Folder UID is required'), - title: z.string().min(1, 'Folder title is required'), -}) - -const grafanaUpdateFolderOutputSchema = z.object({ - id: z.number().nullable(), - uid: z.string().nullable(), - title: z.string().nullable(), - url: z.string().nullable(), - parentUid: z.string().nullable(), - parents: z.array(z.object({ uid: z.string(), title: z.string(), url: z.string() })), - hasAcl: z.boolean().nullable(), - canSave: z.boolean().nullable(), - canEdit: z.boolean().nullable(), - canAdmin: z.boolean().nullable(), - createdBy: z.string().nullable(), - created: z.string().nullable(), - updatedBy: z.string().nullable(), - updated: z.string().nullable(), - version: z.number().nullable(), -}) - -export const grafanaUpdateFolderResponseSchema = z.object({ - success: z.boolean(), - /** Absent on the auth short-circuit, `{}` on handled failures. */ - output: z.union([grafanaUpdateFolderOutputSchema, z.object({})]).optional(), - error: z.string().optional(), - /** untyped-response: Zod issue objects, whose shape is Zod's, not ours to pin. */ - details: z.array(z.unknown()).optional(), -}) - -const grafanaCheckDataSourceHealthBodySchema = z.object({ - apiKey: z.string().min(1, 'Grafana Service Account Token is required'), - baseUrl: z.string().min(1, 'Grafana instance URL is required'), - organizationId: z.string().optional(), - dataSourceUid: z.string().min(1, 'Data source UID is required').max(40, 'UID is too long'), -}) - -const grafanaCheckDataSourceHealthOutputSchema = z.object({ - status: z.string(), - message: z.string().nullable(), - /** untyped-response: health detail is whatever the data source plugin chooses to attach. */ - details: z.unknown().optional(), -}) - -export const grafanaCheckDataSourceHealthResponseSchema = z.object({ - success: z.boolean(), - output: grafanaCheckDataSourceHealthOutputSchema.optional(), - error: z.string().optional(), - /** untyped-response: Zod issue objects, whose shape is Zod's, not ours to pin. */ - details: z.array(z.unknown()).optional(), -}) - -export const grafanaCheckDataSourceHealthContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/grafana/check_data_source_health', - body: grafanaCheckDataSourceHealthBodySchema, - response: { mode: 'json', schema: grafanaCheckDataSourceHealthResponseSchema }, -}) - -export const grafanaUpdateDashboardContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/grafana/update_dashboard', - body: grafanaUpdateDashboardBodySchema, - response: { mode: 'json', schema: grafanaUpdateDashboardResponseSchema }, -}) - -export const grafanaUpdateAlertRuleContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/grafana/update_alert_rule', - body: grafanaUpdateAlertRuleBodySchema, - response: { mode: 'json', schema: grafanaUpdateAlertRuleResponseSchema }, -}) - -export const grafanaUpdateFolderContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/grafana/update_folder', - body: grafanaUpdateFolderBodySchema, - response: { mode: 'json', schema: grafanaUpdateFolderResponseSchema }, -}) - -export { - grafanaUpdateDashboardBodySchema, - grafanaUpdateDashboardOutputSchema, - grafanaUpdateAlertRuleBodySchema, - grafanaUpdateAlertRuleOutputSchema, - grafanaUpdateFolderBodySchema, - grafanaUpdateFolderOutputSchema, -} - -export type GrafanaUpdateDashboardBody = ContractBody -export type GrafanaUpdateDashboardResponse = ContractJsonResponse< - typeof grafanaUpdateDashboardContract -> -export type GrafanaUpdateAlertRuleBody = ContractBody -export type GrafanaUpdateAlertRuleResponse = ContractJsonResponse< - typeof grafanaUpdateAlertRuleContract -> -export type GrafanaUpdateFolderBody = ContractBody -export type GrafanaUpdateFolderResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/tools/index.ts b/apps/sim/lib/api/contracts/tools/index.ts index 94ed2bb939b..69859fe778e 100644 --- a/apps/sim/lib/api/contracts/tools/index.ts +++ b/apps/sim/lib/api/contracts/tools/index.ts @@ -1,39 +1,15 @@ -export * from './a2a' export * from './agiloft' export * from './asana' -export * from './brex' -export * from './clickup' export * from './communication' export * from './crowdstrike' -export * from './cursor' export * from './custom' export * from './databases' -export * from './daytona' -export * from './deployments' export * from './docusign' export * from './file' -export * from './firecrawl' -export * from './fireflies' -export * from './github' export * from './google' -export * from './grafana' export * from './imap' -export * from './latex' -export * from './mail' export * from './media' export * from './microsoft' export * from './onepassword' -export * from './persona' -export * from './pipedrive' -export * from './quiver' -export * from './sap' -export * from './search' -export * from './servicenow' export * from './shared' -export * from './square' -export * from './stagehand' -export * from './thinking' -export * from './typeform' -export * from './vanta' export * from './workday' -export * from './zoom' diff --git a/apps/sim/lib/api/contracts/tools/instagram.ts b/apps/sim/lib/api/contracts/tools/instagram.ts deleted file mode 100644 index 6aaf1fc038c..00000000000 --- a/apps/sim/lib/api/contracts/tools/instagram.ts +++ /dev/null @@ -1,221 +0,0 @@ -import { z } from 'zod' -import { - nonEmptyIdSchema, - userFileSchema, - workflowIdSchema, - workspaceIdSchema, -} from '@/lib/api/contracts/primitives' -import type { ContractBodyInput, ContractJsonResponse } from '@/lib/api/contracts/types' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { RawFileInputArraySchema, RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' - -const MAX_ACCESS_TOKEN_LENGTH = 8192 -const MAX_GRAPH_ID_LENGTH = 256 -const MAX_CAPTION_LENGTH = 2200 -const MAX_ALT_TEXT_LENGTH = 1000 - -const instagramOptionalUserIdSchema = z - .string() - .trim() - .max(MAX_GRAPH_ID_LENGTH, 'Instagram user ID is too long') - .optional() - .nullable() - -const instagramOptionalCaptionSchema = z - .string() - .max(MAX_CAPTION_LENGTH, `Caption cannot exceed ${MAX_CAPTION_LENGTH} characters`) - .optional() - .nullable() - -export const instagramAccessTokenSchema = z - .string() - .min(1, 'Access token is required') - .max(MAX_ACCESS_TOKEN_LENGTH, 'Access token is too long') - -export const instagramDownloadMediaBodySchema = z.object({ - accessToken: instagramAccessTokenSchema, - mediaId: z.string().trim().min(1, 'Media ID is required').max(256, 'Media ID is too long'), - filename: z - .string() - .trim() - .min(1, 'Filename cannot be empty') - .max(180, 'Filename is too long') - .optional(), - workspaceId: workspaceIdSchema.optional(), - workflowId: workflowIdSchema.optional(), - executionId: nonEmptyIdSchema.optional(), -}) - -export const instagramDownloadMediaOutputSchema = z - .object({ - files: z.array(userFileSchema).min(1, 'At least one downloaded file is required').max(10), - mediaId: z.string().min(1).max(MAX_GRAPH_ID_LENGTH), - mediaType: z.string().max(64).nullable(), - downloadedCount: z.number().int().min(1).max(10), - }) - .superRefine((output, context) => { - if (output.downloadedCount !== output.files.length) { - context.addIssue({ - code: 'custom', - path: ['downloadedCount'], - message: 'Downloaded count must match the number of files', - }) - } - }) - -export const instagramDownloadMediaResponseSchema = z.discriminatedUnion('success', [ - z.object({ - success: z.literal(true), - output: instagramDownloadMediaOutputSchema, - }), - z.object({ - success: z.literal(false), - error: z.string().min(1), - }), -]) - -/** Canonical Sim file uploaded in basic mode or referenced from a prior block. */ -export const instagramMediaInputSchema = RawFileInputSchema - -/** Canonical Sim files uploaded in basic mode or referenced from prior blocks. */ -export const instagramCarouselMediaSchema = RawFileInputArraySchema.min( - 2, - 'Carousels require at least 2 items' -).max(10, 'Carousels support at most 10 items') - -export const instagramPublishOutputSchema = z.object({ - containerId: z.string().min(1, 'Container ID is required'), - mediaId: z.string().min(1, 'Media ID is required'), - statusCode: z.string().min(1, 'Status code is required'), -}) - -const instagramFailedPublishOutputSchema = z.object({ - containerId: z.null(), - mediaId: z.null(), - statusCode: z.null(), -}) - -export const instagramPublishResponseSchema = z.discriminatedUnion('success', [ - z.object({ - success: z.literal(true), - output: instagramPublishOutputSchema, - }), - z.object({ - success: z.literal(false), - error: z.string().min(1), - output: instagramFailedPublishOutputSchema.optional(), - }), -]) - -export const instagramPublishImageBodySchema = z.object({ - accessToken: instagramAccessTokenSchema, - igUserId: instagramOptionalUserIdSchema, - image: instagramMediaInputSchema, - caption: instagramOptionalCaptionSchema, - altText: z - .string() - .max(MAX_ALT_TEXT_LENGTH, `Alt text cannot exceed ${MAX_ALT_TEXT_LENGTH} characters`) - .optional() - .nullable(), - isAiGenerated: z.boolean().optional().nullable(), -}) - -export const instagramPublishVideoBodySchema = z.object({ - accessToken: instagramAccessTokenSchema, - igUserId: instagramOptionalUserIdSchema, - video: instagramMediaInputSchema, - cover: instagramMediaInputSchema.optional().nullable(), - caption: instagramOptionalCaptionSchema, -}) - -export const instagramPublishReelBodySchema = z.object({ - accessToken: instagramAccessTokenSchema, - igUserId: instagramOptionalUserIdSchema, - video: instagramMediaInputSchema, - cover: instagramMediaInputSchema.optional().nullable(), - caption: instagramOptionalCaptionSchema, - shareToFeed: z.boolean().optional().nullable(), - thumbOffset: z.number().optional().nullable(), -}) - -export const instagramPublishStoryBodySchema = z.object({ - accessToken: instagramAccessTokenSchema, - igUserId: instagramOptionalUserIdSchema, - media: instagramMediaInputSchema, -}) - -export const instagramPublishCarouselBodySchema = z.object({ - accessToken: instagramAccessTokenSchema, - igUserId: instagramOptionalUserIdSchema, - media: instagramCarouselMediaSchema, - caption: instagramOptionalCaptionSchema, -}) - -export const instagramPublishImageContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/instagram/publish-image', - body: instagramPublishImageBodySchema, - response: { mode: 'json', schema: instagramPublishResponseSchema }, -}) - -export const instagramPublishVideoContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/instagram/publish-video', - body: instagramPublishVideoBodySchema, - response: { mode: 'json', schema: instagramPublishResponseSchema }, -}) - -export const instagramPublishReelContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/instagram/publish-reel', - body: instagramPublishReelBodySchema, - response: { mode: 'json', schema: instagramPublishResponseSchema }, -}) - -export const instagramPublishStoryContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/instagram/publish-story', - body: instagramPublishStoryBodySchema, - response: { mode: 'json', schema: instagramPublishResponseSchema }, -}) - -export const instagramPublishCarouselContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/instagram/publish-carousel', - body: instagramPublishCarouselBodySchema, - response: { mode: 'json', schema: instagramPublishResponseSchema }, -}) - -export const instagramDownloadMediaContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/instagram/download-media', - body: instagramDownloadMediaBodySchema, - response: { mode: 'json', schema: instagramDownloadMediaResponseSchema }, -}) - -export type InstagramDownloadMediaBody = ContractBodyInput -export type InstagramDownloadMediaRouteResponse = ContractJsonResponse< - typeof instagramDownloadMediaContract -> - -export type InstagramPublishImageBody = ContractBodyInput -export type InstagramPublishVideoBody = ContractBodyInput -export type InstagramPublishReelBody = ContractBodyInput -export type InstagramPublishStoryBody = ContractBodyInput -export type InstagramPublishCarouselBody = ContractBodyInput< - typeof instagramPublishCarouselContract -> - -export type InstagramPublishImageResponse = ContractJsonResponse< - typeof instagramPublishImageContract -> -export type InstagramPublishVideoResponse = ContractJsonResponse< - typeof instagramPublishVideoContract -> -export type InstagramPublishReelResponse = ContractJsonResponse -export type InstagramPublishStoryResponse = ContractJsonResponse< - typeof instagramPublishStoryContract -> -export type InstagramPublishCarouselResponse = ContractJsonResponse< - typeof instagramPublishCarouselContract -> diff --git a/apps/sim/lib/api/contracts/tools/latex.ts b/apps/sim/lib/api/contracts/tools/latex.ts deleted file mode 100644 index 665619711e0..00000000000 --- a/apps/sim/lib/api/contracts/tools/latex.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { z } from 'zod' -import { genericToolResponseSchema } from '@/lib/api/contracts/tools/shared' -import { defineRouteContract } from '@/lib/api/contracts/types' - -export const latexCompilers = [ - 'pdflatex', - 'xelatex', - 'lualatex', - 'platex', - 'uplatex', - 'context', -] as const - -const MAX_LATEX_SOURCE_CHARS = 1_000_000 -const MAX_LATEX_RESOURCES = 25 - -const latexResourceSchema = z - .object({ - path: z - .string() - .min(1, 'resource path cannot be empty') - .max(512, 'resource path must be at most 512 characters') - .refine( - (path) => !path.startsWith('/') && path.split(/[/\\]/).every((segment) => segment !== '..'), - 'resource path must be relative and must not contain ".." segments' - ), - content: z - .string() - .min(1, 'resource content cannot be empty') - .max(MAX_LATEX_SOURCE_CHARS, 'resource content must be at most 1,000,000 characters') - .optional(), - file: z - .string() - .min(1, 'resource file cannot be empty') - .max(MAX_LATEX_SOURCE_CHARS, 'resource file must be at most 1,000,000 characters of base64') - .optional(), - url: z - .string() - .url('resource url must be a valid URL') - .max(2048, 'resource url must be at most 2048 characters') - .refine( - (url) => url.startsWith('https://') || url.startsWith('http://'), - 'resource url must use http or https' - ) - .optional(), - }) - .superRefine((resource, ctx) => { - const provided = [resource.content, resource.file, resource.url].filter( - (value) => value !== undefined - ) - if (provided.length !== 1) { - ctx.addIssue({ - code: 'custom', - path: ['path'], - message: `resource "${resource.path}" must provide exactly one of content, file, or url`, - }) - } - }) - -export const latexCompileBodySchema = z.object({ - content: z - .string() - .min(1, 'content cannot be empty') - .max(MAX_LATEX_SOURCE_CHARS, 'content must be at most 1,000,000 characters'), - compiler: z.enum(latexCompilers).optional(), - fileName: z.string().max(255, 'fileName must be at most 255 characters').optional(), - resources: z - .array(latexResourceSchema) - .max(MAX_LATEX_RESOURCES, `resources must contain at most ${MAX_LATEX_RESOURCES} entries`) - .optional(), - workspaceId: z.string().optional(), - workflowId: z.string().optional(), - executionId: z.string().optional(), -}) - -export type LatexCompileBody = z.input - -export const latexCompileContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/latex', - body: latexCompileBodySchema, - response: { - mode: 'json', - schema: genericToolResponseSchema, - }, -}) diff --git a/apps/sim/lib/api/contracts/tools/mail.ts b/apps/sim/lib/api/contracts/tools/mail.ts deleted file mode 100644 index 94bff861dd1..00000000000 --- a/apps/sim/lib/api/contracts/tools/mail.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { z } from 'zod' -import type { - ContractBody, - ContractBodyInput, - ContractJsonResponse, -} from '@/lib/api/contracts/types' -import { defineRouteContract } from '@/lib/api/contracts/types' - -export const mailSendResponseSchema = z.object({ - success: z.literal(true), - message: z.string(), - data: z - .object({ - id: z.string(), - }) - .nullable(), -}) - -export const mailSendBodySchema = z.object({ - fromAddress: z.string().min(1, 'From address is required'), - to: z.string().min(1, 'To email is required'), - subject: z.string().min(1, 'Subject is required'), - body: z.string().min(1, 'Email body is required'), - contentType: z.enum(['text', 'html']).optional().nullable(), - resendApiKey: z.string().min(1, 'Resend API key is required'), - cc: z - .union([z.string().min(1), z.array(z.string().min(1))]) - .optional() - .nullable(), - bcc: z - .union([z.string().min(1), z.array(z.string().min(1))]) - .optional() - .nullable(), - replyTo: z - .union([z.string().min(1), z.array(z.string().min(1))]) - .optional() - .nullable(), - scheduledAt: z.string().datetime().optional().nullable(), - tags: z.string().optional().nullable(), -}) - -export const mailSendContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/mail/send', - body: mailSendBodySchema, - response: { mode: 'json', schema: mailSendResponseSchema }, -}) - -export type MailSendBody = ContractBody -export type MailSendBodyInput = ContractBodyInput -export type MailSendResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/tools/media/document-parse.ts b/apps/sim/lib/api/contracts/tools/media/document-parse.ts index d762e232d8b..39b7e1ed15c 100644 --- a/apps/sim/lib/api/contracts/tools/media/document-parse.ts +++ b/apps/sim/lib/api/contracts/tools/media/document-parse.ts @@ -110,38 +110,6 @@ export const textractAnalyzeIdBodySchema = z } }) -export const reductoParseBodySchema = z.object({ - apiKey: z.string().min(1, 'API key is required'), - filePath: z.string().optional(), - file: RawFileInputSchema.optional(), - pages: z.array(z.number()).optional(), - tableOutputFormat: z.enum(['html', 'md']).optional(), - [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), -}) - -export const pulseParseBodySchema = z.object({ - apiKey: z.string().min(1, 'API key is required'), - filePath: z.string().optional(), - file: RawFileInputSchema.optional(), - pages: z.string().optional(), - extractFigure: z.boolean().optional(), - figureDescription: z.boolean().optional(), - returnHtml: z.boolean().optional(), - chunking: z.string().optional(), - chunkSize: z.number().optional(), - [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), -}) - -export const extendParseBodySchema = z.object({ - apiKey: z.string().min(1, 'API key is required'), - filePath: z.string().optional(), - file: RawFileInputSchema.optional(), - outputFormat: z.enum(['markdown', 'spatial']).optional(), - chunking: z.enum(['page', 'document', 'section']).optional(), - engine: z.enum(['parse_performance', 'parse_light']).optional(), - [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), -}) - export const mistralParseBodySchema = z.object({ apiKey: z.string().min(1, 'API key is required'), filePath: z.string().min(1, 'File path is required').optional(), @@ -176,27 +144,6 @@ export const textractAnalyzeIdContract = defineRouteContract({ response: { mode: 'json', schema: toolJsonResponseSchema }, }) -export const reductoParseContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/reducto/parse', - body: reductoParseBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) - -export const pulseParseContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/pulse/parse', - body: pulseParseBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) - -export const extendParseContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/extend/parse', - body: extendParseBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) - export const mistralParseContract = defineRouteContract({ method: 'POST', path: '/api/tools/mistral/parse', diff --git a/apps/sim/lib/api/contracts/tools/media/elevenlabs.ts b/apps/sim/lib/api/contracts/tools/media/elevenlabs.ts deleted file mode 100644 index f96ea799bc1..00000000000 --- a/apps/sim/lib/api/contracts/tools/media/elevenlabs.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { z } from 'zod' -import { resolvedSecretTraceProvenanceSchema, userFileSchema } from '@/lib/api/contracts/primitives' -import { toolBooleanSchema, toolJsonResponseSchema } from '@/lib/api/contracts/tools/media/shared' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' - -const MISSING_FIELDS_ERROR = 'Missing required fields: operation and apiKey' - -export const elevenLabsAudioFileSchema = userFileSchema.extend({ - type: z.string().optional().default(''), -}) - -export const elevenLabsAudioToolBodySchema = z - .object({ - operation: z.enum(['sound_effects', 'speech_to_speech', 'audio_isolation'], { - error: MISSING_FIELDS_ERROR, - }), - apiKey: z.string({ error: MISSING_FIELDS_ERROR }).min(1, MISSING_FIELDS_ERROR), - voiceId: z.string().optional(), - text: z.string().optional(), - modelId: z.string().optional(), - durationSeconds: z.coerce.number().min(0.5).max(30).optional(), - promptInfluence: z.coerce.number().min(0).max(1).optional(), - loop: toolBooleanSchema.optional(), - removeBackgroundNoise: toolBooleanSchema.optional(), - audioFile: elevenLabsAudioFileSchema.optional(), - workspaceId: z.string().optional(), - workflowId: z.string().optional(), - executionId: z.string().optional(), - [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), - }) - .passthrough() - -export const elevenLabsAudioToolContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/elevenlabs/audio', - body: elevenLabsAudioToolBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) diff --git a/apps/sim/lib/api/contracts/tools/media/image.ts b/apps/sim/lib/api/contracts/tools/media/image.ts index 3429b861b15..3d767062cd0 100644 --- a/apps/sim/lib/api/contracts/tools/media/image.ts +++ b/apps/sim/lib/api/contracts/tools/media/image.ts @@ -1,51 +1,4 @@ import { z } from 'zod' -import { toolBooleanSchema, toolJsonResponseSchema } from '@/lib/api/contracts/tools/media/shared' -import { defineRouteContract } from '@/lib/api/contracts/types' - -export const imageProviders = ['openai', 'gemini', 'falai'] as const -const MISSING_IMAGE_FIELDS_ERROR = 'Missing required fields: provider, apiKey, and prompt' - export const imageProxyQuerySchema = z.object({ url: z.string({ error: 'Missing URL parameter' }).min(1, 'Missing URL parameter'), }) - -export const imageToolBodySchema = z - .object({ - provider: z - .string({ error: MISSING_IMAGE_FIELDS_ERROR }) - .min(1, MISSING_IMAGE_FIELDS_ERROR) - .refine((provider) => imageProviders.includes(provider as (typeof imageProviders)[number]), { - message: `Invalid provider. Must be one of: ${imageProviders.join(', ')}`, - }), - apiKey: z.string({ error: MISSING_IMAGE_FIELDS_ERROR }).min(1, MISSING_IMAGE_FIELDS_ERROR), - model: z.string().optional(), - prompt: z.string({ error: MISSING_IMAGE_FIELDS_ERROR }).min(1, MISSING_IMAGE_FIELDS_ERROR), - size: z.string().optional(), - aspectRatio: z.string().optional(), - resolution: z.string().optional(), - quality: z.string().optional(), - background: z.string().optional(), - outputFormat: z.string().optional(), - moderation: z.string().optional(), - safetyTolerance: z.string().optional(), - numImages: z.coerce.number().int().optional(), - seed: z.coerce.number().int().optional(), - enableSafetyChecker: toolBooleanSchema.optional(), - enableWebSearch: toolBooleanSchema.optional(), - thinkingLevel: z.string().optional(), - workspaceId: z.string().optional(), - workflowId: z.string().optional(), - executionId: z.string().optional(), - userId: z.string().optional(), - useHostedCostTracking: z.boolean().optional(), - }) - .passthrough() - -export type ImageToolBody = z.infer - -export const imageToolContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/image', - body: imageToolBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) diff --git a/apps/sim/lib/api/contracts/tools/media/index.ts b/apps/sim/lib/api/contracts/tools/media/index.ts index 922d0e5651e..d8d3f5817bb 100644 --- a/apps/sim/lib/api/contracts/tools/media/index.ts +++ b/apps/sim/lib/api/contracts/tools/media/index.ts @@ -1,7 +1,5 @@ export * from '@/lib/api/contracts/tools/media/document-parse' export * from '@/lib/api/contracts/tools/media/image' export * from '@/lib/api/contracts/tools/media/shared' -export * from '@/lib/api/contracts/tools/media/stt' export * from '@/lib/api/contracts/tools/media/tts' export * from '@/lib/api/contracts/tools/media/video' -export * from '@/lib/api/contracts/tools/media/vision' diff --git a/apps/sim/lib/api/contracts/tools/media/stt.ts b/apps/sim/lib/api/contracts/tools/media/stt.ts deleted file mode 100644 index 1aef9266069..00000000000 --- a/apps/sim/lib/api/contracts/tools/media/stt.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { z } from 'zod' -import { resolvedSecretTraceProvenanceSchema, userFileSchema } from '@/lib/api/contracts/primitives' -import { toolJsonResponseSchema } from '@/lib/api/contracts/tools/media/shared' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' - -export const sttProviders = ['whisper', 'deepgram', 'elevenlabs', 'assemblyai', 'gemini'] as const -const MISSING_STT_FIELDS_ERROR = 'Missing required fields: provider and apiKey' - -export const sttUserFileSchema = userFileSchema.extend({ - type: z.string().optional().default(''), -}) - -export const sttUserFileInputSchema = z.union([sttUserFileSchema, z.array(sttUserFileSchema)]) - -export const sttToolBodySchema = z - .object({ - provider: z - .string({ error: MISSING_STT_FIELDS_ERROR }) - .min(1, MISSING_STT_FIELDS_ERROR) - .refine((provider) => sttProviders.includes(provider as (typeof sttProviders)[number]), { - message: `Invalid provider. Must be one of: ${sttProviders.join(', ')}`, - }), - apiKey: z.string({ error: MISSING_STT_FIELDS_ERROR }).min(1, MISSING_STT_FIELDS_ERROR), - model: z.string().optional(), - audioFile: sttUserFileInputSchema.optional(), - audioFileReference: sttUserFileInputSchema.optional(), - audioUrl: z.string().optional(), - language: z.string().optional(), - timestamps: z.enum(['none', 'sentence', 'word']).optional(), - diarization: z.boolean().optional(), - translateToEnglish: z.boolean().optional(), - prompt: z.string().optional(), - temperature: z.coerce.number().optional(), - sentiment: z.boolean().optional(), - entityDetection: z.boolean().optional(), - piiRedaction: z.boolean().optional(), - summarization: z.boolean().optional(), - workspaceId: z.string().optional(), - workflowId: z.string().optional(), - executionId: z.string().optional(), - [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), - }) - .passthrough() - -export const sttToolContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/stt', - body: sttToolBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) diff --git a/apps/sim/lib/api/contracts/tools/media/video.ts b/apps/sim/lib/api/contracts/tools/media/video.ts index 90d635c42c4..157bab5d19e 100644 --- a/apps/sim/lib/api/contracts/tools/media/video.ts +++ b/apps/sim/lib/api/contracts/tools/media/video.ts @@ -1,7 +1,6 @@ import { z } from 'zod' import { resolvedSecretTraceProvenanceSchema, userFileSchema } from '@/lib/api/contracts/primitives' -import { toolBooleanSchema, toolJsonResponseSchema } from '@/lib/api/contracts/tools/media/shared' -import { defineRouteContract } from '@/lib/api/contracts/types' +import { toolBooleanSchema } from '@/lib/api/contracts/tools/media/shared' import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' export const videoProviders = ['runway', 'veo', 'luma', 'minimax', 'falai'] as const @@ -34,10 +33,3 @@ export const videoToolBodySchema = z [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), }) .passthrough() - -export const videoToolContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/video', - body: videoToolBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) diff --git a/apps/sim/lib/api/contracts/tools/media/vision.ts b/apps/sim/lib/api/contracts/tools/media/vision.ts deleted file mode 100644 index 80804b3ccbc..00000000000 --- a/apps/sim/lib/api/contracts/tools/media/vision.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { z } from 'zod' -import { resolvedSecretTraceProvenanceSchema } from '@/lib/api/contracts/primitives' -import { toolJsonResponseSchema } from '@/lib/api/contracts/tools/media/shared' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' -import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' - -export const visionAnalyzeBodySchema = z.object({ - apiKey: z.string().min(1, 'API key is required'), - imageUrl: z.string().optional().nullable(), - imageFile: RawFileInputSchema.optional().nullable(), - model: z.string().optional().default('gpt-5.2'), - prompt: z.string().optional().nullable(), - [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), -}) - -export const visionAnalyzeContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/vision/analyze', - body: visionAnalyzeBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) diff --git a/apps/sim/lib/api/contracts/tools/microsoft.ts b/apps/sim/lib/api/contracts/tools/microsoft.ts index dcbb98f3c9d..4b4889b34c4 100644 --- a/apps/sim/lib/api/contracts/tools/microsoft.ts +++ b/apps/sim/lib/api/contracts/tools/microsoft.ts @@ -1,15 +1,7 @@ import { z } from 'zod' import type { ContractBody } from '@/lib/api/contracts/types' import { defineRouteContract } from '@/lib/api/contracts/types' -import { RawFileInputArraySchema, RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' - -const excelCellSchema = z.union([z.string(), z.number(), z.boolean(), z.null()]) -const excelRowSchema = z.array(excelCellSchema) -const excelValuesSchema = z.union([ - z.string(), - z.array(excelRowSchema), - z.array(z.record(z.string(), excelCellSchema)), -]) +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' export const accessTokenSchema = z.string().min(1, 'Access token is required') export const messageIdSchema = z.string().min(1, 'Message ID is required') @@ -40,186 +32,6 @@ export const outlookCopyMoveBodySchema = outlookDeleteBodySchema.extend({ destinationId: destinationIdSchema, }) -export const teamsWriteChannelBodySchema = z.object({ - accessToken: accessTokenSchema, - teamId: z.string().min(1, 'Team ID is required'), - channelId: z.string().min(1, 'Channel ID is required'), - content: z.string().min(1, 'Message content is required'), - files: RawFileInputArraySchema.optional().nullable(), -}) - -export const teamsWriteChatBodySchema = z.object({ - accessToken: accessTokenSchema, - chatId: z.string().min(1, 'Chat ID is required'), - content: z.string().min(1, 'Message content is required'), - files: RawFileInputArraySchema.optional().nullable(), -}) - -export const teamsDeleteChatMessageBodySchema = z.object({ - accessToken: accessTokenSchema, - chatId: z.string().min(1, 'Chat ID is required'), - messageId: messageIdSchema, -}) - -export const onedriveUploadBodySchema = z.object({ - accessToken: accessTokenSchema, - fileName: z.string().min(1, 'File name is required'), - file: RawFileInputSchema.optional(), - folderId: z.string().optional().nullable(), - mimeType: z.string().nullish(), - values: excelValuesSchema.optional().nullable(), - conflictBehavior: z.enum(['fail', 'replace', 'rename']).optional().nullable(), -}) - -export const onedriveDownloadBodySchema = z.object({ - accessToken: accessTokenSchema, - fileId: z.string().min(1, 'File ID is required'), - fileName: z.string().optional().nullable(), -}) - -export const sharepointUploadBodySchema = z.object({ - accessToken: accessTokenSchema, - siteId: z.string().default('root'), - driveId: z.string().optional().nullable(), - folderPath: z.string().optional().nullable(), - fileName: z.string().optional().nullable(), - files: RawFileInputArraySchema.optional().nullable(), -}) - -export const sharepointDownloadFileBodySchema = z.object({ - accessToken: accessTokenSchema, - driveId: z.string().min(1, 'Drive ID is required'), - itemId: z.string().min(1, 'Item ID is required'), - fileName: z.string().optional().nullable(), -}) - -export const dataverseUploadFileBodySchema = z.object({ - accessToken: accessTokenSchema, - environmentUrl: z.string().min(1, 'Environment URL is required'), - entitySetName: z.string().min(1, 'Entity set name is required'), - recordId: z.string().min(1, 'Record ID is required'), - fileColumn: z.string().min(1, 'File column is required'), - fileName: z.string().min(1, 'File name is required'), - file: RawFileInputSchema.optional().nullable(), - fileContent: z.string().optional().nullable(), -}) - -/** - * Ceiling on generated document text. A `.docx` is built in memory before it is - * uploaded, so an unbounded body would be materialized twice — once as text and - * once as a zipped package. - */ -const MAX_DOCUMENT_CONTENT_LENGTH = 2_000_000 - -/** - * Ceiling on the serialized placeholder map. Replacement values are substituted - * into a package that is rewritten in memory, so they need the same kind of - * bound the generated document text has. - */ -const MAX_REPLACEMENTS_LENGTH = 200_000 - -/** Whether a string parses as a JSON object (not an array or scalar). */ -function isPlaceholderMap(value: unknown): boolean { - if (typeof value !== 'object' || value === null || Array.isArray(value)) return false - // A blank placeholder would match at every position in the document, so it is - // rejected here rather than failing mid-rewrite with an unattributable 500. - return Object.keys(value).every((key) => key.trim().length > 0) -} - -/** Whether a string parses as a JSON object whose keys are all usable placeholders. */ -function isPlaceholderMapString(value: string): boolean { - if (!value.trim()) return true - try { - return isPlaceholderMap(JSON.parse(value)) - } catch { - return false - } -} - -/** - * The placeholder map, accepted either as an object from the editor or as the - * JSON string a variable reference resolves to. Both forms are bounded and both - * must describe an object, so malformed caller input is a 400 rather than a 500 - * raised later while the template is being filled. - */ -const wordReplacementsSchema = z - .union([ - z - .string() - .refine( - isPlaceholderMapString, - 'Placeholder values must be a JSON object mapping each non-empty placeholder to its value' - ), - z - .record(z.string(), z.union([z.string(), z.number(), z.boolean(), z.null()])) - .refine(isPlaceholderMap, 'Every placeholder must be a non-empty string'), - ]) - .refine( - (value) => - (typeof value === 'string' ? value.length : JSON.stringify(value).length) <= - MAX_REPLACEMENTS_LENGTH, - 'Placeholder values are too long' - ) - -const wordDocumentIdSchema = z.string().min(1, 'Document ID is required') -const wordDriveIdSchema = z.string().optional().nullable() - -export const microsoftWordCreateBodySchema = z.object({ - accessToken: accessTokenSchema, - name: z.string().min(1, 'Document name is required').max(255, 'Document name is too long'), - content: z - .string() - .max(MAX_DOCUMENT_CONTENT_LENGTH, 'Document content is too long') - .optional() - .nullable(), - folderId: z.string().optional().nullable(), - driveId: wordDriveIdSchema, -}) - -export const microsoftWordReadBodySchema = z.object({ - accessToken: accessTokenSchema, - documentId: wordDocumentIdSchema, - driveId: wordDriveIdSchema, -}) - -export const microsoftWordUpdateBodySchema = z.object({ - accessToken: accessTokenSchema, - documentId: wordDocumentIdSchema, - content: z - .string() - .min(1, 'Document content is required') - .max(MAX_DOCUMENT_CONTENT_LENGTH, 'Document content is too long'), - driveId: wordDriveIdSchema, -}) - -export const microsoftWordAppendBodySchema = microsoftWordUpdateBodySchema - -export const microsoftWordCreateFromTemplateBodySchema = z.object({ - accessToken: accessTokenSchema, - templateDocumentId: z.string().min(1, 'Template document ID is required'), - name: z.string().min(1, 'Document name is required').max(255, 'Document name is too long'), - replacements: wordReplacementsSchema.optional().nullable(), - matchCase: z.boolean().optional().nullable(), - folderId: z.string().optional().nullable(), - driveId: wordDriveIdSchema, -}) - -export const microsoftWordReplaceTextBodySchema = z.object({ - accessToken: accessTokenSchema, - documentId: wordDocumentIdSchema, - findText: z.string().min(1, 'Search text is required').max(4000, 'Search text is too long'), - replaceText: z.string().max(20000, 'Replacement text is too long').optional().nullable(), - matchCase: z.boolean().optional().nullable(), - driveId: wordDriveIdSchema, -}) - -export const microsoftWordExportPdfBodySchema = z.object({ - accessToken: accessTokenSchema, - documentId: wordDocumentIdSchema, - fileName: z.string().optional().nullable(), - driveId: wordDriveIdSchema, -}) - const toolJsonResponseSchema = z.unknown() export const outlookSendContract = defineRouteContract({ @@ -271,111 +83,6 @@ export const outlookMarkUnreadContract = defineRouteContract({ response: { mode: 'json', schema: toolJsonResponseSchema }, }) -export const teamsWriteChannelContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/microsoft_teams/write_channel', - body: teamsWriteChannelBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) - -export const teamsWriteChatContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/microsoft_teams/write_chat', - body: teamsWriteChatBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) - -export const teamsDeleteChatMessageContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/microsoft_teams/delete_chat_message', - body: teamsDeleteChatMessageBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) - -export const onedriveUploadContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/onedrive/upload', - body: onedriveUploadBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) - -export const onedriveDownloadContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/onedrive/download', - body: onedriveDownloadBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) - -export const sharepointUploadContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/sharepoint/upload', - body: sharepointUploadBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) - -export const sharepointDownloadFileContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/sharepoint/download-file', - body: sharepointDownloadFileBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) - -export const dataverseUploadFileContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/microsoft-dataverse/upload-file', - body: dataverseUploadFileBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) - -export const microsoftWordCreateContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/microsoft_word/create', - body: microsoftWordCreateBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) - -export const microsoftWordReadContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/microsoft_word/read', - body: microsoftWordReadBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) - -export const microsoftWordUpdateContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/microsoft_word/update', - body: microsoftWordUpdateBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) - -export const microsoftWordAppendContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/microsoft_word/append', - body: microsoftWordAppendBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) - -export const microsoftWordCreateFromTemplateContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/microsoft_word/create-from-template', - body: microsoftWordCreateFromTemplateBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) - -export const microsoftWordReplaceTextContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/microsoft_word/replace-text', - body: microsoftWordReplaceTextBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) - -export const microsoftWordExportPdfContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/microsoft_word/export-pdf', - body: microsoftWordExportPdfBodySchema, - response: { mode: 'json', schema: toolJsonResponseSchema }, -}) - export type OutlookSendBody = ContractBody export type OutlookDraftBody = ContractBody export type OutlookDeleteBody = ContractBody @@ -383,20 +90,3 @@ export type OutlookCopyBody = ContractBody export type OutlookMoveBody = ContractBody export type OutlookMarkReadBody = ContractBody export type OutlookMarkUnreadBody = ContractBody -export type TeamsWriteChannelBody = ContractBody -export type TeamsWriteChatBody = ContractBody -export type TeamsDeleteChatMessageBody = ContractBody -export type OneDriveUploadBody = ContractBody -export type OneDriveDownloadBody = ContractBody -export type SharepointUploadBody = ContractBody -export type SharepointDownloadFileBody = ContractBody -export type DataverseUploadFileBody = z.output -export type MicrosoftWordCreateBody = ContractBody -export type MicrosoftWordReadBody = ContractBody -export type MicrosoftWordUpdateBody = ContractBody -export type MicrosoftWordAppendBody = ContractBody -export type MicrosoftWordCreateFromTemplateBody = ContractBody< - typeof microsoftWordCreateFromTemplateContract -> -export type MicrosoftWordReplaceTextBody = ContractBody -export type MicrosoftWordExportPdfBody = ContractBody diff --git a/apps/sim/lib/api/contracts/tools/persona.ts b/apps/sim/lib/api/contracts/tools/persona.ts deleted file mode 100644 index 146d321f368..00000000000 --- a/apps/sim/lib/api/contracts/tools/persona.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { z } from 'zod' -import type { ContractBody, ContractJsonResponse } from '@/lib/api/contracts/types' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' - -const personaImporterSchema = z.object({ - id: z.string(), - status: z.string().nullable(), - successfulCount: z.number(), - errorCount: z.number(), - duplicateCount: z.number(), - createdAt: z.string().nullable(), - completedAt: z.string().nullable(), -}) - -const personaImportAccountsResponseSchema = z.object({ - success: z.literal(true), - output: z.object({ - importer: personaImporterSchema, - }), -}) - -export const personaImportAccountsBodySchema = z.object({ - apiKey: z.string().min(1, 'Persona API key is required'), - file: RawFileInputSchema, -}) - -export const personaImportAccountsContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/persona/import-accounts', - body: personaImportAccountsBodySchema, - response: { mode: 'json', schema: personaImportAccountsResponseSchema }, -}) - -export type PersonaImportAccountsBody = ContractBody -export type PersonaImportAccountsRouteResponse = ContractJsonResponse< - typeof personaImportAccountsContract -> diff --git a/apps/sim/lib/api/contracts/tools/pipedrive.ts b/apps/sim/lib/api/contracts/tools/pipedrive.ts deleted file mode 100644 index d1d33f86067..00000000000 --- a/apps/sim/lib/api/contracts/tools/pipedrive.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { z } from 'zod' -import type { - ContractBody, - ContractBodyInput, - ContractJsonResponse, -} from '@/lib/api/contracts/types' -import { defineRouteContract } from '@/lib/api/contracts/types' - -const pipedriveDownloadedFileSchema = z.object({ - name: z.string(), - mimeType: z.string(), - data: z.string(), - size: z.number(), -}) - -export const pipedriveGetFilesResponseSchema = z.object({ - success: z.literal(true), - output: z.object({ - files: z.array(z.unknown()), - downloadedFiles: z.array(pipedriveDownloadedFileSchema).optional(), - total_items: z.number(), - has_more: z.boolean(), - next_start: z.number().nullable(), - success: z.literal(true), - }), -}) - -export const pipedriveGetFilesBodySchema = z.object({ - accessToken: z.string().min(1, 'Access token is required'), - authStyle: z.enum(['x-api-token']).optional(), - sort: z.enum(['id', 'update_time']).optional().nullable(), - limit: z.string().optional().nullable(), - start: z.string().optional().nullable(), - downloadFiles: z.boolean().optional().default(false), -}) - -export const pipedriveGetFilesContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/pipedrive/get-files', - body: pipedriveGetFilesBodySchema, - response: { mode: 'json', schema: pipedriveGetFilesResponseSchema }, -}) - -export type PipedriveGetFilesBody = ContractBody -export type PipedriveGetFilesBodyInput = ContractBodyInput -export type PipedriveGetFilesResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/tools/quiver.ts b/apps/sim/lib/api/contracts/tools/quiver.ts deleted file mode 100644 index c9890ac06ba..00000000000 --- a/apps/sim/lib/api/contracts/tools/quiver.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { z } from 'zod' -import { resolvedSecretTraceProvenanceSchema } from '@/lib/api/contracts/primitives' -import { genericToolResponseSchema } from '@/lib/api/contracts/tools/shared' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' -import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' - -const quiverCommonBodySchema = z.object({ - apiKey: z.string().min(1), - model: z.string().min(1), - temperature: z.number().min(0).max(2).optional().nullable(), - top_p: z.number().min(0).max(1).optional().nullable(), - max_output_tokens: z.number().int().min(1).max(131072).optional().nullable(), - presence_penalty: z.number().min(-2).max(2).optional().nullable(), - [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), -}) - -export const quiverTextToSvgBodySchema = quiverCommonBodySchema.extend({ - prompt: z.string().min(1), - instructions: z.string().optional().nullable(), - references: z - .union([z.array(FileInputSchema), FileInputSchema, z.string()]) - .optional() - .nullable(), - n: z.number().int().min(1).max(16).optional().nullable(), -}) - -export const quiverImageToSvgBodySchema = quiverCommonBodySchema.extend({ - image: z.union([FileInputSchema, z.string()]), - auto_crop: z.boolean().optional().nullable(), - target_size: z.number().int().min(128).max(4096).optional().nullable(), -}) - -export const quiverTextToSvgContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/quiver/text-to-svg', - body: quiverTextToSvgBodySchema, - response: { - mode: 'json', - schema: genericToolResponseSchema, - }, -}) - -export const quiverImageToSvgContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/quiver/image-to-svg', - body: quiverImageToSvgBodySchema, - response: { - mode: 'json', - schema: genericToolResponseSchema, - }, -}) diff --git a/apps/sim/lib/api/contracts/tools/sap.ts b/apps/sim/lib/api/contracts/tools/sap.ts deleted file mode 100644 index dcfa27fcaab..00000000000 --- a/apps/sim/lib/api/contracts/tools/sap.ts +++ /dev/null @@ -1,221 +0,0 @@ -import { isPrivateIpHost } from '@sim/security/ssrf' -import { z } from 'zod' -import { genericToolResponseSchema } from '@/lib/api/contracts/tools/shared' -import { defineRouteContract } from '@/lib/api/contracts/types' - -const sapHttpMethodSchema = z.enum(['GET', 'POST', 'PATCH', 'PUT', 'DELETE', 'MERGE']) -const sapDeploymentTypeSchema = z.enum(['cloud_public', 'cloud_private', 'on_premise']) -const sapAuthTypeSchema = z.enum(['oauth_client_credentials', 'basic']) - -const sapServiceNameSchema = z - .string() - .min(1, 'service is required') - .regex( - /^[A-Z][A-Z0-9_]*(;v=\d+)?$/, - 'service must be an uppercase OData service name optionally suffixed with ";v=NNNN" (e.g., API_BUSINESS_PARTNER, API_OUTBOUND_DELIVERY_SRV;v=0002)' - ) - -const sapServicePathSchema = z - .string() - .min(1, 'path is required') - .refine( - (path) => - !path.split(/[/\\]/).some((segment) => segment === '..' || segment === '.') && - !path.includes('?') && - !path.includes('#') && - !/%(?:2[eEfF]|5[cC]|3[fF]|23)/.test(path), - { - message: - 'path must not contain ".." or "." segments, "?", "#", or percent-encoded path/query/fragment characters', - } - ) - -const sapSubdomainSchema = z - .string() - .regex( - /^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$/i, - 'subdomain must contain only letters, digits, and hyphens (1-63 chars)' - ) - -const FORBIDDEN_SAP_HOSTS = new Set([ - 'localhost', - '0.0.0.0', - '127.0.0.1', - '169.254.169.254', - 'metadata.google.internal', - 'metadata', - '[::1]', - '[::]', - '[::ffff:127.0.0.1]', - '[fd00:ec2::254]', -]) - -export function checkSapExternalUrlSafety( - rawUrl: string, - label: string -): { ok: true; url: URL } | { ok: false; message: string } { - let parsed: URL - try { - parsed = new URL(rawUrl) - } catch { - return { ok: false, message: `${label} must be a valid URL` } - } - if (parsed.protocol !== 'https:') { - return { ok: false, message: `${label} must use https://` } - } - const host = parsed.hostname.toLowerCase() - if (FORBIDDEN_SAP_HOSTS.has(host) || FORBIDDEN_SAP_HOSTS.has(`[${host}]`)) { - return { ok: false, message: `${label} host is not allowed` } - } - if (isPrivateIpHost(host)) { - return { ok: false, message: `${label} host is not allowed (private/loopback range)` } - } - return { ok: true, url: parsed } -} - -export function assertSafeSapExternalUrl(rawUrl: string, label: string): URL { - const result = checkSapExternalUrlSafety(rawUrl, label) - if (!result.ok) throw new Error(result.message) - return result.url -} - -export const sapS4HanaProxyBodySchema = z - .object({ - deploymentType: sapDeploymentTypeSchema.default('cloud_public'), - authType: sapAuthTypeSchema.default('oauth_client_credentials'), - subdomain: sapSubdomainSchema.optional(), - region: z - .string() - .regex(/^[a-z]{2,4}\d{1,3}$/i, 'region must be an SAP BTP region code (e.g., eu10, us30)') - .optional(), - baseUrl: z.string().optional(), - tokenUrl: z.string().optional(), - clientId: z.string().optional(), - clientSecret: z.string().optional(), - username: z.string().optional(), - password: z.string().optional(), - service: sapServiceNameSchema, - path: sapServicePathSchema, - method: sapHttpMethodSchema.default('GET'), - query: z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])).optional(), - body: z.unknown().optional(), - ifMatch: z.string().optional(), - }) - .superRefine((req, ctx) => { - if (req.deploymentType === 'cloud_public') { - if (!req.subdomain) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ['subdomain'], - message: 'subdomain is required for cloud_public deployment', - }) - } - if (!req.region) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ['region'], - message: 'region is required for cloud_public deployment', - }) - } - if (req.authType !== 'oauth_client_credentials') { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ['authType'], - message: 'cloud_public deployment only supports oauth_client_credentials', - }) - } - if (!req.clientId) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ['clientId'], - message: 'clientId is required', - }) - } - if (!req.clientSecret) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ['clientSecret'], - message: 'clientSecret is required', - }) - } - return - } - - if (!req.baseUrl) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ['baseUrl'], - message: 'baseUrl is required for cloud_private and on_premise deployments', - }) - } else { - const baseUrlCheck = checkSapExternalUrlSafety(req.baseUrl, 'baseUrl') - if (!baseUrlCheck.ok) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ['baseUrl'], - message: baseUrlCheck.message, - }) - } - } - - if (req.authType === 'oauth_client_credentials') { - if (!req.tokenUrl) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ['tokenUrl'], - message: 'tokenUrl is required for OAuth on cloud_private/on_premise', - }) - } else { - const tokenUrlCheck = checkSapExternalUrlSafety(req.tokenUrl, 'tokenUrl') - if (!tokenUrlCheck.ok) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ['tokenUrl'], - message: tokenUrlCheck.message, - }) - } - } - if (!req.clientId) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ['clientId'], - message: 'clientId is required for OAuth', - }) - } - if (!req.clientSecret) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ['clientSecret'], - message: 'clientSecret is required for OAuth', - }) - } - return - } - - if (!req.username) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ['username'], - message: 'username is required for Basic auth', - }) - } - if (!req.password) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ['password'], - message: 'password is required for Basic auth', - }) - } - }) - -export type SapS4HanaProxyRequest = z.infer - -export const sapS4HanaProxyContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/sap_s4hana/proxy', - body: sapS4HanaProxyBodySchema, - response: { - mode: 'json', - schema: genericToolResponseSchema, - }, -}) diff --git a/apps/sim/lib/api/contracts/tools/search.ts b/apps/sim/lib/api/contracts/tools/search.ts deleted file mode 100644 index ceee6caae2f..00000000000 --- a/apps/sim/lib/api/contracts/tools/search.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { z } from 'zod' -import type { - ContractBody, - ContractBodyInput, - ContractJsonResponse, -} from '@/lib/api/contracts/types' -import { defineRouteContract } from '@/lib/api/contracts/types' - -const searchResultSchema = z.object({ - title: z.string(), - link: z.string(), - snippet: z.string(), - date: z.string().optional(), - position: z.number(), -}) - -const searchCostSchema = z.object({ - input: z.number(), - output: z.number(), - total: z.number(), - tokens: z.object({ - input: z.number(), - output: z.number(), - total: z.number(), - }), - model: z.string(), - pricing: z.object({ - input: z.number(), - cachedInput: z.number(), - output: z.number(), - updatedAt: z.string(), - }), -}) - -export const searchToolResponseSchema = z.object({ - results: z.array(searchResultSchema), - query: z.string(), - totalResults: z.number(), - source: z.literal('exa'), - cost: searchCostSchema, -}) - -export const searchToolBodySchema = z.object({ - query: z.string().min(1), -}) - -export const searchToolContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/search', - body: searchToolBodySchema, - response: { mode: 'json', schema: searchToolResponseSchema }, -}) - -export type SearchToolBody = ContractBody -export type SearchToolBodyInput = ContractBodyInput -export type SearchToolResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/tools/servicenow.ts b/apps/sim/lib/api/contracts/tools/servicenow.ts deleted file mode 100644 index 49ca5e545dc..00000000000 --- a/apps/sim/lib/api/contracts/tools/servicenow.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { z } from 'zod' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' - -export const servicenowUploadAttachmentBodySchema = z.object({ - instanceUrl: z.string().min(1, 'Instance URL is required'), - username: z.string().min(1, 'Username is required'), - password: z.string().min(1, 'Password is required'), - tableName: z.string().min(1, 'Table name is required'), - recordSysId: z.string().min(1, 'Record sys_id is required'), - fileName: z.string().min(1, 'File name is required'), - file: RawFileInputSchema.optional().nullable(), -}) - -export type ServiceNowUploadAttachmentBody = z.input - -// untyped-response: ServiceNow returns arbitrary attachment metadata wrapped in a success envelope -const servicenowToolResponseSchema = z.unknown() - -export const servicenowUploadAttachmentContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/servicenow/upload-attachment', - body: servicenowUploadAttachmentBodySchema, - response: { mode: 'json', schema: servicenowToolResponseSchema }, -}) diff --git a/apps/sim/lib/api/contracts/tools/square.ts b/apps/sim/lib/api/contracts/tools/square.ts deleted file mode 100644 index 8be19a863cd..00000000000 --- a/apps/sim/lib/api/contracts/tools/square.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { z } from 'zod' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' - -/** - * Internal contract for the Square catalog image upload route. The route - * downloads the referenced file from storage and forwards it to Square's - * multipart `CreateCatalogImage` endpoint, returning the created image object. - * - * The `output.object` is a Square CatalogObject, whose shape is polymorphic by - * `type`, so it is intentionally left opaque while the envelope is typed. - */ -const squareCatalogImageResponseSchema = z.object({ - success: z.boolean(), - output: z - .object({ - object: z.unknown(), - metadata: z.object({ - id: z.string(), - type: z.string().nullable(), - version: z.number().nullable(), - }), - }) - .optional(), - error: z.string().optional(), -}) - -export const squareCatalogImageBodySchema = z.object({ - accessToken: z.string().min(1, 'Access token is required'), - file: FileInputSchema.optional().nullable(), - fileName: z.string().optional().nullable(), - objectId: z.string().optional().nullable(), - caption: z.string().optional().nullable(), - idempotencyKey: z.string().optional().nullable(), -}) - -export type SquareCatalogImageBody = z.input - -export const squareCatalogImageContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/square/catalog-image', - body: squareCatalogImageBodySchema, - response: { mode: 'json', schema: squareCatalogImageResponseSchema }, -}) diff --git a/apps/sim/lib/api/contracts/tools/stagehand.ts b/apps/sim/lib/api/contracts/tools/stagehand.ts deleted file mode 100644 index aff1633f791..00000000000 --- a/apps/sim/lib/api/contracts/tools/stagehand.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { z } from 'zod' -import { unknownRecordSchema } from '@/lib/api/contracts/primitives' -import { defineRouteContract } from '@/lib/api/contracts/types' - -export const stagehandProviderSchema = z.enum(['openai', 'anthropic']) - -export const stagehandAgentBodySchema = z.object({ - task: z.string().min(1), - startUrl: z.string().url(), - outputSchema: z.unknown(), - variables: z.unknown(), - provider: stagehandProviderSchema.optional().default('openai'), - apiKey: z.string(), - mode: z.enum(['dom', 'hybrid', 'cua']).optional().default('dom'), - maxSteps: z.number().int().min(1).max(200).optional().default(20), -}) - -export const stagehandExtractBodySchema = z.object({ - instruction: z.string(), - schema: unknownRecordSchema, - provider: stagehandProviderSchema.optional().default('openai'), - apiKey: z.string(), - url: z.string().url(), -}) - -export const stagehandAgentContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/stagehand/agent', - body: stagehandAgentBodySchema, - response: { - mode: 'json', - schema: unknownRecordSchema, - }, -}) - -export const stagehandExtractContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/stagehand/extract', - body: stagehandExtractBodySchema, - response: { - mode: 'json', - schema: unknownRecordSchema, - }, -}) diff --git a/apps/sim/lib/api/contracts/tools/thinking.ts b/apps/sim/lib/api/contracts/tools/thinking.ts deleted file mode 100644 index 7f5b147ba4a..00000000000 --- a/apps/sim/lib/api/contracts/tools/thinking.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { z } from 'zod' -import { - toolFailureResponseSchema, - toolSuccessResponseSchema, -} from '@/lib/api/contracts/tool-primitives' -import { defineRouteContract } from '@/lib/api/contracts/types' - -export const thinkingToolContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/thinking', - body: z.object({ - thought: z.string().min(1, 'The thought parameter is required and must be a string'), - }), - response: { - mode: 'json', - schema: z.union([ - toolSuccessResponseSchema( - z.object({ - acknowledgedThought: z.string(), - }) - ), - toolFailureResponseSchema, - ]), - }, -}) diff --git a/apps/sim/lib/api/contracts/tools/typeform.ts b/apps/sim/lib/api/contracts/tools/typeform.ts deleted file mode 100644 index ddf263b8e52..00000000000 --- a/apps/sim/lib/api/contracts/tools/typeform.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { z } from 'zod' -import { genericToolResponseSchema } from '@/lib/api/contracts/tools/shared' -import { defineRouteContract } from '@/lib/api/contracts/types' - -export const typeformFilesBodySchema = z.object({ - formId: z.string().min(1, 'Form ID is required'), - responseId: z.string().min(1, 'Response ID is required'), - fieldId: z.string().min(1, 'Field ID is required'), - filename: z.string().min(1, 'Filename is required'), - inline: z.boolean().optional(), - apiKey: z.string().min(1, 'API key is required'), - workspaceId: z.string().optional(), - workflowId: z.string().optional(), - executionId: z.string().optional(), -}) - -export const typeformFilesContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/typeform/files', - body: typeformFilesBodySchema, - response: { mode: 'json', schema: genericToolResponseSchema }, -}) diff --git a/apps/sim/lib/api/contracts/tools/uptimerobot.ts b/apps/sim/lib/api/contracts/tools/uptimerobot.ts deleted file mode 100644 index cf18d6bf868..00000000000 --- a/apps/sim/lib/api/contracts/tools/uptimerobot.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { z } from 'zod' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' - -/** - * Internal contracts for the UptimeRobot public-status-page (PSP) create/update - * routes. Those endpoints accept `multipart/form-data` (for optional logo/icon - * image uploads), so the tools post a JSON envelope to these internal routes, - * which download the referenced files from storage and forward a multipart - * request to UptimeRobot. - */ - -const pspSchema = z.object({ - id: z.number(), - friendlyName: z.string(), - customDomain: z.string().nullable(), - isPasswordSet: z.boolean().nullable(), - monitorIds: z.array(z.number()), - tagIds: z.array(z.number()), - monitorsCount: z.number().nullable(), - status: z.string().nullable(), - urlKey: z.string().nullable(), - homepageLink: z.string().nullable(), - gaCode: z.string().nullable(), - icon: z.string().nullable(), - logo: z.string().nullable(), - noIndex: z.boolean().nullable(), - hideUrlLinks: z.boolean().nullable(), - subscription: z.boolean().nullable(), -}) - -const pspRouteResponseSchema = z.object({ - success: z.boolean(), - output: z.object({ psp: pspSchema }).optional(), - error: z.string().optional(), -}) - -const pspSharedFields = { - apiKey: z.string().min(1, 'API key is required'), - monitorIds: z - .string() - .optional() - .nullable() - .describe('Comma-separated monitor IDs to display on the page'), - status: z.enum(['ENABLED', 'PAUSED']).optional().nullable(), - password: z.string().max(255).optional().nullable(), - customDomain: z.string().max(255).optional().nullable(), - hideUrlLinks: z.boolean().optional().nullable(), - noIndex: z.boolean().optional().nullable(), - logo: FileInputSchema.optional().nullable(), - icon: FileInputSchema.optional().nullable(), -} - -export const uptimeRobotCreatePspBodySchema = z.object({ - ...pspSharedFields, - friendlyName: z.string().min(1, 'friendlyName is required').max(255), -}) - -export type UptimeRobotCreatePspBody = z.input - -export const uptimeRobotCreatePspContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/uptimerobot/create-psp', - body: uptimeRobotCreatePspBodySchema, - response: { mode: 'json', schema: pspRouteResponseSchema }, -}) - -export const uptimeRobotUpdatePspBodySchema = z.object({ - ...pspSharedFields, - pspId: z.number().int().min(1, 'pspId is required'), - friendlyName: z.string().max(255).optional().nullable(), -}) - -export type UptimeRobotUpdatePspBody = z.input - -export const uptimeRobotUpdatePspContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/uptimerobot/update-psp', - body: uptimeRobotUpdatePspBodySchema, - response: { mode: 'json', schema: pspRouteResponseSchema }, -}) diff --git a/apps/sim/lib/api/contracts/tools/vanta.ts b/apps/sim/lib/api/contracts/tools/vanta.ts deleted file mode 100644 index 22ccc188648..00000000000 --- a/apps/sim/lib/api/contracts/tools/vanta.ts +++ /dev/null @@ -1,740 +0,0 @@ -import { z } from 'zod' -import type { - ContractBody, - ContractBodyInput, - ContractJsonResponse, -} from '@/lib/api/contracts/types' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' - -const nullableString = z.string().nullable() -const nullableNumber = z.number().nullable() -const nullableBoolean = z.boolean().nullable() - -const vantaPageInfoSchema = z - .object({ - startCursor: nullableString, - endCursor: nullableString, - hasNextPage: z.boolean(), - hasPreviousPage: z.boolean(), - }) - .nullable() - -const vantaOwnerSchema = z - .object({ - id: nullableString, - displayName: nullableString, - emailAddress: nullableString, - }) - .nullable() - -const vantaCustomFieldsSchema = z.array( - z.object({ - label: nullableString, - value: z.union([z.string(), z.array(z.string())]).nullable(), - }) -) - -const vantaFrameworkSchema = z.object({ - id: nullableString, - displayName: nullableString, - shorthandName: nullableString, - description: nullableString, - numControlsCompleted: nullableNumber, - numControlsTotal: nullableNumber, - numDocumentsPassing: nullableNumber, - numDocumentsTotal: nullableNumber, - numTestsPassing: nullableNumber, - numTestsTotal: nullableNumber, -}) - -const vantaFrameworkDetailSchema = vantaFrameworkSchema.extend({ - requirementCategories: z.array( - z.object({ - id: nullableString, - name: nullableString, - shorthand: nullableString, - requirements: z.array( - z.object({ - id: nullableString, - name: nullableString, - shorthand: nullableString, - description: nullableString, - controls: z.array( - z.object({ - id: nullableString, - externalId: nullableString, - name: nullableString, - description: nullableString, - }) - ), - }) - ), - }) - ), -}) - -const vantaControlSchema = z.object({ - id: nullableString, - externalId: nullableString, - name: nullableString, - description: nullableString, - source: nullableString, - domains: z.array(z.string()), - owner: vantaOwnerSchema, - role: nullableString, - customFields: vantaCustomFieldsSchema, - creationDate: nullableString, - modificationDate: nullableString, -}) - -const vantaControlDetailSchema = vantaControlSchema.extend({ - note: nullableString, - status: nullableString, - numDocumentsPassing: nullableNumber, - numDocumentsTotal: nullableNumber, - numTestsPassing: nullableNumber, - numTestsTotal: nullableNumber, -}) - -const vantaTestSchema = z.object({ - id: nullableString, - name: nullableString, - description: nullableString, - failureDescription: nullableString, - remediationDescription: nullableString, - category: nullableString, - status: nullableString, - integrations: z.array(z.string()), - lastTestRunDate: nullableString, - latestFlipDate: nullableString, - version: z.object({ major: nullableNumber, minor: nullableNumber }).nullable(), - deactivatedStatusInfo: z - .object({ - isDeactivated: nullableBoolean, - deactivatedReason: nullableString, - lastUpdatedDate: nullableString, - }) - .nullable(), - remediationStatusInfo: z - .object({ - status: nullableString, - soonestRemediateByDate: nullableString, - itemCount: nullableNumber, - }) - .nullable(), - owner: vantaOwnerSchema, -}) - -const vantaTestEntitySchema = z.object({ - id: nullableString, - entityStatus: nullableString, - displayName: nullableString, - responseType: nullableString, - deactivatedReason: nullableString, - createdDate: nullableString, - lastUpdatedDate: nullableString, -}) - -const vantaDocumentSchema = z.object({ - id: nullableString, - title: nullableString, - description: nullableString, - category: nullableString, - ownerId: nullableString, - isSensitive: nullableBoolean, - uploadStatus: nullableString, - uploadStatusDate: nullableString, - url: nullableString, -}) - -const vantaDocumentDetailSchema = vantaDocumentSchema.extend({ - note: nullableString, - nextRenewalDate: nullableString, - renewalCadence: nullableString, - reminderWindow: nullableString, - subscribers: z.array(z.string()), - deactivatedStatus: z - .object({ - isDeactivated: nullableBoolean, - reason: nullableString, - creationDate: nullableString, - expiration: nullableString, - }) - .nullable(), -}) - -const vantaUploadedFileSchema = z.object({ - id: nullableString, - fileName: nullableString, - title: nullableString, - description: nullableString, - mimeType: nullableString, - uploadedBy: z.object({ id: nullableString, type: nullableString }).nullable(), - creationDate: nullableString, - updatedDate: nullableString, - deletionDate: nullableString, - effectiveDate: nullableString, - url: nullableString, -}) - -const vantaPersonSchema = z.object({ - id: nullableString, - userId: nullableString, - emailAddress: nullableString, - name: z - .object({ first: nullableString, last: nullableString, display: nullableString }) - .nullable(), - employment: z - .object({ - status: nullableString, - startDate: nullableString, - endDate: nullableString, - jobTitle: nullableString, - }) - .nullable(), - leaveInfo: z - .object({ status: nullableString, startDate: nullableString, endDate: nullableString }) - .nullable(), - groupIds: z.array(z.string()), - tasksSummary: z - .object({ - status: nullableString, - dueDate: nullableString, - completionDate: nullableString, - }) - .nullable(), -}) - -const vantaPolicySchema = z.object({ - id: nullableString, - name: nullableString, - description: nullableString, - status: nullableString, - approvedAtDate: nullableString, - latestVersionStatus: nullableString, - latestApprovedVersion: z - .object({ - versionId: nullableString, - documents: z.array( - z.object({ language: nullableString, slugId: nullableString, url: nullableString }) - ), - }) - .nullable(), -}) - -const vantaVendorSchema = z.object({ - id: nullableString, - name: nullableString, - status: nullableString, - websiteUrl: nullableString, - category: nullableString, - servicesProvided: nullableString, - additionalNotes: nullableString, - accountManagerName: nullableString, - accountManagerEmail: nullableString, - securityOwnerUserId: nullableString, - businessOwnerUserId: nullableString, - inherentRiskLevel: nullableString, - residualRiskLevel: nullableString, - isRiskAutoScored: nullableBoolean, - isVisibleToAuditors: nullableBoolean, - riskAttributeIds: z.array(z.string()), - vendorHeadquarters: nullableString, - contractStartDate: nullableString, - contractRenewalDate: nullableString, - contractTerminationDate: nullableString, - contractAmount: z.object({ amount: nullableNumber, currency: nullableString }).nullable(), - nextSecurityReviewDueDate: nullableString, - lastSecurityReviewCompletionDate: nullableString, - authDetails: z - .object({ - method: nullableString, - passwordMFA: nullableBoolean, - passwordMinimumLength: nullableNumber, - passwordRequiresNumber: nullableBoolean, - passwordRequiresSymbol: nullableBoolean, - }) - .nullable(), - customFields: vantaCustomFieldsSchema, - latestDecision: z.object({ status: nullableString, lastUpdatedAt: nullableString }).nullable(), - linkedTaskTrackerTaskProcurementRequest: z - .object({ url: nullableString, service: nullableString }) - .nullable(), -}) - -const vantaMonitoredComputerSchema = z.object({ - id: nullableString, - integrationId: nullableString, - lastCheckDate: nullableString, - screenlock: nullableString, - diskEncryption: nullableString, - passwordManager: nullableString, - antivirusInstallation: nullableString, - operatingSystem: z.object({ type: nullableString, version: nullableString }).nullable(), - owner: vantaOwnerSchema, - serialNumber: nullableString, - udid: nullableString, -}) - -const vantaVulnerabilitySchema = z.object({ - id: nullableString, - name: nullableString, - description: nullableString, - severity: nullableString, - vulnerabilityType: nullableString, - integrationId: nullableString, - targetId: nullableString, - packageIdentifier: nullableString, - cvssSeverityScore: nullableNumber, - scannerScore: nullableNumber, - isFixable: nullableBoolean, - fixedVersion: nullableString, - remediateByDate: nullableString, - firstDetectedDate: nullableString, - sourceDetectedDate: nullableString, - lastDetectedDate: nullableString, - scanSource: nullableString, - externalURL: nullableString, - relatedVulns: z.array(z.string()), - relatedUrls: z.array(z.string()), - deactivateMetadata: z - .object({ - isVulnDeactivatedIndefinitely: nullableBoolean, - deactivatedUntilDate: nullableString, - deactivationReason: nullableString, - deactivatedOnDate: nullableString, - deactivatedBy: nullableString, - }) - .nullable(), -}) - -const vantaVulnerabilityRemediationSchema = z.object({ - id: nullableString, - vulnerabilityId: nullableString, - vulnerableAssetId: nullableString, - severity: nullableString, - detectedDate: nullableString, - slaDeadlineDate: nullableString, - remediationDate: nullableString, -}) - -const vantaVulnerableAssetSchema = z.object({ - id: nullableString, - name: nullableString, - assetType: nullableString, - hasBeenScanned: nullableBoolean, - imageScanTag: nullableString, - scanners: z.array( - z.object({ - resourceId: nullableString, - integrationId: nullableString, - targetId: nullableString, - imageDigest: nullableString, - imagePushedAtDate: nullableString, - imageTags: z.array(z.string()), - assetTags: z.array(z.object({ key: nullableString, value: nullableString })), - parentAccountOrOrganization: nullableString, - biosUuid: nullableString, - ipv4s: z.array(z.string()), - ipv6s: z.array(z.string()), - macAddresses: z.array(z.string()), - hostnames: z.array(z.string()), - fqdns: z.array(z.string()), - operatingSystems: z.array(z.string()), - }) - ), -}) - -const vantaRiskScenarioSchema = z.object({ - riskId: nullableString, - description: nullableString, - likelihood: nullableNumber, - impact: nullableNumber, - residualLikelihood: nullableNumber, - residualImpact: nullableNumber, - categories: z.array(z.string()), - ciaCategories: z.array(z.string()), - treatment: nullableString, - owner: nullableString, - note: nullableString, - riskRegister: nullableString, - customFields: vantaCustomFieldsSchema, - isArchived: nullableBoolean, - reviewStatus: nullableString, - requiredApprovers: z.array(z.string()), - type: nullableString, - identificationDate: nullableString, -}) - -const VANTA_REGIONS = ['us', 'gov'] as const - -const vantaBaseBodySchema = z.object({ - clientId: z.string().min(1, 'Client ID is required'), - clientSecret: z.string().min(1, 'Client secret is required'), - region: z.enum(VANTA_REGIONS).optional(), -}) - -const vantaPaginationBodySchema = z.object({ - pageSize: z - .number() - .int() - .min(1, 'pageSize must be at least 1') - .max(100, 'pageSize must be at most 100') - .optional(), - pageCursor: z.string().min(1, 'pageCursor cannot be empty').optional(), -}) - -const vantaListBaseBodySchema = vantaBaseBodySchema.extend(vantaPaginationBodySchema.shape) - -const requiredId = (label: string) => z.string().trim().min(1, `${label} is required`) - -const listFrameworksSchema = vantaListBaseBodySchema.extend({ - operation: z.literal('vanta_list_frameworks'), -}) - -const getFrameworkSchema = vantaBaseBodySchema.extend({ - operation: z.literal('vanta_get_framework'), - frameworkId: requiredId('Framework ID'), -}) - -const listFrameworkControlsSchema = vantaListBaseBodySchema.extend({ - operation: z.literal('vanta_list_framework_controls'), - frameworkId: requiredId('Framework ID'), -}) - -const listControlsSchema = vantaListBaseBodySchema.extend({ - operation: z.literal('vanta_list_controls'), - frameworkMatchesAny: z.string().optional(), -}) - -const getControlSchema = vantaBaseBodySchema.extend({ - operation: z.literal('vanta_get_control'), - controlId: requiredId('Control ID'), -}) - -const listControlTestsSchema = vantaListBaseBodySchema.extend({ - operation: z.literal('vanta_list_control_tests'), - controlId: requiredId('Control ID'), -}) - -const listControlDocumentsSchema = vantaListBaseBodySchema.extend({ - operation: z.literal('vanta_list_control_documents'), - controlId: requiredId('Control ID'), -}) - -const listTestsSchema = vantaListBaseBodySchema.extend({ - operation: z.literal('vanta_list_tests'), - statusFilter: z - .enum(['OK', 'DEACTIVATED', 'NEEDS_ATTENTION', 'IN_PROGRESS', 'INVALID', 'NOT_APPLICABLE']) - .optional(), - frameworkFilter: z.string().optional(), - integrationFilter: z.string().optional(), - controlFilter: z.string().optional(), - ownerFilter: z.string().optional(), - categoryFilter: z - .enum([ - 'ACCOUNTS_ACCESS', - 'ACCOUNT_SECURITY', - 'ACCOUNT_SETUP', - 'COMPUTERS', - 'CUSTOM', - 'DATA_STORAGE', - 'EMPLOYEES', - 'INFRASTRUCTURE', - 'IT', - 'LOGGING', - 'MONITORING_ALERTS', - 'PEOPLE', - 'POLICIES', - 'RISK_ANALYSIS', - 'SECURITY_ALERT_MANAGEMENT', - 'SOFTWARE_DEVELOPMENT', - 'VENDORS', - 'VULNERABILITY_MANAGEMENT', - ]) - .optional(), - isInRollout: z.boolean().optional(), -}) - -const getTestSchema = vantaBaseBodySchema.extend({ - operation: z.literal('vanta_get_test'), - testId: requiredId('Test ID'), -}) - -const listTestEntitiesSchema = vantaListBaseBodySchema.extend({ - operation: z.literal('vanta_list_test_entities'), - testId: requiredId('Test ID'), - entityStatus: z.enum(['FAILING', 'DEACTIVATED']).optional(), -}) - -const listDocumentsSchema = vantaListBaseBodySchema.extend({ - operation: z.literal('vanta_list_documents'), - frameworkMatchesAny: z.string().optional(), - statusMatchesAny: z.string().optional(), -}) - -const getDocumentSchema = vantaBaseBodySchema.extend({ - operation: z.literal('vanta_get_document'), - documentId: requiredId('Document ID'), -}) - -const listDocumentUploadsSchema = vantaListBaseBodySchema.extend({ - operation: z.literal('vanta_list_document_uploads'), - documentId: requiredId('Document ID'), -}) - -const submitDocumentSchema = vantaBaseBodySchema.extend({ - operation: z.literal('vanta_submit_document'), - documentId: requiredId('Document ID'), -}) - -const listPeopleSchema = vantaListBaseBodySchema.extend({ - operation: z.literal('vanta_list_people'), - emailAndNameFilter: z.string().optional(), - employmentStatus: z.enum(['UPCOMING', 'CURRENT', 'ON_LEAVE', 'INACTIVE', 'FORMER']).optional(), - groupIdsMatchesAny: z.string().optional(), - tasksSummaryStatusMatchesAny: z.string().optional(), - taskTypeMatchesAny: z.string().optional(), - taskStatusMatchesAny: z.string().optional(), -}) - -const getPersonSchema = vantaBaseBodySchema.extend({ - operation: z.literal('vanta_get_person'), - personId: requiredId('Person ID'), -}) - -const listPoliciesSchema = vantaListBaseBodySchema.extend({ - operation: z.literal('vanta_list_policies'), -}) - -const getPolicySchema = vantaBaseBodySchema.extend({ - operation: z.literal('vanta_get_policy'), - policyId: requiredId('Policy ID'), -}) - -const listVendorsSchema = vantaListBaseBodySchema.extend({ - operation: z.literal('vanta_list_vendors'), - name: z.string().optional(), - statusMatchesAny: z.string().optional(), -}) - -const getVendorSchema = vantaBaseBodySchema.extend({ - operation: z.literal('vanta_get_vendor'), - vendorId: requiredId('Vendor ID'), -}) - -const listMonitoredComputersSchema = vantaListBaseBodySchema.extend({ - operation: z.literal('vanta_list_monitored_computers'), - complianceStatusFilterMatchesAny: z.string().optional(), -}) - -const listVulnerabilitiesSchema = vantaListBaseBodySchema.extend({ - operation: z.literal('vanta_list_vulnerabilities'), - q: z.string().optional(), - severity: z.enum(['LOW', 'MEDIUM', 'HIGH', 'CRITICAL']).optional(), - isFixAvailable: z.boolean().optional(), - isDeactivated: z.boolean().optional(), - includeVulnerabilitiesWithoutSlas: z.boolean().optional(), - packageIdentifier: z.string().optional(), - externalVulnerabilityId: z.string().optional(), - integrationId: z.string().optional(), - vulnerableAssetId: z.string().optional(), - slaDeadlineAfterDate: z.string().optional(), - slaDeadlineBeforeDate: z.string().optional(), -}) - -const listVulnerabilityRemediationsSchema = vantaListBaseBodySchema.extend({ - operation: z.literal('vanta_list_vulnerability_remediations'), - integrationId: z.string().optional(), - severity: z.enum(['LOW', 'MEDIUM', 'HIGH', 'CRITICAL']).optional(), - isRemediatedOnTime: z.boolean().optional(), - remediatedAfterDate: z.string().optional(), - remediatedBeforeDate: z.string().optional(), -}) - -const listVulnerableAssetsSchema = vantaListBaseBodySchema.extend({ - operation: z.literal('vanta_list_vulnerable_assets'), - q: z.string().optional(), - integrationId: z.string().optional(), - assetType: z - .enum([ - 'SERVER', - 'SERVERLESS_FUNCTION', - 'CONTAINER', - 'CONTAINER_REPOSITORY', - 'CONTAINER_REPOSITORY_IMAGE', - 'CODE_REPOSITORY', - 'MANIFEST_FILE', - 'WORKSTATION', - 'OTHER', - ]) - .optional(), - assetExternalAccountId: z.string().optional(), -}) - -const getVulnerableAssetSchema = vantaBaseBodySchema.extend({ - operation: z.literal('vanta_get_vulnerable_asset'), - vulnerableAssetId: requiredId('Vulnerable asset ID'), -}) - -const listRiskScenariosSchema = vantaListBaseBodySchema.extend({ - operation: z.literal('vanta_list_risk_scenarios'), - searchString: z.string().optional(), - includeIgnored: z.boolean().optional(), - type: z.enum(['Risk Scenario', 'Enterprise Risk']).optional(), - ownerMatchesAny: z.string().optional(), - categoryMatchesAny: z.string().optional(), - ciaCategoryMatchesAny: z.string().optional(), - treatmentTypeMatchesAny: z.string().optional(), - inherentScoreGroupMatchesAny: z.string().optional(), - residualScoreGroupMatchesAny: z.string().optional(), - reviewStatusMatchesAny: z.string().optional(), - orderBy: z.enum(['description', 'createdAt']).optional(), -}) - -const getRiskScenarioSchema = vantaBaseBodySchema.extend({ - operation: z.literal('vanta_get_risk_scenario'), - riskScenarioId: requiredId('Risk scenario ID'), -}) - -export const vantaQueryBodySchema = z.discriminatedUnion('operation', [ - listFrameworksSchema, - getFrameworkSchema, - listFrameworkControlsSchema, - listControlsSchema, - getControlSchema, - listControlTestsSchema, - listControlDocumentsSchema, - listTestsSchema, - getTestSchema, - listTestEntitiesSchema, - listDocumentsSchema, - getDocumentSchema, - listDocumentUploadsSchema, - submitDocumentSchema, - listPeopleSchema, - getPersonSchema, - listPoliciesSchema, - getPolicySchema, - listVendorsSchema, - getVendorSchema, - listMonitoredComputersSchema, - listVulnerabilitiesSchema, - listVulnerabilityRemediationsSchema, - listVulnerableAssetsSchema, - getVulnerableAssetSchema, - listRiskScenariosSchema, - getRiskScenarioSchema, -]) - -const vantaQueryOutputSchema = z.union([ - z.object({ frameworks: z.array(vantaFrameworkSchema), pageInfo: vantaPageInfoSchema }), - z.object({ framework: vantaFrameworkDetailSchema }), - z.object({ controls: z.array(vantaControlSchema), pageInfo: vantaPageInfoSchema }), - z.object({ control: vantaControlDetailSchema }), - z.object({ tests: z.array(vantaTestSchema), pageInfo: vantaPageInfoSchema }), - z.object({ test: vantaTestSchema }), - z.object({ entities: z.array(vantaTestEntitySchema), pageInfo: vantaPageInfoSchema }), - z.object({ documents: z.array(vantaDocumentSchema), pageInfo: vantaPageInfoSchema }), - z.object({ document: vantaDocumentDetailSchema }), - z.object({ uploads: z.array(vantaUploadedFileSchema), pageInfo: vantaPageInfoSchema }), - z.object({ documentId: z.string(), submitted: z.boolean() }), - z.object({ people: z.array(vantaPersonSchema), pageInfo: vantaPageInfoSchema }), - z.object({ person: vantaPersonSchema }), - z.object({ policies: z.array(vantaPolicySchema), pageInfo: vantaPageInfoSchema }), - z.object({ policy: vantaPolicySchema }), - z.object({ vendors: z.array(vantaVendorSchema), pageInfo: vantaPageInfoSchema }), - z.object({ vendor: vantaVendorSchema }), - z.object({ computers: z.array(vantaMonitoredComputerSchema), pageInfo: vantaPageInfoSchema }), - z.object({ - vulnerabilities: z.array(vantaVulnerabilitySchema), - pageInfo: vantaPageInfoSchema, - }), - z.object({ - remediations: z.array(vantaVulnerabilityRemediationSchema), - pageInfo: vantaPageInfoSchema, - }), - z.object({ assets: z.array(vantaVulnerableAssetSchema), pageInfo: vantaPageInfoSchema }), - z.object({ asset: vantaVulnerableAssetSchema }), - z.object({ riskScenarios: z.array(vantaRiskScenarioSchema), pageInfo: vantaPageInfoSchema }), - z.object({ riskScenario: vantaRiskScenarioSchema }), -]) - -const vantaQueryResponseSchema = z.object({ - success: z.literal(true), - output: vantaQueryOutputSchema, -}) - -export const vantaQueryContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/vanta/query', - body: vantaQueryBodySchema, - response: { mode: 'json', schema: vantaQueryResponseSchema }, -}) - -const VANTA_MAX_UPLOAD_BYTES = 100 * 1024 * 1024 -/** Base64 length of the largest allowed upload (4 chars per 3 bytes). */ -const VANTA_MAX_UPLOAD_BASE64_LENGTH = Math.ceil(VANTA_MAX_UPLOAD_BYTES / 3) * 4 - -export const vantaUploadBodySchema = vantaBaseBodySchema.extend({ - documentId: requiredId('Document ID'), - file: FileInputSchema.optional().nullable(), - fileContent: z - .string() - .max(VANTA_MAX_UPLOAD_BASE64_LENGTH, 'fileContent exceeds the 100MB upload limit') - .nullish(), - fileName: z.string().nullish(), - mimeType: z.string().nullish(), - description: z.string().nullish(), - effectiveAtDate: z.string().nullish(), -}) - -const vantaUploadResponseSchema = z.object({ - success: z.literal(true), - output: z.object({ upload: vantaUploadedFileSchema }), -}) - -export const vantaUploadContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/vanta/upload', - body: vantaUploadBodySchema, - response: { mode: 'json', schema: vantaUploadResponseSchema }, -}) - -export const vantaDownloadBodySchema = vantaBaseBodySchema.extend({ - documentId: requiredId('Document ID'), - uploadedFileId: requiredId('Uploaded file ID'), -}) - -const vantaDownloadResponseSchema = z.object({ - success: z.literal(true), - output: z.object({ - file: z.object({ - name: z.string(), - mimeType: z.string(), - data: z.string(), - size: z.number(), - }), - name: z.string(), - mimeType: z.string(), - size: z.number(), - }), -}) - -export const vantaDownloadContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/vanta/download', - body: vantaDownloadBodySchema, - response: { mode: 'json', schema: vantaDownloadResponseSchema }, -}) - -export type VantaQueryBody = ContractBody -export type VantaQueryBodyInput = ContractBodyInput -export type VantaQueryResponse = ContractJsonResponse -export type VantaUploadBody = ContractBody -export type VantaUploadBodyInput = ContractBodyInput -export type VantaUploadResponse = ContractJsonResponse -export type VantaDownloadBody = ContractBody -export type VantaDownloadBodyInput = ContractBodyInput -export type VantaDownloadResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/tools/whatsapp.ts b/apps/sim/lib/api/contracts/tools/whatsapp.ts deleted file mode 100644 index 18465dd3c3d..00000000000 --- a/apps/sim/lib/api/contracts/tools/whatsapp.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { z } from 'zod' -import { - nonEmptyIdSchema, - userFileSchema, - workflowIdSchema, - workspaceIdSchema, -} from '@/lib/api/contracts/primitives' -import type { ContractBodyInput, ContractJsonResponse } from '@/lib/api/contracts/types' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' - -const MAX_ACCESS_TOKEN_LENGTH = 8192 -const MAX_GRAPH_ID_LENGTH = 256 - -const whatsappAccessTokenSchema = z - .string() - .min(1, 'Access token is required') - .max(MAX_ACCESS_TOKEN_LENGTH, 'Access token is too long') - -const whatsappPhoneNumberIdSchema = z - .string() - .trim() - .min(1, 'Phone Number ID is required') - .max(MAX_GRAPH_ID_LENGTH, 'Phone Number ID is too long') - -const whatsappMediaIdSchema = z - .string() - .trim() - .min(1, 'Media ID is required') - .max(MAX_GRAPH_ID_LENGTH, 'Media ID is too long') - -const executionContextShape = { - workspaceId: workspaceIdSchema.optional(), - workflowId: workflowIdSchema.optional(), - executionId: nonEmptyIdSchema.optional(), -} - -export const whatsappUploadMediaBodySchema = z.object({ - accessToken: whatsappAccessTokenSchema, - phoneNumberId: whatsappPhoneNumberIdSchema, - file: RawFileInputSchema, -}) - -export const whatsappUploadMediaOutputSchema = z.object({ - mediaId: z.string().min(1).max(MAX_GRAPH_ID_LENGTH), - fileName: z.string().min(1), - mimeType: z.string().min(1), - size: z.number().int().nonnegative(), -}) - -export const whatsappSendMediaBodySchema = z.object({ - accessToken: whatsappAccessTokenSchema, - phoneNumberId: whatsappPhoneNumberIdSchema, - phoneNumber: z.string().trim().min(1, 'Recipient phone number is required').max(64), - mediaType: z.enum(['image', 'document', 'video', 'audio', 'sticker']), - /** Exactly one of file, mediaId, or mediaLink must be supplied. */ - file: RawFileInputSchema.optional().nullable(), - mediaId: whatsappMediaIdSchema.optional().nullable(), - mediaLink: z.string().trim().max(8192).optional().nullable(), - caption: z.string().max(1024, 'Caption cannot exceed 1024 characters').optional().nullable(), - filename: z.string().max(1024).optional().nullable(), -}) - -export const whatsappSendMediaOutputSchema = z.object({ - success: z.literal(true), - messageId: z.string().min(1), - messageStatus: z.string().optional(), - messagingProduct: z.string().optional(), - inputPhoneNumber: z.string().nullable(), - whatsappUserId: z.string().nullable(), - contacts: z.array(z.object({ input: z.string(), wa_id: z.string().nullable() })), - /** Set only when a file was uploaded as part of this send. */ - mediaId: z.string().optional(), -}) - -export const whatsappGetMediaBodySchema = z.object({ - accessToken: whatsappAccessTokenSchema, - mediaId: whatsappMediaIdSchema, - /** Optional ownership scoping check accepted by `GET /{media-id}`. */ - phoneNumberId: whatsappPhoneNumberIdSchema.optional(), - ...executionContextShape, -}) - -export const whatsappGetMediaOutputSchema = z.object({ - file: userFileSchema, - mediaId: z.string().min(1).max(MAX_GRAPH_ID_LENGTH), - mimeType: z.string().min(1), - fileSize: z.number().int().nonnegative(), - sha256: z.string().nullable(), -}) - -const whatsappRouteResponseSchema = (output: TOutput) => - z.discriminatedUnion('success', [ - z.object({ success: z.literal(true), output }), - z.object({ success: z.literal(false), error: z.string().min(1) }), - ]) - -export const whatsappUploadMediaResponseSchema = whatsappRouteResponseSchema( - whatsappUploadMediaOutputSchema -) -export const whatsappSendMediaResponseSchema = whatsappRouteResponseSchema( - whatsappSendMediaOutputSchema -) -export const whatsappGetMediaResponseSchema = whatsappRouteResponseSchema( - whatsappGetMediaOutputSchema -) - -export const whatsappUploadMediaContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/whatsapp/upload-media', - body: whatsappUploadMediaBodySchema, - response: { mode: 'json', schema: whatsappUploadMediaResponseSchema }, -}) - -export const whatsappSendMediaContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/whatsapp/send-media', - body: whatsappSendMediaBodySchema, - response: { mode: 'json', schema: whatsappSendMediaResponseSchema }, -}) - -export const whatsappGetMediaContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/whatsapp/get-media', - body: whatsappGetMediaBodySchema, - response: { mode: 'json', schema: whatsappGetMediaResponseSchema }, -}) - -export type WhatsAppUploadMediaBody = ContractBodyInput -export type WhatsAppSendMediaBody = ContractBodyInput -export type WhatsAppGetMediaBody = ContractBodyInput -export type WhatsAppUploadMediaRouteResponse = ContractJsonResponse< - typeof whatsappUploadMediaContract -> -export type WhatsAppSendMediaRouteResponse = ContractJsonResponse -export type WhatsAppGetMediaRouteResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/tools/zoho-desk.ts b/apps/sim/lib/api/contracts/tools/zoho-desk.ts deleted file mode 100644 index e05e9803efd..00000000000 --- a/apps/sim/lib/api/contracts/tools/zoho-desk.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { z } from 'zod' -import type { ContractBodyInput, ContractJsonResponse } from '@/lib/api/contracts/types' -import { defineRouteContract } from '@/lib/api/contracts/types' - -const zohoAccessTokenSchema = z.string().min(1, 'Access token is required') -const zohoApiDomainSchema = z.string().optional().nullable() -const zohoOrgIdSchema = z.string().min(1, 'Organization ID is required') - -export const zohoDeskGetAttachmentBodySchema = z.object({ - accessToken: zohoAccessTokenSchema, - apiDomain: zohoApiDomainSchema, - orgId: zohoOrgIdSchema, - // The documented download reference from an attachment object (thread/comment - // attachments expose `href`). Absolute Zoho URLs are used as-is; a relative - // path is resolved against the Desk API base. - href: z.string().min(1, 'Attachment href is required'), - fileName: z.string().optional().nullable(), -}) - -const zohoDeskFileSchema = z.object({ - data: z.string(), - mimeType: z.string(), - // FileToolProcessor (ToolFileData) reads the file name from `name`. - name: z.string(), -}) - -export const zohoDeskGetAttachmentResponseSchema = z.object({ - success: z.boolean(), - output: z.object({ file: zohoDeskFileSchema }).optional(), - error: z.string().optional(), -}) - -export const zohoDeskGetAttachmentContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/zoho_desk/attachment', - body: zohoDeskGetAttachmentBodySchema, - response: { mode: 'json', schema: zohoDeskGetAttachmentResponseSchema }, -}) - -export type ZohoDeskGetAttachmentBody = ContractBodyInput -export type ZohoDeskGetAttachmentResponse = ContractJsonResponse< - typeof zohoDeskGetAttachmentContract -> diff --git a/apps/sim/lib/api/contracts/tools/zoom.ts b/apps/sim/lib/api/contracts/tools/zoom.ts deleted file mode 100644 index ed7b0a1079d..00000000000 --- a/apps/sim/lib/api/contracts/tools/zoom.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { z } from 'zod' -import type { - ContractBody, - ContractBodyInput, - ContractJsonResponse, -} from '@/lib/api/contracts/types' -import { defineRouteContract } from '@/lib/api/contracts/types' - -const zoomRecordingFileSchema = z.object({ - id: z.string().optional(), - meeting_id: z.string().optional(), - recording_start: z.string().optional(), - recording_end: z.string().optional(), - file_type: z.string().optional(), - file_extension: z.string().optional(), - file_size: z.number().optional(), - play_url: z.string().optional(), - download_url: z.string().optional(), - status: z.string().optional(), - recording_type: z.string().optional(), -}) - -const zoomDownloadedFileSchema = z.object({ - name: z.string(), - mimeType: z.string(), - data: z.string(), - size: z.number(), -}) - -export const zoomGetRecordingsResponseSchema = z.object({ - success: z.literal(true), - output: z.object({ - recording: z.object({ - uuid: z.string().optional(), - id: z.union([z.string(), z.number()]).optional(), - account_id: z.string().optional(), - host_id: z.string().optional(), - topic: z.string().optional(), - type: z.number().optional(), - start_time: z.string().optional(), - duration: z.number().optional(), - total_size: z.number().optional(), - recording_count: z.number().optional(), - share_url: z.string().optional(), - recording_files: z.array(zoomRecordingFileSchema), - }), - files: z.array(zoomDownloadedFileSchema).optional(), - }), -}) - -export const zoomGetRecordingsBodySchema = z.object({ - accessToken: z.string().min(1, 'Access token is required'), - meetingId: z.string().min(1, 'Meeting ID is required'), - includeFolderItems: z.boolean().optional(), - ttl: z.number().optional(), - downloadFiles: z.boolean().optional().default(false), -}) - -export const zoomGetRecordingsContract = defineRouteContract({ - method: 'POST', - path: '/api/tools/zoom/get-recordings', - body: zoomGetRecordingsBodySchema, - response: { mode: 'json', schema: zoomGetRecordingsResponseSchema }, -}) - -export type ZoomGetRecordingsBody = ContractBody -export type ZoomGetRecordingsBodyInput = ContractBodyInput -export type ZoomGetRecordingsResponse = ContractJsonResponse diff --git a/apps/sim/lib/api/contracts/v1/admin/organizations.ts b/apps/sim/lib/api/contracts/v1/admin/organizations.ts index 463b92fb1c4..418bf7ccf14 100644 --- a/apps/sim/lib/api/contracts/v1/admin/organizations.ts +++ b/apps/sim/lib/api/contracts/v1/admin/organizations.ts @@ -209,6 +209,24 @@ export const adminV1ListOrganizationsContract = defineRouteContract({ }, }) +/** + * Creates the organization and its owner membership, and deliberately nothing else. + * + * Organization settings are reached through a workspace the organization owns, so a + * brand-new organization with none is not yet administrable. Closing that inside + * this call was tried and removed: attaching the owner's existing workspaces cannot + * join the creation transaction (it runs its own, under a lock order that exists to + * avoid deadlocking against invitation acceptance), so it could only ever be + * best-effort — leaving a committed organization, a response that could not honestly + * report the outcome, and a retry blocked by the existing-membership check. + * + * This codebase already solves it properly elsewhere. `AdminMemberOperationView` + * tracks workspace moves with `pending | processing | dead_letter | applied` and + * per-workspace retry, and the enterprise-owner-claim path creates the workspace and + * the organization in one transaction and enqueues an outbox event for the rest. + * Provisioning a workspace for an organization belongs on one of those paths, not + * inline here. + */ export const adminV1CreateOrganizationContract = defineRouteContract({ method: 'POST', path: '/api/v1/admin/organizations', diff --git a/apps/sim/lib/api/server/routes/resource-concealment.test.ts b/apps/sim/lib/api/server/routes/resource-concealment.test.ts index e7c0a5475fe..d29323a9825 100644 --- a/apps/sim/lib/api/server/routes/resource-concealment.test.ts +++ b/apps/sim/lib/api/server/routes/resource-concealment.test.ts @@ -248,12 +248,12 @@ const internalPolicies: Array<{ notFoundMessage: 'Knowledge base not found', }, { - route: 'POST /api/knowledge/[id]/documents', + route: 'Knowledge document upload internal operation', policy: internalKnowledgeErrorPolicies.uploads, notFoundMessage: 'Knowledge base not found', }, { - route: 'POST /api/knowledge/search', + route: 'Knowledge search internal operation', policy: internalKnowledgeErrorPolicies.search, notFoundMessage: 'Knowledge base not found', }, diff --git a/apps/sim/lib/auth/principal.test.ts b/apps/sim/lib/auth/principal.test.ts index ae979fc6a9d..1121a6d9fc5 100644 --- a/apps/sim/lib/auth/principal.test.ts +++ b/apps/sim/lib/auth/principal.test.ts @@ -357,6 +357,45 @@ describe('principal actors', () => { ).toMatchObject({ attributedUserId: 'user-3' }) }) + it('uses the workspace billing owner only for actorless execution attribution', () => { + const principal = { + kind: 'delegated' as const, + serviceId: 'executor' as const, + workspaceId: 'workspace-1', + delegationId: 'delegation-1', + audience: 'sim:tables', + issuedAt: new Date('2026-01-01T00:00:00Z'), + expiresAt: new Date('2026-01-01T00:05:00Z'), + delegationContext: { + kind: 'workflow_execution' as const, + workflowId: 'workflow-1', + principal: { + kind: 'system' as const, + serviceId: 'webhook' as const, + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + webhookId: 'webhook-1', + provider: 'generic', + }, + }, + } + + expect( + resolvePrincipalAttribution(principal, { + workspaceBillingOwnerUserId: 'billing-owner-1', + }) + ).toEqual({ + actor: { + kind: 'delegated', + serviceId: 'executor', + delegationId: 'delegation-1', + }, + attributedUserId: 'billing-owner-1', + }) + expect(resolvePrincipalSubject(principal)).toBeNull() + expect(() => resolvePrincipalAttribution(principal)).toThrow(PrincipalSubjectUserRequiredError) + }) + it('fails fast when workspace-key attribution has no billing owner', () => { expect(() => resolvePrincipalAttribution({ diff --git a/apps/sim/lib/billing/application/organization-usage/authorized-organization-usage-use-case.ts b/apps/sim/lib/billing/application/organization-usage/authorized-organization-usage-use-case.ts new file mode 100644 index 00000000000..edef9a963ec --- /dev/null +++ b/apps/sim/lib/billing/application/organization-usage/authorized-organization-usage-use-case.ts @@ -0,0 +1,107 @@ +import type { Principal } from '@sim/auth/principal' +import type { + OrganizationUsageOperation, + OrganizationUsagePrincipal, +} from '@/lib/billing/application/organization-usage/operations' +import { getOrganizationSubscription } from '@/lib/billing/core/billing' +import { + type ResolvedUsagePeriod, + resolveSubscriptionUsagePeriodOrDefault, +} from '@/lib/billing/core/reporting-period' +import { isOrganizationFeatureEntitled } from '@/lib/billing/core/subscription' +import type { BillingEntity } from '@/lib/billing/core/usage-log' +import { canUserManageBillingEntity } from '@/lib/billing/core/workspace-billing-authority' +import { ForbiddenOperationError, type OperationUseCase } from '@/lib/core/application' +import { isUsageMonitoringEnabled } from '@/lib/core/config/env-flags' + +export interface AuthorizedOrganizationUsageContext { + organizationId: string + billingEntity: BillingEntity + actorUserId: string + /** + * Resolved once per request and shared by every query in the use case. Re-resolving + * per query is how the tiles, the chart, and the event log would come to describe + * three slightly different windows. + */ + period: ResolvedUsagePeriod +} + +interface AuthorizedOrganizationUsageDefinition { + operation: O + organizationId(input: I): string + execute(args: { + principal: OrganizationUsagePrincipal + input: I + context: AuthorizedOrganizationUsageContext + }): Promise +} + +function requireOrganizationUsagePrincipal( + principal: Principal, + operation: OrganizationUsageOperation +): asserts principal is OrganizationUsagePrincipal { + if (!operation.principalKinds.some((kind) => kind === principal.kind)) { + throw new ForbiddenOperationError( + 'PRINCIPAL_KIND_NOT_PERMITTED', + `Principal kind ${principal.kind} cannot perform operation ${operation.id}` + ) + } +} + +/** + * Gate order for every organization usage read. Each step is a distinct refusal so a + * failure says which rule stopped it. + * + * 1. Principal kind — session only. + * 2. Billing authority — organization admin or owner. A workspace `admin` is + * explicitly not sufficient; this is pooled spend across every member. + * 3. Entitlement — enterprise plan on hosted, `USAGE_MONITORING_ENABLED` on + * self-hosted. Reuses audit-logs' error code so the client handles an + * entitlement refusal identically across EE settings. + */ +export function defineAuthorizedOrganizationUsageUseCase< + const O extends OrganizationUsageOperation, + I, + R, +>(definition: AuthorizedOrganizationUsageDefinition): OperationUseCase { + return { + operation: definition.operation, + async execute({ principal, input }) { + requireOrganizationUsagePrincipal(principal, definition.operation) + const actorUserId = principal.userId + const organizationId = definition.organizationId(input) + const billingEntity: BillingEntity = { type: 'organization', id: organizationId } + + if (!(await canUserManageBillingEntity(billingEntity, actorUserId))) { + throw new ForbiddenOperationError( + 'ORGANIZATION_ADMIN_REQUIRED', + 'Organization admin or owner authority is required to read pooled usage' + ) + } + + /** + * One call covers both the plan and the deployment: with billing on it checks + * the enterprise plan; with billing off — a self-hosted deployment, where there + * is no plan to consult — it answers `USAGE_MONITORING_ENABLED`. That is the + * same flag the navigation gate reads, so a section can never be visible here + * and rejected there. Calling `isOrganizationOnEnterprisePlan` directly would + * answer `true` for every self-hosted organization. + */ + if (!(await isOrganizationFeatureEntitled(organizationId, isUsageMonitoringEnabled))) { + throw new ForbiddenOperationError( + 'ENTERPRISE_PLAN_REQUIRED', + 'Active enterprise subscription required' + ) + } + + const subscription = await getOrganizationSubscription(organizationId) + const period = resolveSubscriptionUsagePeriodOrDefault(subscription ?? {}) + + return definition.execute({ + principal, + input, + context: { organizationId, billingEntity, actorUserId, period }, + }) + }, + } +} diff --git a/apps/sim/lib/billing/application/organization-usage/export-organization-usage-events.ts b/apps/sim/lib/billing/application/organization-usage/export-organization-usage-events.ts new file mode 100644 index 00000000000..c5cd4a51ade --- /dev/null +++ b/apps/sim/lib/billing/application/organization-usage/export-organization-usage-events.ts @@ -0,0 +1,117 @@ +import { defineAuthorizedOrganizationUsageUseCase } from '@/lib/billing/application/organization-usage/authorized-organization-usage-use-case' +import { organizationUsageOperations } from '@/lib/billing/application/organization-usage/operations' +import { + resolveUsageAnalyticsWindow, + type UsageWindowPreset, + usageWindowLedgerFilter, +} from '@/lib/billing/core/usage-analytics' +import { getBillingEntityUsageLogs } from '@/lib/billing/core/usage-log' +import { CREDIT_MULTIPLIER } from '@/lib/billing/credits/conversion' +import type { InternalUsageLogSource } from '@/lib/billing/usage-sources' + +/** + * Circuit breaker, not a UX boundary. An enterprise ledger is genuinely large, so + * hitting the cap truncates and says so rather than failing the download. + */ +export const USAGE_EXPORT_SAFETY_CAP = 100_000 +const EXPORT_PAGE_SIZE = 1000 + +export interface OrganizationUsageExportInput { + organizationId: string + preset: UsageWindowPreset + startDate?: Date + endDate?: Date + /** Viewer calendar, so a date-only custom bound means midnight there. */ + timezone?: string + source?: InternalUsageLogSource[] +} + +export interface OrganizationUsageExportRow { + createdAt: string + source: string + description: string + workflowName: string | null + /** + * Unrounded, unlike the event list's integer credits. + * + * A CSV is an analysis surface, and rounding each row to a whole credit printed a + * real sub-credit charge as `0` — the charge disappearing rather than reading as + * small. Full precision here also makes the column summable, which the previous + * `"N credits"` label never was. + */ + credits: number +} + +export interface OrganizationUsageExportResult { + rows: OrganizationUsageExportRow[] + truncated: boolean +} + +/** + * Every event in the window, paged to the cap. + * + * Returns data, not CSV: serialization is presentation and belongs in the route, so + * this module stays free of any HTTP concern. + */ +export const exportOrganizationUsageEvents = defineAuthorizedOrganizationUsageUseCase({ + operation: organizationUsageOperations.exportEvents, + organizationId: (input: OrganizationUsageExportInput) => input.organizationId, + async execute({ input, context }): Promise { + const window = resolveUsageAnalyticsWindow({ + preset: input.preset, + period: context.period, + customStart: input.startDate, + customEnd: input.endDate, + timezone: input.timezone, + }) + const ledgerFilter = usageWindowLedgerFilter(window) + + const rows: OrganizationUsageExportRow[] = [] + let cursor: string | undefined + /** + * The cursor row's timestamp, carried forward from the page that produced it. + * + * Without it `getUsageLogs` resolves the cursor with a lookup on the primary + * before it can read the replica — once per page, up to a hundred times for a + * capped export. This loop is the exact case that option was added for. + */ + let cursorCreatedAt: Date | undefined + let truncated = false + + while (rows.length < USAGE_EXPORT_SAFETY_CAP) { + const page = await getBillingEntityUsageLogs(context.billingEntity, { + // One derivation for both predicates, so the CSV covers exactly the rows the + // summary and breakdowns aggregate over. + ...ledgerFilter, + ...(input.source?.length ? { source: input.source } : {}), + limit: EXPORT_PAGE_SIZE, + ...(cursor ? { cursor } : {}), + ...(cursorCreatedAt ? { cursorCreatedAt } : {}), + // Each page would otherwise repeat the same cursor-independent aggregate + // for a total this export never reads. + includeSummary: false, + }) + + for (const log of page.logs) { + rows.push({ + createdAt: log.createdAt, + source: log.source, + description: log.description, + workflowName: log.workflowName ?? null, + credits: log.cost * CREDIT_MULTIPLIER, + }) + } + + if (!page.pagination.hasMore || !page.pagination.nextCursor) break + const cursorRow = page.logs.find((log) => log.id === page.pagination.nextCursor) + cursor = page.pagination.nextCursor + cursorCreatedAt = cursorRow ? new Date(cursorRow.createdAt) : undefined + if (rows.length >= USAGE_EXPORT_SAFETY_CAP) { + truncated = true + break + } + } + + return { rows: rows.slice(0, USAGE_EXPORT_SAFETY_CAP), truncated } + }, +}) diff --git a/apps/sim/lib/billing/application/organization-usage/get-organization-usage-breakdown.ts b/apps/sim/lib/billing/application/organization-usage/get-organization-usage-breakdown.ts new file mode 100644 index 00000000000..8efa0811cfd --- /dev/null +++ b/apps/sim/lib/billing/application/organization-usage/get-organization-usage-breakdown.ts @@ -0,0 +1,208 @@ +import { defineAuthorizedOrganizationUsageUseCase } from '@/lib/billing/application/organization-usage/authorized-organization-usage-use-case' +import { organizationUsageOperations } from '@/lib/billing/application/organization-usage/operations' +import { + buildUsageAnalyticsScope, + foldUsageBreakdown, + type MergeableRow, + mergeRowsByKey, + resolveUsageAnalyticsWindow, + USAGE_NULL_KEY_LABELS, + type UsageBreakdownDimension, + type UsageWindowPreset, +} from '@/lib/billing/core/usage-analytics' +import { + readUsageBreakdown, + readUsageEntityNames, +} from '@/lib/billing/core/usage-analytics-queries' +import { apportionCredits, dollarsToCredits } from '@/lib/billing/credits/conversion' +import { + BILLING_USAGE_LOG_SOURCE_LABELS, + type InternalUsageLogSource, + toBillingUsageLogSource, +} from '@/lib/billing/usage-sources' +import { getProviderFromModel, PROVIDER_DEFINITIONS } from '@/providers/models' + +export interface OrganizationUsageBreakdownInput { + organizationId: string + dimension: UsageBreakdownDimension + preset: UsageWindowPreset + startDate?: Date + endDate?: Date + /** Viewer calendar, so a date-only custom bound means midnight there. */ + timezone?: string + /** Narrows to one workspace, for the Workspaces drill-down. */ + workspaceId?: string + limit: number +} + +export interface OrganizationUsageBreakdownRow { + id: string + label: string + credits: number + events: number + share: number + providerId?: string + tokens?: number +} + +export interface OrganizationUsageBreakdownResult { + dimension: UsageBreakdownDimension + rows: OrganizationUsageBreakdownRow[] + other: { credits: number; events: number; rowCount: number; tokens: number } + totalCredits: number +} + +/** Entity-backed dimensions need a second lookup to turn ids into names. */ +const NAMED_DIMENSIONS = new Set(['member', 'workspace', 'workflow']) + +/** Provider ids that read better with their conventional casing. */ +/** + * The registry's own display name, not a second hand-written table. + * + * A local map had eleven of the registry's twenty-two providers, so anything newer + * — `zai`, `kimi`, `vertex` — surfaced as a raw lowercase id. This is a server + * module, so reading the registry costs nothing a client bundle would pay for. + */ +function providerLabel(providerId: string): string { + return PROVIDER_DEFINITIONS[providerId]?.name ?? providerId +} + +export const getOrganizationUsageBreakdown = defineAuthorizedOrganizationUsageUseCase({ + operation: organizationUsageOperations.readBreakdown, + organizationId: (input: OrganizationUsageBreakdownInput) => input.organizationId, + async execute({ input, context }): Promise { + const window = resolveUsageAnalyticsWindow({ + preset: input.preset, + period: context.period, + customStart: input.startDate, + customEnd: input.endDate, + timezone: input.timezone, + }) + const scope = buildUsageAnalyticsScope(context.billingEntity, window) + const raw = await readUsageBreakdown(scope, input.dimension, input.workspaceId) + + /** + * Re-key onto what the panel actually displays before ranking. + * + * Two dimensions are coarser than their SQL grouping column: `source` shows one + * "Sim Chat" row for the ledger's `copilot` *and* `workspace-chat`, and `byok` + * shows one row per provider rather than per model. Ranking the raw rows would + * render the same label twice with the total split between them. + */ + const rows: MergeableRow[] = + input.dimension === 'source' + ? mergeRowsByKey(raw, (key) => + key ? toBillingUsageLogSource(key as InternalUsageLogSource) : null + ) + : input.dimension === 'byok' + ? mergeRowsByKey(raw, (key) => (key ? getProviderFromModel(key) : null)) + : raw + + const totalCost = rows.reduce((sum, row) => sum + (Number(row.cost) || 0), 0) + + /** + * Names are hydrated for the surviving keys only — joining inside the aggregate + * would break the index-only scan the member dimension depends on. + * + * Sorted before slicing: the breakdown query only groups, so Postgres returns its + * aggregate in arbitrary order. Slicing that directly hydrated an arbitrary subset + * while the fold below ranks by cost, so a top row whose name was never fetched + * fell through to `?? key` and rendered a raw id. The margin over `limit` covers + * the fold's label tiebreak pulling in a row just past the cut. + */ + const rankedIds = [...rows] + .sort((left, right) => Number(right.cost) - Number(left.cost)) + .slice(0, input.limit * 2) + .map((row) => row.key) + .filter((key): key is string => Boolean(key)) + const names = NAMED_DIMENSIONS.has(input.dimension) + ? await readUsageEntityNames(input.dimension, rankedIds) + : new Map() + + const labelFor = (key: string | null): string => { + /** + * A null key means something different per dimension, and one shared + * "Unattributed" label got both wrong: on Workspaces it is usage owned by no + * workspace, on Workflows it is usage that never came from a workflow — which + * is most of an organization's spend, and reading it as an attribution failure + * is what made that list useless. + */ + if (!key) return USAGE_NULL_KEY_LABELS[input.dimension] + if (input.dimension === 'source') { + return BILLING_USAGE_LOG_SOURCE_LABELS[key as keyof typeof BILLING_USAGE_LOG_SOURCE_LABELS] + } + if (input.dimension === 'byok') return providerLabel(key) + if (input.dimension === 'model') return key + // A deleted workspace or workflow nulls its id on the ledger row, so a key that + // resolves to no name is a live entity we could not read — not a deleted one. + return names.get(key) ?? key + } + + // BYOK is denominated in tokens and every row costs zero, so ranking it by cost + // would order the list alphabetically and call the result "top providers". + const fold = foldUsageBreakdown( + rows, + totalCost, + labelFor, + input.limit, + input.dimension === 'byok' ? 'tokens' : 'cost' + ) + const tokensByKey = new Map( + rows.map((row) => [row.key ?? '', (row.inputTokens ?? 0) + (row.outputTokens ?? 0)]) + ) + const isModelDimension = input.dimension === 'model' || input.dimension === 'byok' + + /** + * One apportionment across the visible rows and the remainder together. + * + * Converting each row independently rounds each one, so with sub-credit + * fractions the rows and `Other` no longer add up — which defeats the entire + * reason `Other` is rendered. Largest-remainder over the whole set is what + * `apportionCredits` is for, and it is the same routine the per-log credit + * costs use, so a row cannot read differently here than in the event list. + * + * Keyed positionally: a row id may be `''` for a null grouping key, and any + * id could otherwise collide with the remainder's own key. + */ + const apportioned = apportionCredits([ + ...fold.rows.map((row, index) => ({ key: `row:${index}` as const, dollars: row.cost })), + { key: 'other' as const, dollars: fold.other.cost }, + ]) + + return { + dimension: input.dimension, + rows: fold.rows.map((row, index) => { + const tokens = tokensByKey.get(row.id) ?? 0 + return { + id: row.id, + label: row.label, + credits: apportioned[`row:${index}`] ?? 0, + events: row.events, + share: row.share, + ...(isModelDimension && tokens > 0 ? { tokens } : {}), + ...(input.dimension === 'byok' ? { providerId: row.id } : {}), + ...(input.dimension === 'model' && row.id + ? { providerId: getProviderFromModel(row.id) } + : {}), + } + }), + other: { + credits: apportioned.other ?? 0, + events: fold.other.events, + rowCount: fold.other.rowCount, + /** + * The omitted rows' tokens, so the BYOK tab still accounts for providers + * past the visible limit instead of hiding them behind an em dash. + */ + tokens: fold.other.tokens, + }, + /** + * The scope total, which the rows need not sum to: the workflow dimension is + * explicitly the workflow-attributed subset of it. Where they do sum to it — + * every other dimension — the apportionment above lands on this exact figure, + * because it derives its target from the same dollar sum. + */ + totalCredits: dollarsToCredits(fold.totalCost), + } + }, +}) diff --git a/apps/sim/lib/billing/application/organization-usage/get-organization-usage-summary.ts b/apps/sim/lib/billing/application/organization-usage/get-organization-usage-summary.ts new file mode 100644 index 00000000000..48616c6f1a9 --- /dev/null +++ b/apps/sim/lib/billing/application/organization-usage/get-organization-usage-summary.ts @@ -0,0 +1,113 @@ +import { defineAuthorizedOrganizationUsageUseCase } from '@/lib/billing/application/organization-usage/authorized-organization-usage-use-case' +import { organizationUsageOperations } from '@/lib/billing/application/organization-usage/operations' +import type { UsagePeriodSource } from '@/lib/billing/core/reporting-period' +import { + buildUsageAnalyticsScope, + densifyUsageSeries, + resolvePreviousPeriod, + resolveUsageAnalyticsWindow, + resolveUsageBucket, + type UsageBucket, + type UsageWindowPreset, + usageWindowBounds, +} from '@/lib/billing/core/usage-analytics' +import { readUsageTimeSeries, readUsageTotals } from '@/lib/billing/core/usage-analytics-queries' +import { apportionCredits, dollarsToCredits } from '@/lib/billing/credits/conversion' + +export interface OrganizationUsageSummaryInput { + organizationId: string + preset: UsageWindowPreset + startDate?: Date + endDate?: Date + timezone: string +} + +export interface OrganizationUsageSummaryResult { + window: { start: string; end: string; source: UsagePeriodSource | 'range' } + bucket: UsageBucket + totals: { credits: number } + previousTotals: { credits: number } | null + series: Array<{ timestamp: string; credits: number; events: number }> +} + +/** + * Everything above the fold in one round trip: headline totals, the delta, and the + * trend series. Every read here is index-covered, so this is cheap enough to be what + * first paint waits on — the breakdowns, which are not, are fetched separately. + */ +export const getOrganizationUsageSummary = defineAuthorizedOrganizationUsageUseCase({ + operation: organizationUsageOperations.readSummary, + organizationId: (input: OrganizationUsageSummaryInput) => input.organizationId, + async execute({ input, context }): Promise { + const window = resolveUsageAnalyticsWindow({ + preset: input.preset, + period: context.period, + customStart: input.startDate, + customEnd: input.endDate, + timezone: input.timezone, + }) + const bucket = resolveUsageBucket(window) + const scope = buildUsageAnalyticsScope(context.billingEntity, window) + + /** + * The comparison window only exists when it is exactly derivable. + * + * `resolvePreviousPeriod` directly, not the `previous-period` preset: that preset + * must always return *something*, because a user who explicitly asks for the + * previous period has to see a window — so for a stripe period it approximates one + * by stepping back the current period's length. That approximation is fine as a + * destination and wrong as a baseline, since Stripe periods are not equal-length + * and the delta would silently compare against a window that is not the previous + * period. No delta beats a delta measured against the wrong window. + */ + const previousPeriod = + input.preset === 'current-period' ? resolvePreviousPeriod(context.period) : null + const comparison = previousPeriod ? { kind: 'period' as const, period: previousPeriod } : null + + const [totals, seriesRows, previous] = await Promise.all([ + readUsageTotals(scope), + readUsageTimeSeries(scope, bucket, input.timezone), + comparison + ? readUsageTotals(buildUsageAnalyticsScope(context.billingEntity, comparison)) + : Promise.resolve(null), + ]) + + const bounds = usageWindowBounds(window) + /** + * One apportionment across the buckets, not a per-bucket round. + * + * A day costing less than half a credit rounds to zero on its own, so an + * organization spending a fraction of a credit a day drew a flat empty chart under + * a positive headline — the reader's conclusion being that the chart is broken. + * Largest-remainder distributes the period's rounded total across its buckets, so + * the bars sum to the headline and a nonzero day is never drawn as zero. + * + * Same routine, and the same reasoning, as the breakdown's rows-plus-remainder. + */ + const densified = densifyUsageSeries(seriesRows, window, bucket, input.timezone) + const apportioned = apportionCredits( + densified.map((point, index) => ({ key: `b:${index}` as const, dollars: point.cost })) + ) + const bucketCredits = densified.map((point, index) => ({ + timestamp: point.timestamp, + credits: apportioned[`b:${index}`] ?? 0, + events: point.events, + })) + + return { + window: { + start: bounds.start.toISOString(), + end: bounds.end.toISOString(), + source: window.kind === 'range' ? 'range' : window.period.source, + }, + bucket, + totals: { credits: dollarsToCredits(totals.cost) }, + previousTotals: previous ? { credits: dollarsToCredits(previous.cost) } : null, + series: bucketCredits.map((point) => ({ + timestamp: point.timestamp, + credits: point.credits, + events: point.events, + })), + } + }, +}) diff --git a/apps/sim/lib/billing/application/organization-usage/list-organization-usage-events.ts b/apps/sim/lib/billing/application/organization-usage/list-organization-usage-events.ts new file mode 100644 index 00000000000..9a1e2b80607 --- /dev/null +++ b/apps/sim/lib/billing/application/organization-usage/list-organization-usage-events.ts @@ -0,0 +1,84 @@ +import { defineAuthorizedOrganizationUsageUseCase } from '@/lib/billing/application/organization-usage/authorized-organization-usage-use-case' +import { organizationUsageOperations } from '@/lib/billing/application/organization-usage/operations' +import { + resolveUsageAnalyticsWindow, + type UsageWindowPreset, + usageWindowLedgerFilter, +} from '@/lib/billing/core/usage-analytics' +import { getBillingEntityUsageLogs } from '@/lib/billing/core/usage-log' +import { dollarsToCredits } from '@/lib/billing/credits/conversion' +import type { InternalUsageLogSource } from '@/lib/billing/usage-sources' + +export interface OrganizationUsageEventsInput { + organizationId: string + preset: UsageWindowPreset + startDate?: Date + endDate?: Date + /** Viewer calendar, so a date-only custom bound means midnight there. */ + timezone?: string + source?: InternalUsageLogSource[] + limit: number + cursor?: string +} + +export interface OrganizationUsageEvent { + id: string + createdAt: string + source: string + description: string + workflowName: string | null + credits: number + hasCost: boolean +} + +export interface OrganizationUsageEventsResult { + events: OrganizationUsageEvent[] + nextCursor?: string + hasMore: boolean +} + +/** + * One page of the organization's raw ledger. + * + * Reuses `getBillingEntityUsageLogs` rather than rebuilding pagination: keyset + * ordering, cursor resolution, and the workflow-name join are already correct there. + * `includeSummary` stays off because the page header reads its total from the summary + * endpoint — recomputing the full-filter aggregate on every scroll would pay for the + * same scan once per page. + */ +export const listOrganizationUsageEvents = defineAuthorizedOrganizationUsageUseCase({ + operation: organizationUsageOperations.listEvents, + organizationId: (input: OrganizationUsageEventsInput) => input.organizationId, + async execute({ input, context }): Promise { + const window = resolveUsageAnalyticsWindow({ + preset: input.preset, + period: context.period, + customStart: input.startDate, + customEnd: input.endDate, + timezone: input.timezone, + }) + const result = await getBillingEntityUsageLogs(context.billingEntity, { + // One derivation for both predicates, so this list covers exactly the rows the + // summary and breakdowns aggregate over. + ...usageWindowLedgerFilter(window), + ...(input.source?.length ? { source: input.source } : {}), + limit: input.limit, + ...(input.cursor ? { cursor: input.cursor } : {}), + includeSummary: false, + }) + + return { + events: result.logs.map((log) => ({ + id: log.id, + createdAt: log.createdAt, + source: log.source, + description: log.description, + workflowName: log.workflowName ?? null, + credits: dollarsToCredits(log.cost), + hasCost: log.cost > 0, + })), + ...(result.pagination.nextCursor ? { nextCursor: result.pagination.nextCursor } : {}), + hasMore: result.pagination.hasMore, + } + }, +}) diff --git a/apps/sim/lib/billing/application/organization-usage/operations.ts b/apps/sim/lib/billing/application/organization-usage/operations.ts new file mode 100644 index 00000000000..a98aa7d9c6f --- /dev/null +++ b/apps/sim/lib/billing/application/organization-usage/operations.ts @@ -0,0 +1,53 @@ +import type { Principal } from '@sim/auth/principal' +import type { ApplicationOperation } from '@/lib/core/application' + +/** + * Session only. + * + * An organization's pooled ledger discloses every member's model spend, which is why + * `workspace-billing-authority` treats organization membership alone as insufficient + * for it. There is no API-key consumer of this surface today, and adding one should + * be a deliberate decision rather than something inherited from a default. + */ +export type OrganizationUsagePrincipal = Extract + +export interface OrganizationUsageOperation + extends ApplicationOperation { + readonly authority: 'organization_billing_admin' + readonly organizationRoles: readonly ['admin', 'owner'] + readonly workspaceApiKey: 'deny' + readonly principalKinds: readonly ['session'] +} + +function defineOrganizationUsageOperation( + operation: OrganizationUsageOperation +): OrganizationUsageOperation { + if ((operation.principalKinds as readonly string[]).some((kind) => kind !== 'session')) { + throw new Error( + `Organization usage operation ${operation.id} may only be performed by a session` + ) + } + Object.freeze(operation.organizationRoles) + Object.freeze(operation.principalKinds) + return Object.freeze(operation) +} + +const BASE = { + authority: 'organization_billing_admin', + organizationRoles: ['admin', 'owner'], + workspaceApiKey: 'deny', + principalKinds: ['session'], +} as const satisfies Omit + +export const organizationUsageOperations = { + readSummary: defineOrganizationUsageOperation({ id: 'organization_usage.summary.read', ...BASE }), + readBreakdown: defineOrganizationUsageOperation({ + id: 'organization_usage.breakdown.read', + ...BASE, + }), + listEvents: defineOrganizationUsageOperation({ id: 'organization_usage.events.list', ...BASE }), + exportEvents: defineOrganizationUsageOperation({ + id: 'organization_usage.events.export', + ...BASE, + }), +} as const diff --git a/apps/sim/lib/billing/application/organization-usage/organization-usage-use-cases.test.ts b/apps/sim/lib/billing/application/organization-usage/organization-usage-use-cases.test.ts new file mode 100644 index 00000000000..173e15826b5 --- /dev/null +++ b/apps/sim/lib/billing/application/organization-usage/organization-usage-use-cases.test.ts @@ -0,0 +1,128 @@ +/** + * @vitest-environment node + */ +import type { PersonalApiKeyPrincipal, SessionPrincipal } from '@sim/auth/principal' +import { setEnvFlags } from '@sim/testing' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + canUserManageBillingEntity: vi.fn(), + isOrganizationFeatureEntitled: vi.fn(), + getOrganizationSubscription: vi.fn(), + readUsageTotals: vi.fn(), + readUsageTimeSeries: vi.fn(), +})) + +vi.mock('@/lib/billing/core/workspace-billing-authority', () => ({ + canUserManageBillingEntity: mocks.canUserManageBillingEntity, +})) +vi.mock('@/lib/billing/core/subscription', () => ({ + isOrganizationFeatureEntitled: mocks.isOrganizationFeatureEntitled, +})) +vi.mock('@/lib/billing/core/billing', () => ({ + getOrganizationSubscription: mocks.getOrganizationSubscription, +})) +vi.mock('@/lib/billing/core/usage-analytics-queries', () => ({ + readUsageTotals: mocks.readUsageTotals, + readUsageTimeSeries: mocks.readUsageTimeSeries, + readUsageBreakdown: vi.fn(), + readUsageEntityNames: vi.fn(), +})) + +import { getOrganizationUsageSummary } from '@/lib/billing/application/organization-usage/get-organization-usage-summary' +import { ForbiddenOperationError } from '@/lib/core/application' + +const ORG = 'org-1' +const session: SessionPrincipal = { kind: 'session', userId: 'admin-1', sessionId: 'session-1' } +const personalKey: PersonalApiKeyPrincipal = { + kind: 'personal_api_key', + userId: 'admin-1', + keyId: 'key-1', +} + +const input = { + organizationId: ORG, + preset: 'current-period' as const, + timezone: 'UTC', +} + +function run(principal: SessionPrincipal | PersonalApiKeyPrincipal = session) { + return getOrganizationUsageSummary.execute({ principal, input }) +} + +async function codeOf(promise: Promise): Promise { + try { + await promise + throw new Error('expected the use case to refuse') + } catch (error) { + expect(error).toBeInstanceOf(ForbiddenOperationError) + return (error as ForbiddenOperationError).detailCode + } +} + +describe('organization usage authorization', () => { + beforeEach(() => { + vi.clearAllMocks() + setEnvFlags({ isBillingEnabled: true, isHosted: true }) + mocks.canUserManageBillingEntity.mockResolvedValue(true) + mocks.isOrganizationFeatureEntitled.mockResolvedValue(true) + mocks.getOrganizationSubscription.mockResolvedValue({ + plan: 'enterprise', + periodStart: new Date('2026-08-01T00:00:00.000Z'), + periodEnd: new Date('2026-09-01T00:00:00.000Z'), + }) + mocks.readUsageTotals.mockResolvedValue({ cost: 1 }) + mocks.readUsageTimeSeries.mockResolvedValue([]) + }) + + afterAll(() => { + setEnvFlags({ isBillingEnabled: false, isHosted: false }) + }) + + it('refuses an API key: pooled usage discloses every member’s spend', async () => { + // The operation names `session` alone. Widening it is a deliberate decision, not + // something that should fall out of a principal shape happening to carry a userId. + expect(await codeOf(run(personalKey))).toBe('PRINCIPAL_KIND_NOT_PERMITTED') + expect(mocks.canUserManageBillingEntity).not.toHaveBeenCalled() + }) + + it('refuses a member who is not an organization admin', async () => { + mocks.canUserManageBillingEntity.mockResolvedValue(false) + + expect(await codeOf(run())).toBe('ORGANIZATION_ADMIN_REQUIRED') + }) + + it('refuses an organization without the entitlement', async () => { + mocks.isOrganizationFeatureEntitled.mockResolvedValue(false) + + expect(await codeOf(run())).toBe('ENTERPRISE_PLAN_REQUIRED') + }) + + it('checks authority before entitlement, so a non-admin learns nothing about the plan', async () => { + mocks.canUserManageBillingEntity.mockResolvedValue(false) + mocks.isOrganizationFeatureEntitled.mockResolvedValue(false) + + expect(await codeOf(run())).toBe('ORGANIZATION_ADMIN_REQUIRED') + expect(mocks.isOrganizationFeatureEntitled).not.toHaveBeenCalled() + }) + + it('asks the entitlement helper, not the plan directly, so self-hosted stays gated', async () => { + // `isOrganizationOnEnterprisePlan` answers true for every organization once billing + // is off, which would hand the feature to every self-hosted deployment. + await run() + + expect(mocks.isOrganizationFeatureEntitled).toHaveBeenCalledWith(ORG, expect.any(Boolean)) + }) + + it('scopes the read to the caller’s organization and resolves its period once', async () => { + await run() + + expect(mocks.getOrganizationSubscription).toHaveBeenCalledTimes(1) + // Both reads share one scope built from one resolved period, which is what keeps the + // totals and the chart describing the same rows. + const [totalsScope] = mocks.readUsageTotals.mock.calls[0] + const [seriesScope] = mocks.readUsageTimeSeries.mock.calls[0] + expect(JSON.stringify(totalsScope)).toContain(ORG) + expect(JSON.stringify(seriesScope)).toBe(JSON.stringify(totalsScope)) + }) +}) diff --git a/apps/sim/lib/billing/core/billing-attribution.test.ts b/apps/sim/lib/billing/core/billing-attribution.test.ts index 41068d71088..e99f4fd2590 100644 --- a/apps/sim/lib/billing/core/billing-attribution.test.ts +++ b/apps/sim/lib/billing/core/billing-attribution.test.ts @@ -44,6 +44,7 @@ import { requireAccountBillingDecisionHeader, requireBillingAttributionHeader, requireBillingRequestIdHeader, + requireWorkspaceBillingAttributionHeader, resolveBillingAttribution, resolveLegacyV0BillingAttribution, resolveSystemBillingAttribution, @@ -426,6 +427,19 @@ describe('serialized attribution boundaries', () => { ).toThrow('Billing attribution header is required') }) + it('restores an executor snapshot by canonical workspace without making its actor authority', () => { + const headers = new Headers({ + 'x-sim-billing-attribution': serializeBillingAttributionHeader(attribution), + }) + + expect( + requireWorkspaceBillingAttributionHeader(headers, { workspaceId: 'workspace-b' }) + ).toEqual(attribution) + expect(() => + requireWorkspaceBillingAttributionHeader(headers, { workspaceId: 'workspace-other' }) + ).toThrow('does not match the authenticated request scope') + }) + it('rejects inconsistent or cross-scope serialized snapshots', () => { expect(() => assertBillingAttributionSnapshot({ diff --git a/apps/sim/lib/billing/core/billing-attribution.ts b/apps/sim/lib/billing/core/billing-attribution.ts index d98a0bb26b9..7ebfc453b93 100644 --- a/apps/sim/lib/billing/core/billing-attribution.ts +++ b/apps/sim/lib/billing/core/billing-attribution.ts @@ -406,7 +406,8 @@ export function requireBillingRequestIdHeader(headers: Pick): st function parseBillingAttributionHeader( headers: Pick, - expected: ResolveBillingAttributionParams + expected: Pick & + Partial> ): BillingAttributionSnapshot | undefined { const encoded = headers.get(BILLING_ATTRIBUTION_HEADER) if (!encoded) return undefined @@ -423,7 +424,7 @@ function parseBillingAttributionHeader( const attribution = assertBillingAttributionSnapshot(parsed) if ( - attribution.actorUserId !== expected.actorUserId || + (expected.actorUserId !== undefined && attribution.actorUserId !== expected.actorUserId) || attribution.workspaceId !== expected.workspaceId ) { throw new Error('Billing attribution header does not match the authenticated request scope') @@ -447,6 +448,22 @@ export function requireBillingAttributionHeader( return attribution } +/** + * Restores the executor's captured billing decision without treating its actor as authorization. + * The authenticated executor is authoritative for the snapshot; the canonical use case supplies + * the workspace scope that must still match. + */ +export function requireWorkspaceBillingAttributionHeader( + headers: Pick, + expected: Pick +): BillingAttributionSnapshot { + const attribution = parseBillingAttributionHeader(headers, expected) + if (!attribution) { + throw new Error('Billing attribution header is required for this internal request') + } + return attribution +} + /** * Compares two independently restored snapshots after canonical validation. */ diff --git a/apps/sim/lib/billing/core/usage-analytics-queries.ts b/apps/sim/lib/billing/core/usage-analytics-queries.ts new file mode 100644 index 00000000000..fec2b8e0480 --- /dev/null +++ b/apps/sim/lib/billing/core/usage-analytics-queries.ts @@ -0,0 +1,230 @@ +import { dbReplica } from '@sim/db' +import { usageLog, user, workflow, workspace } from '@sim/db/schema' +import { and, eq, inArray, isNotNull, type SQL, sql } from 'drizzle-orm' +import type { UsageBreakdownDimension, UsageBucket } from '@/lib/billing/core/usage-analytics' +import { assertValidTimezone } from '@/lib/core/utils/timezone' +import type { DbClient } from '@/lib/db/types' + +/** + * DB half of organization usage analytics. Every read runs on the replica and takes + * a prebuilt scope from `buildUsageAnalyticsScope`, so no query here decides period + * semantics for itself. + * + * Index note (`usage_log_billing_entity_created_at_cost_idx` covers + * `entityType, entityId, createdAt, userId, source, cost`): the series, the totals, + * and the member/source breakdowns are index-only. `workspace_id`, `workflow_id`, + * `description`, `metadata`, and `execution_id` are NOT in it, so the remaining + * reads heap-fetch per row — which is why they are issued only for the dimension + * actually being viewed. + */ + +export interface UsageTimeSeriesRow { + bucketStart: string | null + cost: string + events: number +} + +/** Index-only: reads `created_at` and `cost`, both covered. */ +export async function readUsageTimeSeries( + scope: SQL[], + bucket: UsageBucket, + timezone: string, + executor: DbClient = dbReplica +): Promise { + assertValidTimezone(timezone) + const bucketStart = sql`to_char( + date_trunc(${bucket}, ${usageLog.createdAt} AT TIME ZONE ${timezone}), + 'YYYY-MM-DD"T"HH24:MI:SS' + )` + + return ( + executor + .select({ + bucketStart: bucketStart.as('bucket_start'), + cost: sql`COALESCE(SUM(${usageLog.cost}), 0)`, + events: sql`COUNT(*)`.mapWith(Number), + }) + .from(usageLog) + .where(and(...scope)) + // Group by the output alias, not the expression. Re-rendering the fragment here + // emits a *textually different* one — the select list qualifies the column as + // `created_at`, the group-by as `usage_log.created_at` — and Postgres matches + // group-by expressions syntactically, so it rejects the query outright. It also + // duplicates the bound parameters. + .groupBy(sql`bucket_start`) + ) +} + +export interface UsageTotals { + cost: number +} + +/** + * The headline figure, and the only one first paint waits on. + * + * Deliberately just `SUM(cost)`. A `COUNT(DISTINCT user_id)` alongside it forces a + * sort over every matching row — measured at 830ms of a 909ms query on production's + * largest organization (342k rows in a 30-day window), and the summary runs it twice + * because the delta compares two windows. Without it the same query is 79ms. Any + * per-actor figure added here must earn that cost by actually being displayed. + */ +export async function readUsageTotals( + scope: SQL[], + executor: DbClient = dbReplica +): Promise { + const [row] = await executor + .select({ + cost: sql`COALESCE(SUM(${usageLog.cost}), 0)`, + }) + .from(usageLog) + .where(and(...scope)) + + return { cost: Number.parseFloat(row?.cost ?? '0') || 0 } +} + +export interface UsageBreakdownRow { + key: string | null + cost: string + events: number + /** Model dimensions only — the ledger records tokens only for model categories. */ + inputTokens?: number + outputTokens?: number +} + +/** Dimensions keyed on `description`, which holds a model name for model categories. */ +const MODEL_DIMENSIONS = new Set(['model', 'byok']) + +function breakdownColumn(dimension: UsageBreakdownDimension) { + switch (dimension) { + case 'member': + return usageLog.userId + case 'workspace': + return usageLog.workspaceId + case 'workflow': + return usageLog.workflowId + case 'source': + return usageLog.source + case 'model': + case 'byok': + return usageLog.description + } +} + +/** + * Ranked totals for one dimension. + * + * Aggregate-first: names are hydrated by {@link readUsageEntityNames} for the + * surviving keys only. Joining inside the aggregate would break index-only for + * `member` and force a nested loop across the whole window. + */ +export async function readUsageBreakdown( + scope: SQL[], + dimension: UsageBreakdownDimension, + /** Narrows to one workspace, for the Workspaces drill-down. */ + workspaceId: string | undefined, + executor: DbClient = dbReplica +): Promise { + const column = breakdownColumn(dimension) + const conditions = [...scope] + if (workspaceId) conditions.push(eq(usageLog.workspaceId, workspaceId)) + /** + * `description` holds a model name only for the model categories; a tool or fixed + * row would otherwise appear as a phantom "model". The two model dimensions split + * on who paid: `model` is what Sim charged for, `byok` is the customer's own key. + */ + if (dimension === 'model') conditions.push(eq(usageLog.category, 'model')) + if (dimension === 'byok') conditions.push(eq(usageLog.category, 'model_unbilled')) + /** + * Only `source = 'workflow'` rows ever carry a `workflow_id` — Chat, Agent block, + * Wand, knowledge base, and voice have none by construction, not by omission. So a + * workflow list excludes them outright; bucketing them into an "other" row put most + * of an organization's usage into a list it does not belong in. + */ + if (dimension === 'workflow') conditions.push(isNotNull(usageLog.workflowId)) + + if (!MODEL_DIMENSIONS.has(dimension)) { + return ( + executor + .select({ + key: sql`${column}`, + cost: sql`COALESCE(SUM(${usageLog.cost}), 0)`, + events: sql`COUNT(*)`.mapWith(Number), + }) + .from(usageLog) + .where(and(...conditions)) + .groupBy(column) + /** + * Drops groups whose every row is reporting-only. Unbilled rows carry a user, + * a workspace, a workflow and `source = 'workflow'` like any other, so without + * this a BYOK-only member appeared in a credit-denominated list at 0 credits. + * + * As a `HAVING` on the aggregate rather than a `category` predicate on purpose: + * `category` is not in `usage_log_billing_entity_created_at_cost_idx`, so + * filtering on it would force a heap fetch on `member` and `source` — the two + * dimensions that are index-only today. `cost` is in that index, and only an + * unbilled row can sum to zero, since `recordUsage` admits nothing else at zero. + */ + .having(sql`COALESCE(SUM(${usageLog.cost}), 0) > 0`) + ) + } + + // Already heap-reading `description`, so summing `metadata` costs nothing extra — + // and BYOK rows carry no cost at all, making tokens the only usage they can show. + return executor + .select({ + key: sql`${column}`, + cost: sql`COALESCE(SUM(${usageLog.cost}), 0)`, + events: sql`COUNT(*)`.mapWith(Number), + inputTokens: + sql`COALESCE(SUM((${usageLog.metadata}->>'inputTokens')::bigint), 0)`.mapWith( + Number + ), + outputTokens: + sql`COALESCE(SUM((${usageLog.metadata}->>'outputTokens')::bigint), 0)`.mapWith( + Number + ), + }) + .from(usageLog) + .where(and(...conditions)) + .groupBy(column) +} + +/** + * Display names for the top-N keys of an entity-backed dimension. + * + * Members fall back to email because a user may have no name set, and an empty row + * label is worse than an address. + */ +export async function readUsageEntityNames( + dimension: UsageBreakdownDimension, + ids: string[], + executor: DbClient = dbReplica +): Promise> { + if (ids.length === 0) return new Map() + + if (dimension === 'member') { + const rows = await executor + .select({ id: user.id, name: user.name, email: user.email }) + .from(user) + .where(inArray(user.id, ids)) + return new Map(rows.map((row) => [row.id, row.name?.trim() || row.email])) + } + + if (dimension === 'workspace') { + const rows = await executor + .select({ id: workspace.id, name: workspace.name }) + .from(workspace) + .where(inArray(workspace.id, ids)) + return new Map(rows.map((row) => [row.id, row.name])) + } + + if (dimension === 'workflow') { + const rows = await executor + .select({ id: workflow.id, name: workflow.name }) + .from(workflow) + .where(inArray(workflow.id, ids)) + return new Map(rows.map((row) => [row.id, row.name])) + } + + return new Map() +} diff --git a/apps/sim/lib/billing/core/usage-analytics.test.ts b/apps/sim/lib/billing/core/usage-analytics.test.ts new file mode 100644 index 00000000000..c288c8e62ec --- /dev/null +++ b/apps/sim/lib/billing/core/usage-analytics.test.ts @@ -0,0 +1,542 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import type { ResolvedUsagePeriod } from '@/lib/billing/core/reporting-period' +import { + buildUsageAnalyticsScope, + densifyUsageSeries, + foldUsageBreakdown, + MAX_CUSTOM_RANGE_DAYS, + mergeRowsByKey, + resolvePreviousPeriod, + resolveUsageAnalyticsWindow, + resolveUsageBucket, + UsageWindowRangeInvertedError, + UsageWindowRangeTooLargeError, + usageWindowLedgerFilter, +} from '@/lib/billing/core/usage-analytics' + +const ENTITY = { type: 'organization', id: 'org-1' } as const + +function period(overrides: Partial = {}): ResolvedUsagePeriod { + return { + start: new Date('2026-08-01T00:00:00.000Z'), + end: new Date('2026-09-01T00:00:00.000Z'), + source: 'stripe', + anchorDate: null, + interval: null, + ...overrides, + } +} + +/** The generated predicate as SQL-ish text, for asserting which columns it filters. */ +function scopeShape(window: Parameters[1]): string { + return JSON.stringify(buildUsageAnalyticsScope(ENTITY, window)) +} + +describe('buildUsageAnalyticsScope', () => { + it('matches a reporting period on created_at, not on the stamps', () => { + // A reporting period is derived from an anchor date and is not what rows carry, + // so matching the stamps would return nothing for every enterprise org. + const shape = scopeShape({ + kind: 'period', + period: period({ source: 'reporting', anchorDate: '2026-08-01', interval: 'month' }), + }) + expect(shape).toContain('usageLog.createdAt') + expect(shape).not.toContain('usageLog.billingPeriodStart') + }) + + it('matches a stripe period on the exact stamps', () => { + const shape = scopeShape({ kind: 'period', period: period({ source: 'stripe' }) }) + expect(shape).toContain('usageLog.billingPeriodStart') + expect(shape).toContain('usageLog.billingPeriodEnd') + }) + + it('matches a plain range on created_at', () => { + const shape = scopeShape({ + kind: 'range', + from: new Date('2026-08-01T00:00:00.000Z'), + to: new Date('2026-08-08T00:00:00.000Z'), + }) + expect(shape).toContain('usageLog.createdAt') + expect(shape).not.toContain('usageLog.billingPeriodStart') + }) + + it('always scopes to the billing entity', () => { + const shape = scopeShape({ kind: 'period', period: period() }) + expect(shape).toContain('usageLog.billingEntityType') + expect(shape).toContain('usageLog.billingEntityId') + }) +}) + +describe('usageWindowLedgerFilter', () => { + it('matches a stripe period on the stamps, as the aggregate does', () => { + // Filtering this window on `created_at` instead selects a different set — rows + // created inside the period but stamped to another, and vice versa — so the event + // list and the CSV covered different rows than the totals above them. + const filter = usageWindowLedgerFilter({ kind: 'period', period: period({ source: 'stripe' }) }) + expect(filter).toEqual({ + billingPeriod: { + start: new Date('2026-08-01T00:00:00.000Z'), + end: new Date('2026-09-01T00:00:00.000Z'), + }, + }) + }) + + it('matches a reporting period on created_at, as the aggregate does', () => { + const filter = usageWindowLedgerFilter({ + kind: 'period', + period: period({ source: 'reporting', anchorDate: '2026-08-01', interval: 'month' }), + }) + expect(filter.billingPeriod).toBeUndefined() + expect(filter.endDateExclusive).toBe(true) + }) + + it('keeps a plain range half-open', () => { + const from = new Date('2026-08-01T00:00:00.000Z') + const to = new Date('2026-08-08T00:00:00.000Z') + expect(usageWindowLedgerFilter({ kind: 'range', from, to })).toEqual({ + startDate: from, + endDate: to, + endDateExclusive: true, + }) + }) + + it('branches on the same condition the scope builder does', () => { + // The two predicates are only in step because they read the same discriminant. + for (const source of ['reporting', 'stripe', 'default'] as const) { + const window = { kind: 'period' as const, period: period({ source }) } + const usesStamps = scopeShape(window).includes('usageLog.billingPeriodStart') + expect(usageWindowLedgerFilter(window).billingPeriod !== undefined).toBe(usesStamps) + } + }) +}) + +describe('resolveUsageAnalyticsWindow', () => { + const now = new Date('2026-08-20T12:00:00.000Z') + + it('keeps current-period a period so it matches the billing page', () => { + const window = resolveUsageAnalyticsWindow({ preset: 'current-period', period: period(), now }) + expect(window).toEqual({ kind: 'period', period: period() }) + }) + + it('derives the previous reporting period from its anchor', () => { + const window = resolveUsageAnalyticsWindow({ + preset: 'previous-period', + period: period({ source: 'reporting', anchorDate: '2026-01-15', interval: 'month' }), + now, + }) + expect(window.kind).toBe('period') + }) + + it('falls back to an equal-length range when the previous period is not derivable', () => { + // A stripe period's predecessor lives in Stripe; inventing stamps would match nothing. + const window = resolveUsageAnalyticsWindow({ + preset: 'previous-period', + period: period({ source: 'stripe' }), + now, + }) + expect(window.kind).toBe('range') + if (window.kind === 'range') { + expect(window.to).toEqual(new Date('2026-08-01T00:00:00.000Z')) + expect(window.from).toEqual(new Date('2026-07-01T00:00:00.000Z')) + } + }) + + it('makes a custom range half-open so a single day returns that day', () => { + const window = resolveUsageAnalyticsWindow({ + preset: 'custom', + period: period(), + customStart: new Date('2026-08-04T00:00:00.000Z'), + customEnd: new Date('2026-08-04T00:00:00.000Z'), + now, + }) + expect(window).toEqual({ + kind: 'range', + from: new Date('2026-08-04T00:00:00.000Z'), + to: new Date('2026-08-05T00:00:00.000Z'), + }) + }) + + it('covers exactly the days the picker emitted, from its bare date bounds', () => { + // The picker sends `YYYY-MM-DD`, which parses as UTC midnight. It must not send + // an inclusive `…T23:59:59` wall time: the extra day added below would then land + // on the *following* day, and every custom range would overrun by 24 hours. + const window = resolveUsageAnalyticsWindow({ + preset: 'custom', + period: period(), + customStart: new Date('2026-06-01'), + customEnd: new Date('2026-08-31'), + now, + }) + expect(window).toEqual({ + kind: 'range', + from: new Date('2026-06-01T00:00:00.000Z'), + to: new Date('2026-09-01T00:00:00.000Z'), + }) + }) + + it('accepts a selection of exactly the maximum span', () => { + // June 1 through August 31 inclusive is 92 days. It measured 93 while the end + // bound carried a time of day, so the longest legal pick was rejected. + expect(() => + resolveUsageAnalyticsWindow({ + preset: 'custom', + period: period(), + customStart: new Date('2026-06-01'), + customEnd: new Date('2026-08-31'), + now, + }) + ).not.toThrow() + }) + + it('refuses a custom range beyond the cap rather than scanning the ledger', () => { + expect(() => + resolveUsageAnalyticsWindow({ + preset: 'custom', + period: period(), + customStart: new Date('2026-01-01T00:00:00.000Z'), + customEnd: new Date('2026-08-01T00:00:00.000Z'), + now, + }) + ).toThrow(UsageWindowRangeTooLargeError) + }) + + it('falls back to the period when a custom range is missing a bound', () => { + const window = resolveUsageAnalyticsWindow({ preset: 'custom', period: period(), now }) + expect(window.kind).toBe('period') + }) + + it('anchors custom bounds on midnight in the viewer calendar, not UTC', () => { + // The picker offers calendar days. Anchoring on the UTC instant shifted every + // non-UTC viewer's selection by their offset, and disagreed with the chart, + // whose buckets are already the viewer's calendar days. + const window = resolveUsageAnalyticsWindow({ + preset: 'custom', + period: period(), + customStart: new Date('2026-08-01'), + customEnd: new Date('2026-08-31'), + timezone: 'America/New_York', + now, + }) + expect(window.kind).toBe('range') + if (window.kind === 'range') { + // Midnight on Aug 1 in New York is 04:00 UTC (EDT, UTC-4). + expect(window.from.toISOString()).toBe('2026-08-01T04:00:00.000Z') + expect(window.to.toISOString()).toBe('2026-09-01T04:00:00.000Z') + } + }) + + it('refuses a range that ends before it starts', () => { + // Inverted bounds measured a negative span, passed the cap check, and produced a + // range that matched nothing — indistinguishable from "no usage". + expect(() => + resolveUsageAnalyticsWindow({ + preset: 'custom', + period: period(), + customStart: new Date('2026-08-31'), + customEnd: new Date('2026-08-01'), + now, + }) + ).toThrow(UsageWindowRangeInvertedError) + }) + + it('shows a bounded window for a deployment with no subscription', () => { + // `defaultBillingPeriod()` is the open pair 1970…9999. Rendered as a period it + // produced 1,000 monthly buckets ending in 2053, stopped only by the densifier's + // loop guard — reachable on self-hosted, where the panel opens without a plan. + const window = resolveUsageAnalyticsWindow({ + preset: 'current-period', + period: period({ + source: 'default', + start: new Date(0), + end: new Date(Date.UTC(9999, 11, 31)), + }), + now, + }) + expect(window.kind).toBe('range') + if (window.kind === 'range') { + expect(window.to).toEqual(now) + expect(Math.round((window.to.getTime() - window.from.getTime()) / 86_400_000)).toBe(30) + } + }) + + it('steps an unbounded period back by the display window, not by its own length', () => { + const window = resolveUsageAnalyticsWindow({ + preset: 'previous-period', + period: period({ + source: 'default', + start: new Date(0), + end: new Date(Date.UTC(9999, 11, 31)), + }), + now, + }) + expect(window.kind).toBe('range') + if (window.kind === 'range') { + // Not 1970-minus-eight-millennia, which is what deriving it from the span gave. + expect(window.from.getUTCFullYear()).toBe(2026) + } + }) +}) + +describe('resolvePreviousPeriod', () => { + it('returns null for a period with no derivation rule', () => { + expect(resolvePreviousPeriod(period({ source: 'stripe' }))).toBeNull() + expect(resolvePreviousPeriod(period({ source: 'default' }))).toBeNull() + }) +}) + +describe('resolveUsageBucket', () => { + const from = new Date('2026-01-01T00:00:00.000Z') + const days = (n: number) => new Date(from.getTime() + n * 24 * 60 * 60 * 1000) + + it('scales granularity with the window', () => { + expect(resolveUsageBucket({ kind: 'range', from, to: days(30) })).toBe('day') + expect(resolveUsageBucket({ kind: 'range', from, to: days(200) })).toBe('week') + expect(resolveUsageBucket({ kind: 'range', from, to: days(500) })).toBe('month') + }) + + it('keeps daily bars for the maximum range across a DST transition', () => { + // 92 calendar days spanning the autumn fall-back is 92 days and one hour. Ceiling + // that called it 93 and silently demoted the longest legal custom range to weekly + // bars — a granularity change with no cause the reader could see. + const dstSpan = { + kind: 'range' as const, + from, + to: new Date(from.getTime() + 92 * 86_400_000 + 3_600_000), + } + expect(resolveUsageBucket(dstSpan)).toBe('day') + // A genuinely longer range still steps up. + expect( + resolveUsageBucket({ kind: 'range', from, to: new Date(from.getTime() + 93 * 86_400_000) }) + ).toBe('week') + }) +}) + +describe('densifyUsageSeries', () => { + const window = { + kind: 'range' as const, + from: new Date('2026-08-01T00:00:00.000Z'), + to: new Date('2026-08-04T00:00:00.000Z'), + } + + it('fills empty buckets with zero rather than omitting them', () => { + // A period with no usage must draw a flat zero line; "No data" reads as a failure. + const points = densifyUsageSeries( + [{ bucketStart: '2026-08-02T00:00:00', cost: '1.50', events: 3 }], + window, + 'day', + 'UTC' + ) + expect(points).toHaveLength(3) + expect(points.map((p) => p.cost)).toEqual([0, 1.5, 0]) + expect(points.map((p) => p.events)).toEqual([0, 3, 0]) + }) + + it('preserves the window total', () => { + const points = densifyUsageSeries( + [ + { bucketStart: '2026-08-01T00:00:00', cost: '1.25', events: 1 }, + { bucketStart: '2026-08-03T00:00:00', cost: '2.75', events: 2 }, + ], + window, + 'day', + 'UTC' + ) + expect(points.reduce((sum, p) => sum + p.cost, 0)).toBeCloseTo(4, 8) + }) + + it('emits an empty series for a zero-length window instead of looping', () => { + expect( + densifyUsageSeries([], { kind: 'range', from: window.from, to: window.from }, 'day', 'UTC') + ).toEqual([]) + }) + + it('keys days by the viewer calendar the query grouped by, not UTC', () => { + // Auckland is UTC+12, so the window's final instant is already the next local + // day. Walking UTC dates dropped that bucket: its cost stayed in the headline + // while its bar was never drawn. + const points = densifyUsageSeries( + [{ bucketStart: '2026-08-04T00:00:00', cost: '5.00', events: 1 }], + window, + 'day', + 'Pacific/Auckland' + ) + const keys = points.map((p) => p.timestamp.slice(0, 10)) + expect(keys).toEqual(['2026-08-01', '2026-08-02', '2026-08-03', '2026-08-04']) + expect(points.at(-1)?.cost).toBe(5) + }) + + it('aligns week buckets to Monday, as `date_trunc` does', () => { + // A period starting mid-week previously produced keys Postgres never emits, so + // every bar read zero while the total was correct. Reachable through an annual + // enterprise period, which resolves to `week`. + const points = densifyUsageSeries( + [{ bucketStart: '2026-08-10T00:00:00', cost: '9.00', events: 4 }], + // 2026-08-15 is a Saturday; its ISO week starts Monday 2026-08-10. + { + kind: 'range', + from: new Date('2026-08-15T00:00:00.000Z'), + to: new Date('2026-08-29T00:00:00.000Z'), + }, + 'week', + 'UTC' + ) + expect(points.map((p) => p.timestamp.slice(0, 10))).toEqual([ + '2026-08-10', + '2026-08-17', + '2026-08-24', + ]) + expect(points[0].cost).toBe(9) + }) + + it('aligns month buckets to the first, as `date_trunc` does', () => { + const points = densifyUsageSeries( + [{ bucketStart: '2026-09-01T00:00:00', cost: '3.00', events: 2 }], + { + kind: 'range', + from: new Date('2026-08-15T00:00:00.000Z'), + to: new Date('2026-10-15T00:00:00.000Z'), + }, + 'month', + 'UTC' + ) + expect(points.map((p) => p.timestamp.slice(0, 10))).toEqual([ + '2026-08-01', + '2026-09-01', + '2026-10-01', + ]) + expect(points[1].cost).toBe(3) + }) +}) + +describe('foldUsageBreakdown', () => { + const rows = [ + { key: 'a', cost: '5', events: 5 }, + { key: 'b', cost: '3', events: 3 }, + { key: 'c', cost: '2', events: 2 }, + ] + const labelFor = (key: string | null) => key ?? 'Unattributed' + + it('reconciles visible rows plus the remainder to the total', () => { + const fold = foldUsageBreakdown(rows, 10, labelFor, 2) + const visible = fold.rows.reduce((sum, row) => sum + row.cost, 0) + expect(visible + fold.other.cost).toBeCloseTo(fold.totalCost, 8) + expect(fold.other.rowCount).toBe(1) + }) + + it('ranks by cost descending', () => { + expect(foldUsageBreakdown(rows, 10, labelFor, 3).rows.map((r) => r.id)).toEqual(['a', 'b', 'c']) + }) + + it('computes share against the window total, not the visible subset', () => { + // Sharing against the visible rows would make a truncated list read as 100%. + const fold = foldUsageBreakdown(rows, 10, labelFor, 1) + expect(fold.rows[0].share).toBeCloseTo(0.5, 8) + }) + + it('labels a null grouping key rather than dropping the row', () => { + const fold = foldUsageBreakdown([{ key: null, cost: '4', events: 1 }], 4, labelFor, 5) + expect(fold.rows[0].label).toBe('Unattributed') + }) + + it('yields zero shares when nothing was spent, without dividing by zero', () => { + const fold = foldUsageBreakdown([{ key: 'a', cost: '0', events: 2 }], 0, labelFor, 5) + expect(fold.rows[0].share).toBe(0) + }) + + it('measures share in the ranking unit, so a zero-cost list still draws bars', () => { + // BYOK ranks by a cost that is zero for every row, so a cost-based share is 0/0 + // for all of them and every bar renders at the same minimum width. + const byokRows = [ + { key: 'gpt-4o', cost: '0', events: 1, inputTokens: 100, outputTokens: 50 }, + { key: 'claude', cost: '0', events: 1, inputTokens: 700, outputTokens: 300 }, + { key: 'gemini', cost: '0', events: 1, inputTokens: 20, outputTokens: 5 }, + ] + const fold = foldUsageBreakdown(byokRows, 0, labelFor, 3, 'tokens') + // Share is measured in the ranking unit, or every BYOK bar renders identical: + // a cost-based share is 0/0 for every provider. + expect(fold.rows.map((row) => row.share)).toEqual([1000 / 1175, 150 / 1175, 25 / 1175]) + }) + + it('carries the omitted rows tokens, so a zero-cost dimension still adds up', () => { + const byokRows = [ + { key: 'gpt-4o', cost: '0', events: 1, inputTokens: 100, outputTokens: 50 }, + { key: 'claude', cost: '0', events: 1, inputTokens: 700, outputTokens: 300 }, + { key: 'gemini', cost: '0', events: 1, inputTokens: 20, outputTokens: 5 }, + ] + const fold = foldUsageBreakdown(byokRows, 0, labelFor, 1, 'tokens') + // Ranked by tokens, so the biggest provider is the one shown... + expect(fold.rows.map((row) => row.id)).toEqual(['claude']) + // ...and the tail's tokens are still accounted for rather than hidden. + expect(fold.other.rowCount).toBe(2) + expect(fold.other.tokens).toBe(175) + }) +}) + +describe('MAX_CUSTOM_RANGE_DAYS', () => { + it('is the documented cap', () => { + expect(MAX_CUSTOM_RANGE_DAYS).toBe(92) + }) +}) + +describe('mergeRowsByKey', () => { + it('collapses sources that share a display label', () => { + // The ledger stores `copilot` and `workspace-chat` separately but both render as + // "Sim Chat", so grouping on the raw column alone shipped the label twice with the + // usage split between the rows — and ranked both below their true position. + const merged = mergeRowsByKey( + [ + { key: 'copilot', cost: '0.0295', events: 2 }, + { key: 'workspace-chat', cost: '23.3967', events: 40 }, + { key: 'workflow', cost: '0.9333', events: 130 }, + ], + (key) => (key === 'copilot' || key === 'workspace-chat' ? 'sim-chat' : key) + ) + + expect(merged).toHaveLength(2) + const simChat = merged.find((row) => row.key === 'sim-chat') + expect(Number(simChat?.cost)).toBeCloseTo(23.4262, 8) + expect(simChat?.events).toBe(42) + }) + + it('sums token columns alongside cost', () => { + const merged = mergeRowsByKey( + [ + { key: 'claude-opus-4.8', cost: '1', events: 1, inputTokens: 100, outputTokens: 10 }, + { key: 'claude-sonnet-4', cost: '2', events: 3, inputTokens: 50, outputTokens: 5 }, + ], + () => 'anthropic' + ) + + expect(merged).toEqual([ + { key: 'anthropic', cost: 3, events: 4, inputTokens: 150, outputTokens: 15 }, + ]) + }) + + it('preserves a null key rather than merging it into a named row', () => { + const merged = mergeRowsByKey( + [ + { key: null, cost: '1', events: 1 }, + { key: 'a', cost: '2', events: 1 }, + ], + (key) => key + ) + expect(merged.map((row) => row.key)).toEqual([null, 'a']) + }) + + it('leaves totals unchanged', () => { + const rows = [ + { key: 'a', cost: '1.5', events: 1 }, + { key: 'b', cost: '2.5', events: 2 }, + { key: 'c', cost: '3', events: 3 }, + ] + const before = rows.reduce((sum, row) => sum + Number(row.cost), 0) + const after = mergeRowsByKey(rows, (key) => (key === 'c' ? 'c' : 'ab')).reduce( + (sum, row) => sum + Number(row.cost), + 0 + ) + expect(after).toBeCloseTo(before, 8) + }) +}) diff --git a/apps/sim/lib/billing/core/usage-analytics.ts b/apps/sim/lib/billing/core/usage-analytics.ts new file mode 100644 index 00000000000..07533fe0ac6 --- /dev/null +++ b/apps/sim/lib/billing/core/usage-analytics.ts @@ -0,0 +1,571 @@ +import { usageLog } from '@sim/db/schema' +import { eq, gte, lt, type SQL } from 'drizzle-orm' +import { MAX_CUSTOM_RANGE_DAYS } from '@/lib/api/contracts/organization-usage' +import { + type ResolvedUsagePeriod, + resolveEnterpriseReportingPeriod, +} from '@/lib/billing/core/reporting-period' +import type { BillingEntity } from '@/lib/billing/core/usage-log' +import { zonedWallClockToUtc } from '@/lib/core/utils/timezone' + +/** + * Pure half of organization usage analytics: window resolution, the ledger scope + * every query is built from, bucket granularity, and the folds that turn sparse + * rows into dense series and ranked lists. + * + * No DB access, so the parts most likely to be wrong — period semantics and + * reconciliation arithmetic — are directly testable. + */ + +export const USAGE_WINDOW_PRESETS = [ + 'current-period', + 'previous-period', + '7d', + '30d', + 'custom', +] as const +export type UsageWindowPreset = (typeof USAGE_WINDOW_PRESETS)[number] + +export const USAGE_BREAKDOWN_DIMENSIONS = [ + 'member', + 'workspace', + 'workflow', + 'model', + 'byok', + 'source', +] as const +export type UsageBreakdownDimension = (typeof USAGE_BREAKDOWN_DIMENSIONS)[number] + +export type UsageBucket = 'day' | 'week' | 'month' + +/** + * A custom range is capped because three of the five breakdown dimensions are not + * index-covered and heap-fetch per row; an unbounded range over a large ledger is a + * table scan. Longer look-back goes through `previous-period`, which is stamped. + * + * Declared on the contract so the picker and this resolver state one number. + */ +export { MAX_CUSTOM_RANGE_DAYS } + +const DAY_MS = 24 * 60 * 60 * 1000 + +export type UsageAnalyticsWindow = + | { kind: 'period'; period: ResolvedUsagePeriod } + | { kind: 'range'; from: Date; to: Date } + +/** + * The ledger predicate every usage query is built from. + * + * This is the one place the `reporting` branch is written for this feature, and it + * mirrors `getBillingPeriodUsageCost` deliberately: a reporting period is derived + * from an anchor date and is *not* what rows are stamped with, so it matches on + * `created_at`; a stripe/default period matches the stamps exactly. Diverging here + * is how the usage panel would come to disagree with the billing page about the + * same period. + */ +export function buildUsageAnalyticsScope( + entity: BillingEntity, + window: UsageAnalyticsWindow +): SQL[] { + const conditions: SQL[] = [ + eq(usageLog.billingEntityType, entity.type), + eq(usageLog.billingEntityId, entity.id), + ] + + if (window.kind === 'range') { + conditions.push(gte(usageLog.createdAt, window.from), lt(usageLog.createdAt, window.to)) + return conditions + } + + if (window.period.source === 'reporting') { + conditions.push( + gte(usageLog.createdAt, window.period.start), + lt(usageLog.createdAt, window.period.end) + ) + return conditions + } + + conditions.push( + eq(usageLog.billingPeriodStart, window.period.start), + eq(usageLog.billingPeriodEnd, window.period.end) + ) + return conditions +} + +/** The instants a window covers, for labelling and for deriving bucket granularity. */ +export function usageWindowBounds(window: UsageAnalyticsWindow): { start: Date; end: Date } { + return window.kind === 'range' + ? { start: window.from, end: window.to } + : { start: window.period.start, end: window.period.end } +} + +export interface UsageWindowLedgerFilter { + startDate?: Date + endDate?: Date + endDateExclusive?: boolean + billingPeriod?: { start: Date; end: Date } +} + +/** + * The same window, expressed for the ledger listing query. + * + * `getBillingEntityUsageLogs` filters rows while {@link buildUsageAnalyticsScope} + * aggregates them, and the two must select the same set or the event list and CSV + * describe different rows than the totals above them. That is not hypothetical for a + * stripe or default period: those are matched on the *stamps* rows carry, and + * filtering the same window on `created_at` picks up rows created inside it but + * stamped to another period, while missing the reverse. + * + * Deriving both from one function is what keeps the predicates in step — the branches + * below mirror `buildUsageAnalyticsScope` case for case. + */ +export function usageWindowLedgerFilter(window: UsageAnalyticsWindow): UsageWindowLedgerFilter { + if (window.kind === 'range' || window.period.source === 'reporting') { + const { start, end } = usageWindowBounds(window) + // Half-open, so a row on the boundary belongs to the next window — as it does + // for the aggregate, whose `lt` says the same thing. + return { startDate: start, endDate: end, endDateExclusive: true } + } + return { billingPeriod: { start: window.period.start, end: window.period.end } } +} + +export class UsageWindowRangeTooLargeError extends Error { + constructor(days: number) { + super(`Custom range spans ${days} days; the maximum is ${MAX_CUSTOM_RANGE_DAYS}.`) + this.name = 'UsageWindowRangeTooLargeError' + } +} + +export class UsageWindowRangeInvertedError extends Error { + constructor() { + super('Custom range ends before it starts.') + this.name = 'UsageWindowRangeInvertedError' + } +} + +/** + * How much of an unbounded period to show. + * + * A deployment with no subscription resolves to `defaultBillingPeriod()`, which is + * the open pair `1970-01-01 … 9999-12-31`. Rendered as a period that is 1,000 + * monthly buckets ending in 2053, stopped only by the densifier's loop guard — the + * chart was unusable and the label claimed it was a billing period. Self-hosted is + * exactly where this is reachable, since `USAGE_MONITORING_ENABLED` opens the panel + * on deployments that have no subscription at all. + */ +const UNBOUNDED_PERIOD_DISPLAY_DAYS = 30 + +/** True for the open sentinel period a deployment without a subscription resolves to. */ +function isUnboundedPeriod(period: ResolvedUsagePeriod): boolean { + return period.source === 'default' +} + +interface ResolveUsageWindowArgs { + preset: UsageWindowPreset + /** The payer's current period, already resolved from its subscription. */ + period: ResolvedUsagePeriod + customStart?: Date + customEnd?: Date + /** + * The viewer's calendar, used to resolve date-only custom bounds. The picker + * offers calendar days, so "Aug 31" has to mean midnight-to-midnight *there*. + */ + timezone?: string + now?: Date +} + +/** The civil date an already-UTC-parsed `YYYY-MM-DD` bound represents. */ +function civilBoundKey(bound: Date): string { + return bound.toISOString().slice(0, 10) +} + +/** Exact whole-day distance between two civil dates; unaffected by DST. */ +function civilDaysBetween(fromKey: string, toKey: string): number { + return Math.round((civilDate(toKey).getTime() - civilDate(fromKey).getTime()) / DAY_MS) +} + +/** + * Maps a picker selection to a window the ledger can actually match. + * + * `current-period` and `previous-period` stay *periods* so they use the same + * predicate the billing page does; the rolling and custom presets are plain + * `created_at` ranges. + */ +export function resolveUsageAnalyticsWindow({ + preset, + period, + customStart, + customEnd, + timezone = 'UTC', + now = new Date(), +}: ResolveUsageWindowArgs): UsageAnalyticsWindow { + switch (preset) { + case 'current-period': + return isUnboundedPeriod(period) + ? { + kind: 'range', + from: new Date(now.getTime() - UNBOUNDED_PERIOD_DISPLAY_DAYS * DAY_MS), + to: now, + } + : { kind: 'period', period } + case 'previous-period': { + const previous = resolvePreviousPeriod(period) + if (previous) return { kind: 'period', period: previous } + // An open period has no meaningful predecessor — deriving one from its length + // reaches back eight millennia — so it steps back by the display window instead. + if (isUnboundedPeriod(period)) { + const to = new Date(now.getTime() - UNBOUNDED_PERIOD_DISPLAY_DAYS * DAY_MS) + return { + kind: 'range', + from: new Date(to.getTime() - UNBOUNDED_PERIOD_DISPLAY_DAYS * DAY_MS), + to, + } + } + // A stripe period carries no rule for deriving its predecessor, so fall back to + // a range of the same length rather than inventing stamps that would match + // nothing. This is an approximation, which is why it is never used as the + // summary's comparison window — see `resolvePreviousPeriod` there. + return { + kind: 'range', + from: new Date(period.start.getTime() - (period.end.getTime() - period.start.getTime())), + to: period.start, + } + } + case '7d': + return { kind: 'range', from: new Date(now.getTime() - 7 * DAY_MS), to: now } + case '30d': + return { kind: 'range', from: new Date(now.getTime() - 30 * DAY_MS), to: now } + case 'custom': { + // A partial selection is not a range, so it falls back to the current period — + // through the same branch, which is what keeps an unbounded period from being + // scanned in full here as well. + if (!customStart || !customEnd) { + return resolveUsageAnalyticsWindow({ preset: 'current-period', period, now }) + } + /** + * The picker offers calendar days and sends `YYYY-MM-DD`, which arrives here + * parsed as UTC midnight. Anchoring the window on those instants shifted every + * non-UTC viewer's selection by their offset — a range labelled "Aug 1–31" + * covered half of Jul 31 and half of Aug 31 for a viewer twelve hours east, + * and disagreed with the chart, whose buckets are already the viewer's + * calendar days. Reinterpret the same civil dates as midnight *there*. + */ + const startKey = civilBoundKey(customStart) + const endKey = civilBoundKey(customEnd) + // Guarded before the span check, which would otherwise measure a negative + // number of days, pass the cap, and return an inverted range that matches no + // rows — reading as "no usage" rather than as a bad request. + if (endKey < startKey) throw new UsageWindowRangeInvertedError() + + const days = civilDaysBetween(startKey, endKey) + 1 + if (days > MAX_CUSTOM_RANGE_DAYS) throw new UsageWindowRangeTooLargeError(days) + + const exclusiveEndKey = civilKey( + (() => { + const cursor = civilDate(endKey) + cursor.setUTCDate(cursor.getUTCDate() + 1) + return cursor + })() + ) + return { + kind: 'range', + from: zonedWallClockToUtc(`${startKey}T00:00`, timezone), + to: zonedWallClockToUtc(`${exclusiveEndKey}T00:00`, timezone), + } + } + } +} + +/** + * The period immediately before this one, or `null` when it is not exactly + * derivable. Only a reporting period has a rule (its anchor); a stripe period's + * predecessor lives in Stripe, and guessing it would silently compare against the + * wrong window. + */ +export function resolvePreviousPeriod(period: ResolvedUsagePeriod): ResolvedUsagePeriod | null { + if (period.source !== 'reporting' || !period.anchorDate || !period.interval) return null + return resolveEnterpriseReportingPeriod( + period.anchorDate, + period.interval, + new Date(period.start.getTime() - 1) + ) +} + +/** + * Bucket width derived from the window rather than requested. + * + * Calendar-aligned on purpose: a billing period starts at an arbitrary instant, so + * epoch-modulo buckets would cut every day mid-afternoon and each bar would straddle + * two calendar days. Spend is read against the calendar. + */ +export function resolveUsageBucket(window: UsageAnalyticsWindow): UsageBucket { + const { start, end } = usageWindowBounds(window) + // Rounded, not ceiled: a 92-day range spanning the autumn transition is 92 days and + // one hour, which `ceil` called 93 — quietly demoting the longest legal custom range + // from daily bars to weekly ones. + const days = Math.max(1, Math.round((end.getTime() - start.getTime()) / DAY_MS)) + if (days <= 92) return 'day' + if (days <= 400) return 'week' + return 'month' +} + +export interface UsageSeriesPoint { + timestamp: string + cost: number + events: number +} + +interface SparseBucketRow { + bucketStart: string | null + cost: string | number | null + events: number | string | null +} + +function toNumber(value: string | number | null | undefined): number { + if (typeof value === 'number') return Number.isFinite(value) ? value : 0 + const parsed = Number.parseFloat(value ?? '0') + return Number.isFinite(parsed) ? parsed : 0 +} + +/** The `YYYY-MM-DD` an instant falls on in the viewer's calendar — what `AT TIME ZONE` produced. */ +function localCalendarDate(instant: Date, timezone: string): string { + return new Intl.DateTimeFormat('en-CA', { + timeZone: timezone, + year: 'numeric', + month: '2-digit', + day: '2-digit', + }).format(instant) +} + +/** + * Civil-date arithmetic on a `YYYY-MM-DD` key. + * + * UTC is used purely as a proleptic calendar here — these values are never converted + * back to an instant, so no offset or DST transition can shift them. Doing the same + * arithmetic on a real local instant is what would break across a DST boundary. + */ +function civilDate(key: string): Date { + return new Date(`${key}T00:00:00.000Z`) +} + +function civilKey(date: Date): string { + return date.toISOString().slice(0, 10) +} + +/** + * Mirrors Postgres `date_trunc(bucket, …)`: an ISO week starts Monday, a month on + * the 1st. The series keys have to land on the same boundaries the SQL emitted or + * no lookup below will ever hit. + */ +function truncateToBucket(key: string, bucket: UsageBucket): string { + const date = civilDate(key) + if (bucket === 'week') date.setUTCDate(date.getUTCDate() - ((date.getUTCDay() + 6) % 7)) + else if (bucket === 'month') date.setUTCDate(1) + return civilKey(date) +} + +/** + * Fills every bucket in the window, because SQL only returns buckets that have rows. + * + * A period with no usage must render a flat zero line, not the chart's "No data" + * branch — zero is information, "No data" reads as a failure. + * + * The keys are generated in the *same calendar the query grouped by*: + * `readUsageTimeSeries` truncates `created_at AT TIME ZONE `, so a viewer + * east or west of UTC buckets rows by their own calendar date. Walking UTC dates + * here instead dropped the edge buckets of every non-UTC window — their cost stayed + * in the headline while their bar read zero. Week and month were worse than an edge + * case: Postgres aligns those to Monday and the 1st, so a cursor stepping from an + * arbitrary period start shared no key with the query at all and the whole chart + * came back zeroed. That is reachable today through an annual enterprise period, + * which resolves to `week`. + */ +export function densifyUsageSeries( + rows: SparseBucketRow[], + window: UsageAnalyticsWindow, + bucket: UsageBucket, + timezone: string +): UsageSeriesPoint[] { + const byBucket = new Map() + for (const row of rows) { + if (row.bucketStart) byBucket.set(row.bucketStart.slice(0, 10), row) + } + + const { start, end } = usageWindowBounds(window) + const first = truncateToBucket(localCalendarDate(start, timezone), bucket) + // The window is half-open, so the last bucket is the one holding its final instant. + const last = truncateToBucket(localCalendarDate(new Date(end.getTime() - 1), timezone), bucket) + + const points: UsageSeriesPoint[] = [] + const cursor = civilDate(first) + let guard = 0 + + // `YYYY-MM-DD` sorts lexicographically in calendar order, so this compares dates. + while (civilKey(cursor) <= last && guard < 1000) { + guard += 1 + const key = civilKey(cursor) + const row = byBucket.get(key) + points.push({ + timestamp: `${key}T00:00:00`, + cost: toNumber(row?.cost), + events: Math.round(toNumber(row?.events)), + }) + if (bucket === 'day') cursor.setUTCDate(cursor.getUTCDate() + 1) + else if (bucket === 'week') cursor.setUTCDate(cursor.getUTCDate() + 7) + else cursor.setUTCMonth(cursor.getUTCMonth() + 1) + } + + return points +} + +export interface UsageBreakdownEntry { + id: string + label: string + cost: number + events: number + /** Share of the window total, 0..1 — of everything, not just the visible rows. */ + share: number +} + +export interface UsageBreakdownFold { + rows: UsageBreakdownEntry[] + /** `tokens` is the omitted rows' total, so a token-denominated tab still adds up. */ + other: { cost: number; events: number; rowCount: number; tokens: number } + totalCost: number +} + +interface RankedRow { + key: string | null + cost: string | number | null + events: number | string | null + inputTokens?: number + outputTokens?: number +} + +/** + * Ranks a dimension and closes it with an explicit remainder. + * + * The remainder is not cosmetic: five lists that do not add up to the headline + * number is the classic "the numbers are wrong" bug, and the only way a truncated + * ranking can reconcile is by naming what it left out. + * + * `rankBy` exists because BYOK has no cost to rank by — every row is zero by + * definition, so a cost sort fell through to its alphabetical tiebreak and the + * "top providers" were whichever ones sorted first. A tab denominated in tokens + * has to rank by tokens. + */ +export function foldUsageBreakdown( + rows: RankedRow[], + totalCost: number, + labelFor: (key: string | null) => string, + limit: number, + rankBy: 'cost' | 'tokens' = 'cost' +): UsageBreakdownFold { + const ranked = rows + .map((row) => ({ + id: row.key ?? '', + label: labelFor(row.key), + cost: toNumber(row.cost), + events: Math.round(toNumber(row.events)), + tokens: (row.inputTokens ?? 0) + (row.outputTokens ?? 0), + })) + .sort( + (left, right) => + (rankBy === 'tokens' ? right.tokens - left.tokens : right.cost - left.cost) || + left.label.localeCompare(right.label) + ) + + const visible = ranked.slice(0, limit) + const hidden = ranked.slice(limit) + /** + * Share is measured in whatever the list is ranked by, because it is what draws the + * bar. On BYOK every row costs zero, so a cost-based share made every provider's bar + * identical — the minimum width — and the ranking above became invisible. + */ + const shareTotal = + rankBy === 'tokens' ? ranked.reduce((sum, row) => sum + row.tokens, 0) : totalCost + const share = (row: { cost: number; tokens: number }) => + shareTotal > 0 ? (rankBy === 'tokens' ? row.tokens : row.cost) / shareTotal : 0 + + return { + rows: visible.map(({ tokens: _tokens, ...row }) => ({ + ...row, + share: share({ cost: row.cost, tokens: _tokens }), + })), + other: { + cost: hidden.reduce((sum, row) => sum + row.cost, 0), + events: hidden.reduce((sum, row) => sum + row.events, 0), + rowCount: hidden.length, + // BYOK ranks by cost, which is zero for every row, so the visible slice is + // effectively arbitrary — omitting the tail's tokens would hide real volume. + tokens: hidden.reduce((sum, row) => sum + row.tokens, 0), + }, + totalCost, + } +} + +/** + * What a null grouping key means, per dimension. + * + * A single "Unattributed" label was wrong in both directions: on Workspaces it means + * usage that belongs to no workspace, and on Workflows it means usage that never came + * from a workflow — which is most of an organization's spend, and reading that as an + * attribution failure is what made the workflow list useless. + */ +export const USAGE_NULL_KEY_LABELS: Record = { + member: 'Unknown member', + workspace: 'No workspace', + // Unreachable: the workflow dimension filters null ids out entirely. + workflow: 'Unknown workflow', + model: 'Unknown model', + byok: 'Unknown provider', + source: 'Other', +} + +export interface MergeableRow { + key: string | null + cost: string | number | null + events: number | string | null + inputTokens?: number + outputTokens?: number +} + +/** + * Re-keys rows onto a coarser identity and sums the collisions. + * + * Needed wherever the SQL grouping column is finer than what the panel shows. The + * ledger stores `copilot` and `workspace-chat` as distinct sources but both display + * as "Sim Chat", so grouping by the raw column alone renders the same label twice + * with the usage split across the two rows — which reads as a bug and makes the + * ranking wrong. Models collapse to a provider the same way. + */ +export function mergeRowsByKey( + rows: T[], + resolveKey: (key: string | null) => string | null +): MergeableRow[] { + const merged = new Map() + for (const row of rows) { + const key = resolveKey(row.key) + const mapKey = key ?? '' + const existing = merged.get(mapKey) + if (!existing) { + merged.set(mapKey, { + key, + cost: toNumber(row.cost), + events: Math.round(toNumber(row.events)), + ...(row.inputTokens !== undefined ? { inputTokens: row.inputTokens } : {}), + ...(row.outputTokens !== undefined ? { outputTokens: row.outputTokens } : {}), + }) + continue + } + existing.cost = toNumber(existing.cost) + toNumber(row.cost) + existing.events = Math.round(toNumber(existing.events) + toNumber(row.events)) + if (row.inputTokens !== undefined) { + existing.inputTokens = (existing.inputTokens ?? 0) + row.inputTokens + } + if (row.outputTokens !== undefined) { + existing.outputTokens = (existing.outputTokens ?? 0) + row.outputTokens + } + } + return [...merged.values()] +} diff --git a/apps/sim/lib/billing/core/usage-log.test.ts b/apps/sim/lib/billing/core/usage-log.test.ts index 6568fd4dacc..1098fddc277 100644 --- a/apps/sim/lib/billing/core/usage-log.test.ts +++ b/apps/sim/lib/billing/core/usage-log.test.ts @@ -159,6 +159,53 @@ describe('recordUsage', () => { expect(mockGetHighestPrioritySubscription).not.toHaveBeenCalled() expect(mockInsert).not.toHaveBeenCalled() }) + + it('keeps zero-cost unbilled rows and still drops every other zero-cost entry', async () => { + await recordUsage({ + userId: 'user-1', + billingEntity: { type: 'organization', id: 'org-1' }, + billingPeriod: { + start: new Date('2026-05-01T00:00:00.000Z'), + end: new Date('2026-06-01T00:00:00.000Z'), + }, + executionId: 'execution-1', + entries: [ + { + category: 'model_unbilled', + source: 'workflow', + description: 'claude-sonnet-4', + cost: 0, + metadata: { inputTokens: 1200, outputTokens: 340 }, + }, + // A billed category at zero cost is still noise, and stays filtered. + { category: 'model', source: 'workflow', description: 'gpt-4', cost: 0 }, + { category: 'tool', source: 'workflow', description: 'exa_search', cost: 0 }, + ], + }) + + const values = mockValues.mock.calls[0][0] + expect(values).toHaveLength(1) + expect(values[0]).toMatchObject({ + category: 'model_unbilled', + cost: '0', + description: 'claude-sonnet-4', + metadata: { inputTokens: 1200, outputTokens: 340 }, + }) + }) + + it('writes nothing when every entry is zero-cost and billable', async () => { + await recordUsage({ + userId: 'user-1', + billingEntity: { type: 'user', id: 'user-1' }, + billingPeriod: { + start: new Date('2026-05-01T00:00:00.000Z'), + end: new Date('2026-06-01T00:00:00.000Z'), + }, + entries: [{ category: 'model', source: 'workflow', description: 'gpt-4', cost: 0 }], + }) + + expect(mockInsert).not.toHaveBeenCalled() + }) }) describe('resolveCumulativeTopUp', () => { diff --git a/apps/sim/lib/billing/core/usage-log.ts b/apps/sim/lib/billing/core/usage-log.ts index 552f15b4e2f..ed0609b2e01 100644 --- a/apps/sim/lib/billing/core/usage-log.ts +++ b/apps/sim/lib/billing/core/usage-log.ts @@ -4,7 +4,7 @@ import { usageLog, workflow } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' -import { and, desc, eq, gte, inArray, lt, lte, or, sql } from 'drizzle-orm' +import { and, desc, eq, gte, inArray, lt, lte, notInArray, or, sql } from 'drizzle-orm' import { defaultBillingPeriod } from '@/lib/billing/core/billing-period' import { getHighestPrioritySubscription } from '@/lib/billing/core/plan' import { @@ -21,9 +21,12 @@ import type { DbClient, DbOrTx } from '@/lib/db/types' const logger = createLogger('UsageLog') /** - * Usage log category types + * Usage log category types. + * + * `model_unbilled` is reporting-only — see {@link UNBILLED_USAGE_CATEGORIES}. Code + * that means "a charge" must not treat it as one. */ -export type UsageLogCategory = 'model' | 'fixed' | 'tool' +export type UsageLogCategory = 'model' | 'fixed' | 'tool' | 'model_unbilled' /** * Usage log source types @@ -42,6 +45,26 @@ export const COPILOT_USAGE_SOURCES: UsageLogSource[] = [ 'mothership_block', ] +/** + * Categories that record usage Sim does not charge for. Their `cost` is always `0` + * and their value is the token counts in `metadata`, so usage reporting can show + * volume that the billing ledger has no reason to know about. + * + * These are the only categories exempt from {@link recordUsage}'s `cost > 0` filter. + * Every billing aggregate over usage_log is `SUM(cost)`, so zero-cost rows leave + * every existing total unchanged. + */ +export const UNBILLED_USAGE_CATEGORIES = [ + 'model_unbilled', +] as const satisfies readonly UsageLogCategory[] + +const UNBILLED_USAGE_CATEGORY_SET: ReadonlySet = new Set(UNBILLED_USAGE_CATEGORIES) + +/** True for a category whose rows are recorded for reporting rather than billing. */ +export function isUnbilledUsageCategory(category: UsageLogCategory): boolean { + return UNBILLED_USAGE_CATEGORY_SET.has(category) +} + /** * Metadata for 'model' category charges */ @@ -219,6 +242,12 @@ export async function getBillingPeriodUsageCost( * Counts distinct workflow executions that produced billable ledger entries in * an attributed billing period. Multiple line items for one execution count as * one run; executions with no billable usage are intentionally excluded. + * + * The category filter is what keeps that last clause true now that unbilled rows + * exist. A BYOK-only run whose base charge is zero writes nothing but a + * `model_unbilled` row, and without this it would newly appear in a count that + * feeds the enterprise billing preview — a customer-facing number moving because + * of a reporting-only row. */ export async function getBillingPeriodWorkflowRunCount( billingEntity: BillingEntity, @@ -227,8 +256,15 @@ export async function getBillingPeriodWorkflowRunCount( ): Promise { const [row] = await executor .select({ + /** + * The exclusion goes through `notInArray`, not `<> ALL(${array})`. Interpolating + * a JavaScript array into a `sql` template emits parenthesized scalar binds — + * `ALL(($1))` — which Postgres rejects outright with "op ANY/ALL (array) + * requires array on right side". Unit tests cannot catch it, because `@sim/db` + * is mocked and no statement is ever rendered. + */ workflowRuns: - sql`COUNT(DISTINCT ${usageLog.executionId}) FILTER (WHERE ${usageLog.source} = 'workflow')`.mapWith( + sql`COUNT(DISTINCT ${usageLog.executionId}) FILTER (WHERE ${usageLog.source} = 'workflow' AND ${notInArray(usageLog.category, [...UNBILLED_USAGE_CATEGORIES])})`.mapWith( Number ), }) @@ -397,7 +433,12 @@ export async function recordUsage(params: RecordUsageParams): Promise { tx, } = params - const validEntries = entries.filter((e) => e.cost > 0) + // An unbilled row is admitted only at exactly zero cost. The category's whole + // safety argument is that every billing aggregate is `SUM(cost)` and these rows + // contribute nothing; a nonzero one would quietly move real money. + const validEntries = entries.filter((e) => + isUnbilledUsageCategory(e.category) ? e.cost === 0 : e.cost > 0 + ) if (validEntries.length === 0) { return @@ -721,17 +762,52 @@ interface UsageLogFilter { source?: UsageLogSource | UsageLogSource[] workspaceId?: string startDate?: Date + /** + * Inclusive by default, which is what the personal credit-usage surfaces have + * always meant by it. + */ endDate?: Date + /** + * Treat {@link endDate} as exclusive instead. + * + * Analytics windows are half-open `[start, end)` — a billing period's end + * instant is the next period's start — so a row stamped exactly on the + * boundary belongs to the next window. Without this the event list and the + * export counted it while the summary and breakdowns did not, and the two + * disagreed by one row at exactly the moment a period rolls over. + */ + endDateExclusive?: boolean + /** + * Match the stamped billing period instead of a `created_at` range. + * + * A stripe or default period is what rows are *stamped* with, and that is the + * predicate every analytics read uses for it. Filtering the same window on + * `created_at` selects a different set — a row created inside the period but + * stamped to another, or the reverse — so the event list and the CSV covered + * different rows than the totals above them. Mutually exclusive with + * {@link startDate}/{@link endDate}; a reporting period and a plain range still + * use those, exactly as `buildUsageAnalyticsScope` does. + */ + billingPeriod?: { start: Date; end: Date } } -type UsageLogScope = { kind: 'user'; userId: string } | { kind: 'workspace'; workspaceId: string } +type UsageLogScope = + | { kind: 'user'; userId: string } + | { kind: 'workspace'; workspaceId: string } + /** Every event billed to a payer, regardless of which actor or workspace produced it. */ + | { kind: 'billingEntity'; entity: BillingEntity } + +function scopeCondition(scope: UsageLogScope) { + if (scope.kind === 'user') return eq(usageLog.userId, scope.userId) + if (scope.kind === 'workspace') return eq(usageLog.workspaceId, scope.workspaceId) + return and( + eq(usageLog.billingEntityType, scope.entity.type), + eq(usageLog.billingEntityId, scope.entity.id) + ) +} function buildUsageLogConditions(scope: UsageLogScope, filter: UsageLogFilter) { - const conditions = [ - scope.kind === 'user' - ? eq(usageLog.userId, scope.userId) - : eq(usageLog.workspaceId, scope.workspaceId), - ] + const conditions = [scopeCondition(scope)] if (filter.source) { conditions.push( Array.isArray(filter.source) @@ -740,8 +816,21 @@ function buildUsageLogConditions(scope: UsageLogScope, filter: UsageLogFilter) { ) } if (filter.workspaceId) conditions.push(eq(usageLog.workspaceId, filter.workspaceId)) + if (filter.billingPeriod) { + conditions.push( + eq(usageLog.billingPeriodStart, filter.billingPeriod.start), + eq(usageLog.billingPeriodEnd, filter.billingPeriod.end) + ) + return conditions + } if (filter.startDate) conditions.push(gte(usageLog.createdAt, filter.startDate)) - if (filter.endDate) conditions.push(lte(usageLog.createdAt, filter.endDate)) + if (filter.endDate) { + conditions.push( + filter.endDateExclusive + ? lt(usageLog.createdAt, filter.endDate) + : lte(usageLog.createdAt, filter.endDate) + ) + } return conditions } @@ -823,8 +912,19 @@ export interface GetUsageLogsOptions { workspaceId?: string /** Start date (inclusive) */ startDate?: Date - /** End date (inclusive) */ + /** End date (inclusive, unless {@link endDateExclusive}) */ endDate?: Date + /** + * Treat {@link endDate} as exclusive, matching a half-open analytics window. + * See {@link UsageLogFilter.endDateExclusive}. + */ + endDateExclusive?: boolean + /** + * Match the stamped billing period instead of a `created_at` range, so a + * ledger listing covers the same rows an analytics read of the same window + * does. See {@link UsageLogFilter.billingPeriod}. + */ + billingPeriod?: { start: Date; end: Date } /** Maximum number of results */ limit?: number /** Cursor for pagination (log ID) */ @@ -891,6 +991,8 @@ async function getUsageLogs( workspaceId, startDate, endDate, + endDateExclusive, + billingPeriod, limit = 50, cursor, cursorCreatedAt, @@ -898,7 +1000,14 @@ async function getUsageLogs( } = options try { - const conditions = buildUsageLogConditions(scope, { source, workspaceId, startDate, endDate }) + const conditions = buildUsageLogConditions(scope, { + source, + workspaceId, + startDate, + endDate, + endDateExclusive, + billingPeriod, + }) if (cursor) { let resolvedCursorCreatedAt = cursorCreatedAt @@ -972,6 +1081,8 @@ async function getUsageLogs( workspaceId, startDate, endDate, + endDateExclusive, + billingPeriod, }) const summaryResult = await dbReplica @@ -1029,6 +1140,20 @@ export function getUserUsageLogs( return getUsageLogs({ kind: 'user', userId }, options) } +/** + * Gets every usage event billed to a payer, regardless of actor or workspace. + * + * This is the organization-wide ledger the usage panel pages through. It reuses this + * module's keyset pagination, cursor handling, and workflow-name join rather than + * reimplementing them — the only thing it adds is the scope predicate. + */ +export function getBillingEntityUsageLogs( + entity: BillingEntity, + options: GetUsageLogsOptions = {} +): Promise { + return getUsageLogs({ kind: 'billingEntity', entity }, options) +} + /** Gets usage logs attributed to the selected workspace, regardless of actor. */ export function getWorkspaceUsageLogs( workspaceId: string, diff --git a/apps/sim/lib/billing/core/usage.ts b/apps/sim/lib/billing/core/usage.ts index b9ec280ec81..b0c5045af01 100644 --- a/apps/sim/lib/billing/core/usage.ts +++ b/apps/sim/lib/billing/core/usage.ts @@ -708,9 +708,15 @@ export async function maybeSendUsageThresholdEmail(params: { const upgradeCreditsLink = params.workspaceId ? `${baseUrl}${buildUpgradeHref(params.workspaceId, 'credits')}` : `${baseUrl}/workspace` + /** + * Organization billing is reached through the workspace the usage occurred in + * — that is the only plane that serves it. Without a workspace there is no such + * link to build, so the account page is the honest fallback rather than a guess + * at which workspace the recipient would want. + */ const billingSettingsLink = - params.scope === 'organization' && params.organizationId - ? `${baseUrl}/organization/${params.organizationId}/settings/billing` + params.scope === 'organization' && params.workspaceId + ? `${baseUrl}/workspace/${params.workspaceId}/settings/billing` : `${baseUrl}/account/settings/billing` // Check for 80% threshold crossing — used for paid users (budget warning) and free users (upgrade nudge) diff --git a/apps/sim/lib/billing/storage/payer-transfer.test.ts b/apps/sim/lib/billing/storage/payer-transfer.test.ts index 8113e236e5f..a9518c25add 100644 --- a/apps/sim/lib/billing/storage/payer-transfer.test.ts +++ b/apps/sim/lib/billing/storage/payer-transfer.test.ts @@ -769,7 +769,7 @@ describe('changeOrganizationWorkspaceBilledAccountsInTx', () => { expect(returning).toHaveBeenCalledWith({ id: 'workspace.id' }) expect(select).toHaveBeenCalledWith({ id: 'workspace.id' }) expect(orderBy).toHaveBeenCalledTimes(1) - expect(lock).toHaveBeenCalledWith('update') + expect(lock).toHaveBeenCalledWith('no key update') expect(lock.mock.invocationCallOrder[0]).toBeLessThan(update.mock.invocationCallOrder[0]) expect(execute).not.toHaveBeenCalled() }) diff --git a/apps/sim/lib/billing/storage/payer-transfer.ts b/apps/sim/lib/billing/storage/payer-transfer.ts index 14687a4df56..049070dfd8b 100644 --- a/apps/sim/lib/billing/storage/payer-transfer.ts +++ b/apps/sim/lib/billing/storage/payer-transfer.ts @@ -125,7 +125,9 @@ async function getExactWorkspaceStorageBytes(tx: DbOrTx, workspaceId: string): P /** * Locks a payer row and returns its current aggregate. A missing source can be * historical drift and is represented as `null`; callers must reject a - * missing destination. + * missing destination. `FOR NO KEY UPDATE` avoids upgrading the implicit + * foreign-key `FOR KEY SHARE` this transaction may already hold; see the + * module header of `lib/billing/storage/tracking.ts`. */ async function lockStoragePayer(tx: DbOrTx, payer: BillingEntity): Promise { if (payer.type === 'organization') { @@ -133,7 +135,7 @@ async function lockStoragePayer(tx: DbOrTx, payer: BillingEntity): Promise [row.id, row])) for (const workspaceId of workspaceIds) { @@ -562,7 +564,7 @@ export async function changeOrganizationWorkspaceBilledAccountsInTx( ) ) .orderBy(asc(workspace.id)) - .for('update') + .for('no key update') const rows = await tx .update(workspace) @@ -604,7 +606,7 @@ export async function changeWorkspaceStoragePayerInTx( }) .from(workspace) .where(eq(workspace.id, params.workspaceId)) - .for('update') + .for('no key update') .limit(1) if (!lockedWorkspace) { diff --git a/apps/sim/lib/billing/storage/tracking.test.ts b/apps/sim/lib/billing/storage/tracking.test.ts index f8d36e8202d..a7b6f481ab9 100644 --- a/apps/sim/lib/billing/storage/tracking.test.ts +++ b/apps/sim/lib/billing/storage/tracking.test.ts @@ -13,6 +13,7 @@ const { mockMaybeNotifyLimit, mockOrderedLockRows, mockSql, + mockTxFor, mockTxFrom, mockTxLimit, mockTxOrderBy, @@ -32,6 +33,7 @@ const { mockMaybeNotifyLimit: vi.fn(), mockOrderedLockRows: { queue: [] as unknown[][] }, mockSql: vi.fn((strings: TemplateStringsArray, ...values: unknown[]) => ({ strings, values })), + mockTxFor: vi.fn(), mockTxFrom: vi.fn(), mockTxLimit: vi.fn(), mockTxOrderBy: vi.fn(), @@ -120,6 +122,42 @@ const ORG_CONTEXT: StorageBillingContext = { customStorageLimitGB: null, } +const USER_CONTEXT: StorageBillingContext = { + workspaceId: 'workspace-1', + billedAccountUserId: 'workspace-owner', + billingEntity: { type: 'user', id: 'workspace-owner' }, + plan: 'pro', + customStorageLimitGB: null, +} + +/** + * Both payer kinds. The workspace lock is shared, but the payer lock branches + * to a different table per kind, so a lock-mode regression on only one of them + * has to fail a test. + */ +const PAYER_CASES = [ + { + label: 'organization', + context: ORG_CONTEXT, + workspaceRow: { + billedAccountUserId: 'workspace-owner', + organizationId: 'workspace-org' as string | null, + storageUsedBytes: 1_000, + }, + payerLockRows: [{ id: 'workspace-org', storageUsedBytes: 1_000 }], + }, + { + label: 'user', + context: USER_CONTEXT, + workspaceRow: { + billedAccountUserId: 'workspace-owner', + organizationId: null as string | null, + storageUsedBytes: 1_000, + }, + payerLockRows: [{ id: 'workspace-owner', storageUsedBytes: 1_000 }], + }, +] as const + beforeAll(() => { setEnvFlags({ isBillingEnabled: true }) }) @@ -138,9 +176,10 @@ describe('workspace storage counter mutations', () => { mockOrderedLockRows.queue = [] mockTxSelect.mockReturnValue({ from: mockTxFrom }) + mockTxFor.mockReturnValue({ limit: mockTxLimit }) mockTxFrom.mockReturnValue({ where: vi.fn(() => ({ - for: vi.fn(() => ({ limit: mockTxLimit })), + for: mockTxFor, limit: mockTxLimit, orderBy: mockTxOrderBy, })), @@ -183,6 +222,39 @@ describe('workspace storage counter mutations', () => { expect(mockMaybeNotifyLimit).not.toHaveBeenCalled() }) + /** + * `FOR UPDATE` on these rows deadlocked in production: `workspace`, + * `organization`, and `user_stats` are foreign-key parents, so the calling + * transaction already holds an implicit `FOR KEY SHARE` on them from the + * billable child row it just wrote, and the stronger lock is an upgrade that + * two concurrent uploads take on each other. `FOR NO KEY UPDATE` still + * conflicts with itself, so the ledgers stay serialized. + */ + it.each(PAYER_CASES)( + 'locks the workspace and its $label payer as FOR NO KEY UPDATE', + async ({ context, workspaceRow }) => { + mockWorkspaceRow.current = { ...workspaceRow } + + await incrementStorageUsageForBillingContextInTx(mockTx as unknown as DbOrTx, context, 100) + + expect(mockTxFor.mock.calls).toEqual([['no key update'], ['no key update']]) + } + ) + + it.each(PAYER_CASES)( + 'locks batched workspace and $label payer ledgers as FOR NO KEY UPDATE', + async ({ context, workspaceRow, payerLockRows }) => { + mockOrderedLockRows.queue = [[{ id: 'workspace-1', ...workspaceRow }], [...payerLockRows]] + + await applyStorageUsageDeltasInTx(mockTx as unknown as DbOrTx, { + workspaceDeltas: [{ context, deltaBytes: 100 }], + legacyDeltas: [], + }) + + expect(mockTxOrderedFor.mock.calls).toEqual([['no key update'], ['no key update']]) + } + ) + it('serializes quota admission on the locked payer ledger', async () => { mockGetStorageLimitForBillingContext.mockReturnValue(1_050) mockTxLimit diff --git a/apps/sim/lib/billing/storage/tracking.ts b/apps/sim/lib/billing/storage/tracking.ts index 0de9b03507b..7ae702a94c3 100644 --- a/apps/sim/lib/billing/storage/tracking.ts +++ b/apps/sim/lib/billing/storage/tracking.ts @@ -1,5 +1,20 @@ /** * Storage usage tracking for durable workspace and payer ledgers. + * + * Every row lock here is `FOR NO KEY UPDATE`, never `FOR UPDATE`. The + * `workspace`, `organization`, and `user_stats` rows these transactions lock + * are foreign-key parents (49 tables reference `workspace` alone), so any + * insert or update of a child row — a `workspace_files` row in this very + * transaction — implicitly takes `FOR KEY SHARE` on the parent first. A later + * `FOR UPDATE` on the same row is then a lock upgrade, and two concurrent + * uploads or deletes in one workspace deadlock on it. `FOR NO KEY UPDATE` + * does not conflict with `FOR KEY SHARE`, yet still conflicts with itself and + * with `FOR UPDATE`, so writers remain serialized against each other and + * against payer transfers. It is exactly the lock a plain `UPDATE` of these + * non-key counters takes anyway. The only key columns on these tables are + * `workspace.id`, `workspace.inbox_provider_id`, `organization.id`, + * `user_stats.id`, and `user_stats.user_id`, and no path under these locks + * writes any of them or deletes a locked row. */ import { organization, userStats, workspace } from '@sim/db/schema' @@ -124,6 +139,7 @@ async function mutateStorageUsage( /** * Locks and reads the payer ledger after the workspace row has been locked. + * `FOR NO KEY UPDATE` for the reason documented at the top of this module. */ async function lockStorageUsageForMutation( tx: DbOrTx, @@ -134,7 +150,7 @@ async function lockStorageUsageForMutation( .select({ storageUsedBytes: organization.storageUsedBytes }) .from(organization) .where(eq(organization.id, billingEntity.id)) - .for('update') + .for('no key update') .limit(1) if (!row) throw new Error(`Storage payer organization:${billingEntity.id} not found`) return row.storageUsedBytes @@ -144,7 +160,7 @@ async function lockStorageUsageForMutation( .select({ storageUsedBytes: userStats.storageUsedBytes }) .from(userStats) .where(eq(userStats.userId, billingEntity.id)) - .for('update') + .for('no key update') .limit(1) if (!row) throw new Error(`Storage payer user:${billingEntity.id} not found`) return row.storageUsedBytes @@ -242,7 +258,7 @@ export async function applyStorageUsageDeltasInTx( .from(workspace) .where(inArray(workspace.id, workspaceIds)) .orderBy(asc(workspace.id)) - .for('update') + .for('no key update') : [] const workspaceById = new Map(lockedWorkspaces.map((row) => [row.id, row])) @@ -318,7 +334,7 @@ export async function applyStorageUsageDeltasInTx( .from(userStats) .where(inArray(userStats.userId, userIds)) .orderBy(asc(userStats.userId)) - .for('update') + .for('no key update') for (const row of rows) { payerUsageByKey.set(getPayerKey({ type: 'user', id: row.id }), row.storageUsedBytes) } @@ -329,7 +345,7 @@ export async function applyStorageUsageDeltasInTx( .from(organization) .where(inArray(organization.id, organizationIds)) .orderBy(asc(organization.id)) - .for('update') + .for('no key update') for (const row of rows) { payerUsageByKey.set(getPayerKey({ type: 'organization', id: row.id }), row.storageUsedBytes) } @@ -439,7 +455,7 @@ async function mutateWorkspaceStorageUsage( }) .from(workspace) .where(eq(workspace.id, workspaceId)) - .for('update') + .for('no key update') .limit(1) if (!workspacePayer) { diff --git a/apps/sim/lib/billing/usage-sources.ts b/apps/sim/lib/billing/usage-sources.ts index f74bf99730a..056285e0a30 100644 --- a/apps/sim/lib/billing/usage-sources.ts +++ b/apps/sim/lib/billing/usage-sources.ts @@ -61,6 +61,15 @@ const BILLING_TO_INTERNAL_SOURCES = { 'voice-output': ['voice-output'], } as const satisfies Record +/** + * What each source is called wherever usage is shown. + * + * `workflow` covers everything one run consumed — the models it called on Sim's + * hosted keys, any hosted-key tool calls, and the per-run execution fee — so the + * label stays the broad "Workflow". Naming it after the execution fee would + * understate it by orders of magnitude; the fee is a rounding error beside the + * model cost. + */ export const BILLING_USAGE_LOG_SOURCE_LABELS = { workflow: 'Workflow', wand: 'Wand', diff --git a/apps/sim/lib/compare/data/sim.ts b/apps/sim/lib/compare/data/sim.ts index 4f680b962de..0f2951b5ae0 100644 --- a/apps/sim/lib/compare/data/sim.ts +++ b/apps/sim/lib/compare/data/sim.ts @@ -752,7 +752,7 @@ export const simProfile: CompetitorProfile = { }, mcpPublishing: { value: - 'Yes: any deployed workflow can be published as a tool on an MCP server (private, API-key protected, or public/no-auth), with ready-to-paste client config generated for Cursor, Claude Code, Claude Desktop, and VS Code', + 'Yes: any deployed workflow can be published as a tool on an MCP server (private, API-key protected, or public/no-auth), with ready-to-paste client config generated for Codex, Cursor, Claude Code, Claude Desktop, and VS Code', shortValue: 'Deployed workflows publish as MCP server tools', confidence: 'verified', sources: [ diff --git a/apps/sim/lib/copilot/application/execute-custom-tool-use-case.test.ts b/apps/sim/lib/copilot/application/execute-custom-tool-use-case.test.ts new file mode 100644 index 00000000000..b5089227bf4 --- /dev/null +++ b/apps/sim/lib/copilot/application/execute-custom-tool-use-case.test.ts @@ -0,0 +1,66 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it, vi } from 'vitest' +import { executeCopilotCustomToolUseCase } from '@/lib/copilot/application/execute-custom-tool-use-case' +import { customToolOperations } from '@/lib/custom-tools/application/operations' + +const trustedContext = { + userId: 'user-1', + workspaceId: 'workspace-1', + chatId: 'chat-1', + executionId: 'execution-1', + toolCallId: 'tool-call-1', + copilotToolExecution: true, +} as const + +describe('executeCopilotCustomToolUseCase', () => { + it('normalizes trusted Copilot authority for available custom-tool lookup', async () => { + const execute = vi.fn().mockResolvedValue({ tool: { id: 'tool-1' } }) + const useCase = { + operation: customToolOperations.readAvailableByIdOrTitle, + execute, + } + const input = { + workspaceId: trustedContext.workspaceId, + identifier: 'tool-1', + lookup: 'id' as const, + } + + await expect(executeCopilotCustomToolUseCase(trustedContext, useCase, input)).resolves.toEqual({ + tool: { id: 'tool-1' }, + }) + expect(execute).toHaveBeenCalledWith({ + principal: expect.objectContaining({ + kind: 'delegated', + serviceId: 'copilot', + subjectUserId: trustedContext.userId, + workspaceId: trustedContext.workspaceId, + delegationId: `copilot-tool:${trustedContext.toolCallId}`, + audience: 'sim:custom-tools', + resourceScope: expect.objectContaining({ + chatId: trustedContext.chatId, + executionId: trustedContext.executionId, + }), + }), + input, + }) + }) + + it('rejects an untrusted Copilot marker before application execution', () => { + const execute = vi.fn() + const useCase = { + operation: customToolOperations.readAvailableByIdOrTitle, + execute, + } + + expect(() => + executeCopilotCustomToolUseCase({ ...trustedContext, copilotToolExecution: false }, useCase, { + workspaceId: trustedContext.workspaceId, + identifier: 'tool-1', + lookup: 'id', + }) + ).toThrow('trusted Copilot execution context') + expect(execute).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/copilot/application/execute-log-use-case.ts b/apps/sim/lib/copilot/application/execute-log-use-case.ts new file mode 100644 index 00000000000..216b00eb3c7 --- /dev/null +++ b/apps/sim/lib/copilot/application/execute-log-use-case.ts @@ -0,0 +1,34 @@ +import { createCopilotApplicationAdapter } from '@/lib/copilot/application/application-adapter' +import { + COPILOT_APPLICATION_DELEGATION_TTL_MS, + type CopilotExecutionContext, +} from '@/lib/copilot/auth/application-delegation' +import type { OperationUseCase } from '@/lib/core/application' +import { logDelegationPolicy } from '@/lib/logs/application/authorization' +import { logOperations } from '@/lib/logs/application/operations' + +const copilotLogOperations = { + list: logOperations.list, + readDetail: logOperations.readDetail, +} as const + +type CopilotLogOperation = (typeof copilotLogOperations)[keyof typeof copilotLogOperations] + +const executeLogUseCase = createCopilotApplicationAdapter({ + domain: 'logs', + delegation: { + audience: logDelegationPolicy.audience, + ttlMs: COPILOT_APPLICATION_DELEGATION_TTL_MS, + createDelegationId: (context) => `copilot-tool:${context.toolCallId}`, + }, + operations: copilotLogOperations, +}) + +/** Enters a registered Logs use case with trusted Copilot identity. */ +export function executeCopilotLogUseCase( + context: CopilotExecutionContext | undefined, + useCase: OperationUseCase, + input: I +): Promise { + return executeLogUseCase(context, useCase, input) +} diff --git a/apps/sim/lib/copilot/generated/docs-manifest.ts b/apps/sim/lib/copilot/generated/docs-manifest.ts index dfad9b4f730..39e1656d64d 100644 --- a/apps/sim/lib/copilot/generated/docs-manifest.ts +++ b/apps/sim/lib/copilot/generated/docs-manifest.ts @@ -369,6 +369,7 @@ export const DOCS_MANIFEST: readonly string[] = [ 'platform/enterprise/self-hosted.mdx', 'platform/enterprise/session-policies.mdx', 'platform/enterprise/sso.mdx', + 'platform/enterprise/usage-tracking.mdx', 'platform/enterprise/verified-domains.mdx', 'platform/enterprise/whitelabeling.mdx', 'platform/organization.mdx', diff --git a/apps/sim/lib/copilot/integration-tools.ts b/apps/sim/lib/copilot/integration-tools.ts index cae7704fb7f..c8190886bc2 100644 --- a/apps/sim/lib/copilot/integration-tools.ts +++ b/apps/sim/lib/copilot/integration-tools.ts @@ -3,7 +3,7 @@ import type { IsToolAllowed } from '@/lib/permission-groups/operation-access' import { BLOCK_REGISTRY } from '@/blocks/registry-maps' import { isHiddenUnder } from '@/blocks/visibility/context' import { tools as toolRegistry } from '@/tools/registry' -import type { ToolConfig } from '@/tools/types' +import type { ExecutableToolConfig } from '@/tools/types' import { getLatestVersionTools, stripVersionSuffix } from '@/tools/utils' export interface ExposedIntegrationToolOwner { @@ -19,7 +19,7 @@ export interface ExposedIntegrationTool extends ExposedIntegrationToolOwner { * callable id are all this exact value, matching the block's tools.access. */ toolId: string - config: ToolConfig + config: ExecutableToolConfig /** Service directory name, e.g. "gmail". */ service: string /** Operation stem within the service (used for the VFS path filename), e.g. "read". */ diff --git a/apps/sim/lib/copilot/request/tools/executor.ts b/apps/sim/lib/copilot/request/tools/executor.ts index 127267555dc..f96192e579d 100644 --- a/apps/sim/lib/copilot/request/tools/executor.ts +++ b/apps/sim/lib/copilot/request/tools/executor.ts @@ -61,7 +61,10 @@ import { setTerminalToolCallState, } from '@/lib/copilot/request/tool-call-state' import { maybeWriteOutputToFile } from '@/lib/copilot/request/tools/files' -import { inspectToolResultForCopilot } from '@/lib/copilot/request/tools/resolved-secret-result' +import { + describeWithholdingCause, + inspectToolResultForCopilot, +} from '@/lib/copilot/request/tools/resolved-secret-result' import { handleResourceSideEffects } from '@/lib/copilot/request/tools/resources' import { maybeWriteOutputToTable, @@ -737,15 +740,20 @@ async function executeToolAndReportInner( toolSpan.attributes = { ...toolSpan.attributes, ...summarizeToolResultForSpan(copilotResult), - ...(projection.safe ? {} : { resultWithheld: true }), + ...(projection.safe + ? {} + : { resultWithheld: true, ...describeWithholdingCause(projection.cause) }), } if (!projection.safe) { // A withheld SUCCESS otherwise leaves no trace anywhere: the span reads // ok and the model just sees a bare `{success: true}` with no output. + // The cause is what says whether a guard latched, no catalog was built, + // or the payload itself was unprojectable — three different fixes. logger.warn('Tool result withheld by egress projection', { toolCallId: toolCall.id, toolName: toolCall.name, runtimeSucceeded: result.success, + ...describeWithholdingCause(projection.cause), }) } diff --git a/apps/sim/lib/copilot/request/tools/resolved-secret-result.test.ts b/apps/sim/lib/copilot/request/tools/resolved-secret-result.test.ts index 26285b0d7d5..7bd4eaa5109 100644 --- a/apps/sim/lib/copilot/request/tools/resolved-secret-result.test.ts +++ b/apps/sim/lib/copilot/request/tools/resolved-secret-result.test.ts @@ -4,6 +4,8 @@ import { describe, expect, it } from 'vitest' import { RunCode, RunFunction } from '@/lib/copilot/generated/tool-catalog-v1' import { + describeWithholdingCause, + inspectToolResultForCopilot, projectToolResultForCopilot, READ_TOOL_RESULT_UNAVAILABLE_ERROR, TOOL_RESULT_UNAVAILABLE_ERROR, @@ -457,3 +459,105 @@ describe('projectToolResultForCopilot', () => { expect(toolResultUnavailableError(undefined)).toBe(TOOL_RESULT_UNAVAILABLE_ERROR) }) }) + +describe('effect disclosure on a withheld result', () => { + const EXECUTION_ID = '0f4d5a4c-6a1e-4c2f-9b7d-2c8f1a3e5d90' + + it('carries nothing extra for a tool that declared no effect', () => { + expect(projectToolResultForCopilot({ success: true, output: { a: 1 } }, undefined)).toEqual({ + success: true, + }) + expect(projectToolResultForCopilot({ success: false, error: 'why' }, undefined)).toEqual({ + success: false, + error: TOOL_RESULT_UNAVAILABLE_ERROR, + }) + }) + + /** + * The exemption is what makes the disclosure trustworthy, so it has to be all or + * nothing: a disclosure that silently dropped the id it could not vouch for would + * read exactly like one that never had a run to name. + */ + it('voids the whole disclosure when an id is not a shape this system mints', () => { + expect( + projectToolResultForCopilot( + { + success: false, + error: 'why', + effect: { phase: 'performed', ids: { executionId: 'not-a-server-minted-id' } }, + }, + undefined, + 'run_workflow' + ) + ).toEqual({ success: false, error: TOOL_RESULT_UNAVAILABLE_ERROR }) + }) + + it.each(['effect', 'resultWithheld'])( + 'voids the disclosure when an id would take the reserved key %s', + (reserved) => { + expect( + projectToolResultForCopilot( + { + success: false, + error: 'why', + effect: { phase: 'performed', ids: { [reserved]: EXECUTION_ID } }, + }, + undefined, + 'run_workflow' + ) + ).toEqual({ success: false, error: TOOL_RESULT_UNAVAILABLE_ERROR }) + } + ) + + it('reports the phase and ids when every id is vouchable', () => { + expect( + projectToolResultForCopilot( + { + success: false, + error: 'why', + effect: { phase: 'attempted', ids: { executionId: EXECUTION_ID } }, + }, + undefined, + 'run_workflow' + ) + ).toEqual({ + success: false, + output: { resultWithheld: true, effect: 'attempted', executionId: EXECUTION_ID }, + error: expect.stringContaining('At most one run exists'), + }) + }) + + it('never leaks the disclosure into a result that projected cleanly', () => { + const registry = new ResolvedSecretTraceRegistry() + + expect( + projectToolResultForCopilot( + { + success: true, + output: { executionId: EXECUTION_ID }, + effect: { phase: 'performed', ids: { executionId: EXECUTION_ID } }, + }, + registry, + 'run_workflow' + ) + ).toEqual({ success: true, output: { executionId: EXECUTION_ID } }) + }) + + it('names why the content was withheld, for the surface about to log it', () => { + const latched = createRegistry() + latched.markIncomplete('source-provenance-incomplete', { origin: 'test.origin' }) + + const projection = inspectToolResultForCopilot({ success: false }, latched, 'run_workflow') + expect(projection.safe).toBe(false) + // The per-call fork adds its own propagation reason; the guard that originally + // tripped has to survive alongside it, or a refusal names only the messenger. + expect(projection.safe === false && describeWithholdingCause(projection.cause)).toEqual({ + withheldCause: 'registry-incomplete', + withheldReasons: expect.arrayContaining(['source-provenance-incomplete']), + withheldOrigins: ['test.origin'], + }) + + const absent = inspectToolResultForCopilot({ success: false }, undefined) + expect(absent.safe === false && absent.cause).toEqual({ kind: 'registry-absent' }) + }) +}) diff --git a/apps/sim/lib/copilot/request/tools/resolved-secret-result.ts b/apps/sim/lib/copilot/request/tools/resolved-secret-result.ts index f6785f60a8d..65b704a06d0 100644 --- a/apps/sim/lib/copilot/request/tools/resolved-secret-result.ts +++ b/apps/sim/lib/copilot/request/tools/resolved-secret-result.ts @@ -1,6 +1,10 @@ -import type { ToolExecutionResult } from '@/lib/copilot/tool-executor/types' +import type { ToolCallEffect, ToolExecutionResult } from '@/lib/copilot/tool-executor/types' +import { TOOL_EFFECT_PHASE } from '@/lib/copilot/tool-executor/types' import { projectResolvedSecretModelJsonContent } from '@/executor/utils/resolved-secret-content-projection' -import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' +import type { + ResolvedSecretIncompletenessReason, + ResolvedSecretTraceRegistry, +} from '@/executor/utils/resolved-secret-trace-registry' export const TOOL_RESULT_UNAVAILABLE_ERROR = 'Tool execution settled, but its result could not be returned safely. Do not retry a mutation automatically.' @@ -13,8 +17,38 @@ export const TOOL_RESULT_UNAVAILABLE_ERROR = export const READ_TOOL_RESULT_UNAVAILABLE_ERROR = 'Tool executed, but its result could not be returned safely. The call was read-only, so you may retry it or continue without the result.' +/** + * Withheld-result wording for a call that disclosed how far its side effect got. + * + * The generic message above has to cover both "nothing happened" and "it happened, + * you just cannot see it", which is why a caller could not build a retry policy from + * it: a rejected call and a completed mutation read identically. A tool that declares + * its {@link ToolCallEffect} gets the phrasing its phase actually warrants. + */ +const WITHHELD_ERROR_BY_EFFECT_PHASE: Record = { + [TOOL_EFFECT_PHASE.notAttempted]: + 'Tool call was rejected before it ran, so nothing was created or changed. The reason could not be returned safely — correct the call and try again.', + [TOOL_EFFECT_PHASE.attempted]: + 'Tool execution was dispatched but its outcome could not be returned safely. At most one run exists for the ids in this result — resolve it before retrying a mutation.', + [TOOL_EFFECT_PHASE.performed]: + 'Tool execution completed but its result could not be returned safely. Do not retry — read the outcome using the ids in this result.', +} + const READ_ONLY_RESULT_TOOLS = new Set(['read', 'glob', 'grep']) +/** + * The shape of an identifier this system mints — `generateId`'s UUID, and the + * database ids that share it. Effect ids bypass secret projection, so the set of + * values that may occupy one is pinned to a syntax no credential we issue or store + * takes. A caller with a differently shaped id has to widen this deliberately, + * where the exemption is reviewed, rather than by passing it. + */ +const SERVER_MINTED_ID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i + +/** Field names the disclosure record owns; an id may not take one. */ +const RESERVED_DISCLOSURE_KEYS = new Set(['resultWithheld', 'effect']) + /** Chooses the withheld-result message a tool's caller should surface. */ export function toolResultUnavailableError(toolId?: string): string { return toolId && READ_ONLY_RESULT_TOOLS.has(toolId) @@ -22,24 +56,101 @@ export function toolResultUnavailableError(toolId?: string): string { : TOOL_RESULT_UNAVAILABLE_ERROR } +/** + * Why complete content could not cross, for the caller that is about to log a refusal. + * + * The three causes need different fixes — a latched registry names the guard that tripped, + * an absent one means the surface never built a catalog, and a content refusal means the + * registry was fine and the payload itself was unprojectable — so they are not collapsed. + */ +export type ToolResultWithholdingCause = + | { + kind: 'registry-incomplete' + reasons: readonly ResolvedSecretIncompletenessReason[] + origins: readonly string[] + } + | { kind: 'registry-absent' } + | { kind: 'content-refused' } + +export type CopilotToolResultProjection = + | { safe: true; result: ToolExecutionResult } + | { safe: false; result: ToolExecutionResult; cause: ToolResultWithholdingCause } + function structuralResult(result: ToolExecutionResult): ToolExecutionResult { return { success: result.success === true } } +/** + * Reduces a withheld result to the facts the tool asserted about the call itself. + * + * Content is dropped because nothing here can prove it secret-free. The effect + * disclosure survives because it is not derived from content: the phase is a + * code-defined literal and every id is checked against {@link SERVER_MINTED_ID_PATTERN}. + * An id that fails that check voids the whole disclosure rather than being dropped + * on its own — a partially honoured exemption is the one shape a reader would + * misread as complete. + */ function omittedResult(result: ToolExecutionResult, toolId?: string): ToolExecutionResult { - if (result.success) return { success: true } - return { success: false, error: toolResultUnavailableError(toolId) } + const effect = vouchableEffect(result.effect) + if (!effect) { + return result.success + ? { success: true } + : { success: false, error: toolResultUnavailableError(toolId) } + } + + return { + success: result.success === true, + output: { resultWithheld: true, effect: effect.phase, ...effect.ids }, + ...(result.success ? {} : { error: WITHHELD_ERROR_BY_EFFECT_PHASE[effect.phase] }), + } } -export type CopilotToolResultProjection = - | { safe: true; result: ToolExecutionResult } - | { safe: false; result: ToolExecutionResult } +/** + * Returns the disclosure only when every id it carries is a shape this system mints and none + * of them would displace the record's own fields. An id named `effect` overwriting the phase + * would corrupt exactly the field the retry decision reads, so a collision voids the + * disclosure on the same all-or-nothing terms as an unvouchable id. + */ +function vouchableEffect(effect: ToolCallEffect | undefined): ToolCallEffect | undefined { + if (!effect) return undefined + for (const [key, value] of Object.entries(effect.ids ?? {})) { + if (RESERVED_DISCLOSURE_KEYS.has(key)) return undefined + if (typeof value !== 'string' || !SERVER_MINTED_ID_PATTERN.test(value)) return undefined + } + return effect +} + +function withholdingCause( + registry: ResolvedSecretTraceRegistry | undefined +): ToolResultWithholdingCause { + if (!registry) return { kind: 'registry-absent' } + const diagnostics = registry.getIncompletenessDiagnostics() + return diagnostics + ? { + kind: 'registry-incomplete', + reasons: diagnostics.reasons, + origins: diagnostics.origins, + } + : { kind: 'content-refused' } +} + +function withheld( + result: ToolExecutionResult, + registry: ResolvedSecretTraceRegistry | undefined, + toolId: string | undefined +): CopilotToolResultProjection { + return { + safe: false, + result: omittedResult(result, toolId), + cause: withholdingCause(registry), + } +} /** * Projects terminal tool content and reports whether the complete content was safe to cross. * Callers that isolate provenance per tool call may merge that child registry only when `safe` * is true and the child is complete. The returned result is always safe to expose: an unsafe - * projection is reduced to a structural success or failure. + * projection is reduced to a structural success or failure, plus any effect the tool disclosed. */ export function inspectToolResultForCopilot( result: ToolExecutionResult, @@ -54,7 +165,7 @@ export function inspectToolResultForCopilot( if (Object.hasOwn(result, 'error')) content.error = result.error const projection = projectResolvedSecretModelJsonContent(content, resultRegistry) if (!projection.safe || !projection.value || typeof projection.value !== 'object') { - return { safe: false, result: omittedResult(result, toolId) } + return withheld(result, resultRegistry, toolId) } const projectedContent = projection.value as Record @@ -62,7 +173,7 @@ export function inspectToolResultForCopilot( if (Object.hasOwn(projectedContent, 'output')) projected.output = projectedContent.output if (Object.hasOwn(projectedContent, 'error')) { if (typeof projectedContent.error !== 'string') { - return { safe: false, result: omittedResult(result, toolId) } + return withheld(result, resultRegistry, toolId) } projected.error = projectedContent.error } @@ -74,7 +185,7 @@ export function inspectToolResultForCopilot( } return { safe: true, result: projected } } catch { - return { safe: false, result: omittedResult(result, toolId) } + return withheld(result, registry, toolId) } } @@ -98,3 +209,16 @@ export function projectToolErrorMessageForCopilot( ): string { return projectToolResultForCopilot({ success: false, error }, registry, toolId).error ?? '' } + +/** Flattens a withholding cause into log/span fields, so every surface reports it alike. */ +export function describeWithholdingCause( + cause: ToolResultWithholdingCause +): Record { + return cause.kind === 'registry-incomplete' + ? { + withheldCause: cause.kind, + withheldReasons: [...cause.reasons], + ...(cause.origins.length > 0 ? { withheldOrigins: [...cause.origins] } : {}), + } + : { withheldCause: cause.kind } +} diff --git a/apps/sim/lib/copilot/tool-executor/executor.test.ts b/apps/sim/lib/copilot/tool-executor/executor.test.ts index 772cccc5ca7..47411cefe67 100644 --- a/apps/sim/lib/copilot/tool-executor/executor.test.ts +++ b/apps/sim/lib/copilot/tool-executor/executor.test.ts @@ -140,16 +140,62 @@ describe('copilot tool executor fallback', () => { enforceCredentialAccess: true, }), }), - { - internalExecutorDelegation: { - subjectUserId: 'user-1', + expect.objectContaining({ + operationContext: expect.objectContaining({ + userId: 'user-1', workflowId: 'workflow-1', - }, - } + workspaceId: 'ws-1', + }), + }) ) expect(result).toEqual({ success: true, output: { emails: [] } }) }) + it('forwards trusted authority and cancellation to dynamic custom tools', async () => { + isKnownTool.mockReturnValue(false) + isSimExecuted.mockReturnValue(false) + executeAppTool.mockResolvedValue({ success: true, output: { result: 'custom output' } }) + const abortController = new AbortController() + + const result = await executeTool( + 'custom_weather-tool', + { + location: 'San Francisco', + _context: { userId: 'attacker', workspaceId: 'evil-workspace' }, + }, + { + userId: 'user-1', + workflowId: 'workflow-1', + workspaceId: 'ws-1', + executionId: 'execution-1', + abortSignal: abortController.signal, + } + ) + + expect(executeAppTool).toHaveBeenCalledWith( + 'custom_weather-tool', + expect.objectContaining({ + location: 'San Francisco', + _context: expect.objectContaining({ + userId: 'user-1', + workspaceId: 'ws-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }), + }), + expect.objectContaining({ + signal: abortController.signal, + operationContext: expect.objectContaining({ + userId: 'user-1', + workspaceId: 'ws-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }), + }) + ) + expect(result).toEqual({ success: true, output: { result: 'custom output' } }) + }) + it('threads billing attribution into _context for dynamic tools (MCP)', async () => { isKnownTool.mockReturnValue(false) isSimExecuted.mockReturnValue(false) @@ -184,6 +230,13 @@ describe('copilot tool executor fallback', () => { workspaceId: 'ws-1', billingAttribution, }), + }), + expect.objectContaining({ + operationContext: expect.objectContaining({ + userId: 'user-1', + workspaceId: 'ws-1', + billingAttribution, + }), }) ) }) @@ -221,13 +274,14 @@ describe('copilot tool executor fallback', () => { query: 'hello', _context: expect.not.objectContaining({ resolvedSecretTraceRegistry: expect.anything() }), }), - { + expect.objectContaining({ resolvedSecretTraceRegistry: registry, - internalExecutorDelegation: { - subjectUserId: 'user-1', + operationContext: expect.objectContaining({ + userId: 'user-1', workflowId: 'workflow-1', - }, - } + resolvedSecretTraceRegistry: registry, + }), + }) ) const appParams = executeAppTool.mock.calls[0]?.[1] expect(JSON.stringify(appParams)).not.toContain('resolvedSecretTraceRegistry') @@ -295,7 +349,11 @@ describe('copilot tool executor fallback', () => { await executeTool('unknown_client_tool', {}, { userId: 'user-1' }) - expect(executeAppTool).toHaveBeenCalledWith('unknown_client_tool', expect.any(Object)) + expect(executeAppTool).toHaveBeenCalledWith( + 'unknown_client_tool', + expect.any(Object), + expect.objectContaining({ operationContext: expect.any(Object) }) + ) }) it('converts run_function timeout from seconds to milliseconds for copilot calls', async () => { @@ -322,12 +380,14 @@ describe('copilot tool executor fallback', () => { copilotToolExecution: true, }), }), - { - internalExecutorDelegation: { - subjectUserId: 'user-1', + expect.objectContaining({ + operationContext: expect.objectContaining({ + userId: 'user-1', workflowId: 'workflow-1', - }, - } + workspaceId: 'ws-1', + copilotToolExecution: true, + }), + }) ) }) @@ -377,12 +437,14 @@ describe('copilot tool executor fallback', () => { expect.objectContaining({ timeout: 10_000, }), - { - internalExecutorDelegation: { - subjectUserId: 'user-1', + expect.objectContaining({ + operationContext: expect.objectContaining({ + userId: 'user-1', workflowId: 'workflow-1', - }, - } + workspaceId: 'ws-1', + copilotToolExecution: true, + }), + }) ) }) @@ -407,12 +469,14 @@ describe('copilot tool executor fallback', () => { expect.objectContaining({ timeout: 10_000, }), - { - internalExecutorDelegation: { - subjectUserId: 'user-1', + expect.objectContaining({ + operationContext: expect.objectContaining({ + userId: 'user-1', workflowId: 'workflow-1', - }, - } + workspaceId: 'ws-1', + copilotToolExecution: true, + }), + }) ) }) @@ -437,12 +501,14 @@ describe('copilot tool executor fallback', () => { expect.objectContaining({ timeout: DEFAULT_EXECUTION_TIMEOUT_MS, }), - { - internalExecutorDelegation: { - subjectUserId: 'user-1', + expect.objectContaining({ + operationContext: expect.objectContaining({ + userId: 'user-1', workflowId: 'workflow-1', - }, - } + workspaceId: 'ws-1', + copilotToolExecution: true, + }), + }) ) }) diff --git a/apps/sim/lib/copilot/tool-executor/executor.ts b/apps/sim/lib/copilot/tool-executor/executor.ts index e4f7ba3a3ee..ba0cfe9ad73 100644 --- a/apps/sim/lib/copilot/tool-executor/executor.ts +++ b/apps/sim/lib/copilot/tool-executor/executor.ts @@ -81,15 +81,24 @@ export async function executeTool( ...(context.resolvedSecretTraceRegistry ? { resolvedSecretTraceRegistry: context.resolvedSecretTraceRegistry } : {}), - ...(context.workflowId - ? { - internalExecutorDelegation: { - subjectUserId: context.userId, - workflowId: context.workflowId, - ...(context.executionId ? { executionId: context.executionId } : {}), - }, - } - : {}), + ...(context.abortSignal ? { signal: context.abortSignal } : {}), + operationContext: { + userId: context.userId, + workflowId: context.workflowId, + workspaceId: context.workspaceId, + executionId: context.executionId, + chatId: context.chatId, + toolCallId: context.toolCallId, + executorDelegationOrigin: { + subjectUserId: context.userId, + workflowId: context.workflowId, + ...(context.executionId ? { executionId: context.executionId } : {}), + }, + copilotToolExecution: context.copilotToolExecution, + copilotInteractionMode: context.copilotInteractionMode, + billingAttribution: context.billingAttribution, + resolvedSecretTraceRegistry: context.resolvedSecretTraceRegistry, + }, } try { return await (Object.keys(options).length > 0 diff --git a/apps/sim/lib/copilot/tool-executor/types.ts b/apps/sim/lib/copilot/tool-executor/types.ts index d774ca0999e..47db6aa0d95 100644 --- a/apps/sim/lib/copilot/tool-executor/types.ts +++ b/apps/sim/lib/copilot/tool-executor/types.ts @@ -45,11 +45,52 @@ export interface ToolExecutionContext { resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry } +/** + * How far a tool call got in performing its side effect. + * + * This is a property of the call, not of the content it produced, which is why it + * can still be reported when the content itself cannot cross the model boundary. + * It is the only thing that lets a caller decide about retry: a rejected call and a + * completed mutation are otherwise indistinguishable once their payloads are withheld. + */ +export const TOOL_EFFECT_PHASE = { + /** Rejected before anything could happen. Correcting the call and retrying is safe. */ + notAttempted: 'not_attempted', + /** + * Dispatched; zero or one effects may exist. Resolve by id before retrying. + * + * Zero is a legitimate outcome here, not a defect: the id is a correlation key, not a + * promise that a row exists. Narrowing this to "a run definitely exists" would take + * per-block instrumentation across every execution in the product to spare one caller a + * lookup that answers the question definitively either way. + */ + attempted: 'attempted', + /** The effect ran to completion, whatever its outcome. Never retry blind. */ + performed: 'performed', +} as const +export type ToolEffectPhase = (typeof TOOL_EFFECT_PHASE)[keyof typeof TOOL_EFFECT_PHASE] + +export interface ToolCallEffect { + phase: ToolEffectPhase + /** + * Server-minted identifiers naming the effect, so an unreadable result stays + * resolvable. Values must be identifiers this system issues; the egress + * projection rejects the whole disclosure otherwise. + */ + ids?: Readonly> +} + export interface ToolExecutionResult { success: boolean output?: unknown error?: string resources?: MothershipResource[] + /** + * Declared by tools whose failure a caller cannot otherwise act on. Consumed by + * the egress projection and never returned to the model as-is — on a withheld + * result it becomes the disclosure record that replaces the dropped content. + */ + effect?: ToolCallEffect } export type ToolHandler = ( diff --git a/apps/sim/lib/copilot/tools/descriptions.ts b/apps/sim/lib/copilot/tools/descriptions.ts index 5b89d1e4871..a2b21613b0c 100644 --- a/apps/sim/lib/copilot/tools/descriptions.ts +++ b/apps/sim/lib/copilot/tools/descriptions.ts @@ -1,5 +1,5 @@ import type { HostedApiKeySupport } from '@/tools/hosted-api-key' -import type { ToolConfig } from '@/tools/types' +import type { ToolDefinition } from '@/tools/types' const HOSTED_API_KEY_NOTE = 'API key is hosted by Sim.' const CONDITIONAL_HOSTED_API_KEY_NOTE = @@ -16,7 +16,7 @@ const EMAIL_TAGLINE_TOOL_IDS = new Set(['gmail_send', 'gmail_send_v2', 'outlook_ * argument keeps one branch here and lets either source supply it. */ export function getCopilotToolDescription( - tool: Pick, + tool: Pick, options?: { isHosted?: boolean hostedApiKey?: HostedApiKeySupport diff --git a/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts b/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts index b7b1074db0f..c9893ad2b58 100644 --- a/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts +++ b/apps/sim/lib/copilot/tools/handlers/function-execute.test.ts @@ -239,7 +239,10 @@ describe('executeFunctionExecute trace-secret provenance', () => { mountedSecrets: ['API_KEY'], _context: expect.not.objectContaining({ resolvedSecretTraceRegistry: expect.anything() }), }), - { resolvedSecretTraceRegistry: expect.any(ResolvedSecretTraceRegistry) } + expect.objectContaining({ + resolvedSecretTraceRegistry: expect.any(ResolvedSecretTraceRegistry), + operationContext: expect.objectContaining({ userId: 'u1', workspaceId: 'ws_1' }), + }) ) const appParams = mockExecuteTool.mock.calls[0]?.[1] as Record expect(JSON.stringify(appParams)).not.toContain('resolvedSecretTraceRegistry') @@ -259,7 +262,10 @@ describe('executeFunctionExecute trace-secret provenance', () => { expect(mockExecuteTool).toHaveBeenCalledWith( 'function_execute', expect.objectContaining({ envVars: {}, secretScope: 'selected', mountedSecrets: [] }), - { resolvedSecretTraceRegistry: expect.any(ResolvedSecretTraceRegistry) } + expect.objectContaining({ + resolvedSecretTraceRegistry: expect.any(ResolvedSecretTraceRegistry), + operationContext: expect.objectContaining({ userId: 'u1', workspaceId: 'ws_1' }), + }) ) }) @@ -292,7 +298,10 @@ describe('executeFunctionExecute trace-secret provenance', () => { expect(mockExecuteTool).toHaveBeenCalledWith( 'function_execute', expect.objectContaining({ code, language, mountedSecrets: names }), - { resolvedSecretTraceRegistry: expect.any(ResolvedSecretTraceRegistry) } + expect.objectContaining({ + resolvedSecretTraceRegistry: expect.any(ResolvedSecretTraceRegistry), + operationContext: expect.objectContaining({ userId: 'u1', workspaceId: 'ws_1' }), + }) ) } ) @@ -319,11 +328,12 @@ describe('executeFunctionExecute trace-secret provenance', () => { expect(mockExecuteTool).toHaveBeenCalledWith( 'function_execute', expect.objectContaining({ code, language: 'shell', mountedSecrets: ['CLI_TOKEN'] }), - { + expect.objectContaining({ resolvedSecretTraceRegistry: expect.any(ResolvedSecretTraceRegistry), + operationContext: expect.objectContaining({ userId: 'u1', workspaceId: 'ws_1' }), internalSandboxProfile: 'mothership', signal: abortController.signal, - } + }) ) }) @@ -342,10 +352,11 @@ describe('executeFunctionExecute trace-secret provenance', () => { expect.objectContaining({ _context: expect.not.objectContaining({ sandboxProfile: expect.anything() }), }), - { + expect.objectContaining({ resolvedSecretTraceRegistry: expect.any(ResolvedSecretTraceRegistry), + operationContext: expect.objectContaining({ userId: 'u1', workspaceId: 'ws_1' }), internalSandboxProfile: 'mothership', - } + }) ) expect(mockExecuteTool.mock.calls[0]?.[1]).not.toHaveProperty('sandboxProfile') }) diff --git a/apps/sim/lib/copilot/tools/handlers/function-execute.ts b/apps/sim/lib/copilot/tools/handlers/function-execute.ts index 0d40949bc7e..9f619b0677e 100644 --- a/apps/sim/lib/copilot/tools/handlers/function-execute.ts +++ b/apps/sim/lib/copilot/tools/handlers/function-execute.ts @@ -717,6 +717,20 @@ export async function executeFunctionExecute( */ const result = await executeAppTool('function_execute', enrichedParams, { resolvedSecretTraceRegistry: mountedRegistry, + operationContext: { + userId: context.userId, + workflowId: context.workflowId, + workspaceId: context.workspaceId, + executionId: context.executionId, + executorDelegationOrigin: { + subjectUserId: context.userId, + workflowId: context.workflowId, + ...(context.executionId ? { executionId: context.executionId } : {}), + }, + copilotToolExecution: context.copilotToolExecution, + billingAttribution: context.billingAttribution, + resolvedSecretTraceRegistry: mountedRegistry, + }, ...(context.abortSignal ? { signal: context.abortSignal } : {}), ...(context.sandboxProfile ? { internalSandboxProfile: context.sandboxProfile } : {}), }) diff --git a/apps/sim/lib/copilot/tools/handlers/materialize-file.ts b/apps/sim/lib/copilot/tools/handlers/materialize-file.ts index 755c8b718d8..3ebfb5e2941 100644 --- a/apps/sim/lib/copilot/tools/handlers/materialize-file.ts +++ b/apps/sim/lib/copilot/tools/handlers/materialize-file.ts @@ -140,7 +140,8 @@ async function executeSave( try { transition = await db.transaction(async (tx) => { - await tx.execute(sql`SELECT 1 FROM workspace WHERE id = ${workspaceId} FOR UPDATE`) + /** `FOR NO KEY UPDATE`: see the module header of `lib/billing/storage/tracking.ts`. */ + await tx.execute(sql`SELECT 1 FROM workspace WHERE id = ${workspaceId} FOR NO KEY UPDATE`) const [updated] = await tx .update(workspaceFiles) diff --git a/apps/sim/lib/copilot/tools/handlers/workflow/mutations.test.ts b/apps/sim/lib/copilot/tools/handlers/workflow/mutations.test.ts index 37f6860601b..9e11b107c6a 100644 --- a/apps/sim/lib/copilot/tools/handlers/workflow/mutations.test.ts +++ b/apps/sim/lib/copilot/tools/handlers/workflow/mutations.test.ts @@ -9,6 +9,7 @@ const { mocks } = vi.hoisted(() => ({ apiKey: vi.fn(), executeWorkflowUseCase: vi.fn(), hasExecutionResult: vi.fn(), + readAttemptedExecutionId: vi.fn(), }, })) @@ -28,6 +29,7 @@ vi.mock('@/lib/workflows/sanitization/json-sanitizer', () => ({ vi.mock('@/executor/utils/errors', () => ({ hasExecutionResult: mocks.hasExecutionResult, + readAttemptedExecutionId: mocks.readAttemptedExecutionId, })) vi.mock('@/lib/core/telemetry', () => ({ @@ -57,6 +59,7 @@ describe('workflow mutation Copilot adapters', () => { beforeEach(() => { vi.clearAllMocks() mocks.hasExecutionResult.mockReturnValue(false) + mocks.readAttemptedExecutionId.mockReturnValue(undefined) }) it('maps encoded folder aliases into one create application command', async () => { @@ -259,6 +262,66 @@ describe('workflow mutation Copilot adapters', () => { const result = await executeRunWorkflow({ workflowId: 'workflow-1' }, context) - expect(result).toEqual({ success: false, error: 'Workflow execution failed' }) + expect(result).toEqual({ + success: false, + error: 'Workflow execution failed', + effect: { phase: 'not_attempted' }, + }) + }) + + /** + * How far the run got is the only thing a caller can act on once the egress boundary + * withholds the payload, so each of these must reach the projection distinguishable. + */ + it.each([ + { + label: 'refused on its own arguments', + arrange: () => {}, + run: () => executeRunWorkflow({}, { ...context, workflowId: undefined }), + effect: { phase: 'not_attempted' }, + }, + { + label: 'failed before dispatch', + arrange: () => mocks.executeWorkflowUseCase.mockRejectedValueOnce(new Error('denied')), + run: () => executeRunWorkflow({ workflowId: 'workflow-1' }, context), + effect: { phase: 'not_attempted' }, + }, + { + label: 'failed after dispatch', + arrange: () => { + mocks.executeWorkflowUseCase.mockRejectedValueOnce(new Error('crashed')) + mocks.readAttemptedExecutionId.mockReturnValue('execution-1') + }, + run: () => executeRunWorkflow({ workflowId: 'workflow-1' }, context), + effect: { phase: 'attempted', ids: { executionId: 'execution-1' } }, + }, + { + label: 'cancelled before it could finish', + arrange: () => + mocks.executeWorkflowUseCase.mockResolvedValueOnce({ + success: false, + output: {}, + logs: [], + status: 'cancelled', + metadata: { executionId: 'execution-1' }, + }), + run: () => executeRunWorkflow({ workflowId: 'workflow-1' }, context), + effect: { phase: 'attempted', ids: { executionId: 'execution-1' } }, + }, + { + label: 'completed', + arrange: () => + mocks.executeWorkflowUseCase.mockResolvedValueOnce({ + success: true, + output: {}, + logs: [], + metadata: { executionId: 'execution-1' }, + }), + run: () => executeRunWorkflow({ workflowId: 'workflow-1' }, context), + effect: { phase: 'performed', ids: { executionId: 'execution-1' } }, + }, + ])('states that a run $label', async ({ arrange, run, effect }) => { + arrange() + expect((await run()).effect).toEqual(effect) }) }) diff --git a/apps/sim/lib/copilot/tools/handlers/workflow/mutations.ts b/apps/sim/lib/copilot/tools/handlers/workflow/mutations.ts index 164574a84cb..61a417d148d 100644 --- a/apps/sim/lib/copilot/tools/handlers/workflow/mutations.ts +++ b/apps/sim/lib/copilot/tools/handlers/workflow/mutations.ts @@ -7,6 +7,11 @@ import { messageForCopilotWorkflowError, } from '@/lib/copilot/application/execute-workflow-use-case' import type { ExecutionContext, ToolCallResult } from '@/lib/copilot/request/types' +import { + TOOL_EFFECT_PHASE, + type ToolCallEffect, + type ToolEffectPhase, +} from '@/lib/copilot/tool-executor/types' import { requireCopilotWorkspace } from '@/lib/copilot/tools/server/workspace-scope' import { decodeVfsPathSegments, encodeVfsPathSegments } from '@/lib/copilot/vfs/path-utils' import { PlatformEvents } from '@/lib/core/telemetry' @@ -24,7 +29,7 @@ import { setWorkflowBlockEnabled, } from '@/lib/workflows/application/update-workflow-content' import { sanitizeForCopilot } from '@/lib/workflows/sanitization/json-sanitizer' -import { hasExecutionResult } from '@/executor/utils/errors' +import { hasExecutionResult, readAttemptedExecutionId } from '@/executor/utils/errors' import type { WorkflowState } from '@/stores/workflows/workflow/types' function stripBinaryFields(value: unknown): unknown { @@ -39,6 +44,41 @@ function stripBinaryFields(value: unknown): unknown { return out } +/** + * States how far a run got, so the answer survives a result the egress boundary withholds. + * + * Without it a withheld run reduces to a bare success or an opaque failure and takes the + * execution id with it, which is what left a caller unable to tell a rejected call from a + * completed run — and with nothing to look either one up by. + */ +function executionEffect(phase: ToolEffectPhase, executionId?: string): ToolCallEffect { + return { phase, ...(executionId ? { ids: { executionId } } : {}) } +} + +/** A run refused on its own arguments, before anything could be created. */ +function runRejected(error: string): ToolCallResult { + return { success: false, error, effect: executionEffect(TOOL_EFFECT_PHASE.notAttempted) } +} + +/** + * The phase of a run whose result came back, from how that run ended. + * + * A result in hand means the executor reached a terminal state and recorded it, so the + * caller can read the whole story by id — `performed`. Cancelled and paused stopped partway + * and may have run every block, one, or none, which is exactly what `attempted` says. + * + * Deliberately does not separate "ran no blocks" from "ran some". Establishing that would + * take a callback on every block of every execution in the product, and buys the caller + * nothing it cannot get by resolving the id it was already handed. + */ +function settledPhase(status: ExecutionResultStatus): ToolEffectPhase { + return status === 'cancelled' || status === 'paused' + ? TOOL_EFFECT_PHASE.attempted + : TOOL_EFFECT_PHASE.performed +} + +type ExecutionResultStatus = 'completed' | 'paused' | 'cancelled' | undefined + function buildExecutionOutput( result: { success: boolean @@ -46,7 +86,9 @@ function buildExecutionOutput( output?: unknown logs?: unknown[] error?: string + status?: ExecutionResultStatus }, + phase: ToolEffectPhase, extra?: Record ): ToolCallResult { return { @@ -59,21 +101,34 @@ function buildExecutionOutput( logs: stripBinaryFields(result.logs), }, error: result.success ? undefined : result.error || 'Workflow execution failed', + effect: executionEffect(phase, result.metadata?.executionId), } } function buildExecutionError(error: unknown): ToolCallResult { if (hasExecutionResult(error)) { - return buildExecutionOutput({ - ...error.executionResult, - success: false, - error: error.executionResult.error || 'Workflow execution failed', - }) + return buildExecutionOutput( + { + ...error.executionResult, + success: false, + error: error.executionResult.error || 'Workflow execution failed', + }, + settledPhase(error.executionResult.status) + ) } logger.error('Copilot workflow execution command failed', { error }) + /** + * Only failures raised after dispatch carry the id, so its absence is the positive + * statement that nothing was created rather than an admission of not knowing. + */ + const attemptedExecutionId = readAttemptedExecutionId(error) return { success: false, error: messageForCopilotWorkflowError(error, 'Workflow execution failed'), + effect: executionEffect( + attemptedExecutionId ? TOOL_EFFECT_PHASE.attempted : TOOL_EFFECT_PHASE.notAttempted, + attemptedExecutionId + ), } } @@ -204,7 +259,7 @@ export async function executeRunWorkflow( try { const workflowId = params.workflowId || context.workflowId if (!workflowId) { - return { success: false, error: 'workflowId is required' } + return runRejected('workflowId is required') } const useDraftState = !params.useDeployedState @@ -221,7 +276,7 @@ export async function executeRunWorkflow( lifecycle: copilotRunLifecycle(context), }) - return buildExecutionOutput(result) + return buildExecutionOutput(result, settledPhase(result.status)) } catch (error) { return buildExecutionError(error) } @@ -322,10 +377,10 @@ export async function executeRunWorkflowUntilBlock( try { const workflowId = params.workflowId || context.workflowId if (!workflowId) { - return { success: false, error: 'workflowId is required' } + return runRejected('workflowId is required') } if (!params.stopAfterBlockId) { - return { success: false, error: 'stopAfterBlockId is required' } + return runRejected('stopAfterBlockId is required') } const useDraftState = !params.useDeployedState @@ -343,7 +398,9 @@ export async function executeRunWorkflowUntilBlock( lifecycle: copilotRunLifecycle(context), }) - return buildExecutionOutput(result, { stoppedAfterBlockId: params.stopAfterBlockId }) + return buildExecutionOutput(result, settledPhase(result.status), { + stoppedAfterBlockId: params.stopAfterBlockId, + }) } catch (error) { return buildExecutionError(error) } @@ -401,10 +458,10 @@ export async function executeRunFromBlock( try { const workflowId = params.workflowId || context.workflowId if (!workflowId) { - return { success: false, error: 'workflowId is required' } + return runRejected('workflowId is required') } if (!params.startBlockId) { - return { success: false, error: 'startBlockId is required' } + return runRejected('startBlockId is required') } const useDraftState = !params.useDeployedState @@ -418,7 +475,9 @@ export async function executeRunFromBlock( lifecycle: copilotRunLifecycle(context), }) - return buildExecutionOutput(result, { startBlockId: params.startBlockId }) + return buildExecutionOutput(result, settledPhase(result.status), { + startBlockId: params.startBlockId, + }) } catch (error) { return buildExecutionError(error) } @@ -487,10 +546,10 @@ export async function executeRunBlock( try { const workflowId = params.workflowId || context.workflowId if (!workflowId) { - return { success: false, error: 'workflowId is required' } + return runRejected('workflowId is required') } if (!params.blockId) { - return { success: false, error: 'blockId is required' } + return runRejected('blockId is required') } const useDraftState = !params.useDeployedState @@ -504,7 +563,7 @@ export async function executeRunBlock( lifecycle: copilotRunLifecycle(context), }) - return buildExecutionOutput(result, { blockId: params.blockId }) + return buildExecutionOutput(result, settledPhase(result.status), { blockId: params.blockId }) } catch (error) { return buildExecutionError(error) } diff --git a/apps/sim/lib/copilot/tools/handlers/workflow/withheld-run-result.test.ts b/apps/sim/lib/copilot/tools/handlers/workflow/withheld-run-result.test.ts new file mode 100644 index 00000000000..793a3185a66 --- /dev/null +++ b/apps/sim/lib/copilot/tools/handlers/workflow/withheld-run-result.test.ts @@ -0,0 +1,229 @@ +/** + * @vitest-environment node + * + * What a caller can learn about a workflow run whose result the secret-egress boundary + * withholds. + * + * The registry is latched the way production latches one — a child run that returned no + * provenance envelope — rather than by asserting an "unsafe" flag, so these fail for the + * same reason the incident did. Every outcome the copilot run path can produce is driven + * through the real handler and the real projection and asserted on two axes: the retry + * decision a caller can reach, which is the point of the disclosure, and that no run + * content crosses, which is the point of the boundary. + * + * The phases are deliberately coarse. `attempted` and `performed` both mean "an execution + * exists under this id". Separating "ran no blocks" from "ran some" would take a callback + * on every block of every execution in the product, and buys a caller nothing it cannot get + * by resolving the id it was handed. + */ +import { getErrorMessage } from '@sim/utils/errors' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { inspectToolResultForCopilot } from '@/lib/copilot/request/tools/resolved-secret-result' +import type { ExecutionContext } from '@/lib/copilot/request/types' +import type { ToolExecutionResult } from '@/lib/copilot/tool-executor/types' +import { attachAttemptedExecutionId } from '@/executor/utils/errors' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' + +const { mocks } = vi.hoisted(() => ({ mocks: { executeWorkflowUseCase: vi.fn() } })) + +vi.mock('@/lib/copilot/application/execute-workflow-use-case', () => ({ + executeCopilotWorkflowUseCase: mocks.executeWorkflowUseCase, + /** Passthrough, so a masked message reads as masking rather than as a fallback. */ + messageForCopilotWorkflowError: (error: unknown, fallback = 'Workflow operation failed') => + getErrorMessage(error, fallback), +})) + +vi.mock('@/lib/workflows/sanitization/json-sanitizer', () => ({ + sanitizeForCopilot: vi.fn((state) => state), +})) + +vi.mock('@/lib/core/telemetry', () => ({ PlatformEvents: { apiKeyGenerated: vi.fn() } })) + +import { executeRunWorkflow } from '@/lib/copilot/tools/handlers/workflow/mutations' + +const EXECUTION_ID = '0f4d5a4c-6a1e-4c2f-9b7d-2c8f1a3e5d90' +/** + * Above the eight-character substitution floor, and deliberately not shaped like a real + * provider credential — a realistic fixture makes secret scanners flag this file. + */ +const SECRET = 'fake-secret-for-test-only' + +const context = { + userId: 'user-1', + workspaceId: 'workspace-1', + toolCallId: 'tool-call-1', +} as ExecutionContext + +/** A registry latched exactly as `importCrossingProvenance` latches one in production. */ +async function latchedRegistry(): Promise { + const registry = new ResolvedSecretTraceRegistry([ + { name: 'API_KEY', plaintext: SECRET, encryptedValue: 'ciphertext' }, + ]) + registry.recordResolved('API_KEY', SECRET, { propagated: true }) + await registry.importCrossingProvenance( + undefined, + { output: {} }, + { trusted: true, origin: 'copilotWorkflowMutation.runCrossing' } + ) + expect(registry.isPermanentlyIncomplete()).toBe(true) + return registry +} + +/** A run dense with the active secret, so a leak cannot pass unnoticed. */ +function secretBearingResult(extra: Record = {}) { + return { + success: true, + output: { report: `PASS ${SECRET}`, nested: { key: SECRET } }, + logs: [{ blockName: 'report', output: SECRET }], + metadata: { executionId: EXECUTION_ID, duration: 2800 }, + ...extra, + } +} + +function dispatchFailure(): Error { + const error = new Error(`crashed reading ${SECRET}`) + // What `executeCopilotRun` does once the run has been handed to the executor. + attachAttemptedExecutionId(error, EXECUTION_ID) + return error +} + +interface Outcome { + label: string + arrange: () => void + effect: string + /** Whether the caller may re-issue the call without resolving anything first. */ + safeToRetry: boolean + succeeded: boolean +} + +const OUTCOMES: Outcome[] = [ + { + label: 'refused before the executor was handed the run', + arrange: () => mocks.executeWorkflowUseCase.mockRejectedValue(new Error('Access denied')), + effect: 'not_attempted', + safeToRetry: true, + succeeded: false, + }, + { + label: 'failed after the executor was handed the run', + arrange: () => mocks.executeWorkflowUseCase.mockRejectedValue(dispatchFailure()), + effect: 'attempted', + safeToRetry: false, + succeeded: false, + }, + { + label: 'cancelled partway', + arrange: () => + mocks.executeWorkflowUseCase.mockResolvedValue( + secretBearingResult({ success: false, status: 'cancelled' }) + ), + effect: 'attempted', + safeToRetry: false, + succeeded: false, + }, + { + label: 'paused partway', + arrange: () => + mocks.executeWorkflowUseCase.mockResolvedValue( + secretBearingResult({ success: false, status: 'paused' }) + ), + effect: 'attempted', + safeToRetry: false, + succeeded: false, + }, + { + label: 'ran and failed', + arrange: () => + mocks.executeWorkflowUseCase.mockResolvedValue( + secretBearingResult({ success: false, error: `Block failed with ${SECRET}` }) + ), + effect: 'performed', + safeToRetry: false, + succeeded: false, + }, + { + label: 'ran and completed', + arrange: () => mocks.executeWorkflowUseCase.mockResolvedValue(secretBearingResult()), + effect: 'performed', + safeToRetry: false, + succeeded: true, + }, +] + +async function withhold(): Promise { + const settled = await executeRunWorkflow({ workflowId: 'wf-1' }, context) + const projection = inspectToolResultForCopilot(settled, await latchedRegistry(), 'run_workflow') + expect(projection.safe).toBe(false) + return projection.result +} + +describe('a withheld run_workflow result', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.executeWorkflowUseCase.mockReset() + }) + + it('says nothing was created when the call never reached the use case', async () => { + const rejected = await executeRunWorkflow({}, { ...context, workflowId: undefined }) + expect(mocks.executeWorkflowUseCase).not.toHaveBeenCalled() + + const { result } = inspectToolResultForCopilot( + rejected, + await latchedRegistry(), + 'run_workflow' + ) + + expect(result.output).toEqual({ resultWithheld: true, effect: 'not_attempted' }) + expect(result.error).toContain('nothing was created') + }) + + it.each(OUTCOMES)('discloses a run that was $label', async ({ arrange, effect, succeeded }) => { + arrange() + const result = await withhold() + + expect(result.success).toBe(succeeded) + expect(result.output).toEqual({ + resultWithheld: true, + effect, + // An id is present exactly when there is something to resolve. + ...(effect === 'not_attempted' ? {} : { executionId: EXECUTION_ID }), + }) + }) + + it.each(OUTCOMES)('never leaks run content for a run that was $label', async ({ arrange }) => { + arrange() + const serialized = JSON.stringify(await withhold()) + + expect(serialized).not.toContain(SECRET) + expect(serialized).not.toContain('PASS') + expect(serialized).not.toContain('Block failed') + expect(serialized).not.toContain('crashed') + }) + + /** + * The property the disclosure exists for: a caller can decide about retry from the + * response alone, and can never conclude "nothing happened" about a run that exists. + */ + it('lets a caller decide retry safety without resolving anything', async () => { + for (const outcome of OUTCOMES) { + mocks.executeWorkflowUseCase.mockReset() + outcome.arrange() + const output = (await withhold()).output as Record + + expect(output.effect === 'not_attempted', outcome.label).toBe(outcome.safeToRetry) + expect(Object.hasOwn(output, 'executionId'), outcome.label).toBe(!outcome.safeToRetry) + } + }) + + /** The defect this replaced: every one of these arrived as the same sentence. */ + it('distinguishes outcomes that need different decisions', async () => { + const seen = new Set() + for (const outcome of OUTCOMES) { + mocks.executeWorkflowUseCase.mockReset() + outcome.arrange() + seen.add(JSON.stringify(await withhold())) + } + // Retry, resolve-then-decide, and read-the-result are the three distinct answers. + expect(seen.size).toBeGreaterThanOrEqual(3) + }) +}) diff --git a/apps/sim/lib/copilot/tools/server/workflow/query-logs.test.ts b/apps/sim/lib/copilot/tools/server/workflow/query-logs.test.ts index 984432e7ebd..55e10d1596f 100644 --- a/apps/sim/lib/copilot/tools/server/workflow/query-logs.test.ts +++ b/apps/sim/lib/copilot/tools/server/workflow/query-logs.test.ts @@ -12,6 +12,9 @@ const { toFullMock, toTraceMock, grepSpansMock, + executeLogUseCaseMock, + listLogsUseCase, + readLogDetailUseCase, } = vi.hoisted(() => ({ listLogsMock: vi.fn(), statsLogsMock: vi.fn(), @@ -20,11 +23,17 @@ const { toFullMock: vi.fn(), toTraceMock: vi.fn(), grepSpansMock: vi.fn(), + executeLogUseCaseMock: vi.fn(), + listLogsUseCase: { kind: 'list' }, + readLogDetailUseCase: { kind: 'detail' }, })) -vi.mock('@/lib/logs/list-logs', () => ({ listLogs: listLogsMock })) +vi.mock('@/lib/copilot/application/execute-log-use-case', () => ({ + executeCopilotLogUseCase: executeLogUseCaseMock, +})) +vi.mock('@/lib/logs/application/list-logs', () => ({ listLogsUseCase })) +vi.mock('@/lib/logs/application/read-log-detail', () => ({ readLogDetailUseCase })) vi.mock('@/lib/logs/stats-logs', () => ({ statsLogs: statsLogsMock })) -vi.mock('@/lib/logs/fetch-log-detail', () => ({ fetchLogDetail: fetchLogDetailMock })) vi.mock('@/lib/logs/log-views', () => ({ toOverview: toOverviewMock, toFull: toFullMock, @@ -39,7 +48,12 @@ vi.mock('@/lib/execution/payloads/large-execution-value', () => ({ import type { ServerToolContext } from '@/lib/copilot/tools/server/base-tool' import { queryLogsServerTool } from './query-logs' -const ctx: ServerToolContext = { userId: 'user-1', workspaceId: 'ws-1' } +const ctx: ServerToolContext = { + userId: 'user-1', + workspaceId: 'ws-1', + toolCallId: 'tool-call-1', + copilotToolExecution: true, +} type QueryLogsArgs = Parameters[0] @@ -62,6 +76,12 @@ function detail(overrides: Record = {}) { beforeEach(() => { vi.clearAllMocks() + executeLogUseCaseMock.mockImplementation(async (_context, useCase, input) => { + if (useCase === listLogsUseCase) return listLogsMock(input) + const detail = await fetchLogDetailMock(input) + if (!detail) throw new Error('Not found') + return { detail } + }) }) describe('queryLogsServerTool', () => { @@ -74,8 +94,7 @@ describe('queryLogsServerTool', () => { ) expect(listLogsMock).toHaveBeenCalledTimes(1) - const [params, userId] = listLogsMock.mock.calls[0] - expect(userId).toBe('user-1') + const [params] = listLogsMock.mock.calls[0] expect(params.workspaceId).toBe('ws-1') expect(params.includeTotal).toBe(true) expect(params).not.toHaveProperty('view') diff --git a/apps/sim/lib/copilot/tools/server/workflow/query-logs.ts b/apps/sim/lib/copilot/tools/server/workflow/query-logs.ts index a832bb3d8ae..2c5316c97a4 100644 --- a/apps/sim/lib/copilot/tools/server/workflow/query-logs.ts +++ b/apps/sim/lib/copilot/tools/server/workflow/query-logs.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { z } from 'zod' +import { executeCopilotLogUseCase } from '@/lib/copilot/application/execute-log-use-case' import { QueryLogs } from '@/lib/copilot/generated/tool-catalog-v1' import type { BaseServerTool, ServerToolContext } from '@/lib/copilot/tools/server/base-tool' import { requireCopilotWorkspace } from '@/lib/copilot/tools/server/workspace-scope' @@ -7,8 +8,9 @@ import { collectLargeValueExecutionIds, collectLargeValueKeys, } from '@/lib/execution/payloads/large-execution-value' -import { fetchLogDetail } from '@/lib/logs/fetch-log-detail' -import { type ListLogsParams, listLogs } from '@/lib/logs/list-logs' +import { listLogsUseCase } from '@/lib/logs/application/list-logs' +import { readLogDetailUseCase } from '@/lib/logs/application/read-log-detail' +import type { ListLogsParams } from '@/lib/logs/list-logs' import { grepSpans, type LogViewContext, toFull, toOverview, toTrace } from '@/lib/logs/log-views' import { statsLogs } from '@/lib/logs/stats-logs' import type { TraceSpan } from '@/lib/logs/types' @@ -174,7 +176,11 @@ export const queryLogsServerTool: BaseServerTool = { const { view: _view, title: _title, ...rest } = args const params = { ...rest, workspaceId, includeTotal: true } as ListLogsParams logger.info('query_logs list', { workspaceId, sortBy: params.sortBy }) - const { data, nextCursor, total } = await listLogs(params, userId) + const { data, nextCursor, total } = await executeCopilotLogUseCase( + context, + listLogsUseCase, + params + ) // Cursor and total lead the payload so a truncated render still shows them. return { total, nextCursor, data } } @@ -186,13 +192,19 @@ export const queryLogsServerTool: BaseServerTool = { } // overview / full / grep — single execution by id - const detail = await fetchLogDetail({ - userId, - workspaceId, - lookupColumn: 'executionId', - lookupValue: args.executionId, - }) - if (!detail) { + let detail + try { + ;({ detail } = await executeCopilotLogUseCase(context, readLogDetailUseCase, { + workspaceId, + lookupColumn: 'executionId', + lookupValue: args.executionId, + })) + } catch (error) { + if (!(error instanceof Error && error.message === 'Not found')) throw error + return { ok: false, error: `Execution not found: ${args.executionId}` } + } + const detailExecutionId = detail.executionId + if (!detailExecutionId) { return { ok: false, error: `Execution not found: ${args.executionId}` } } @@ -212,7 +224,11 @@ export const queryLogsServerTool: BaseServerTool = { } } - const viewCtx = buildLogViewContext(detail, workspaceId, userId) + const viewCtx = buildLogViewContext( + { ...detail, executionId: detailExecutionId }, + workspaceId, + userId + ) if (args.pattern) { logger.info('query_logs grep', { workspaceId, executionId: args.executionId }) diff --git a/apps/sim/lib/copilot/vfs/serializers.ts b/apps/sim/lib/copilot/vfs/serializers.ts index 212a8ad674d..647684e7e78 100644 --- a/apps/sim/lib/copilot/vfs/serializers.ts +++ b/apps/sim/lib/copilot/vfs/serializers.ts @@ -27,7 +27,7 @@ import { SIM_AUTO_MODEL_ID, } from '@/providers/models' import { deriveHostedApiKeySupport } from '@/tools/hosted-api-key' -import type { ToolConfig, ToolHostingCondition } from '@/tools/types' +import type { ExecutableToolConfig, ToolHostingCondition } from '@/tools/types' import { buildSlackManifest, SLACK_CAPABILITIES } from '@/triggers/slack/capabilities' import { buildSlackCustomBotRequestUrl } from '@/triggers/webhook-url' @@ -85,7 +85,7 @@ export function describeServiceAccountForOAuthProvider( export interface ComponentSerializationOptions { hosted?: boolean - toolConfigs?: ReadonlyMap + toolConfigs?: ReadonlyMap ownerBlockType?: string /** Product-gated inputs removed from both subBlocks and the input schema. */ hiddenInputIds?: ReadonlySet @@ -116,7 +116,7 @@ export interface ComponentSerializationOptions { * ToolConfig.hosting remains the source of truth for every hosted-key integration. */ export function serializeToolAuth( - tool: ToolConfig, + tool: ExecutableToolConfig, hosted = isHosted, ownerBlockType?: string ): VfsToolAuth | undefined { @@ -831,7 +831,7 @@ export function serializeApiKeys( } interface ApiKeyIntegrationTool { - config: ToolConfig + config: ExecutableToolConfig service: string operation: string } @@ -1156,7 +1156,7 @@ export function serializeSandboxCatalog(strategy: 'prebuilt' | 'runtime'): strin * Serialize an integration/tool schema for VFS components/integrations/{service}/{operation}.json */ export function serializeIntegrationSchema( - tool: ToolConfig, + tool: ExecutableToolConfig, options?: Pick & { oauthAvailable?: boolean } diff --git a/apps/sim/lib/copilot/vfs/workspace-vfs.ts b/apps/sim/lib/copilot/vfs/workspace-vfs.ts index 6f562aa93cf..83722e4330a 100644 --- a/apps/sim/lib/copilot/vfs/workspace-vfs.ts +++ b/apps/sim/lib/copilot/vfs/workspace-vfs.ts @@ -211,7 +211,7 @@ import { isForkingAvailableForWorkspace } from '@/ee/workspace-forking/lib/linea import { getForkChildren, getForkParent } from '@/ee/workspace-forking/lib/lineage/lineage' import { loadForkBlockMap } from '@/ee/workspace-forking/lib/mapping/block-map-store' import { getEdgeMappingRows } from '@/ee/workspace-forking/lib/mapping/mapping-store' -import type { ToolConfig } from '@/tools/types' +import type { ExecutableToolConfig } from '@/tools/types' import { TRIGGER_REGISTRY } from '@/triggers/registry' const logger = createLogger('WorkspaceVFS') @@ -441,11 +441,11 @@ function isBinaryDocBuffer(buffer: Buffer, ext: string): boolean { * process. Shared by the one-time static build and the per-viewer re-projection * of a block whose operations are partly denied. */ -let staticToolConfigs: ReadonlyMap | null = null +let staticToolConfigs: ReadonlyMap | null = null -function getStaticToolConfigs(): ReadonlyMap { +function getStaticToolConfigs(): ReadonlyMap { if (staticToolConfigs) return staticToolConfigs - const configs = new Map() + const configs = new Map() for (const { toolId, config } of getExposedIntegrationTools()) { configs.set(toolId, config) configs.set(config.id, config) diff --git a/apps/sim/lib/core/application/workspace-authorization.test.ts b/apps/sim/lib/core/application/workspace-authorization.test.ts index ac6c4ab6886..c53e266ca76 100644 --- a/apps/sim/lib/core/application/workspace-authorization.test.ts +++ b/apps/sim/lib/core/application/workspace-authorization.test.ts @@ -1,7 +1,11 @@ /** * @vitest-environment node */ -import type { SessionPrincipal, WorkspaceApiKeyPrincipal } from '@sim/auth/principal' +import type { + DelegatedPrincipal, + SessionPrincipal, + WorkspaceApiKeyPrincipal, +} from '@sim/auth/principal' import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ @@ -52,6 +56,43 @@ const workspaceKeyPrincipal: WorkspaceApiKeyPrincipal = { keyId: 'key-1', } +const executorOperation = defineWorkspaceOperation({ + id: 'test.executor-write', + minimumRole: 'write', + workspaceApiKey: 'deny', + principalKinds: ['delegated'], + delegatedServices: ['executor'], +}) + +function executorPrincipal( + originalPrincipal: NonNullable['principal'], + currentWorkflow?: NonNullable['currentWorkflow'] +): DelegatedPrincipal { + return { + kind: 'delegated', + serviceId: 'executor', + workspaceId: 'workspace-1', + delegationId: 'delegation-1', + audience: 'sim:test', + issuedAt: new Date('2026-01-01T00:00:00.000Z'), + expiresAt: new Date('2099-01-01T00:00:00.000Z'), + resourceScope: { executionId: 'execution-1' }, + delegationContext: { + kind: 'workflow_execution', + workflowId: 'root-workflow-1', + principal: originalPrincipal, + ...(currentWorkflow ? { currentWorkflow } : {}), + }, + } +} + +const executorAuthorization = { + delegation: { + audience: 'sim:test', + isWithinScope: () => true, + }, +} + const context = { workspaceId: 'workspace-1', workspaceOrganizationId: 'organization-1', @@ -115,4 +156,111 @@ describe('authorizeWorkspaceOperation', () => { authorizeWorkspaceOperation(principal, workspaceKeyOperation, context) ).rejects.toBeInstanceOf(PrincipalKindAuthorizationError) }) + + it.each([ + { + name: 'generic webhook', + principal: { + kind: 'system' as const, + serviceId: 'webhook' as const, + workspaceId: 'workspace-1', + workflowId: 'root-workflow-1', + webhookId: 'webhook-1', + provider: 'generic', + }, + }, + { + name: 'Slack webhook', + principal: { + kind: 'system' as const, + serviceId: 'webhook' as const, + workspaceId: 'workspace-1', + workflowId: 'root-workflow-1', + webhookId: 'webhook-1', + provider: 'slack', + subject: { + kind: 'external_user' as const, + provider: 'slack', + tenantId: 'tenant-1', + subjectId: 'subject-1', + }, + }, + }, + { + name: 'schedule', + principal: { + kind: 'system' as const, + serviceId: 'schedule' as const, + workspaceId: 'workspace-1', + workflowId: 'root-workflow-1', + }, + }, + ])('authorizes a $name by its bound deployed workflow', async ({ principal }) => { + await expect( + authorizeWorkspaceOperation( + executorPrincipal(principal, { + workflowId: 'current-workflow-1', + mode: 'deployment', + deploymentVersionId: 'deployment-1', + }), + executorOperation, + context, + executorAuthorization + ) + ).resolves.toBeUndefined() + expect(mocks.resolvePermission).not.toHaveBeenCalled() + }) + + it.each([ + { name: 'missing', currentWorkflow: undefined }, + { + name: 'draft', + currentWorkflow: { workflowId: 'current-workflow-1', mode: 'draft' as const }, + }, + ])('rejects actorless execution with a $name workflow authority', async ({ currentWorkflow }) => { + await expect( + authorizeWorkspaceOperation( + executorPrincipal( + { + kind: 'system', + serviceId: 'webhook', + workspaceId: 'workspace-1', + workflowId: 'root-workflow-1', + webhookId: 'webhook-1', + provider: 'generic', + }, + currentWorkflow + ), + executorOperation, + context, + executorAuthorization + ) + ).rejects.toMatchObject({ name: 'DelegatedWorkspaceAuthorizationError' }) + }) + + it('keeps a real human execution on the human workspace-role path in draft mode', async () => { + mocks.resolvePermission.mockResolvedValue('write') + + await expect( + authorizeWorkspaceOperation( + { + ...executorPrincipal( + { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + { workflowId: 'current-workflow-1', mode: 'draft' } + ), + subjectUserId: 'user-1', + }, + executorOperation, + context, + executorAuthorization + ) + ).resolves.toBeUndefined() + expect(mocks.resolvePermission).toHaveBeenCalledWith( + 'user-1', + 'workspace-1', + 'organization-1', + undefined, + { forUpdate: undefined } + ) + }) }) diff --git a/apps/sim/lib/core/application/workspace-authorization.ts b/apps/sim/lib/core/application/workspace-authorization.ts index dee740f955c..afbd6c5342f 100644 --- a/apps/sim/lib/core/application/workspace-authorization.ts +++ b/apps/sim/lib/core/application/workspace-authorization.ts @@ -212,7 +212,10 @@ export async function authorizeWorkspaceOperation { orgIds: ['Org_1', 'org_1', 'org_2'], }) }) + + it('normalizes the workspaceIds allowlist', () => { + expect(normalizeRule({ workspaceIds: [' ws_1 ', 'ws_1', ''] })).toEqual({ + workspaceIds: ['ws_1'], + }) + expect(normalizeRule({ workspaceIds: 'ws_1' })).toEqual({}) + }) }) describe('parseGateConfig', () => { @@ -61,6 +68,12 @@ describe('matchesRule', () => { expect(matchesRule({ orgIds: ['o1'] }, {}, false)).toBe(false) }) + it('matches the workspaceId allowlist', () => { + expect(matchesRule({ workspaceIds: ['w1'] }, { workspaceId: 'w1' }, false)).toBe(true) + expect(matchesRule({ workspaceIds: ['w1'] }, { workspaceId: 'w2' }, false)).toBe(false) + expect(matchesRule({ workspaceIds: ['w1'] }, {}, false)).toBe(false) + }) + it('matches the admin clause only with the supplied isAdmin', () => { expect(matchesRule({ adminEnabled: true }, { userId: 'u1' }, true)).toBe(true) expect(matchesRule({ adminEnabled: true }, { userId: 'u1' }, false)).toBe(false) diff --git a/apps/sim/lib/core/config/appconfig-rules.ts b/apps/sim/lib/core/config/appconfig-rules.ts index 34ceec5173a..786a5ec617f 100644 --- a/apps/sim/lib/core/config/appconfig-rules.ts +++ b/apps/sim/lib/core/config/appconfig-rules.ts @@ -10,11 +10,12 @@ /** * A single gating rule. A gate is open for a context when ANY clause matches: - * the global `enabled` default, the org/user allowlists, or `adminEnabled` for - * platform admins. An absent clause never matches. + * the global `enabled` default, the workspace/org/user allowlists, or + * `adminEnabled` for platform admins. An absent clause never matches. */ export interface AppConfigGateRule { enabled?: boolean + workspaceIds?: string[] orgIds?: string[] userIds?: string[] adminEnabled?: boolean @@ -28,6 +29,7 @@ export interface AppConfigGateRule { export interface AppConfigGateContext { userId?: string | null orgId?: string | null + workspaceId?: string | null isAdmin?: boolean } @@ -44,6 +46,8 @@ export function normalizeRule(value: unknown): AppConfigGateRule | null { const rule: AppConfigGateRule = {} if (typeof obj.enabled === 'boolean') rule.enabled = obj.enabled if (typeof obj.adminEnabled === 'boolean') rule.adminEnabled = obj.adminEnabled + const workspaceIds = normalizeIds(obj.workspaceIds) + if (workspaceIds) rule.workspaceIds = workspaceIds const orgIds = normalizeIds(obj.orgIds) if (orgIds) rule.orgIds = orgIds const userIds = normalizeIds(obj.userIds) @@ -75,6 +79,7 @@ export function matchesRule( if (rule.enabled) return true if (ctx.userId && rule.userIds?.includes(ctx.userId)) return true if (ctx.orgId && rule.orgIds?.includes(ctx.orgId)) return true + if (ctx.workspaceId && rule.workspaceIds?.includes(ctx.workspaceId)) return true if (rule.adminEnabled && isAdmin) return true return false } diff --git a/apps/sim/lib/core/config/enterprise-entitlements.ts b/apps/sim/lib/core/config/enterprise-entitlements.ts index 3c30d68b0fa..f48fd8ca9a0 100644 --- a/apps/sim/lib/core/config/enterprise-entitlements.ts +++ b/apps/sim/lib/core/config/enterprise-entitlements.ts @@ -37,6 +37,7 @@ export type EnterpriseFeature = | 'sandboxes' | 'sessionPolicies' | 'sso' + | 'usageMonitoring' | 'whitelabeling' /** @@ -59,6 +60,9 @@ export type EnterpriseFeature = * was always writable when billing was off and stays that way; only the * delete pass is gated here. Defaulting it on would start expiring logs on * upgrade against plan defaults the operator never chose. + * - `usageMonitoring` had no prior behavior at all — it ships with this flag — + * and it discloses every member's spend, so it stays opt-in rather than + * appearing unannounced on upgrade. * * `sandboxes` is deliberately `false`. A remote Function provider and immutable * base are operational prerequisites, so a billing-free deployment must opt in @@ -80,6 +84,7 @@ export const ENTERPRISE_FEATURE_LEGACY_DEFAULTS: Readonly { expect(await isFeatureEnabled('credential-groups')).toBe(true) }) - it('uses only the global AppConfig clause', async () => { + it('uses the global AppConfig clause', async () => { withAppConfig({ 'credential-groups': { enabled: true } }) expect(await isFeatureEnabled('credential-groups')).toBe(true) }) + + it('opens for an allowlisted workspace only', async () => { + withAppConfig({ 'credential-groups': { workspaceIds: ['ws-1'] } }) + expect(await isFeatureEnabled('credential-groups', { workspaceId: 'ws-1' })).toBe(true) + expect(await isFeatureEnabled('credential-groups', { workspaceId: 'ws-2' })).toBe(false) + expect(await isFeatureEnabled('credential-groups')).toBe(false) + }) + }) + + it('matches the workspaceIds clause', async () => { + withAppConfig({ f: { workspaceIds: ['ws-1'] } }) + expect(await enabled('f', { workspaceId: 'ws-1' })).toBe(true) + expect(await enabled('f', { workspaceId: 'ws-2' })).toBe(false) + expect(await enabled('f', { userId: 'ws-1' })).toBe(false) }) it('returns false for an unknown flag', async () => { diff --git a/apps/sim/lib/core/config/feature-flags.ts b/apps/sim/lib/core/config/feature-flags.ts index b9f75143ff0..b610508b7ee 100644 --- a/apps/sim/lib/core/config/feature-flags.ts +++ b/apps/sim/lib/core/config/feature-flags.ts @@ -13,8 +13,8 @@ const FEATURE_FLAGS_PROFILE = 'feature-flags' /** * A single flag's gating rule. A flag is ON for a context when ANY clause matches: - * the global `enabled` default, the org/user allowlists, or `adminEnabled` for - * platform admins. An absent clause never matches. Shape shared with the other + * the global `enabled` default, the workspace/org/user allowlists, or + * `adminEnabled` for platform admins. An absent clause never matches. Shape shared with the other * AppConfig gating documents via {@link AppConfigGateRule}. */ export type FeatureFlagRule = AppConfigGateRule @@ -33,7 +33,7 @@ export type FeatureFlagContext = AppConfigGateContext * AppConfig is not the source of truth (self-hosted/OSS, local dev, or hosted * without APPCONFIG_*). A truthy secret turns the flag on globally. * - * Gating by org/user/admin is available ONLY through the hosted AppConfig document + * Gating by workspace/org/user/admin is available ONLY through the hosted AppConfig document * — it deliberately cannot be expressed here, so no environment can grant (e.g.) * admin access from a code literal. To add a flag, register its name and the secret * to fall back on. @@ -44,7 +44,7 @@ export type FeatureFlagContext = AppConfigGateContext * `fallback` secret consulted when AppConfig isn't the source of truth (truthy ⇒ on * globally). * - * Gating by org/user/admin is deliberately NOT part of a definition — it lives only + * Gating by workspace/org/user/admin is deliberately NOT part of a definition — it lives only * in the hosted AppConfig document, so no environment can grant access from a code * literal. */ @@ -75,7 +75,8 @@ const FEATURE_FLAGS = { 'credential-groups': { description: 'Workspace-owned collections that gather managed OAuth credentials from external users. ' + - 'Global on/off only; hosted workspaces must also have an Enterprise subscription.', + 'Gated by workspaceId via AppConfig (or globally); hosted workspaces must also have an ' + + 'Enterprise subscription. Off-AppConfig falls back to CREDENTIAL_GROUPS.', fallback: 'CREDENTIAL_GROUPS', }, } satisfies Record @@ -108,8 +109,8 @@ async function resolveAdmin(userId: string): Promise { } /** - * The admin clause is resolved last and lazily: a global/userId/orgId match - * short-circuits before any DB read, a rule without `adminEnabled` never queries, + * The admin clause is resolved last and lazily: a global/userId/orgId/workspaceId + * match short-circuits before any DB read, a rule without `adminEnabled` never queries, * and a missing `userId` resolves to `false` without a query. */ async function evaluate( diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts index 08cd4ad311e..7a2046fdb5a 100644 --- a/apps/sim/lib/core/security/input-validation.server.ts +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -372,6 +372,8 @@ export interface SecureFetchOptions { stripAuthOnRedirect?: boolean /** Omit for the historical behavior used by existing workflows. */ redirectPolicy?: HttpRedirectPolicy + /** Rejects a redirect target before DNS resolution or a follow-up request is attempted. */ + assertRedirectTarget?: (url: string) => void /** * Pre-validated, IP-pinned `http://` proxy URL (see {@link validateAndPinProxyUrl}). * When set, the connection routes through this proxy and target-IP pinning is @@ -1059,6 +1061,12 @@ export async function secureFetchWithPinnedIP( res.resume() const redirectUrl = resolveRedirectUrl(url, location) + try { + options.assertRedirectTarget?.(redirectUrl) + } catch (error) { + settledReject(error) + return + } validateUrlWithDNS(redirectUrl, 'redirectUrl', { allowHttp: options.allowHttp }) .then((validation) => { if (!validation.isValid) { diff --git a/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts b/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts index 5d0141add32..cccf9ddfa8d 100644 --- a/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts +++ b/apps/sim/lib/core/security/pinned-redirect-replay.server.test.ts @@ -56,6 +56,29 @@ async function startRecordingServer(hops: RecordedHop[]): Promise { } describe('secureFetchWithPinnedIP redirect replay', () => { + it('rejects a redirect target before following it', async () => { + const hops: RecordedHop[] = [] + const target = await startRecordingServer(hops) + const origin = await startServer((req, res) => { + req.resume() + res.writeHead(302, { location: `${target}/after` }) + res.end() + }) + const assertRedirectTarget = vi.fn((url: string) => { + if (url === `${target}/after`) throw new Error('redirect target rejected') + }) + + await expect( + secureFetchWithPinnedIP(origin, '127.0.0.1', { + allowHttp: true, + assertRedirectTarget, + }) + ).rejects.toThrow('redirect target rejected') + + expect(assertRedirectTarget).toHaveBeenCalledWith(`${target}/after`) + expect(hops).toEqual([]) + }) + it('preserves historical replay when no redirect policy is present', async () => { const hops: RecordedHop[] = [] const target = await startRecordingServer(hops) diff --git a/apps/sim/lib/core/utils/stream-limits.ts b/apps/sim/lib/core/utils/stream-limits.ts index 246983ab906..18af9773f79 100644 --- a/apps/sim/lib/core/utils/stream-limits.ts +++ b/apps/sim/lib/core/utils/stream-limits.ts @@ -242,7 +242,7 @@ export async function readNodeStreamToBufferWithLimit( const onAbort = () => { if ('destroy' in stream && typeof stream.destroy === 'function') { - stream.destroy(toError(options.signal?.reason ?? new Error('Aborted'))) + stream.destroy() } finish(() => reject(toError(options.signal?.reason ?? new Error('Aborted')))) } diff --git a/apps/sim/lib/core/utils/timezone.ts b/apps/sim/lib/core/utils/timezone.ts index 5118061bb4d..297c0a65ec8 100644 --- a/apps/sim/lib/core/utils/timezone.ts +++ b/apps/sim/lib/core/utils/timezone.ts @@ -1,3 +1,5 @@ +import { truncate } from '@sim/utils/string' + /** * A curated fallback for runtimes without `Intl.supportedValuesOf` (e.g. Safari * < 15.4), so the timezone picker is never an empty dead-end. @@ -26,6 +28,37 @@ export function getBrowserTimezone(): string { return Intl.DateTimeFormat().resolvedOptions().timeZone } +/** Whether the runtime recognizes `timezone` as an IANA name. */ +export function isValidTimezone(timezone: string): boolean { + try { + new Intl.DateTimeFormat('en-US', { timeZone: timezone }) + return true + } catch { + return false + } +} + +/** + * Rejects a timezone that is not an IANA name. + * + * Timezones reach SQL through `AT TIME ZONE`, which takes an identifier rather than + * a bound parameter, so an unvalidated value off a query string would be + * interpolated into the statement. Every bucketed aggregate calls this first. + * + * Request boundaries should reject the value earlier, through {@link isValidTimezone} + * in their contract, so a bad query param is a 400 rather than the 500 this throw + * projects to. This stays as the backstop for every non-HTTP caller. + */ +export function assertValidTimezone(timezone: string): void { + if (!isValidTimezone(timezone)) { + // Echoed back trimmed and stripped of line breaks: the rejected value came off + // a query string, and a raw one carrying newlines or U+2028/U+2029 would forge + // extra lines in whatever log or error surface renders the message. + const safe = truncate(timezone.replace(/[\p{Cc}\p{Zl}\p{Zp}]/gu, ' '), 64) + throw new Error(`Invalid timezone: ${safe}. Use an IANA name like "America/Los_Angeles".`) + } +} + /** * Every IANA timezone identifier the runtime knows, for populating a picker; * falls back to a curated common set on runtimes without `Intl.supportedValuesOf`. diff --git a/apps/sim/lib/credential-groups/application/context.ts b/apps/sim/lib/credential-groups/application/context.ts index 3329e7a3b9d..7ca35722a39 100644 --- a/apps/sim/lib/credential-groups/application/context.ts +++ b/apps/sim/lib/credential-groups/application/context.ts @@ -10,7 +10,7 @@ import { loadActiveWorkspaceApplicationContext } from '@/lib/workspaces/applicat export async function requireCredentialGroupsAvailable(workspaceId: string): Promise { const ownerBilling = await getWorkspaceOwnerSubscriptionAccess(workspaceId) - const availability = await resolveCredentialGroupsAvailability(ownerBilling) + const availability = await resolveCredentialGroupsAvailability({ workspaceId, ownerBilling }) if (!availability.available) { const message = availability.reason === 'enterprise_plan_required' @@ -22,7 +22,7 @@ export async function requireCredentialGroupsAvailable(workspaceId: string): Pro export async function requireCredentialGroupSettingsAvailable(workspaceId: string): Promise { const ownerBilling = await getWorkspaceOwnerSubscriptionAccess(workspaceId) - if (!(await isCredentialGroupsAvailable(ownerBilling))) { + if (!(await isCredentialGroupsAvailable({ workspaceId, ownerBilling }))) { throw new OrchestrationError('not_found', 'Credential Groups are not available') } } diff --git a/apps/sim/lib/credential-groups/application/list-credentials.test.ts b/apps/sim/lib/credential-groups/application/list-credentials.test.ts index 32b62578b11..6739b8ee581 100644 --- a/apps/sim/lib/credential-groups/application/list-credentials.test.ts +++ b/apps/sim/lib/credential-groups/application/list-credentials.test.ts @@ -99,6 +99,11 @@ function executorPrincipal(credentialGroupId = 'group-1'): WorkflowExecutionDele kind: 'workflow_execution', workflowId: 'workflow-1', principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + currentWorkflow: { + workflowId: 'workflow-1', + mode: 'deployment', + deploymentVersionId: 'deployment-version-1', + }, }, } } diff --git a/apps/sim/lib/credential-groups/application/slack-managed-users.ts b/apps/sim/lib/credential-groups/application/slack-managed-users.ts index a448edbd284..8480195f87d 100644 --- a/apps/sim/lib/credential-groups/application/slack-managed-users.ts +++ b/apps/sim/lib/credential-groups/application/slack-managed-users.ts @@ -18,7 +18,7 @@ import { loadActiveWorkspaceApplicationContext } from '@/lib/workspaces/applicat async function requireCredentialGroups(workspaceId: string): Promise { const ownerBilling = await getWorkspaceOwnerSubscriptionAccess(workspaceId) - if (!(await isCredentialGroupsAvailable(ownerBilling))) { + if (!(await isCredentialGroupsAvailable({ workspaceId, ownerBilling }))) { throw new OrchestrationError('not_found', 'Credential Groups are not available') } } diff --git a/apps/sim/lib/credential-groups/availability.test.ts b/apps/sim/lib/credential-groups/availability.test.ts index 51960c8184d..2a51f7c6225 100644 --- a/apps/sim/lib/credential-groups/availability.test.ts +++ b/apps/sim/lib/credential-groups/availability.test.ts @@ -25,7 +25,12 @@ describe('resolveCredentialGroupsAvailability', () => { it('attributes a disabled feature flag before considering the plan', async () => { mockIsFeatureEnabled.mockResolvedValue(false) - await expect(resolveCredentialGroupsAvailability({ isEnterprise: false })).resolves.toEqual({ + await expect( + resolveCredentialGroupsAvailability({ + workspaceId: 'ws-1', + ownerBilling: { isEnterprise: false }, + }) + ).resolves.toEqual({ available: false, reason: 'feature_disabled', }) @@ -34,16 +39,37 @@ describe('resolveCredentialGroupsAvailability', () => { it('requires Enterprise when the hosted feature is enabled', async () => { mockIsFeatureEnabled.mockResolvedValue(true) - await expect(resolveCredentialGroupsAvailability({ isEnterprise: false })).resolves.toEqual({ + await expect( + resolveCredentialGroupsAvailability({ + workspaceId: 'ws-1', + ownerBilling: { isEnterprise: false }, + }) + ).resolves.toEqual({ available: false, reason: 'enterprise_plan_required', }) }) + it('evaluates the flag against the workspace id', async () => { + mockIsFeatureEnabled.mockResolvedValue(true) + + await resolveCredentialGroupsAvailability({ + workspaceId: 'ws-1', + ownerBilling: { isEnterprise: true }, + }) + + expect(mockIsFeatureEnabled).toHaveBeenCalledWith('credential-groups', { workspaceId: 'ws-1' }) + }) + it('allows Enterprise workspaces when the hosted feature is enabled', async () => { mockIsFeatureEnabled.mockResolvedValue(true) - await expect(resolveCredentialGroupsAvailability({ isEnterprise: true })).resolves.toEqual({ + await expect( + resolveCredentialGroupsAvailability({ + workspaceId: 'ws-1', + ownerBilling: { isEnterprise: true }, + }) + ).resolves.toEqual({ available: true, }) }) diff --git a/apps/sim/lib/credential-groups/availability.ts b/apps/sim/lib/credential-groups/availability.ts index cc56832f87c..76852800476 100644 --- a/apps/sim/lib/credential-groups/availability.ts +++ b/apps/sim/lib/credential-groups/availability.ts @@ -5,10 +5,21 @@ export type CredentialGroupsAvailability = | { available: true } | { available: false; reason: 'feature_disabled' | 'enterprise_plan_required' } -export async function resolveCredentialGroupsAvailability(ownerBilling: { - isEnterprise: boolean -}): Promise { - if (!(await isFeatureEnabled('credential-groups'))) { +/** + * The workspace the gate is evaluated for. `workspaceId` is required so no call + * site can silently fall back to the global clause and reveal the feature to a + * workspace the AppConfig `credential-groups` allowlist does not name. + */ +export interface CredentialGroupsAvailabilityInput { + workspaceId: string + ownerBilling: { isEnterprise: boolean } +} + +export async function resolveCredentialGroupsAvailability({ + workspaceId, + ownerBilling, +}: CredentialGroupsAvailabilityInput): Promise { + if (!(await isFeatureEnabled('credential-groups', { workspaceId }))) { return { available: false, reason: 'feature_disabled' } } if (isHosted && !ownerBilling.isEnterprise) { @@ -17,9 +28,12 @@ export async function resolveCredentialGroupsAvailability(ownerBilling: { return { available: true } } -/** Credential Groups are globally gated and restricted to Enterprise workspaces on Sim Cloud. */ -export async function isCredentialGroupsAvailable(ownerBilling: { - isEnterprise: boolean -}): Promise { - return (await resolveCredentialGroupsAvailability(ownerBilling)).available +/** + * Credential Groups are gated per workspace (globally or by the AppConfig + * `workspaceIds` allowlist) and restricted to Enterprise workspaces on Sim Cloud. + */ +export async function isCredentialGroupsAvailable( + input: CredentialGroupsAvailabilityInput +): Promise { + return (await resolveCredentialGroupsAvailability(input)).available } diff --git a/apps/sim/lib/credential-groups/enrollments.ts b/apps/sim/lib/credential-groups/enrollments.ts index c389d884b8b..093ddaad70a 100644 --- a/apps/sim/lib/credential-groups/enrollments.ts +++ b/apps/sim/lib/credential-groups/enrollments.ts @@ -227,7 +227,8 @@ async function resolvePublicEnrollmentRowByIdentity( if (row.enrollment.invitationExpiresAt.getTime() <= Date.now()) return null const ownerBilling = await getWorkspaceOwnerSubscriptionAccess(row.workspaceId) - if (!(await isCredentialGroupsAvailable(ownerBilling))) return null + if (!(await isCredentialGroupsAvailable({ workspaceId: row.workspaceId, ownerBilling }))) + return null return row } diff --git a/apps/sim/lib/credentials/environment.ts b/apps/sim/lib/credentials/environment.ts index ad808306d30..9756c4659e7 100644 --- a/apps/sim/lib/credentials/environment.ts +++ b/apps/sim/lib/credentials/environment.ts @@ -78,8 +78,9 @@ export async function getCredentialCreationWorkspaceContext(params: { }) .from(workspace) .where(and(eq(workspace.id, params.workspaceId), isNull(workspace.archivedAt))) + /** `FOR NO KEY UPDATE`: see the module header of `lib/billing/storage/tracking.ts`. */ const [workspaceRow] = params.forUpdate - ? await workspaceQuery.for('update').limit(1) + ? await workspaceQuery.for('no key update').limit(1) : await workspaceQuery.limit(1) if (!workspaceRow) return null diff --git a/apps/sim/lib/credentials/managed-oauth.ts b/apps/sim/lib/credentials/managed-oauth.ts index 4edca04150e..134b1f711e6 100644 --- a/apps/sim/lib/credentials/managed-oauth.ts +++ b/apps/sim/lib/credentials/managed-oauth.ts @@ -290,7 +290,7 @@ export async function resolveManagedOAuthToken( } const ownerBilling = await getWorkspaceOwnerSubscriptionAccess(initial.workspaceId) - if (!(await isCredentialGroupsAvailable(ownerBilling))) { + if (!(await isCredentialGroupsAvailable({ workspaceId: initial.workspaceId, ownerBilling }))) { throw new ManagedOAuthCredentialError( 'MANAGED_CREDENTIAL_UNAVAILABLE', 'Managed credentials are not available for this workspace', diff --git a/apps/sim/lib/custom-tools/application/operations.ts b/apps/sim/lib/custom-tools/application/operations.ts index 2bee97b7cf8..fbeace41633 100644 --- a/apps/sim/lib/custom-tools/application/operations.ts +++ b/apps/sim/lib/custom-tools/application/operations.ts @@ -28,6 +28,13 @@ export const customToolOperations = { workspaceApiKey: 'allow', ...ALL_PRINCIPAL_POLICY, }), + readAvailableByIdOrTitle: defineWorkspaceOperation({ + id: 'custom_tools.read_available_by_id_or_title', + minimumRole: 'read', + workspaceApiKey: 'deny', + principalKinds: ['delegated'], + delegatedServices: ['copilot', 'executor'], + }), create: defineWorkspaceOperation({ id: 'custom_tools.create', minimumRole: 'write', diff --git a/apps/sim/lib/custom-tools/application/use-cases.test.ts b/apps/sim/lib/custom-tools/application/use-cases.test.ts index b7f7136c421..98acac58fe4 100644 --- a/apps/sim/lib/custom-tools/application/use-cases.test.ts +++ b/apps/sim/lib/custom-tools/application/use-cases.test.ts @@ -1,12 +1,14 @@ /** * @vitest-environment node */ +import type { DelegatedPrincipal } from '@sim/auth/principal' import { beforeEach, describe, expect, it, vi } from 'vitest' const { mocks } = vi.hoisted(() => ({ mocks: { loadContext: vi.fn(), resolvePermission: vi.fn(), + getAvailableTool: vi.fn(), getByTitle: vi.fn(), getWorkspaceTool: vi.fn(), updateWorkspaceTool: vi.fn(), @@ -35,6 +37,7 @@ vi.mock('@sim/audit', () => ({ vi.mock('@/lib/workflows/custom-tools/operations', () => ({ deleteCustomTool: vi.fn(), deleteWorkspaceCustomTool: vi.fn(), + getAvailableCustomTool: mocks.getAvailableTool, getCustomToolById: vi.fn(), getWorkspaceCustomTool: mocks.getWorkspaceTool, getWorkspaceCustomToolByTitle: mocks.getByTitle, @@ -46,8 +49,10 @@ vi.mock('@/lib/workflows/custom-tools/operations', () => ({ })) import { CUSTOM_TOOL_DELEGATION_AUDIENCE } from '@/lib/custom-tools/application/authorization' +import { customToolOperations } from '@/lib/custom-tools/application/operations' import { createWorkspaceCustomToolUseCase, + readAvailableCustomToolByIdOrTitleUseCase, saveWorkspaceCustomToolUseCase, updateWorkspaceCustomToolUseCase, } from '@/lib/custom-tools/application/use-cases' @@ -75,9 +80,153 @@ describe('custom tool application use cases', () => { mocks.loadContext.mockResolvedValue(workspace) mocks.resolvePermission.mockResolvedValue('write') mocks.getByTitle.mockResolvedValue(null) + mocks.getAvailableTool.mockResolvedValue(tool) mocks.upsert.mockResolvedValue([tool]) }) + describe('delegated custom-tool resolution', () => { + function executorPrincipal(overrides: Partial = {}): DelegatedPrincipal { + return { + kind: 'delegated' as const, + serviceId: 'executor' as const, + subjectUserId: 'user-1', + workspaceId: workspace.workspaceId, + delegationId: 'execution-1', + audience: CUSTOM_TOOL_DELEGATION_AUDIENCE, + issuedAt: new Date(Date.now() - 1_000), + expiresAt: new Date(Date.now() + 60_000), + ...overrides, + } + } + + it('declares the executor and Copilot read policy explicitly', () => { + expect(customToolOperations.readAvailableByIdOrTitle).toMatchObject({ + id: 'custom_tools.read_available_by_id_or_title', + minimumRole: 'read', + workspaceApiKey: 'deny', + principalKinds: ['delegated'], + delegatedServices: ['copilot', 'executor'], + }) + }) + + it('authorizes the current subject and preserves workspace-first personal fallback lookup', async () => { + mocks.resolvePermission.mockResolvedValueOnce('read') + + const result = await readAvailableCustomToolByIdOrTitleUseCase.execute({ + principal: executorPrincipal(), + input: { + workspaceId: workspace.workspaceId, + identifier: tool.id, + lookup: 'id_or_title', + }, + }) + + expect(result).toEqual({ tool }) + expect(mocks.getAvailableTool).toHaveBeenCalledWith({ + identifier: tool.id, + userId: 'user-1', + workspaceId: workspace.workspaceId, + lookup: 'id_or_title', + }) + expect(mocks.audit).not.toHaveBeenCalled() + }) + + it('authorizes an actorless deployment without enabling personal fallback', async () => { + const result = await readAvailableCustomToolByIdOrTitleUseCase.execute({ + principal: executorPrincipal({ + subjectUserId: undefined, + delegationContext: { + kind: 'workflow_execution', + workflowId: 'workflow-1', + executionId: 'execution-1', + principal: { + kind: 'system', + serviceId: 'schedule', + workspaceId: workspace.workspaceId, + workflowId: 'workflow-1', + }, + currentWorkflow: { + workflowId: 'workflow-1', + mode: 'deployment', + deploymentVersionId: 'version-1', + }, + }, + }), + input: { + workspaceId: workspace.workspaceId, + identifier: tool.id, + lookup: 'id_or_title', + }, + }) + + expect(result).toEqual({ tool }) + expect(mocks.resolvePermission).not.toHaveBeenCalled() + expect(mocks.getAvailableTool).toHaveBeenCalledWith({ + identifier: tool.id, + workspaceId: workspace.workspaceId, + lookup: 'id_or_title', + }) + }) + + it('conceals a workspace assertion outside the delegated workspace before lookup', async () => { + mocks.loadContext.mockResolvedValueOnce({ ...workspace, workspaceId: 'workspace-2' }) + + await expect( + readAvailableCustomToolByIdOrTitleUseCase.execute({ + principal: executorPrincipal(), + input: { + workspaceId: 'workspace-2', + identifier: tool.id, + lookup: 'id', + }, + }) + ).rejects.toMatchObject({ code: 'forbidden' }) + + expect(mocks.resolvePermission).not.toHaveBeenCalled() + expect(mocks.getAvailableTool).not.toHaveBeenCalled() + }) + + it('rejects the wrong delegation audience before lookup', async () => { + await expect( + readAvailableCustomToolByIdOrTitleUseCase.execute({ + principal: executorPrincipal({ audience: 'sim:other' }), + input: { + workspaceId: workspace.workspaceId, + identifier: tool.id, + lookup: 'id', + }, + }) + ).rejects.toMatchObject({ code: 'forbidden' }) + + expect(mocks.resolvePermission).not.toHaveBeenCalled() + expect(mocks.getAvailableTool).not.toHaveBeenCalled() + }) + + it('re-checks the delegated subject current workspace access before lookup', async () => { + mocks.resolvePermission.mockResolvedValueOnce(null) + + await expect( + readAvailableCustomToolByIdOrTitleUseCase.execute({ + principal: executorPrincipal(), + input: { + workspaceId: workspace.workspaceId, + identifier: tool.id, + lookup: 'id', + }, + }) + ).rejects.toMatchObject({ code: 'forbidden' }) + + expect(mocks.resolvePermission).toHaveBeenCalledWith( + 'user-1', + workspace.workspaceId, + workspace.workspaceOrganizationId, + undefined, + { forUpdate: undefined } + ) + expect(mocks.getAvailableTool).not.toHaveBeenCalled() + }) + }) + it('uses compatibility attribution without impersonating a workspace-key audit actor', async () => { const principal = { kind: 'workspace_api_key' as const, diff --git a/apps/sim/lib/custom-tools/application/use-cases.ts b/apps/sim/lib/custom-tools/application/use-cases.ts index 7da69ade2c4..177b6e379b1 100644 --- a/apps/sim/lib/custom-tools/application/use-cases.ts +++ b/apps/sim/lib/custom-tools/application/use-cases.ts @@ -3,6 +3,7 @@ import { type Principal, requirePrincipalSubjectUserId, resolvePrincipalAttribution, + resolvePrincipalSubject, } from '@sim/auth/principal' import type { customTools } from '@sim/db/schema' import { getErrorMessage, getPostgresErrorCode } from '@sim/utils/errors' @@ -20,6 +21,7 @@ import { type CustomToolSortBy, deleteCustomTool, deleteWorkspaceCustomTool, + getAvailableCustomTool, getCustomToolById, getWorkspaceCustomTool, getWorkspaceCustomToolByTitle, @@ -149,6 +151,29 @@ export const getWorkspaceCustomToolUseCase = defineAuthorizedWorkspaceUseCase({ }, }) +export interface ReadAvailableCustomToolByIdOrTitleInput { + workspaceId: string + identifier: string + lookup: 'id' | 'id_or_title' +} + +export const readAvailableCustomToolByIdOrTitleUseCase = defineAuthorizedWorkspaceUseCase({ + operation: customToolOperations.readAvailableByIdOrTitle, + resolveContext: ({ input }: { input: ReadAvailableCustomToolByIdOrTitleInput }) => + resolveWorkspaceContext(input.workspaceId), + authorizationOptions, + async execute({ principal, input, context }) { + const subject = resolvePrincipalSubject(principal) + const tool = await getAvailableCustomTool({ + identifier: input.identifier, + ...(subject?.kind === 'sim_user' ? { userId: subject.userId } : {}), + workspaceId: context.workspaceId, + lookup: input.lookup, + }) + return { tool } + }, +}) + export interface CreateWorkspaceCustomToolInput { workspaceId: string title: string diff --git a/apps/sim/lib/embeddings/client.ts b/apps/sim/lib/embeddings/client.ts index b761f8297af..6e1682ba3b2 100644 --- a/apps/sim/lib/embeddings/client.ts +++ b/apps/sim/lib/embeddings/client.ts @@ -434,7 +434,8 @@ async function callEmbeddingAPI( * native size is a 400, not a no-op. */ requestedDimensions: number | undefined, - expectedDimensions: number | undefined + expectedDimensions: number | undefined, + signal?: AbortSignal ): Promise<{ embeddings: number[][]; totalTokens: number; dimensions: number }> { return retryWithExponentialBackoff( async () => { @@ -448,7 +449,11 @@ async function callEmbeddingAPI( dimensions: requestedDimensions, }) + signal?.throwIfAborted() const controller = new AbortController() + const onAbort = () => controller.abort(signal?.reason) + signal?.addEventListener('abort', onAbort, { once: true }) + if (signal?.aborted) onAbort() const timeout = setTimeout(() => controller.abort(), EMBEDDING_REQUEST_TIMEOUT_MS) const response = await fetch(request.apiUrl, { @@ -456,7 +461,10 @@ async function callEmbeddingAPI( headers: request.headers, body: JSON.stringify(request.body), signal: controller.signal, - }).finally(() => clearTimeout(timeout)) + }).finally(() => { + clearTimeout(timeout) + signal?.removeEventListener('abort', onAbort) + }) if (!response.ok) { const classificationBody = await readEmbeddingErrorBody(response) @@ -531,7 +539,8 @@ async function callEmbeddingAPI( initialDelayMs: 1000, maxDelayMs: EMBEDDING_MAX_RETRY_DELAY_MS, retryBudgetMs: EMBEDDING_RETRY_BUDGET_MS, - retryCondition: isWorthRetrying, + retryCondition: (error) => !signal?.aborted && isWorthRetrying(error), + signal, } ) } @@ -602,8 +611,10 @@ async function embedWithProvider( model: string, taskType: EmbeddingTaskType, requestedDimensions: number | undefined, - provider: ResolvedProvider + provider: ResolvedProvider, + signal?: AbortSignal ): Promise { + signal?.throwIfAborted() assertEmbeddingAggregateResponseWithinLimit(boundedInputs.length, provider.dimensions) const batches = createEmbeddingBatches( boundedInputs, @@ -618,6 +629,7 @@ async function embedWithProvider( MAX_CONCURRENT_BATCHES, async (batch, i) => { try { + signal?.throwIfAborted() return await callEmbeddingAPI( batch, provider.adapter, @@ -626,7 +638,8 @@ async function embedWithProvider( provider.providerId, provider.quotaCircuitIdentity, requestedDimensions, - provider.dimensions + provider.dimensions, + signal ) } catch (error) { const message = `Failed to generate embeddings for batch ${i + 1}/${batches.length}:` @@ -748,6 +761,7 @@ function combineEmbeddingBatches( * per-provider item caps, bounded concurrency, and retry on transient failures. */ export async function embed(texts: string[], options: EmbedOptions): Promise { + options.signal?.throwIfAborted() const model = options.model ?? DEFAULT_EMBEDDING_MODEL const taskType = options.taskType ?? 'document' const provider = await resolveProvider(model, options) @@ -757,7 +771,14 @@ export async function embed(texts: string[], options: EmbedOptions): Promise - callOpenRouterBatch(batch, firstResult.dimensions) - ) + const remainingResults = await mapWithConcurrency(batches, MAX_CONCURRENT_BATCHES, (batch) => { + options.signal?.throwIfAborted() + return callOpenRouterBatch(batch, firstResult.dimensions) + }) batchResults = [firstResult, ...remainingResults] } else { assertEmbeddingAggregateResponseWithinLimit(boundedInputs.length, options.dimensions) @@ -827,9 +851,10 @@ export async function embedOpenRouter( adapter.maxItemsPerRequest, options.dimensions ) - batchResults = await mapWithConcurrency(batches, MAX_CONCURRENT_BATCHES, (batch) => - callOpenRouterBatch(batch, options.dimensions) - ) + batchResults = await mapWithConcurrency(batches, MAX_CONCURRENT_BATCHES, (batch) => { + options.signal?.throwIfAborted() + return callOpenRouterBatch(batch, options.dimensions) + }) } const result = combineEmbeddingBatches(batchResults) diff --git a/apps/sim/lib/embeddings/openrouter-model-catalog.server.test.ts b/apps/sim/lib/embeddings/openrouter-model-catalog.server.test.ts index 8dd24a150a7..1e6ed46e09d 100644 --- a/apps/sim/lib/embeddings/openrouter-model-catalog.server.test.ts +++ b/apps/sim/lib/embeddings/openrouter-model-catalog.server.test.ts @@ -20,12 +20,11 @@ describe('OpenRouter embedding model catalog', () => { }) it('resolves a prefixed model with its live input ceiling', async () => { - fetchMock.mockResolvedValue({ - ok: true, - json: async () => ({ + fetchMock.mockResolvedValue( + Response.json({ data: [{ id: 'qwen/qwen3-embedding-8b', context_length: 32768 }], - }), - }) + }) + ) await expect( getOpenRouterEmbeddingModelMetadata('openrouter/qwen/qwen3-embedding-8b') @@ -36,7 +35,7 @@ describe('OpenRouter embedding model catalog', () => { }) it('rejects a model absent from the live embedding catalog', async () => { - fetchMock.mockResolvedValue({ ok: true, json: async () => ({ data: [] }) }) + fetchMock.mockResolvedValue(Response.json({ data: [] })) await expect( getOpenRouterEmbeddingModelMetadata('openrouter/example/missing') @@ -44,10 +43,7 @@ describe('OpenRouter embedding model catalog', () => { }) it('fails fast when OpenRouter omits a model context length', async () => { - fetchMock.mockResolvedValue({ - ok: true, - json: async () => ({ data: [{ id: 'example/missing-context' }] }), - }) + fetchMock.mockResolvedValue(Response.json({ data: [{ id: 'example/missing-context' }] })) await expect( getOpenRouterEmbeddingModelMetadata('openrouter/example/missing-context') diff --git a/apps/sim/lib/embeddings/openrouter-model-catalog.server.ts b/apps/sim/lib/embeddings/openrouter-model-catalog.server.ts index f26ee8697c5..b65f499b892 100644 --- a/apps/sim/lib/embeddings/openrouter-model-catalog.server.ts +++ b/apps/sim/lib/embeddings/openrouter-model-catalog.server.ts @@ -1,10 +1,12 @@ import { openRouterEmbeddingModelsUpstreamResponseSchema } from '@/lib/api/contracts/providers' +import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { toOpenRouterEmbeddingModelId, toOpenRouterWireEmbeddingModelId, } from '@/lib/embeddings/openrouter-models' const OPENROUTER_EMBEDDING_MODELS_URL = 'https://openrouter.ai/api/v1/embeddings/models' +const MAX_OPENROUTER_EMBEDDING_CATALOG_BYTES = 4 * 1024 * 1024 export interface OpenRouterEmbeddingModelMetadata { id: string @@ -19,12 +21,13 @@ export class OpenRouterEmbeddingModelNotFoundError extends Error { } /** Loads OpenRouter's current embedding-only catalog with its input ceilings. */ -export async function fetchOpenRouterEmbeddingModelCatalog(): Promise< - OpenRouterEmbeddingModelMetadata[] -> { +export async function fetchOpenRouterEmbeddingModelCatalog( + signal?: AbortSignal +): Promise { const response = await fetch(OPENROUTER_EMBEDDING_MODELS_URL, { headers: { 'Content-Type': 'application/json' }, next: { revalidate: 300 }, + signal, }) if (!response.ok) { throw new Error( @@ -32,7 +35,13 @@ export async function fetchOpenRouterEmbeddingModelCatalog(): Promise< ) } - const data = openRouterEmbeddingModelsUpstreamResponseSchema.parse(await response.json()) + const data = openRouterEmbeddingModelsUpstreamResponseSchema.parse( + await readResponseJsonWithLimit(response, { + maxBytes: MAX_OPENROUTER_EMBEDDING_CATALOG_BYTES, + label: 'OpenRouter embedding model catalog', + signal, + }) + ) const models = new Map() for (const model of data.data) { const id = toOpenRouterEmbeddingModelId(model.id) @@ -43,10 +52,11 @@ export async function fetchOpenRouterEmbeddingModelCatalog(): Promise< /** Resolves and validates one selected model against OpenRouter's live catalog. */ export async function getOpenRouterEmbeddingModelMetadata( - model: string + model: string, + signal?: AbortSignal ): Promise { const normalizedId = toOpenRouterEmbeddingModelId(toOpenRouterWireEmbeddingModelId(model)) - const metadata = (await fetchOpenRouterEmbeddingModelCatalog()).find( + const metadata = (await fetchOpenRouterEmbeddingModelCatalog(signal)).find( (candidate) => candidate.id === normalizedId ) if (!metadata) throw new OpenRouterEmbeddingModelNotFoundError(model) diff --git a/apps/sim/lib/embeddings/types.ts b/apps/sim/lib/embeddings/types.ts index 33a7677d539..f7b6b647110 100644 --- a/apps/sim/lib/embeddings/types.ts +++ b/apps/sim/lib/embeddings/types.ts @@ -78,6 +78,8 @@ export type EmbeddingAdapterFactory EmbeddingProviderAdapter export interface EmbedOptions { + /** Cancels provider requests, retry waits, and remaining batches. */ + signal?: AbortSignal /** Catalog model id. Defaults to the platform default when omitted. */ model?: string /** Transport override for catalog models exposed through another provider. */ @@ -121,6 +123,8 @@ export interface EmbedResult { } export interface OpenRouterEmbedOptions { + /** Cancels provider requests, retry waits, and remaining batches. */ + signal?: AbortSignal apiKey: string model?: string /** Per-input ceiling reported by OpenRouter's embedding model catalog. */ diff --git a/apps/sim/lib/environment/utils.ts b/apps/sim/lib/environment/utils.ts index ea10b6d1546..065d610c0b0 100644 --- a/apps/sim/lib/environment/utils.ts +++ b/apps/sim/lib/environment/utils.ts @@ -151,6 +151,15 @@ export async function getPersonalAndWorkspaceEnv( let workspaceCanAdmin = false if (workspaceId) { const access = options?.workspaceAccess ?? (await checkWorkspaceAccess(workspaceId, userId)) + /** + * A workspace that no longer exists and one the caller may not read are different facts + * and take different corrections — stop using the id versus ask for access. Collapsing + * them sent every deleted-workspace call down the access-denied path, where it read as a + * permissions problem nobody could reproduce. + */ + if (!access.exists) { + throw new Error(`Workspace ${workspaceId} does not exist`) + } if (!access.hasAccess) { throw new Error(`Access denied to workspace ${workspaceId}`) } diff --git a/apps/sim/lib/execution/durable-secret-provenance.test.ts b/apps/sim/lib/execution/durable-secret-provenance.test.ts index 00a29654067..500dd5b386a 100644 --- a/apps/sim/lib/execution/durable-secret-provenance.test.ts +++ b/apps/sim/lib/execution/durable-secret-provenance.test.ts @@ -104,6 +104,16 @@ describe('private durable provenance scope admission', () => { }) }) + it('admits workspace-scoped execution without fabricating a destination user', () => { + expect( + durableSecretProvenanceFromPrivateBundle( + privateBundle({ userId: 'workflow-owner', workspaceId: 'workspace-1' }), + 'value', + { workspaceId: 'workspace-1' } + ) + ).toMatchObject({ status: 'exact' }) + }) + it('rejects a source from another or no workspace', () => { expect( durableSecretProvenanceFromPrivateBundle( diff --git a/apps/sim/lib/execution/durable-secret-provenance.ts b/apps/sim/lib/execution/durable-secret-provenance.ts index d6918c764b9..e4424a47ec7 100644 --- a/apps/sim/lib/execution/durable-secret-provenance.ts +++ b/apps/sim/lib/execution/durable-secret-provenance.ts @@ -126,7 +126,9 @@ export function durableSecretProvenanceFromRegistry( */ export function isPrivateSecretProvenanceScopeCompatible( sourceScope: ResolvedSecretTraceScopeV1 | undefined, - destinationScope: { userId: string; workspaceId?: string } + destinationScope: + | { workspaceId: string; userId?: string } + | { userId: string; workspaceId?: undefined } ): sourceScope is ResolvedSecretTraceScopeV1 { if (!sourceScope) return false if (destinationScope.workspaceId !== undefined) { @@ -139,7 +141,9 @@ export function isPrivateSecretProvenanceScopeCompatible( export function durableSecretProvenanceFromPrivateBundle( value: unknown, selectionKey: string, - destinationScope: { userId: string; workspaceId?: string } + destinationScope: + | { workspaceId: string; userId?: string } + | { userId: string; workspaceId?: undefined } ): DurableSecretProvenance | undefined { if (!isPrivateSecretProvenanceBundleV1(value)) return undefined const bundle: PrivateSecretProvenanceBundleV1 = value diff --git a/apps/sim/lib/execution/payloads/materialization.server.test.ts b/apps/sim/lib/execution/payloads/materialization.server.test.ts index c2a649e2e9d..7b55b117813 100644 --- a/apps/sim/lib/execution/payloads/materialization.server.test.ts +++ b/apps/sim/lib/execution/payloads/materialization.server.test.ts @@ -3,10 +3,12 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockDownloadServableFileFromStorage, mockVerifyFileAccess } = vi.hoisted(() => ({ - mockDownloadServableFileFromStorage: vi.fn(), - mockVerifyFileAccess: vi.fn(), -})) +const { mockDownloadServableFileFromStorage, mockReadWorkspaceFileByKey, mockVerifyFileAccess } = + vi.hoisted(() => ({ + mockDownloadServableFileFromStorage: vi.fn(), + mockReadWorkspaceFileByKey: vi.fn(), + mockVerifyFileAccess: vi.fn(), + })) vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ downloadServableFileFromStorage: mockDownloadServableFileFromStorage, @@ -16,6 +18,10 @@ vi.mock('@/app/api/files/authorization', () => ({ verifyFileAccess: mockVerifyFileAccess, })) +vi.mock('@/lib/workspace-files/application/read-workspace-file-content-by-key', () => ({ + readWorkspaceFileRecordByKey: { execute: mockReadWorkspaceFileByKey }, +})) + import { readUserFileContent } from '@/lib/execution/payloads/materialization.server' import type { UserFile } from '@/executor/types' @@ -36,6 +42,7 @@ describe('readUserFileContent', () => { vi.clearAllMocks() generatedPdf.size = PDF_SOURCE.length mockVerifyFileAccess.mockResolvedValue(true) + mockReadWorkspaceFileByKey.mockResolvedValue({ file: { id: 'file-1' } }) mockDownloadServableFileFromStorage.mockResolvedValue({ buffer: PDF_BYTES, contentType: 'application/pdf', @@ -53,4 +60,144 @@ describe('readUserFileContent', () => { expect(content).not.toBe(PDF_SOURCE.toString('base64')) expect(generatedPdf.size).toBe(PDF_BYTES.length) }) + + it('authorizes execution-scoped files without inventing a human subject', async () => { + const executionFile: UserFile = { + id: 'file-2', + name: 'result.txt', + url: '', + size: 6, + type: 'text/plain', + key: 'execution/workspace-1/workflow-1/execution-1/result.txt', + context: 'execution', + } + mockDownloadServableFileFromStorage.mockResolvedValueOnce({ buffer: Buffer.from('result') }) + + await expect( + readUserFileContent(executionFile, { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + encoding: 'text', + }) + ).resolves.toBe('result') + + expect(mockVerifyFileAccess).not.toHaveBeenCalled() + }) + + it.each(['profile-pictures', 'og-images', 'workspace-logos'] as const)( + 'authorizes actorless reads from the trusted public %s context', + async (context) => { + const publicFile: UserFile = { + id: 'public-file', + name: 'public.png', + url: '', + size: 6, + type: 'image/png', + key: `${context}/public.png`, + context, + } + mockDownloadServableFileFromStorage.mockResolvedValueOnce({ buffer: Buffer.from('public') }) + + await expect(readUserFileContent(publicFile, { encoding: 'text' })).resolves.toBe('public') + + expect(mockVerifyFileAccess).not.toHaveBeenCalled() + expect(mockReadWorkspaceFileByKey).not.toHaveBeenCalled() + } + ) + + it('does not let an actorless caller relabel a private key as public', async () => { + const relabeledFile: UserFile = { + id: 'private-file', + name: 'private.txt', + url: '', + size: 7, + type: 'text/plain', + key: 'workspace/workspace-1/private.txt', + context: 'og-images', + } + + await expect(readUserFileContent(relabeledFile, { encoding: 'text' })).rejects.toThrow( + 'File context does not match its storage key.' + ) + + expect(mockDownloadServableFileFromStorage).not.toHaveBeenCalled() + expect(mockVerifyFileAccess).not.toHaveBeenCalled() + }) + + it('authorizes workspace files with the preserved actorless deployment principal', async () => { + const principal = { + kind: 'delegated' as const, + serviceId: 'executor' as const, + workspaceId: 'workspace-1', + delegationId: 'function-1', + audience: 'sim:function-executions', + issuedAt: new Date(Date.now() - 1_000), + expiresAt: new Date(Date.now() + 60_000), + delegationContext: { + kind: 'workflow_execution' as const, + workflowId: 'workflow-1', + executionId: 'execution-1', + principal: { + kind: 'system' as const, + serviceId: 'schedule' as const, + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + }, + currentWorkflow: { + workflowId: 'workflow-1', + mode: 'deployment' as const, + deploymentVersionId: 'deployment-1', + }, + }, + } + + await readUserFileContent(generatedPdf, { + principal, + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + requestId: 'request-1', + encoding: 'base64', + }) + + expect(mockVerifyFileAccess).not.toHaveBeenCalled() + expect(mockReadWorkspaceFileByKey).toHaveBeenCalledWith( + expect.objectContaining({ + input: { + key: generatedPdf.key, + assertedWorkspaceId: 'workspace-1', + }, + principal: expect.objectContaining({ + audience: 'sim:workspace-files', + delegationContext: principal.delegationContext, + }), + }) + ) + }) + + it('authorizes an exact workspace storage key with the workspace-key principal', async () => { + const principal = { + kind: 'workspace_api_key' as const, + workspaceId: 'workspace-1', + keyId: 'key-1', + } + + await readUserFileContent(generatedPdf, { + principal, + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + encoding: 'base64', + }) + + expect(mockVerifyFileAccess).not.toHaveBeenCalled() + expect(mockReadWorkspaceFileByKey).toHaveBeenCalledWith({ + principal, + input: { + key: generatedPdf.key, + assertedWorkspaceId: 'workspace-1', + }, + }) + }) }) diff --git a/apps/sim/lib/execution/payloads/materialization.server.ts b/apps/sim/lib/execution/payloads/materialization.server.ts index ec7c7bfb82e..ec3f7ef2509 100644 --- a/apps/sim/lib/execution/payloads/materialization.server.ts +++ b/apps/sim/lib/execution/payloads/materialization.server.ts @@ -1,5 +1,7 @@ +import type { Principal } from '@sim/auth/principal' import { createLogger, type Logger } from '@sim/logger' import { toError } from '@sim/utils/errors' +import { OrchestrationError } from '@/lib/core/orchestration/types' import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { isUserFileWithMetadata } from '@/lib/core/utils/user-file' import { @@ -21,13 +23,17 @@ import { bufferToBase64, inferContextFromKey, isGeneratedDocumentSourceType, + isPublicStorageContext, } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { rebindWorkspaceFileDelegatedPrincipal } from '@/lib/workspace-files/application/delegated-principal' +import { readWorkspaceFileRecordByKey } from '@/lib/workspace-files/application/read-workspace-file-content-by-key' import type { UserFile } from '@/executor/types' const logger = createLogger('ExecutionPayloadMaterialization') export interface ExecutionMaterializationContext { + principal?: Principal workflowId?: string workspaceId?: string executionId?: string @@ -244,7 +250,7 @@ function assertExecutionFileScope(key: string, options: ExecutionMaterialization } } -function getVerifiedStorageContext(file: UserFile): StorageContext { +function getVerifiedStorageContext(file: Pick): StorageContext { if (!file.key) { throw new Error('File content requires a storage key.') } @@ -258,13 +264,48 @@ function getVerifiedStorageContext(file: UserFile): StorageContext { } export async function assertUserFileContentAccess( - file: UserFile, + file: Pick, options: ExecutionMaterializationContext ): Promise { const context = getVerifiedStorageContext(file) if (context === 'execution') { assertExecutionFileScope(file.key, options) + return + } + + if (isPublicStorageContext(context)) { + return + } + + if (context === 'workspace' && options.principal && options.workspaceId) { + const principal = + options.principal.kind === 'delegated' + ? rebindWorkspaceFileDelegatedPrincipal({ + principal: options.principal, + workspaceId: options.workspaceId, + delegationId: `execution-file-read:${options.requestId ?? 'unknown'}`, + ...(options.principal.resourceScope?.fileId + ? { fileId: options.principal.resourceScope.fileId } + : {}), + ...(options.principal.resourceScope?.chatId + ? { chatId: options.principal.resourceScope.chatId } + : {}), + ...(options.executionId ? { executionId: options.executionId } : {}), + }) + : options.principal + try { + await readWorkspaceFileRecordByKey.execute({ + principal, + input: { + key: file.key, + assertedWorkspaceId: options.workspaceId, + }, + }) + return + } catch (error) { + if (!(error instanceof OrchestrationError && error.code === 'not_found')) throw error + } } if (!options.userId) { diff --git a/apps/sim/lib/function-execution/application/execute-function.test.ts b/apps/sim/lib/function-execution/application/execute-function.test.ts new file mode 100644 index 00000000000..156ee2d7542 --- /dev/null +++ b/apps/sim/lib/function-execution/application/execute-function.test.ts @@ -0,0 +1,157 @@ +/** + * @vitest-environment node + */ +import type { WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + executeRequest: vi.fn(), + loadWorkspace: vi.fn(), + resolvePermission: vi.fn(), +})) + +vi.mock('@/lib/function-execution/execute-request', () => ({ + executeFunctionRequest: mocks.executeRequest, +})) + +vi.mock('@/lib/workspaces/application/workspace-context', () => ({ + resolveActiveWorkspaceApplicationContext: mocks.loadWorkspace, +})) + +vi.mock('@sim/platform-authz/workspace', () => ({ + permissionSatisfies: (actual: string | null, required: string) => + actual === 'admin' || actual === required || (actual === 'write' && required === 'read'), + resolveEffectiveWorkspacePermission: mocks.resolvePermission, +})) + +import { FUNCTION_EXECUTION_DELEGATION_AUDIENCE } from '@/lib/function-execution/application/authorization' +import { executeFunction } from '@/lib/function-execution/application/execute-function' + +const principal: WorkflowExecutionDelegatedPrincipal = { + kind: 'delegated', + serviceId: 'executor', + workspaceId: 'workspace-1', + delegationId: 'delegation-1', + audience: FUNCTION_EXECUTION_DELEGATION_AUDIENCE, + issuedAt: new Date(Date.now() - 1_000), + expiresAt: new Date(Date.now() + 60_000), + resourceScope: { executionId: 'execution-1' }, + delegationContext: { + kind: 'workflow_execution', + workflowId: 'workflow-1', + executionId: 'execution-1', + principal: { + kind: 'system', + serviceId: 'schedule', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + }, + currentWorkflow: { + workflowId: 'workflow-1', + mode: 'deployment', + deploymentVersionId: 'deployment-1', + }, + }, +} + +describe('executeFunction', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.loadWorkspace.mockResolvedValue({ + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + billedAccountUserId: 'workspace-owner', + allowPersonalApiKeys: true, + }) + mocks.executeRequest.mockResolvedValue(Response.json({ success: true })) + mocks.resolvePermission.mockResolvedValue('write') + }) + + it('uses only the real workflow subject for legacy file contexts', async () => { + const humanPrincipal: WorkflowExecutionDelegatedPrincipal = { + ...principal, + subjectUserId: 'invoking-user', + delegationContext: { + ...principal.delegationContext!, + principal: { + kind: 'session', + userId: 'invoking-user', + sessionId: 'session-1', + }, + }, + } + + await executeFunction.execute({ + principal: humanPrincipal, + input: { + workspaceId: 'workspace-1', + body: { + code: 'return 1', + workspaceId: 'workspace-1', + executionId: 'execution-1', + }, + headers: new Headers(), + }, + }) + + expect(mocks.executeRequest).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + expect.objectContaining({ + attributedUserId: 'invoking-user', + fileAccessUserId: 'invoking-user', + principal: humanPrincipal, + }) + ) + }) + + it('keeps an actorless deployed principal authoritative and attributes legacy work afterward', async () => { + const headers = new Headers() + const signal = new AbortController().signal + const response = await executeFunction.execute({ + principal, + input: { + workspaceId: 'workspace-1', + body: { + code: 'return 1', + workflowId: 'workflow-1', + executionId: 'execution-1', + workspaceId: 'workspace-1', + }, + headers, + signal, + }, + }) + + expect(response.status).toBe(200) + expect(mocks.resolvePermission).not.toHaveBeenCalled() + expect(mocks.executeRequest).toHaveBeenCalledWith( + { headers, signal }, + expect.objectContaining({ + code: 'return 1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }), + { + attributedUserId: 'workspace-owner', + principal, + } + ) + }) + + it('rejects a body workspace that differs from the trusted operation scope', async () => { + await expect( + executeFunction.execute({ + principal, + input: { + workspaceId: 'workspace-1', + body: { code: 'return 1', workspaceId: 'workspace-victim' }, + headers: new Headers(), + }, + }) + ).rejects.toMatchObject({ code: 'not_found' }) + + expect(mocks.loadWorkspace).not.toHaveBeenCalled() + expect(mocks.executeRequest).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/function-execution/application/execute-function.ts b/apps/sim/lib/function-execution/application/execute-function.ts index 502922e19f7..da18f303abe 100644 --- a/apps/sim/lib/function-execution/application/execute-function.ts +++ b/apps/sim/lib/function-execution/application/execute-function.ts @@ -1,5 +1,4 @@ -import { requirePrincipalSubjectUserId } from '@sim/auth/principal' -import { type NextRequest, NextRequest as ServerRequest } from 'next/server' +import { resolvePrincipalAttribution, resolvePrincipalSubject } from '@sim/auth/principal' import { type FunctionExecuteBody, functionExecuteBodySchema } from '@/lib/api/contracts' import { defineAuthorizedWorkspaceUseCase } from '@/lib/core/application' import { OrchestrationError } from '@/lib/core/orchestration/types' @@ -36,7 +35,7 @@ export const executeFunction = defineAuthorizedWorkspaceUseCase({ authorizationOptions: { delegation: functionExecutionDelegationPolicy, }, - execute: async ({ principal, input }): Promise => { + execute: async ({ principal, input, context }): Promise => { const parsedBody = functionExecuteBodySchema.safeParse(input.body) if (!parsedBody.success) { throw new OrchestrationError( @@ -44,15 +43,23 @@ export const executeFunction = defineAuthorizedWorkspaceUseCase({ parsedBody.error.issues[0]?.message ?? 'Function execution input is invalid' ) } - const request = new ServerRequest('http://sim.internal/api/function/execute', { - method: 'POST', - headers: input.headers, - ...(input.signal ? { signal: input.signal } : {}), - }) as NextRequest const { executeFunctionRequest } = await import('@/lib/function-execution/execute-request') - return executeFunctionRequest(request, parsedBody.data, { - userId: requirePrincipalSubjectUserId(principal), - ...(input.sandboxProfile ? { sandboxProfile: input.sandboxProfile } : {}), + const { attributedUserId } = resolvePrincipalAttribution(principal, { + workspaceBillingOwnerUserId: context.billedAccountUserId, }) + const subject = resolvePrincipalSubject(principal) + return executeFunctionRequest( + { + headers: input.headers, + signal: input.signal ?? new AbortController().signal, + }, + parsedBody.data, + { + attributedUserId, + principal, + ...(subject?.kind === 'sim_user' ? { fileAccessUserId: subject.userId } : {}), + ...(input.sandboxProfile ? { sandboxProfile: input.sandboxProfile } : {}), + } + ) }, }) diff --git a/apps/sim/lib/function-execution/execute-request.test.ts b/apps/sim/lib/function-execution/execute-request.test.ts new file mode 100644 index 00000000000..2fd1b35eaa5 --- /dev/null +++ b/apps/sim/lib/function-execution/execute-request.test.ts @@ -0,0 +1,3149 @@ +/** + * @vitest-environment node + */ +import { spawnSync } from 'node:child_process' +import { readFileSync } from 'node:fs' +import { resolve } from 'node:path' +import { + createMockRequest, + envFlagsMock, + hybridAuthMockFns, + resetEnvFlagsMock, + workflowsUtilsMock, +} from '@sim/testing' +import { NextRequest } from 'next/server' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { functionExecuteBodySchema } from '@/lib/api/contracts' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { INTERNAL_EXECUTION_DEADLINE_HEADER } from '@/lib/execution/execution-deadline-header' +import { + MOUNTED_WORKSPACE_FILES_PROVENANCE_KEY, + PRIVATE_SECRET_PROVENANCE_BUNDLE_V1, + PRIVATE_SECRET_PROVENANCE_FIELD, + PRIVATE_SECRET_PROVENANCE_HEADER, +} from '@/lib/execution/private-tool-metadata' +import { + MAX_SANDBOX_OUTPUT_BYTES, + SandboxOutputFileError, + SandboxOutputLimitError, +} from '@/lib/execution/remote-sandbox/output-limits' + +const { + mockExecuteInSandbox, + mockExecuteInIsolatedVM, + mockExecuteShellInSandbox, + mockFetchWorkspaceFileBuffer, + mockDecryptSecret, + mockEncryptSecret, + mockGetWorkspaceFile, + mockResolveWorkspaceFileReference, + mockUpdateWorkspaceFileContent, + mockUploadFile, + mockValidateWorkspaceFileWriteTarget, + mockWriteWorkspaceFileByPath, +} = vi.hoisted(() => ({ + mockExecuteInSandbox: vi.fn(), + mockExecuteInIsolatedVM: vi.fn(), + mockExecuteShellInSandbox: vi.fn(), + mockFetchWorkspaceFileBuffer: vi.fn(), + mockDecryptSecret: vi.fn(async (value: string) => ({ + decrypted: value === 'encrypted:mounted-secret' ? 'mounted-secret' : value, + })), + mockEncryptSecret: vi.fn(async (value: string) => ({ + encrypted: `encrypted:${value}`, + iv: 'iv', + })), + mockGetWorkspaceFile: vi.fn(), + mockResolveWorkspaceFileReference: vi.fn(), + mockUpdateWorkspaceFileContent: vi.fn(), + mockUploadFile: vi.fn(), + mockValidateWorkspaceFileWriteTarget: vi.fn(), + mockWriteWorkspaceFileByPath: vi.fn(), +})) + +vi.mock('@/lib/core/security/encryption', () => ({ + decryptSecret: mockDecryptSecret, + encryptSecret: mockEncryptSecret, +})) + +vi.mock('@/lib/execution/isolated-vm', () => ({ + executeInIsolatedVM: mockExecuteInIsolatedVM, +})) + +vi.mock('@/lib/execution/remote-sandbox', () => ({ + executeInSandbox: mockExecuteInSandbox, + executeShellInSandbox: mockExecuteShellInSandbox, + SIM_RESULT_PREFIX: '__SIM_RESULT__=', +})) + +vi.mock('@/lib/copilot/request/tools/files', () => ({ + FORMAT_TO_CONTENT_TYPE: { + json: 'application/json', + csv: 'text/csv', + txt: 'text/plain', + md: 'text/markdown', + html: 'text/html', + }, + normalizeOutputWorkspaceFileName: vi.fn((p: string) => p.replace(/^files\//, '')), + resolveOutputFormat: vi.fn(() => 'json'), + getOutputFileDeclarations: vi.fn((params: Record) => { + if (Array.isArray(params.outputs?.files)) { + return params.outputs.files.map((file: Record) => ({ + path: file.path, + mode: file.mode === 'overwrite' ? 'overwrite' : 'create', + sandboxPath: file.sandboxPath, + mimeType: file.mimeType, + format: file.format, + })) + } + return params.outputPath + ? [ + { + path: params.overwriteFileId || params.outputPath, + mode: params.overwriteFileId ? 'overwrite' : 'create', + sandboxPath: params.outputSandboxPath, + mimeType: params.outputMimeType, + format: params.outputFormat, + formatPath: params.outputPath, + overwriteFileId: params.overwriteFileId, + }, + ] + : [] + }), +})) + +vi.mock('@/lib/copilot/vfs/resource-writer', () => ({ + validateWorkspaceFileWriteTarget: mockValidateWorkspaceFileWriteTarget, + writeWorkspaceFileByPath: mockWriteWorkspaceFileByPath, +})) + +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ + fetchWorkspaceFileBuffer: mockFetchWorkspaceFileBuffer, + getWorkspaceFile: mockGetWorkspaceFile, + resolveWorkspaceFileReference: mockResolveWorkspaceFileReference, + updateWorkspaceFileContent: mockUpdateWorkspaceFileContent, + uploadWorkspaceFile: vi.fn(), +})) + +vi.mock('@/lib/workspace-files/application/resolve-workspace-file-reference', () => ({ + resolveWorkspaceFileReference: mockResolveWorkspaceFileReference, +})) + +vi.mock('@/lib/workspace-files/application/read-workspace-file-content', () => ({ + readWorkspaceFileContent: { + execute: vi.fn(async () => ({ content: await mockFetchWorkspaceFileBuffer() })), + }, +})) + +vi.mock('@/lib/uploads', () => ({ + StorageService: { + uploadFile: mockUploadFile, + }, +})) + +vi.mock('@/lib/workflows/utils', () => workflowsUtilsMock) + +import { validateProxyUrl } from '@/lib/core/security/input-validation' +import { clearLargeValueCacheForTests } from '@/lib/execution/payloads/cache' +import { isLargeArrayManifest } from '@/lib/execution/payloads/large-array-manifest-metadata' +import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref' +import { executeFunctionRequest } from '@/lib/function-execution/execute-request' + +async function POST(request: NextRequest): Promise { + const auth = await hybridAuthMockFns.mockCheckInternalAuth(request) + if (!auth.success || !auth.userId) { + return Response.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) + } + + let body: unknown + try { + body = await request.json() + } catch { + return Response.json({ error: 'Request body must be valid JSON' }, { status: 400 }) + } + const parsed = functionExecuteBodySchema.safeParse(body) + if (!parsed.success) { + return Response.json( + { error: 'Invalid request data', details: parsed.error.issues }, + { status: 400 } + ) + } + + return executeFunctionRequest({ headers: request.headers, signal: request.signal }, parsed.data, { + attributedUserId: auth.userId, + principal: { + kind: 'delegated', + serviceId: 'executor', + subjectUserId: auth.userId, + workspaceId: parsed.data.workspaceId ?? 'workspace-test', + delegationId: 'function-test', + audience: 'sim:function-executions', + issuedAt: new Date(Date.now() - 1_000), + expiresAt: new Date(Date.now() + 60_000), + delegationContext: { + kind: 'workflow_execution', + workflowId: parsed.data.workflowId ?? 'workflow-test', + ...(parsed.data.executionId ? { executionId: parsed.data.executionId } : {}), + }, + }, + ...(auth.sandboxProfile === 'mothership' ? { sandboxProfile: 'mothership' } : {}), + }) +} + +afterAll(resetEnvFlagsMock) + +describe('Function execution request', () => { + beforeEach(() => { + vi.clearAllMocks() + envFlagsMock.isRemoteSandboxEnabled = false + envFlagsMock.isMothershipSandboxEnabled = false + + hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ + success: true, + userId: 'user-123', + authType: 'internal_jwt', + }) + + mockExecuteInIsolatedVM.mockResolvedValue({ result: 'test', stdout: '' }) + mockUploadFile.mockImplementation(async ({ customKey }) => ({ key: customKey })) + clearLargeValueCacheForTests() + + mockExecuteInSandbox.mockResolvedValue({ + result: 'e2b success', + stdout: 'e2b output', + sandboxId: 'test-sandbox-id', + }) + mockExecuteShellInSandbox.mockResolvedValue({ + result: null, + stdout: '', + sandboxId: 'test-shell-sandbox-id', + }) + mockGetWorkspaceFile.mockResolvedValue({ + id: 'wf_existing', + name: 'existing.png', + size: 10, + type: 'image/png', + url: '/api/files/view/existing', + key: 'workspace/existing.png', + }) + mockUpdateWorkspaceFileContent.mockResolvedValue({ + id: 'wf_existing', + name: 'existing.png', + size: 20, + type: 'image/png', + url: '/api/files/view/existing', + key: 'workspace/existing.png', + }) + mockResolveWorkspaceFileReference.mockResolvedValue({ + id: 'wf_existing', + workspaceId: 'workspace-1', + name: 'existing.txt', + size: 0, + key: 'workspace/existing.txt', + }) + mockFetchWorkspaceFileBuffer.mockResolvedValue(Buffer.alloc(0)) + mockValidateWorkspaceFileWriteTarget.mockImplementation(async ({ target }) => ({ + mode: target.mode, + vfsPath: target.path, + })) + mockWriteWorkspaceFileByPath.mockImplementation(async ({ target, buffer }) => ({ + id: `wf_${String(target.path).split('/').pop()?.replace(/\W+/g, '_') || 'file'}`, + name: String(target.path).split('/').pop() || 'file', + vfsPath: target.path, + downloadUrl: `/api/files/view/${encodeURIComponent(target.path)}`, + mode: target.mode, + size: buffer.length, + contentType: target.mimeType || 'application/octet-stream', + })) + }) + + describe('Security Tests', () => { + it('should reject unauthorized requests', async () => { + hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValueOnce({ + success: false, + error: 'Unauthorized', + }) + + const req = createMockRequest('POST', { + code: 'return "test"', + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(401) + expect(data).toHaveProperty('error', 'Unauthorized') + }) + + it('rejects a sandbox output export through the workspace-file application policy', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'done', + stdout: 'ok', + sandboxId: 'sandbox-123', + exportedFiles: { '/tmp/out.txt': 'owned by attacker' }, + }) + mockWriteWorkspaceFileByPath.mockRejectedValueOnce( + new OrchestrationError('forbidden', 'Insufficient workspace permissions') + ) + + const req = createMockRequest('POST', { + code: 'print("done")', + language: 'python', + workspaceId: 'workspace-victim', + outputs: { + files: [{ path: 'files/README.md', mode: 'overwrite', sandboxPath: '/tmp/out.txt' }], + }, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(403) + expect(data).toHaveProperty('error', 'Insufficient workspace permissions') + expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledTimes(1) + }) + + it('runs import-free JavaScript in isolated-vm without a remote provider', async () => { + const req = createMockRequest('POST', { + code: 'return "test"', + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.success).toBe(true) + expect(data.output.result).toBe('test') + expect(mockExecuteInIsolatedVM).toHaveBeenCalledTimes(1) + expect(mockExecuteInSandbox).not.toHaveBeenCalled() + expect(mockExecuteShellInSandbox).not.toHaveBeenCalled() + }) + + it('does not accept a Mothership sandbox profile from the request body', async () => { + const req = createMockRequest('POST', { + code: 'return "test"', + sandboxProfile: 'mothership', + }) + + const response = await POST(req) + + expect(response.status).toBe(200) + expect(mockExecuteInIsolatedVM).toHaveBeenCalledTimes(1) + expect(mockExecuteInSandbox).not.toHaveBeenCalled() + }) + + it('fails closed when a trusted Mothership call has no configured image', async () => { + hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValueOnce({ + success: true, + userId: 'user-123', + authType: 'internal_jwt', + sandboxProfile: 'mothership', + }) + + const response = await POST( + createMockRequest('POST', { code: 'return "test"', language: 'javascript' }) + ) + + expect(response.status).toBe(503) + await expect(response.json()).resolves.toMatchObject({ + success: false, + error: 'Mothership code sandbox is not configured', + }) + expect(mockExecuteInIsolatedVM).not.toHaveBeenCalled() + expect(mockExecuteInSandbox).not.toHaveBeenCalled() + }) + + it.each([ + { language: 'javascript', code: 'return 42' }, + { language: 'python', code: '__sim_result__ = 42' }, + ])( + 'runs trusted Mothership $language in the Mothership sandbox image', + async ({ language, code }) => { + envFlagsMock.isMothershipSandboxEnabled = true + hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValueOnce({ + success: true, + userId: 'user-123', + authType: 'internal_jwt', + sandboxProfile: 'mothership', + }) + + const response = await POST(createMockRequest('POST', { code, language })) + + expect(response.status).toBe(200) + expect(mockExecuteInSandbox).toHaveBeenCalledWith( + expect.objectContaining({ + language, + sandboxKind: 'mothership', + }) + ) + expect(mockExecuteInIsolatedVM).not.toHaveBeenCalled() + } + ) + + it('runs trusted Mothership Shell in the Mothership sandbox image', async () => { + envFlagsMock.isMothershipSandboxEnabled = true + hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValueOnce({ + success: true, + userId: 'user-123', + authType: 'internal_jwt', + sandboxProfile: 'mothership', + }) + + const response = await POST( + createMockRequest('POST', { code: 'echo ready', language: 'shell' }) + ) + + expect(response.status).toBe(200) + expect(mockExecuteShellInSandbox).toHaveBeenCalledWith( + expect.objectContaining({ sandboxKind: 'mothership' }) + ) + }) + + it.each([ + { language: 'javascript', code: 'return 42' }, + { language: 'python', code: '__sim_result__ = 42' }, + ])( + 'runs trusted Mothership $language in the selected Function-based Sim sandbox', + async ({ language, code }) => { + envFlagsMock.isRemoteSandboxEnabled = true + hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValueOnce({ + success: true, + userId: 'user-123', + authType: 'internal_jwt', + sandboxProfile: 'mothership', + }) + + const response = await POST( + createMockRequest('POST', { + code, + language, + workspaceId: 'workspace-1', + sandboxId: 'sandbox-1', + }) + ) + + expect(response.status).toBe(200) + const request = mockExecuteInSandbox.mock.calls.at(-1)?.[0] + expect(request).toMatchObject({ + language, + workspaceId: 'workspace-1', + sandboxId: 'sandbox-1', + }) + expect(request).not.toHaveProperty('sandboxKind') + } + ) + + it('runs trusted Mothership Shell in the selected Sim sandbox', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValueOnce({ + success: true, + userId: 'user-123', + authType: 'internal_jwt', + sandboxProfile: 'mothership', + }) + + const response = await POST( + createMockRequest('POST', { + code: 'kubectl version --client', + language: 'shell', + workspaceId: 'workspace-1', + sandboxId: 'sandbox-1', + }) + ) + + expect(response.status).toBe(200) + const request = mockExecuteShellInSandbox.mock.calls.at(-1)?.[0] + expect(request).toMatchObject({ + workspaceId: 'workspace-1', + sandboxId: 'sandbox-1', + }) + expect(request).not.toHaveProperty('sandboxKind') + }) + + it('does not treat the Mothership base as a fallback for a selected Sim sandbox', async () => { + envFlagsMock.isMothershipSandboxEnabled = true + hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValueOnce({ + success: true, + userId: 'user-123', + authType: 'internal_jwt', + sandboxProfile: 'mothership', + }) + + const response = await POST( + createMockRequest('POST', { + code: 'return 42', + language: 'javascript', + workspaceId: 'workspace-1', + sandboxId: 'sandbox-1', + }) + ) + + expect(response.status).toBe(503) + await expect(response.json()).resolves.toMatchObject({ + error: 'The Function code sandbox is not configured', + }) + expect(mockExecuteInSandbox).not.toHaveBeenCalled() + expect(mockExecuteInIsolatedVM).not.toHaveBeenCalled() + }) + + it('forces import-free JavaScript into the remote runtime when a Sim sandbox is selected', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + + const response = await POST( + createMockRequest('POST', { + code: 'return 42', + language: 'javascript', + workspaceId: 'workspace-1', + sandboxId: 'sandbox-1', + }) + ) + + expect(response.status).toBe(200) + expect(mockExecuteInSandbox).toHaveBeenCalledWith( + expect.objectContaining({ + language: 'javascript', + workspaceId: 'workspace-1', + sandboxId: 'sandbox-1', + }) + ) + expect(mockExecuteInIsolatedVM).not.toHaveBeenCalled() + }) + + it('should prevent VM escape via constructor chain', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: undefined, stdout: '' }) + + const req = createMockRequest('POST', { + code: 'return this.constructor.constructor("return process")().env', + }) + + const response = await POST(req) + const data = await response.json() + + if (response.status === 422 || response.status === 500) { + expect(data.success).toBe(false) + } else { + const result = data.output?.result + expect(result === undefined || result === null).toBe(true) + } + }) + + it.concurrent('should prevent access to require via constructor chain', async () => { + const req = createMockRequest('POST', { + code: ` + const proc = this.constructor.constructor("return process")(); + const fs = proc.mainModule.require("fs"); + return fs.readFileSync("/etc/passwd", "utf8"); + `, + }) + + const response = await POST(req) + const data = await response.json() + + if (response.status === 200) { + const result = data.output?.result + if (result !== undefined && result !== null && typeof result === 'string') { + expect(result).not.toContain('root:') + } + } + }) + + it('should not expose process object', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'undefined', stdout: '' }) + + const req = createMockRequest('POST', { + code: 'return typeof process', + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.output.result).toBe('undefined') + }) + + it('should not expose require function', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'undefined', stdout: '' }) + + const req = createMockRequest('POST', { + code: 'return typeof require', + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.output.result).toBe('undefined') + }) + + it.concurrent('should block SSRF attacks through secure fetch wrapper', async () => { + expect(validateProxyUrl('http://169.254.169.254/latest/meta-data/').isValid).toBe(false) + expect(validateProxyUrl('http://127.0.0.1:8080/admin').isValid).toBe(true) + expect(validateProxyUrl('http://192.168.1.1/config').isValid).toBe(false) + expect(validateProxyUrl('http://10.0.0.1/internal').isValid).toBe(false) + }) + + it.concurrent('should allow legitimate external URLs', async () => { + expect(validateProxyUrl('https://api.github.com/user').isValid).toBe(true) + expect(validateProxyUrl('https://httpbin.org/get').isValid).toBe(true) + expect(validateProxyUrl('https://example.com/api').isValid).toBe(true) + }) + + it.concurrent('should block dangerous protocols', async () => { + expect(validateProxyUrl('file:///etc/passwd').isValid).toBe(false) + expect(validateProxyUrl('ftp://internal.server/files').isValid).toBe(false) + expect(validateProxyUrl('gopher://old.server/menu').isValid).toBe(false) + }) + }) + + describe('Basic Function Execution', () => { + it.concurrent('should execute simple JavaScript code successfully', async () => { + const req = createMockRequest('POST', { + code: 'return "Hello World"', + timeout: 5000, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.success).toBe(true) + expect(data.output).toHaveProperty('result') + expect(data.output).toHaveProperty('executionTime') + }) + + it('compacts large array result fields to manifests when execution context is durable', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ + result: { + rows: Array.from({ length: 120_000 }, (_, index) => ({ + key: `SIM-${index}`, + payload: 'x'.repeat(100), + })), + }, + stdout: '', + }) + + const req = createMockRequest('POST', { + code: 'return rows', + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionId: 'execution-1', + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.success).toBe(true) + expect(isLargeArrayManifest(data.output.result.rows)).toBe(true) + expect(data.output.result.rows).toMatchObject({ + __simLargeArrayManifest: true, + kind: 'array', + totalCount: 120_000, + }) + }) + + it('keeps large string result fields as generic large value refs', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ + result: { + text: 'x'.repeat(9 * 1024 * 1024), + }, + stdout: '', + }) + + const req = createMockRequest('POST', { + code: 'return text', + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionId: 'execution-1', + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.success).toBe(true) + expect(isLargeValueRef(data.output.result.text)).toBe(true) + }) + + it('captures secret provenance before a large result is compacted', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ + result: { text: `${'x'.repeat(9 * 1024 * 1024)}secret-at-the-end` }, + stdout: '', + }) + + const response = await POST( + createMockRequest( + 'POST', + { + code: 'return {{API_KEY}}', + envVars: { API_KEY: 'secret-at-the-end' }, + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionId: 'execution-1', + }, + { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } + ) + ) + const data = await response.json() + + expect(isLargeValueRef(data.output.result.text)).toBe(true) + expect(data.__resolvedSecretNames).toEqual(['API_KEY']) + }) + + it('exports multiple declared sandbox output files', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'done', + stdout: 'ok', + sandboxId: 'sandbox-123', + exportedFiles: { + '/home/user/chart.png': 'iVBORw0KGgo=', + '/home/user/summary.json': '{"ok":true}', + }, + }) + + const req = createMockRequest('POST', { + code: 'print("done")', + language: 'python', + workspaceId: 'workspace-1', + outputs: { + files: [ + { + path: 'files/reports/chart.png', + mode: 'create', + sandboxPath: '/home/user/chart.png', + mimeType: 'image/png', + }, + { + path: 'files/reports/summary.json', + mode: 'overwrite', + sandboxPath: '/home/user/summary.json', + mimeType: 'application/json', + }, + ], + }, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.success).toBe(true) + expect(mockExecuteInSandbox).toHaveBeenCalledWith( + expect.objectContaining({ + outputSandboxPaths: ['/home/user/chart.png', '/home/user/summary.json'], + }) + ) + expect(mockValidateWorkspaceFileWriteTarget).toHaveBeenCalledTimes(2) + expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledTimes(2) + expect(mockWriteWorkspaceFileByPath).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + target: expect.objectContaining({ path: 'files/reports/chart.png', mode: 'create' }), + }) + ) + expect(mockWriteWorkspaceFileByPath).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + target: expect.objectContaining({ + path: 'files/reports/summary.json', + mode: 'overwrite', + }), + }) + ) + expect(data.output.result.files).toHaveLength(2) + expect(data.resources).toEqual([ + expect.objectContaining({ path: 'files/reports/chart.png' }), + expect.objectContaining({ path: 'files/reports/summary.json' }), + ]) + }) + + it('atomically classifies text exports and acknowledges the durable v2 capability', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'done', + stdout: '', + sandboxId: 'sandbox-123', + exportedFiles: { '/home/user/secret.txt': 'Bearer secret-value' }, + }) + + const response = await POST( + createMockRequest( + 'POST', + { + code: 'print("{{API_KEY}}")', + language: 'python', + workspaceId: 'workspace-1', + envVars: { API_KEY: 'secret-value' }, + outputs: { + files: [ + { + path: 'files/secret.txt', + sandboxPath: '/home/user/secret.txt', + mimeType: 'text/plain', + }, + ], + }, + }, + { + 'x-sim-request-private-tool-metadata': 'resolved-secret-names-durable-files-v2', + } + ) + ) + const data = await response.json() + + expect(response.status).toBe(200) + expect(response.headers.get('x-sim-private-tool-metadata')).toBe( + 'resolved-secret-names-durable-files-v2' + ) + expect(data).not.toHaveProperty('__resolvedSecretFileNames') + expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( + expect.objectContaining({ + secretProvenance: { + status: 'exact', + entries: [ + { + name: 'API_KEY', + encryptedValue: 'encrypted:secret-value', + sourceUserId: 'user-123', + sourceWorkspaceId: 'workspace-1', + }, + ], + }, + }) + ) + }) + + it('classifies exports exact-empty when the only compiled secret is exempt, still reporting its name', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'done', + stdout: '', + sandboxId: 'sandbox-123', + exportedFiles: { + '/home/user/secret.txt': 'Bearer secret-value', + '/home/user/small.jpg': '/9j/4AAQ', + }, + }) + + const response = await POST( + createMockRequest( + 'POST', + { + code: 'print("{{API_KEY}}")', + language: 'python', + workspaceId: 'workspace-1', + envVars: { API_KEY: 'secret-value' }, + unredactedSecretNames: ['API_KEY'], + outputs: { + files: [ + { + path: 'files/secret.txt', + sandboxPath: '/home/user/secret.txt', + mimeType: 'text/plain', + }, + { + path: 'files/small.jpg', + sandboxPath: '/home/user/small.jpg', + mimeType: 'image/jpeg', + }, + ], + }, + }, + { + 'x-sim-request-private-tool-metadata': 'resolved-secret-names-durable-files-v2', + } + ) + ) + const data = await response.json() + + expect(response.status).toBe(200) + // The text export carries the exempt plaintext yet records no entry for it. + expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( + expect.objectContaining({ + target: expect.objectContaining({ path: 'files/secret.txt' }), + secretProvenance: { status: 'exact', entries: [] }, + }) + ) + // With only exempt material in scope the binary export must not lock as unknown. + expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( + expect.objectContaining({ + target: expect.objectContaining({ path: 'files/small.jpg' }), + secretProvenance: { status: 'exact', entries: [] }, + }) + ) + // The exemption changes file classification only — the usage trail still sees the name. + expect(data.__resolvedSecretNames).toEqual(['API_KEY']) + }) + + it('keeps recording the non-exempt owner when an exempt name shares its plaintext', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'done', + stdout: '', + sandboxId: 'sandbox-123', + exportedFiles: { '/home/user/secret.txt': 'Bearer shared-value' }, + }) + + const response = await POST( + createMockRequest('POST', { + code: 'print("{{EXEMPT_KEY}}", "{{OTHER_KEY}}")', + language: 'python', + workspaceId: 'workspace-1', + envVars: { EXEMPT_KEY: 'shared-value', OTHER_KEY: 'shared-value' }, + unredactedSecretNames: ['EXEMPT_KEY'], + outputs: { + files: [ + { + path: 'files/secret.txt', + sandboxPath: '/home/user/secret.txt', + mimeType: 'text/plain', + }, + ], + }, + }) + ) + + expect(response.status).toBe(200) + expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( + expect.objectContaining({ + secretProvenance: { + status: 'exact', + entries: [ + { + name: 'OTHER_KEY', + encryptedValue: 'encrypted:shared-value', + sourceUserId: 'user-123', + sourceWorkspaceId: 'workspace-1', + }, + ], + }, + }) + ) + }) + + it('classifies text exports against private mounted-file provenance', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'done', + stdout: '', + sandboxId: 'sandbox-123', + exportedFiles: { '/home/user/copied.txt': 'Bearer mounted-secret' }, + }) + + const response = await POST( + createMockRequest( + 'POST', + { + code: 'print("done")', + language: 'python', + workspaceId: 'workspace-1', + outputs: { + files: [ + { + path: 'files/copied.txt', + sandboxPath: '/home/user/copied.txt', + mimeType: 'text/plain', + }, + ], + }, + [PRIVATE_SECRET_PROVENANCE_FIELD]: { + version: 1, + complete: true, + selections: [ + { + key: MOUNTED_WORKSPACE_FILES_PROVENANCE_KEY, + provenance: { + version: 1, + complete: true, + entries: [{ encryptedValue: 'encrypted:mounted-secret' }], + scope: { userId: 'user-123', workspaceId: 'workspace-1' }, + }, + }, + ], + }, + }, + { + 'x-sim-request-private-tool-metadata': 'resolved-secret-names-durable-files-v2', + [PRIVATE_SECRET_PROVENANCE_HEADER]: PRIVATE_SECRET_PROVENANCE_BUNDLE_V1, + } + ) + ) + + expect(response.status).toBe(200) + expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( + expect.objectContaining({ + secretProvenance: { + status: 'exact', + entries: [ + { + name: 'MOUNTED_FILE_SECRET', + encryptedValue: 'encrypted:mounted-secret', + sourceUserId: 'user-123', + sourceWorkspaceId: 'workspace-1', + }, + ], + }, + }) + ) + }) + + it('rejects a partial mounted-file provenance envelope before execution', async () => { + const response = await POST( + createMockRequest('POST', { + code: 'return 1', + [PRIVATE_SECRET_PROVENANCE_FIELD]: { + version: 1, + complete: true, + selections: [ + { + key: MOUNTED_WORKSPACE_FILES_PROVENANCE_KEY, + provenance: { version: 1, complete: true, entries: [] }, + }, + ], + }, + }) + ) + + expect(response.status).toBe(400) + expect(await response.json()).toEqual({ + success: false, + error: 'Mounted file secret provenance is invalid', + }) + expect(mockExecuteInIsolatedVM).not.toHaveBeenCalled() + expect(mockExecuteInSandbox).not.toHaveBeenCalled() + }) + + it('runs with authenticated incomplete mount provenance and marks exported bytes unknown', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'raw result', + stdout: '', + sandboxId: 'sandbox-123', + exportedFiles: { '/home/user/output.txt': 'raw output' }, + }) + + const response = await POST( + createMockRequest( + 'POST', + { + code: 'print("done")', + language: 'python', + workspaceId: 'workspace-1', + outputs: { + files: [ + { + path: 'files/output.txt', + sandboxPath: '/home/user/output.txt', + mimeType: 'text/plain', + }, + ], + }, + [PRIVATE_SECRET_PROVENANCE_FIELD]: { + version: 1, + complete: false, + selections: [], + }, + }, + { [PRIVATE_SECRET_PROVENANCE_HEADER]: PRIVATE_SECRET_PROVENANCE_BUNDLE_V1 } + ) + ) + + expect(response.status).toBe(200) + expect((await response.json()).output.result).toEqual( + expect.objectContaining({ fileId: 'wf_output_txt', vfsPath: 'files/output.txt' }) + ) + expect(mockExecuteInSandbox).toHaveBeenCalledOnce() + expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( + expect.objectContaining({ + buffer: Buffer.from('raw output'), + secretProvenance: { status: 'unknown' }, + }) + ) + }) + + it('does not rewrite a static export path that happens to equal a resolved secret', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'done', + stdout: '', + sandboxId: 'sandbox-123', + exportedFiles: { '/home/user/report.txt': 'safe content' }, + }) + + const response = await POST( + createMockRequest('POST', { + code: 'print("done")', + language: 'python', + workspaceId: 'workspace-1', + envVars: { API_KEY: 'secret-value' }, + outputs: { + files: [ + { + path: 'files/report-secret-value.txt', + sandboxPath: '/home/user/report.txt', + mimeType: 'text/plain', + }, + ], + }, + }) + ) + const data = await response.json() + + expect(response.status).toBe(200) + expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( + expect.objectContaining({ + target: expect.objectContaining({ path: 'files/report-secret-value.txt' }), + secretProvenance: { status: 'exact', entries: [] }, + }) + ) + expect(JSON.stringify(data)).toContain('files/report-secret-value.txt') + }) + + it('classifies a binary export exact-empty when no secret was in scope', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'done', + stdout: '', + sandboxId: 'sandbox-123', + exportedFiles: { '/home/user/small.jpg': '/9j/4AAQ' }, + }) + + const response = await POST( + createMockRequest('POST', { + code: 'print("done")', + language: 'python', + workspaceId: 'workspace-1', + outputs: { + files: [ + { + path: 'files/small.jpg', + sandboxPath: '/home/user/small.jpg', + mimeType: 'image/jpeg', + }, + ], + }, + }) + ) + + expect(response.status).toBe(200) + expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( + expect.objectContaining({ secretProvenance: { status: 'exact', entries: [] } }) + ) + }) + + it('classifies a binary export exact-empty when ordinary files were mounted without secret provenance', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'done', + stdout: '', + sandboxId: 'sandbox-123', + exportedFiles: { '/home/user/small.jpg': '/9j/4AAQ' }, + }) + + const response = await POST( + createMockRequest('POST', { + code: 'print("done")', + language: 'python', + workspaceId: 'workspace-1', + _sandboxFiles: [{ path: '/home/user/in.bin', content: 'mounted bytes' }], + outputs: { + files: [ + { + path: 'files/small.jpg', + sandboxPath: '/home/user/small.jpg', + mimeType: 'image/jpeg', + }, + ], + }, + }) + ) + + expect(response.status).toBe(200) + expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( + expect.objectContaining({ secretProvenance: { status: 'exact', entries: [] } }) + ) + }) + + it('keeps a binary export unknown when a mounted input file carried a secret', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'done', + stdout: '', + sandboxId: 'sandbox-123', + exportedFiles: { '/home/user/small.jpg': '/9j/4AAQ' }, + }) + + const response = await POST( + createMockRequest( + 'POST', + { + code: 'print("done")', + language: 'python', + workspaceId: 'workspace-1', + outputs: { + files: [ + { + path: 'files/small.jpg', + sandboxPath: '/home/user/small.jpg', + mimeType: 'image/jpeg', + }, + ], + }, + [PRIVATE_SECRET_PROVENANCE_FIELD]: { + version: 1, + complete: true, + selections: [ + { + key: MOUNTED_WORKSPACE_FILES_PROVENANCE_KEY, + provenance: { + version: 1, + complete: true, + entries: [{ encryptedValue: 'encrypted:mounted-secret' }], + scope: { userId: 'user-123', workspaceId: 'workspace-1' }, + }, + }, + ], + }, + }, + { + [PRIVATE_SECRET_PROVENANCE_HEADER]: PRIVATE_SECRET_PROVENANCE_BUNDLE_V1, + } + ) + ) + + expect(response.status).toBe(200) + expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( + expect.objectContaining({ secretProvenance: { status: 'unknown' } }) + ) + }) + + it('marks binary exports unknown without failing the Function execution', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'done', + stdout: '', + sandboxId: 'sandbox-123', + exportedFiles: { '/home/user/archive.zip': 'UEsDBA==' }, + }) + + const response = await POST( + createMockRequest('POST', { + code: 'print("{{API_KEY}}")', + language: 'python', + workspaceId: 'workspace-1', + envVars: { API_KEY: 'secret-value' }, + outputs: { + files: [ + { + path: 'files/archive.zip', + sandboxPath: '/home/user/archive.zip', + mimeType: 'application/zip', + }, + ], + }, + }) + ) + + expect(response.status).toBe(200) + expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( + expect.objectContaining({ secretProvenance: { status: 'unknown' } }) + ) + }) + + it('rejects one oversized sandbox output before creating a workspace file buffer', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'done', + stdout: 'ok', + sandboxId: 'sandbox-123', + exportedFiles: { + '/home/user/report.json': 'x'.repeat(MAX_SANDBOX_OUTPUT_BYTES + 1), + }, + }) + + const req = createMockRequest('POST', { + code: 'print("done")', + language: 'python', + workspaceId: 'workspace-1', + outputs: { + files: [ + { + path: 'files/report.json', + sandboxPath: '/home/user/report.json', + mimeType: 'application/json', + }, + ], + }, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(400) + expect(data.error).toBe(`Sandbox output files exceed ${MAX_SANDBOX_OUTPUT_BYTES} bytes total`) + expect(mockValidateWorkspaceFileWriteTarget).not.toHaveBeenCalled() + expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() + }) + + it('rejects cumulative sandbox output size before validating workspace destinations', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + const fileSize = MAX_SANDBOX_OUTPUT_BYTES / 2 + 1 + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'done', + stdout: 'ok', + sandboxId: 'sandbox-123', + exportedFiles: { + '/home/user/first.json': 'x'.repeat(fileSize), + '/home/user/second.json': 'y'.repeat(fileSize), + }, + }) + + const req = createMockRequest('POST', { + code: 'print("done")', + language: 'python', + workspaceId: 'workspace-1', + outputs: { + files: [ + { + path: 'files/first.json', + sandboxPath: '/home/user/first.json', + mimeType: 'application/json', + }, + { + path: 'files/second.json', + sandboxPath: '/home/user/second.json', + mimeType: 'application/json', + }, + ], + }, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(400) + expect(data.error).toBe(`Sandbox output files exceed ${MAX_SANDBOX_OUTPUT_BYTES} bytes total`) + expect(mockValidateWorkspaceFileWriteTarget).not.toHaveBeenCalled() + expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() + }) + + it('preserves output-limit classification from provider-side size inspection', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockRejectedValueOnce( + new SandboxOutputLimitError(MAX_SANDBOX_OUTPUT_BYTES + 1) + ) + + const req = createMockRequest('POST', { + code: 'print("done")', + language: 'python', + workspaceId: 'workspace-1', + outputs: { + files: [ + { + path: 'files/report.json', + sandboxPath: '/home/user/report.json', + }, + ], + }, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(400) + expect(data.error).toBe(`Sandbox output files exceed ${MAX_SANDBOX_OUTPUT_BYTES} bytes total`) + expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() + }) + + it('rejects non-regular sandbox output paths as a client error', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockRejectedValueOnce(new SandboxOutputFileError('/out/link.json')) + + const response = await POST( + createMockRequest('POST', { + code: 'print("done")', + language: 'python', + workspaceId: 'workspace-1', + outputs: { + files: [{ path: 'files/report.json', sandboxPath: '/out/link.json' }], + }, + }) + ) + const data = await response.json() + + expect(response.status).toBe(400) + expect(data.error).toContain('must reference a regular file') + expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() + }) + + it('prevalidates all sandbox output destinations before writing any files', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'done', + stdout: 'ok', + sandboxId: 'sandbox-123', + exportedFiles: { + '/home/user/first.json': '{"first":true}', + '/home/user/second.json': '{"second":true}', + }, + }) + mockValidateWorkspaceFileWriteTarget + .mockResolvedValueOnce({ mode: 'create', vfsPath: 'files/first.json' }) + .mockRejectedValueOnce(new Error('Directory not yet created: files/missing')) + + const req = createMockRequest('POST', { + code: 'print("done")', + language: 'python', + workspaceId: 'workspace-1', + outputs: { + files: [ + { + path: 'files/first.json', + mode: 'create', + sandboxPath: '/home/user/first.json', + }, + { + path: 'files/missing/second.json', + mode: 'create', + sandboxPath: '/home/user/second.json', + }, + ], + }, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(400) + expect(data.success).toBe(false) + expect(data.error).toContain('Directory not yet created') + expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() + }) + + it('rejects duplicate sandbox output destinations before writing files', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'done', + stdout: 'ok', + sandboxId: 'sandbox-123', + exportedFiles: { + '/home/user/first.json': '{"first":true}', + '/home/user/second.json': '{"second":true}', + }, + }) + mockValidateWorkspaceFileWriteTarget.mockResolvedValue({ + mode: 'create', + vfsPath: 'files/dupe.json', + }) + + const req = createMockRequest('POST', { + code: 'print("done")', + language: 'python', + workspaceId: 'workspace-1', + outputs: { + files: [ + { + path: 'files/dupe.json', + mode: 'create', + sandboxPath: '/home/user/first.json', + }, + { + path: 'files/dupe.json', + mode: 'create', + sandboxPath: '/home/user/second.json', + }, + ], + }, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(400) + expect(data.success).toBe(false) + expect(data.error).toContain('Duplicate sandbox output destination') + expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() + }) + + it('returns a targeted error when a declared sandbox output is missing', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'done', + stdout: 'ok', + sandboxId: 'sandbox-123', + exportedFiles: {}, + }) + + const req = createMockRequest('POST', { + code: 'print("done")', + language: 'python', + workspaceId: 'workspace-1', + outputs: { + files: [ + { + path: 'files/missing.json', + mode: 'create', + sandboxPath: '/home/user/missing.json', + }, + ], + }, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(500) + expect(data.success).toBe(false) + expect(data.error).toContain('Sandbox file "/home/user/missing.json" was not found') + expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() + }) + + it('rejects sandboxPath outputs when the call would run in isolated-vm (E2B enabled, JS without imports)', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + + const req = createMockRequest('POST', { + code: 'return "content"', + language: 'javascript', + workspaceId: 'workspace-1', + outputs: { + files: [ + { + path: 'files/doc.md', + mode: 'overwrite', + sandboxPath: '/home/user/doc.md', + }, + ], + }, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(422) + expect(data.success).toBe(false) + expect(data.error).toContain('no sandbox filesystem') + expect(mockExecuteInIsolatedVM).not.toHaveBeenCalled() + expect(mockExecuteInSandbox).not.toHaveBeenCalled() + expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled() + }) + + it('rejects sandbox file mounts when the call would run in isolated-vm', async () => { + const req = createMockRequest('POST', { + code: 'return 1', + language: 'javascript', + workspaceId: 'workspace-1', + _sandboxFiles: [{ path: '/home/user/files/data.csv', content: 'a,b\n1,2' }], + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(422) + expect(data.success).toBe(false) + // No remote sandbox is enabled in this test, so the remediation must name + // that cause instead of suggesting python (which would also fail without one). + expect(data.error).toContain('No remote code sandbox is enabled') + expect(mockExecuteInIsolatedVM).not.toHaveBeenCalled() + }) + + it('flags an overwrite export whose bytes are identical to the current file content as unchanged', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + const staleContent = '# doc\nunchanged mounted content\n' + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'done', + stdout: 'ok', + sandboxId: 'sandbox-123', + exportedFiles: { '/home/user/doc.md': staleContent }, + }) + mockResolveWorkspaceFileReference.mockResolvedValue({ + id: 'wf_doc', + name: 'doc.md', + size: Buffer.byteLength(staleContent, 'utf-8'), + key: 'workspace/doc.md', + }) + mockFetchWorkspaceFileBuffer.mockResolvedValue(Buffer.from(staleContent, 'utf-8')) + + const req = createMockRequest('POST', { + code: 'print("done")', + language: 'python', + workspaceId: 'workspace-1', + outputs: { + files: [ + { + path: 'files/doc.md', + mode: 'overwrite', + sandboxPath: '/home/user/doc.md', + mimeType: 'text/markdown', + }, + ], + }, + }) + + const response = await POST(req) + const data = await response.json() + + // Idempotent overwrites (retries, unchanged regenerations) must not fail; + // the write proceeds and the receipt carries the loud unchanged signal so + // the model can tell its "new content" never reached the sandbox file. + expect(response.status).toBe(200) + expect(data.success).toBe(true) + expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledTimes(1) + expect(data.output.result.unchanged).toBe(true) + expect(data.output.result.message).toContain('byte-identical to the previous version') + expect(data.output.result.message).toContain('/home/user/doc.md') + }) + + it('continues an overwrite when the advisory comparison fails', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + const newContent = '# doc\nnew content\n' + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'done', + stdout: 'ok', + sandboxId: 'sandbox-123', + exportedFiles: { '/home/user/doc.md': newContent }, + }) + mockResolveWorkspaceFileReference.mockRejectedValueOnce( + new Error('comparison storage unavailable') + ) + + const response = await POST( + createMockRequest('POST', { + code: 'print("done")', + language: 'python', + workspaceId: 'workspace-1', + outputs: { + files: [ + { + path: 'files/doc.md', + mode: 'overwrite', + sandboxPath: '/home/user/doc.md', + mimeType: 'text/markdown', + }, + ], + }, + }) + ) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.success).toBe(true) + expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledTimes(1) + expect(data.output.result).toMatchObject({ unchanged: false }) + expect(data.output.result).not.toHaveProperty('previousSize') + }) + + it('reports size, previousSize, and sha256 receipts on a successful overwrite export', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + const newContent = '# doc\nnew content\n' + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'done', + stdout: 'ok', + sandboxId: 'sandbox-123', + exportedFiles: { '/home/user/doc.md': newContent }, + }) + mockResolveWorkspaceFileReference.mockResolvedValue({ + id: 'wf_doc', + name: 'doc.md', + size: 36728, + key: 'workspace/doc.md', + }) + + const req = createMockRequest('POST', { + code: 'print("done")', + language: 'python', + workspaceId: 'workspace-1', + outputs: { + files: [ + { + path: 'files/doc.md', + mode: 'overwrite', + sandboxPath: '/home/user/doc.md', + mimeType: 'text/markdown', + }, + ], + }, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.success).toBe(true) + // Sizes differ, so the current content is never downloaded for comparison. + expect(mockFetchWorkspaceFileBuffer).not.toHaveBeenCalled() + expect(data.output.result.size).toBe(Buffer.byteLength(newContent, 'utf-8')) + expect(data.output.result.previousSize).toBe(36728) + expect(data.output.result.sha256).toMatch(/^[0-9a-f]{64}$/) + expect(data.output.result.unchanged).toBe(false) + expect(data.output.result.message).toContain('replaced 36728 bytes') + expect(data.output.result.message).toContain('sha256:') + // The python wrapper prints the marker with a leading \n so it always + // starts a fresh line even after non-newline-terminated user output. + const e2bCode = mockExecuteInSandbox.mock.calls[0][0].code as string + expect(e2bCode).toContain("print('\\n__SIM_RESULT__=' + json.dumps(__sim_result__))") + }) + + it('runs complete Python modules without nesting their main guard inside a function', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + const source = [ + 'import subprocess', + '', + 'def main():', + ' subprocess.run(["bq", "version"], check=True)', + '', + 'if __name__ == "__main__":', + ' main()', + ].join('\n') + + const response = await POST( + createMockRequest('POST', { + code: source, + language: 'python', + workspaceId: 'workspace-1', + }) + ) + + expect(response.status).toBe(200) + const e2bCode = mockExecuteInSandbox.mock.calls[0][0].code as string + expect(e2bCode).toContain('compile(__sim_source__, "", "exec")') + expect(e2bCode).toContain('__sim_exec_globals__["__name__"] = "__main__"') + expect(e2bCode).toContain(JSON.stringify(source)) + expect(e2bCode).not.toContain('def __sim_main__():\n import subprocess') + }) + + it('supports a Fellows-style Python module that invokes bq and exports a deterministic archive', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + const archiveBase64 = + 'UEsDBBQAAAAIAAAAIQAcWyFBIAAAAB8AAAAMAAAAcHJldmlldy5odG1ss8kwtHNLzcnJLy9WcM4vzUvOzFEIT03Nzqm00QdKAQBQSwECFAMUAAAACAAAACEAHFshQSAAAAAfAAAADAAAAAAAAAAAAAAAgAEAAAAAcHJldmlldy5odG1sUEsFBgAAAAABAAEAOgAAAEoAAAAAAA==' + const source = readFileSync( + resolve(process.cwd(), 'lib/execution/remote-sandbox/fixtures/fellows-council-weekly.py'), + 'utf8' + ) + mockExecuteInSandbox.mockResolvedValueOnce({ + result: null, + stdout: 'generated 1 preview', + sandboxId: 'sandbox-123', + exportedFiles: { '/tmp/fellows-previews.zip': archiveBase64 }, + }) + + const response = await POST( + createMockRequest('POST', { + code: source, + language: 'python', + workspaceId: 'workspace-1', + sandboxId: 'fellows-sandbox', + envVars: { + AIRTABLE_PAT: 'stub-airtable-token', + ANTHROPIC_API_KEY: 'stub-anthropic-key', + GOOGLE_SERVICE_ACCOUNT_JSON: + '{"type":"service_account","project_id":"fixture-project"}', + NCBI_API_KEY: 'stub-ncbi-key', + }, + outputs: { + files: [ + { + path: 'files/fellows-previews.zip', + sandboxPath: '/tmp/fellows-previews.zip', + mimeType: 'application/zip', + }, + ], + }, + }) + ) + + expect(response.status).toBe(200) + const sandboxRequest = mockExecuteInSandbox.mock.calls[0][0] + expect(sandboxRequest.code).toContain("['bq', 'query'") + expect(sandboxRequest.code).toContain('__sim_exec_globals__["__name__"] = "__main__"') + expect(sandboxRequest.sandboxId).toBe('fellows-sandbox') + expect(sandboxRequest.outputSandboxPaths).toEqual(['/tmp/fellows-previews.zip']) + expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledWith( + expect.objectContaining({ + buffer: Buffer.from(archiveBase64, 'base64'), + target: expect.objectContaining({ path: 'files/fellows-previews.zip' }), + }) + ) + }) + + it('retains Function-body return semantics for Python snippets', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + + await POST( + createMockRequest('POST', { + code: 'value = 41\nreturn value + 1', + language: 'python', + workspaceId: 'workspace-1', + }) + ) + + const e2bCode = mockExecuteInSandbox.mock.calls[0][0].code as string + expect(e2bCode).toContain('"outside function" not in str(__sim_compile_error__)') + expect(e2bCode).toContain('__sim_result__ = __sim_exec_globals__["__sim_main__"]()') + }) + + it.each([ + { reason: 'timeout', status: 408, message: 'timed out' }, + { reason: 'user', status: 499, message: 'cancelled' }, + ])('keeps $reason aborts distinct', async ({ reason, status, message }) => { + envFlagsMock.isRemoteSandboxEnabled = true + const controller = new AbortController() + const req = new NextRequest('http://localhost:3000/internal/function-execution', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + code: 'print("running")', + language: 'python', + workspaceId: 'workspace-1', + timeout: 30_000, + }), + signal: controller.signal, + }) + mockExecuteInSandbox.mockImplementationOnce(async () => { + controller.abort(new DOMException(reason, 'AbortError')) + throw controller.signal.reason + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(status) + expect(data.error).toContain(message) + }) + + it.each([ + { + termination: 'timeout' as const, + errorName: 'TimeoutError', + status: 408, + message: 'timed out', + }, + { + termination: 'cancelled' as const, + errorName: 'AbortError', + status: 499, + message: 'cancelled', + }, + ])( + 'classifies trusted isolated-vm $termination results consistently with remote runtimes', + async ({ termination, errorName, status, message }) => { + const partialStdout = `partial output before ${termination}` + mockExecuteInIsolatedVM.mockResolvedValueOnce({ + result: null, + stdout: partialStdout, + error: { name: errorName, message: `${errorName} from isolated-vm` }, + termination, + }) + + const response = await POST( + createMockRequest('POST', { + code: 'return true', + language: 'javascript', + timeout: 30_000, + }) + ) + const data = await response.json() + + expect(response.status).toBe(status) + expect(data.error).toContain(message) + expect(data.output.stdout).toBe(partialStdout) + } + ) + + it.each(['TimeoutError', 'AbortError'])( + 'keeps a user-thrown %s as an ordinary code error', + async (errorName) => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ + result: null, + stdout: 'partial output before user error', + error: { name: errorName, message: `User threw ${errorName}` }, + }) + + const response = await POST( + createMockRequest('POST', { + code: `const error = new Error('user error'); error.name = '${errorName}'; throw error`, + language: 'javascript', + timeout: 30_000, + }) + ) + const data = await response.json() + + expect(response.status).toBe(422) + expect(data.output.stdout).toBe('partial output before user error') + expect(data.debug.errorType).toBe(errorName) + } + ) + + it('enforces the explicit Function timeout with a server-owned abort signal', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockImplementationOnce( + ({ signal }: { signal: AbortSignal }) => + new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => reject(signal.reason), { once: true }) + }) + ) + + const response = await POST( + createMockRequest('POST', { + code: 'print("running")', + language: 'python', + workspaceId: 'workspace-1', + timeout: 1, + }) + ) + const data = await response.json() + + expect(response.status).toBe(408) + expect(data.error).toContain('timed out after 1ms') + }) + + it('uses the remaining workflow deadline when no block timeout is supplied', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + const remainingBudgetMs = 10 * 60_000 + const req = new NextRequest('http://localhost:3000/internal/function-execution', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + [INTERNAL_EXECUTION_DEADLINE_HEADER]: String(Date.now() + remainingBudgetMs), + }, + body: JSON.stringify({ + code: 'print("running")', + language: 'python', + workspaceId: 'workspace-1', + }), + }) + + const response = await POST(req) + + expect(response.status).toBe(200) + const sandboxRequest = mockExecuteInSandbox.mock.calls[0][0] + expect(sandboxRequest.timeoutMs).toBeGreaterThan(9 * 60_000) + expect(sandboxRequest.timeoutMs).toBeLessThanOrEqual(remainingBudgetMs) + }) + + it('classifies a client abort at the propagated execution deadline as a timeout', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + const controller = new AbortController() + const req = new NextRequest('http://localhost:3000/internal/function-execution', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + [INTERNAL_EXECUTION_DEADLINE_HEADER]: String(Date.now() - 1_000), + }, + body: JSON.stringify({ + code: 'print("running")', + language: 'python', + workspaceId: 'workspace-1', + timeout: 30_000, + }), + signal: controller.signal, + }) + mockExecuteInSandbox.mockImplementationOnce(async (sandboxRequest) => { + expect(sandboxRequest.timeoutMs).toBe(1) + controller.abort(new DOMException('The operation was aborted.', 'AbortError')) + throw controller.signal.reason + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(408) + expect(data.error).toContain('timed out') + }) + + it('should return computed result for multi-line code', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 10, stdout: '' }) + + const req = createMockRequest('POST', { + code: 'const a = 1;\nconst b = 2;\nconst c = 3;\nconst d = 4;\nreturn a + b + c + d;', + timeout: 5000, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.success).toBe(true) + expect(data.output.result).toBe(10) + }) + + it.concurrent('should handle missing code parameter', async () => { + const req = createMockRequest('POST', { + timeout: 5000, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(400) + expect(data).toHaveProperty('error') + }) + + it.concurrent('should use default timeout when not provided', async () => { + const req = createMockRequest('POST', { + code: 'return "test"', + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.success).toBe(true) + }) + + it('rejects large refs in runtimes without ref-native helpers', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + const req = createMockRequest('POST', { + code: 'echo "$__blockRef_0"', + language: 'shell', + contextVariables: { + __blockRef_0: { + __simLargeValueRef: true, + version: 1, + id: 'lv_ABCDEFGHIJKL', + kind: 'array', + size: 12 * 1024 * 1024, + executionId: 'execution-1', + }, + }, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(500) + expect(data.success).toBe(false) + expect(data.error).toContain( + 'Large execution values require the JavaScript isolated-vm runtime' + ) + }) + + it('registers manifest array read broker for isolated-vm execution', async () => { + const req = createMockRequest('POST', { + code: 'return await sim.values.readArray(__blockRef_0)', + language: 'javascript', + contextVariables: { + __blockRef_0: { + __simLargeArrayManifest: true, + version: 2, + kind: 'array', + totalCount: 1, + chunkCount: 1, + byteSize: 16, + chunks: [ + { + ref: { + __simLargeValueRef: true, + version: 1, + id: 'lv_ABCDEFGHIJKL', + kind: 'array', + size: 16, + executionId: 'execution-1', + }, + count: 1, + byteSize: 16, + }, + ], + preview: [{ id: 1 }], + }, + }, + }) + + const response = await POST(req) + const data = await response.json() + const [, options] = mockExecuteInIsolatedVM.mock.calls.at(-1) ?? [] + + expect(response.status).toBe(200) + expect(data.success).toBe(true) + expect(options?.brokers).toHaveProperty('sim.values.readArray') + }) + }) + + describe('Template Variable Resolution', () => { + it('should resolve environment variables with {{var_name}} syntax', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'secret-key-123', stdout: '' }) + const req = createMockRequest( + 'POST', + { + code: 'return {{API_KEY}}', + envVars: { + API_KEY: 'secret-key-123', + }, + }, + { + 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', + } + ) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.__resolvedSecretNames).toEqual(['API_KEY']) + }) + + it('keeps an exact-name/exact-value JavaScript secret out of source and returns its raw runtime value with private provenance', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'Test', stdout: '' }) + + const response = await POST( + createMockRequest( + 'POST', + { + code: 'return {{Test}}', + language: 'javascript', + envVars: { Test: 'Test' }, + }, + { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } + ) + ) + + const data = await response.json() + const [request] = mockExecuteInIsolatedVM.mock.calls.at(-1) ?? [] + const bindingEntries = Object.entries(request.contextVariables) + + expect(response.status).toBe(200) + expect(request.code).not.toContain('Test') + expect(request.code).not.toContain('{{Test}}') + expect(request.code).not.toContain('__var_') + expect(bindingEntries).toHaveLength(1) + expect(bindingEntries[0]?.[0]).toMatch(/^__sim_code_\d+_binding_\d+$/) + expect(bindingEntries[0]?.[1]).toBe('Test') + expect(data.output.result).toBe('Test') + expect(data.__resolvedSecretNames).toEqual(['Test']) + expect(JSON.stringify(data)).not.toContain('__sim_code_') + expect(JSON.stringify(data)).not.toContain('__var_') + }) + + it('keeps an exact-name/exact-value Python secret out of source and supplies it only through private runtime input', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteInSandbox.mockResolvedValueOnce({ + result: 'Test', + stdout: '', + sandboxId: 'test-sandbox-id', + }) + + const response = await POST( + createMockRequest( + 'POST', + { + code: 'return {{Test}}', + language: 'python', + envVars: { Test: 'Test' }, + }, + { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } + ) + ) + + const data = await response.json() + const [request] = mockExecuteInSandbox.mock.calls.at(-1) ?? [] + const runtimeInput = request.privateInputs.find( + (input: { environmentVariable: string }) => + input.environmentVariable === '__SIM_RUNTIME_PAYLOAD_PATH' + ) + const runtimePayload = JSON.parse(runtimeInput?.content ?? '{}') + const secretBinding = runtimePayload.contextVariables.find( + (entry: { value?: unknown }) => entry.value === 'Test' + ) + + expect(response.status).toBe(200) + expect(request.code).not.toContain('Test') + expect(request.code).not.toContain('{{Test}}') + expect(request.code).not.toContain('__var_') + expect(runtimePayload.environmentVariables).toEqual({ Test: 'Test' }) + expect(secretBinding).toMatchObject({ kind: 'json', value: 'Test' }) + expect(secretBinding.name).toMatch(/^__sim_code_\d+_binding_\d+__$/) + expect(request.code).toContain(secretBinding.name) + expect(data.output.result).toBe('Test') + expect(data.__resolvedSecretNames).toEqual(['Test']) + expect(JSON.stringify(data)).not.toContain('__sim_code_') + expect(JSON.stringify(data)).not.toContain('__var_') + }) + + it('compiles legacy bare and quoted Custom Tool placeholders into opaque VM bindings', async () => { + const secret = 'quote" slash\\ newline\n{{OTHER}} true 123' + mockExecuteInIsolatedVM.mockResolvedValueOnce({ + result: [secret, secret, `Bearer ${secret}`], + stdout: '', + }) + const response = await POST( + createMockRequest( + 'POST', + { + code: [ + 'const bare = {{API_KEY}}', + 'const quoted = "{{API_KEY}}"', + 'return [bare, quoted, "Bearer {{API_KEY}}"]', + ].join('\n'), + isCustomTool: true, + envVars: { API_KEY: secret, OTHER: 'must-not-resolve' }, + }, + { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } + ) + ) + + const [request] = mockExecuteInIsolatedVM.mock.calls.at(-1) ?? [] + expect(response.status).toBe(200) + expect((await response.json()).__resolvedSecretNames).toEqual(['API_KEY']) + expect(request.code).not.toContain(secret) + expect(request.code).not.toContain('__var_') + expect(request.code).toContain('__sim_code_') + expect(request.code).not.toContain('globalThis[') + expect(Object.values(request.contextVariables)).toContain(secret) + expect(Object.keys(request.contextVariables)).not.toContain('API_KEY') + }) + + it('installs regex constructors as opaque runtime bindings before isolated user code', async () => { + const response = await POST( + createMockRequest('POST', { + code: [ + 'RegExp.prototype.constructor = null', + 'return /^{{PATTERN}}$/.test("candidate")', + ].join('\n'), + envVars: { PATTERN: 'secret' }, + }) + ) + + const [request] = mockExecuteInIsolatedVM.mock.calls.at(-1) ?? [] + const [runtimeBinding] = request.runtimeBindings + expect(response.status).toBe(200) + expect(runtimeBinding.kind).toBe('javascript-runtime') + expect(request.code).toContain(`new ${runtimeBinding.name}.RegExp`) + expect(request.code).not.toContain('secret') + }) + + it('captures regex constructors in the remote preload before static imports execute', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + const response = await POST( + createMockRequest('POST', { + code: ['import "side-effect-module"', 'return /^{{PATTERN}}$/.test("candidate")'].join( + '\n' + ), + language: 'javascript', + envVars: { PATTERN: 'secret' }, + }) + ) + + const [sandboxRequest] = mockExecuteInSandbox.mock.calls.at(-1) ?? [] + const runtimeBindingName = /new (__sim_code_\d+_runtime_\d+)\.RegExp/.exec( + sandboxRequest.code + )?.[1] + expect(response.status).toBe(200) + expect(runtimeBindingName).toBeDefined() + expect(sandboxRequest.runtimeBindings).toContainEqual({ + name: runtimeBindingName, + kind: 'javascript-runtime', + }) + expect(JSON.stringify(sandboxRequest.runtimeBindings)).not.toContain('secret') + }) + + it('allocates remote runtime helpers against decoded JavaScript identifiers', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + const escapedAlias = String.raw`\u005f\u005fsim_runtime_read_0` + + const response = await POST( + createMockRequest('POST', { + code: [ + `import { basename as ${escapedAlias} } from "node:path"`, + `return ["{{KEY}}", ${escapedAlias}("/tmp/file.txt")]`, + ].join('\n'), + language: 'javascript', + envVars: { KEY: 'secret' }, + }) + ) + + const [sandboxRequest] = mockExecuteInSandbox.mock.calls.at(-1) ?? [] + const syntaxCheck = spawnSync(process.execPath, ['--input-type=module', '--check'], { + encoding: 'utf8', + input: sandboxRequest.code, + }) + expect(response.status).toBe(200) + expect(sandboxRequest.code).toContain('readFileSync as __sim_runtime_read_1') + expect(sandboxRequest.code).not.toContain('readFileSync as __sim_runtime_read_0') + expect(syntaxCheck.stderr).toBe('') + expect(syntaxCheck.status).toBe(0) + }) + + it('keeps comments and missing placeholders unchanged without secret provenance', async () => { + const response = await POST( + createMockRequest( + 'POST', + { + code: '// {{COMMENT_ONLY}}\nreturn "{{MISSING}}"', + envVars: { COMMENT_ONLY: 'must-not-bind' }, + }, + { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } + ) + ) + + const [request] = mockExecuteInIsolatedVM.mock.calls.at(-1) ?? [] + expect(response.status).toBe(200) + expect((await response.json()).__resolvedSecretNames).toEqual([]) + expect(request.code).toContain('// {{COMMENT_ONLY}}') + expect(request.code).toContain('"{{MISSING}}"') + expect(Object.values(request.contextVariables)).not.toContain('must-not-bind') + }) + + it('does not infer provenance from an unused low-entropy environment value', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'Box eSign', stdout: '' }) + + const response = await POST( + createMockRequest( + 'POST', + { + code: 'return "Box eSign"', + envVars: { SERVICENOW_PASSWORD: 'x' }, + }, + { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } + ) + ) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.output.result).toBe('Box eSign') + expect(data.__resolvedSecretNames).toEqual([]) + }) + + it('does not build provenance matchers for unused oversized environment values', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'safe', stdout: '' }) + + const response = await POST( + createMockRequest( + 'POST', + { + code: 'return "safe"', + envVars: { UNUSED: 'x'.repeat(65 * 1024) }, + }, + { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } + ) + ) + + expect(response.status).toBe(200) + expect((await response.json()).__resolvedSecretNames).toEqual([]) + }) + + it('conservatively reports only compiled secrets when bounded output classification is exceeded', async () => { + const result = Array.from({ length: 100_001 }, () => 'ordinary') + mockExecuteInIsolatedVM.mockResolvedValueOnce({ result, stdout: '' }) + + const response = await POST( + createMockRequest( + 'POST', + { + code: 'const key = {{API_KEY}}; return params.items', + params: { items: result }, + envVars: { API_KEY: 'secret-value', UNUSED: 'x' }, + }, + { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } + ) + ) + const data = await response.json() + + expect(response.status).toBe(200) + expect(response.headers.get('x-sim-private-tool-metadata')).toBe('resolved-secret-names-v1') + expect(data.output.result).toHaveLength(100_001) + expect(data.output.result[0]).toBe('ordinary') + expect(data.__resolvedSecretNames).toEqual(['API_KEY']) + }) + + it('conservatively reports a compiled secret whose value exceeds matcher capacity', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'ordinary', stdout: '' }) + + const response = await POST( + createMockRequest( + 'POST', + { + code: 'const key = {{OVERSIZED_SECRET}}; return "ordinary"', + envVars: { OVERSIZED_SECRET: 's'.repeat(64 * 1024 + 1), UNUSED: 'x' }, + }, + { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } + ) + ) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.output.result).toBe('ordinary') + expect(data.__resolvedSecretNames).toEqual(['OVERSIZED_SECRET']) + }) + + it('tracks only compiled names when configured secrets share the same value', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'true', stdout: '' }) + const oneResponse = await POST( + createMockRequest( + 'POST', + { + code: 'return {{SECOND}}', + envVars: { FIRST: 'true', SECOND: 'true' }, + }, + { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } + ) + ) + + mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'true', stdout: '' }) + const bothResponse = await POST( + createMockRequest( + 'POST', + { + code: 'const first = {{FIRST}}; return {{SECOND}}', + envVars: { FIRST: 'true', SECOND: 'true' }, + }, + { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } + ) + ) + + expect((await oneResponse.json()).__resolvedSecretNames).toEqual(['SECOND']) + expect((await bothResponse.json()).__resolvedSecretNames).toEqual(['FIRST', 'SECOND']) + }) + + it('lowers missing shell placeholders while preserving comments and heredoc delimiters', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + const response = await POST( + createMockRequest( + 'POST', + { + code: [ + '# {{COMMENT_ONLY}}', + 'printf \'%s\\n\' "before{{MISSING}}after"', + "cat <<'{{DELIMITER}}'", + 'literal body', + '{{DELIMITER}}', + ].join('\n'), + language: 'shell', + envVars: { COMMENT_ONLY: 'must-not-bind' }, + }, + { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } + ) + ) + + const [request] = mockExecuteShellInSandbox.mock.calls.at(-1) ?? [] + expect(response.status).toBe(200) + expect((await response.json()).__resolvedSecretNames).toEqual([]) + expect(request.code).toContain('# {{COMMENT_ONLY}}') + expect(request.code).toContain('"beforeafter"') + expect(request.code).toContain("cat <<'{{DELIMITER}}'") + expect(request.code).toContain('\n{{DELIMITER}}') + expect(request.code).not.toContain('{{MISSING}}') + }) + + it.each([ + { + language: 'javascript', + code: 'import path from "node:path"\nreturn "{{API_KEY}}"', + }, + { language: 'python', code: 'return "{{API_KEY}}"' }, + ])( + 'keeps $language runtime values out of remote generated source', + async ({ language, code }) => { + envFlagsMock.isRemoteSandboxEnabled = true + const secret = 'remote"\\\nsecret' + + const response = await POST( + createMockRequest('POST', { + code, + language, + envVars: { API_KEY: secret }, + params: { input: 'value' }, + contextVariables: { __blockRef_0: 'context' }, + }) + ) + + const [sandboxRequest] = mockExecuteInSandbox.mock.calls.at(-1) ?? [] + expect(response.status).toBe(200) + expect(sandboxRequest.code).not.toContain(secret) + expect(sandboxRequest.code).not.toContain('__var_') + expect(sandboxRequest.privateInputs).toHaveLength(1) + const payload = JSON.parse(sandboxRequest.privateInputs[0].content) + expect(payload.environmentVariables.API_KEY).toBe(secret) + expect(payload.params.input).toBe('value') + expect(payload.contextVariables).toContainEqual({ + name: '__blockRef_0', + kind: 'json', + value: 'context', + }) + } + ) + + it('routes quoted shell heredocs through private sandbox input files', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + const secret = 'shell"\\\n{{OTHER}}' + mockExecuteShellInSandbox.mockResolvedValueOnce({ + result: null, + stdout: `Bearer ${secret}\n$UNRELATED \`touch /tmp/nope\``, + sandboxId: 'test-shell-sandbox-id', + }) + + const response = await POST( + createMockRequest( + 'POST', + { + code: [ + "cat <<'PAYLOAD'", + 'Bearer {{API_KEY}}', + '$UNRELATED `touch /tmp/nope`', + 'PAYLOAD', + ].join('\n'), + language: 'shell', + envVars: { API_KEY: secret, OTHER: 'must-not-resolve' }, + }, + { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } + ) + ) + + const [sandboxRequest] = mockExecuteShellInSandbox.mock.calls.at(-1) ?? [] + expect(response.status).toBe(200) + expect((await response.json()).__resolvedSecretNames).toEqual(['API_KEY']) + expect(sandboxRequest.code).not.toContain(secret) + expect(sandboxRequest.code).not.toContain('$UNRELATED') + expect(sandboxRequest.privateInputs).toHaveLength(1) + expect(sandboxRequest.privateInputs[0].content).toContain(secret) + expect(sandboxRequest.privateInputs[0].content).toContain('$UNRELATED `touch /tmp/nope`') + }) + + /** + * The founding scenario of the usage trail: code that reads a secret and emits it only in + * transformed form. No output ever matches the value, so an output-gated report said + * "never used" for exactly the run an admin needs to see. A referenced secret reports + * whether or not its value surfaces. + */ + it('reports a secret exfiltrated character by character', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ + result: 's|e|c|r|e|t|-|v|a|l|u|e|-|1|2|3|4', + stdout: '', + }) + const response = await POST( + createMockRequest( + 'POST', + { + code: "const k = '{{API_KEY}}'; return k.split('').join('|')", + envVars: { API_KEY: 'secret-value-1234' }, + }, + { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } + ) + ) + + expect(response.status).toBe(200) + expect((await response.json()).__resolvedSecretNames).toEqual(['API_KEY']) + }) + + /** The ordinary silent use: the key authenticates a call and never appears in output. */ + it('reports a secret used without appearing in the output', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: { status: 200 }, stdout: '' }) + const response = await POST( + createMockRequest( + 'POST', + { + code: "await fetch('https://api.example.com', { headers: { auth: environmentVariables['API_KEY'] } }); return { status: 200 }", + envVars: { API_KEY: 'secret-value-1234' }, + }, + { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } + ) + ) + + expect(response.status).toBe(200) + expect((await response.json()).__resolvedSecretNames).toEqual(['API_KEY']) + }) + + it('does not report a reference when validation rejects before code resolution', async () => { + const response = await POST( + createMockRequest( + 'POST', + { + code: 'return {{API_KEY}}', + envVars: { API_KEY: 'secret-value' }, + outputs: { + files: Array.from({ length: 21 }, (_, index) => ({ + path: `files/output-${index}.json`, + sandboxPath: `/home/user/output-${index}.json`, + })), + }, + }, + { + 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', + } + ) + ) + const data = await response.json() + + expect(response.status).toBe(400) + expect(data.error).toContain('Too many sandbox output files requested') + expect(data.__resolvedSecretNames).toEqual([]) + expect(mockExecuteInIsolatedVM).not.toHaveBeenCalled() + expect(mockExecuteInSandbox).not.toHaveBeenCalled() + }) + + /** + * A direct read is a factual reference to the environment binding, not the value-coincidence + * inference #6374 removed — that one claimed a secret because its plaintext happened to equal + * an unrelated output. Reporting it is what activates execution-log masking for the value. + */ + it('reports secrets reached through placeholders and through direct environment reads', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ + result: 'secret-valueother-secret', + stdout: '', + }) + const envResponse = await POST( + createMockRequest( + 'POST', + { + code: 'return {{SHARED}} + {{ENV_ONLY}} + {{MISSING}}', + params: { SHARED: 'param-value', MISSING: 'ordinary-param' }, + envVars: { SHARED: 'secret-value', ENV_ONLY: 'other-secret' }, + }, + { + 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', + } + ) + ) + const envData = await envResponse.json() + + mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'secret-value', stdout: '' }) + const directResponse = await POST( + createMockRequest( + 'POST', + { + code: 'return environmentVariables.API_KEY + params.API_KEY', + params: { API_KEY: 'ordinary-param' }, + envVars: { API_KEY: 'secret-value' }, + }, + { + 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', + } + ) + ) + const directData = await directResponse.json() + + expect(envData.__resolvedSecretNames).toEqual(['ENV_ONLY', 'SHARED']) + expect(directData.output.result).toBe('secret-value') + expect(directData.__resolvedSecretNames).toEqual(['API_KEY']) + }) + + it.each([ + { name: 'numeric', secret: '123', result: 123 }, + { name: 'boolean', secret: 'true', result: true }, + ])( + 'preserves a typed $name value returned through a direct environment read while reporting it', + async ({ secret, result }) => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ result, stdout: '' }) + + const response = await POST( + createMockRequest( + 'POST', + { + code: 'return environmentVariables.API_KEY', + envVars: { API_KEY: secret }, + }, + { + 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', + } + ) + ) + const data = await response.json() + + /** The typed value survives: a secret this short is never substitutable. */ + expect(data.output.result).toBe(result) + expect(data.__resolvedSecretNames).toEqual(['API_KEY']) + } + ) + + it('reports placeholder output and a shell environment expansion alike', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + mockExecuteShellInSandbox.mockResolvedValueOnce({ + result: null, + stdout: 'secret-value', + sandboxId: 'test-shell-sandbox-id', + }) + + const referencedResponse = await POST( + createMockRequest( + 'POST', + { + code: 'printf "%s" "{{API_KEY}}"', + language: 'shell', + envVars: { API_KEY: 'secret-value' }, + }, + { + 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', + } + ) + ) + const referencedData = await referencedResponse.json() + + mockExecuteShellInSandbox.mockResolvedValueOnce({ + result: null, + stdout: 'secret-value', + sandboxId: 'test-shell-sandbox-id', + }) + const directResponse = await POST( + createMockRequest( + 'POST', + { + code: 'printf "%s" "$API_KEY"', + language: 'shell', + envVars: { API_KEY: 'secret-value' }, + }, + { + 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', + } + ) + ) + const directData = await directResponse.json() + + expect(referencedData.__resolvedSecretNames).toEqual(['API_KEY']) + expect(directData.output.stdout).toBe('secret-value') + expect(directData.__resolvedSecretNames).toEqual(['API_KEY']) + }) + + it('returns nonzero shell stderr as a visible 422 error and diagnostic output', async () => { + envFlagsMock.isRemoteSandboxEnabled = true + const stderr = "error: unknown flag: --short\nSee 'kubectl version --help' for usage." + mockExecuteShellInSandbox.mockResolvedValueOnce({ + result: null, + stdout: stderr, + error: stderr, + sandboxId: 'test-shell-sandbox-id', + }) + + const response = await POST( + createMockRequest('POST', { + code: 'kubectl version --client --short', + language: 'shell', + }) + ) + const data = await response.json() + + expect(response.status).toBe(422) + expect(data).toMatchObject({ + success: false, + error: stderr, + output: { result: null, stdout: stderr }, + }) + }) + + it('keeps execution available when the scoped catalog exceeds provenance matcher bounds', async () => { + const response = await POST( + createMockRequest( + 'POST', + { + code: 'return "ok"', + envVars: { OVERSIZED_SECRET: 's'.repeat(64 * 1024 + 1) }, + }, + { + 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', + } + ) + ) + + expect(response.status).toBe(200) + expect((await response.json()).__resolvedSecretNames).toEqual([]) + expect(response.headers.get('x-sim-private-tool-metadata')).toBe('resolved-secret-names-v1') + expect(mockExecuteInIsolatedVM).toHaveBeenCalled() + }) + + it('reports only substitutions allowed by the Function secret scope', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'allowed-secret', stdout: '' }) + const response = await POST( + createMockRequest( + 'POST', + { + code: 'return {{ALLOWED}} + {{BLOCKED}}', + envVars: { ALLOWED: 'allowed-secret', BLOCKED: 'blocked-secret' }, + secretScope: 'selected', + mountedSecrets: ['ALLOWED'], + }, + { + 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', + } + ) + ) + + expect((await response.json()).__resolvedSecretNames).toEqual(['ALLOWED']) + }) + + it('resolves a selected __proto__ secret as an own environment key', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'secret-value', stdout: '' }) + const response = await POST( + createMockRequest( + 'POST', + { + code: 'return "{{__proto__}}"', + envVars: Object.fromEntries([['__proto__', 'secret-value']]), + secretScope: 'selected', + mountedSecrets: ['__proto__'], + }, + { + 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1', + } + ) + ) + + expect(response.status).toBe(200) + expect((await response.json()).__resolvedSecretNames).toEqual(['__proto__']) + }) + + /** + * Previously asserted the inverse: a referenced secret whose value stayed out of the + * result reported nothing. That gate made the trail miss silent use — the ordinary + * API-call case and the transformed-exfiltration case alike — so activation now follows + * the referenced set. The value never appearing costs nothing downstream; the masking + * matcher simply never fires on it. + */ + it('activates a referenced secret even when its value never crosses the result', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: 'safe-result', stdout: '' }) + + const response = await POST( + createMockRequest( + 'POST', + { + code: 'const key = {{API_KEY}}; return "safe-result"', + envVars: { API_KEY: 'secret-value' }, + }, + { 'x-sim-request-private-tool-metadata': 'resolved-secret-names-v1' } + ) + ) + + expect((await response.json()).__resolvedSecretNames).toEqual(['API_KEY']) + }) + + it.concurrent('should resolve tag variables with syntax', async () => { + const req = createMockRequest('POST', { + code: 'return ', + blockData: { + 'block-123': { id: '123', subject: 'Test Email' }, + }, + blockNameMapping: { + email: 'block-123', + }, + }) + + const response = await POST(req) + + expect(response.status).toBe(200) + }) + + it.concurrent('should NOT treat email addresses as template variables', async () => { + const req = createMockRequest('POST', { + code: 'return "Email sent to user"', + params: { + email: { + from: 'Dr. Shaw ', + to: 'User ', + }, + }, + }) + + const response = await POST(req) + + expect(response.status).toBe(200) + }) + + it.concurrent('should only match valid variable names in angle brackets', async () => { + const req = createMockRequest('POST', { + code: 'return + "" + ', + blockData: { + 'block-1': 'hello', + 'block-2': 'world', + }, + blockNameMapping: { + validvar: 'block-1', + another_valid: 'block-2', + }, + }) + + const response = await POST(req) + + expect(response.status).toBe(200) + }) + }) + + describe('Gmail Email Data Handling', () => { + it.concurrent( + 'should handle Gmail webhook data with email addresses containing angle brackets', + async () => { + const emailData = { + id: '123', + from: 'Dr. Shaw ', + to: 'User ', + subject: 'Test Email', + bodyText: 'Hello world', + } + + const req = createMockRequest('POST', { + code: 'return ', + blockData: { + 'block-email': emailData, + }, + blockNameMapping: { + email: 'block-email', + }, + }) + + const response = await POST(req) + + expect(response.status).toBe(200) + const data = await response.json() + expect(data.success).toBe(true) + } + ) + + it.concurrent( + 'should properly serialize complex email objects with special characters', + async () => { + const emailData = { + from: 'Test User ', + bodyHtml: '
HTML content with "quotes" and \'apostrophes\'
', + bodyText: 'Text with\nnewlines\tand\ttabs', + } + + const req = createMockRequest('POST', { + code: 'return ', + blockData: { + 'block-email': emailData, + }, + blockNameMapping: { + email: 'block-email', + }, + }) + + const response = await POST(req) + + expect(response.status).toBe(200) + } + ) + }) + + describe('Custom Tools', () => { + it.concurrent('should handle custom tool execution with direct parameter access', async () => { + const req = createMockRequest('POST', { + code: 'return location + " weather is sunny"', + params: { + location: 'San Francisco', + }, + isCustomTool: true, + }) + + const response = await POST(req) + + expect(response.status).toBe(200) + }) + }) + + describe('Security and Edge Cases', () => { + it.concurrent('should handle malformed JSON in request body', async () => { + const req = new NextRequest('http://localhost:3000/internal/function-execution', { + method: 'POST', + body: 'invalid json{', + headers: { 'Content-Type': 'application/json' }, + }) + + const response = await POST(req) + + expect(response.status).toBe(400) + }) + + it.concurrent('should handle timeout parameter', async () => { + const req = createMockRequest('POST', { + code: 'return "test"', + timeout: 10000, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.success).toBe(true) + expect(mockExecuteInIsolatedVM).toHaveBeenCalledWith( + expect.objectContaining({ timeoutMs: 10000 }), + expect.any(Object) + ) + }) + + it.concurrent('should handle empty parameters object', async () => { + const req = createMockRequest('POST', { + code: 'return "no params"', + params: {}, + }) + + const response = await POST(req) + + expect(response.status).toBe(200) + }) + }) + + describe('Enhanced Error Handling', () => { + it('should provide detailed syntax error with line content', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ + result: null, + stdout: '', + error: { message: 'Unexpected end of input', name: 'SyntaxError' }, + }) + + const req = createMockRequest('POST', { + code: 'const obj = {\n name: "test",\n description: "This has a missing closing quote\n};\nreturn obj;', + timeout: 5000, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(422) + expect(data.success).toBe(false) + expect(data.error).toBeTruthy() + }) + + it('should provide detailed runtime error with line and column', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ + result: null, + stdout: '', + error: { + message: "Cannot read properties of null (reading 'someMethod')", + name: 'TypeError', + }, + }) + + const req = createMockRequest('POST', { + code: 'const obj = null;\nreturn obj.someMethod();', + timeout: 5000, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(422) + expect(data.success).toBe(false) + expect(data.error).toContain('Type Error') + expect(data.error).toContain('Cannot read properties of null') + }) + + it('should handle ReferenceError with enhanced details', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ + result: null, + stdout: '', + error: { message: 'undefinedVariable is not defined', name: 'ReferenceError' }, + }) + + const req = createMockRequest('POST', { + code: 'const x = 42;\nreturn undefinedVariable + x;', + timeout: 5000, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(422) + expect(data.success).toBe(false) + expect(data.error).toContain('Reference Error') + expect(data.error).toContain('undefinedVariable is not defined') + }) + + it('should show original source code when resolved block references cause syntax errors', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ + result: null, + stdout: '', + error: { + message: 'Unexpected identifier "globalThis"', + name: 'SyntaxError', + line: 1, + column: 7, + lineContent: 'retur globalThis["__blockRef_0"]', + }, + }) + + const req = createMockRequest('POST', { + code: 'retur globalThis["__blockRef_0"]', + sourceCode: 'retur ', + contextVariables: { __blockRef_0: 'value' }, + timeout: 5000, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(422) + expect(data.success).toBe(false) + expect(data.error).toContain('Line 1: `retur `') + expect(data.error).not.toContain('globalThis') + expect(data.debug.lineContent).toBe('retur ') + }) + + it('should handle thrown errors gracefully', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ + result: null, + stdout: '', + error: { message: 'Custom error message', name: 'Error' }, + }) + + const req = createMockRequest('POST', { + code: 'throw new Error("Custom error message");', + timeout: 5000, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(422) + expect(data.success).toBe(false) + expect(data.error).toContain('Custom error message') + }) + + it('should provide helpful suggestions for common syntax errors', async () => { + mockExecuteInIsolatedVM.mockResolvedValueOnce({ + result: null, + stdout: '', + error: { message: 'Unexpected end of input', name: 'SyntaxError' }, + }) + + const req = createMockRequest('POST', { + code: 'const obj = {\n name: "test"\n// Missing closing brace', + timeout: 5000, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(422) + expect(data.success).toBe(false) + expect(data.error).toBeTruthy() + }) + }) + + describe('Utility Functions', () => { + it.concurrent('should properly escape regex special characters', async () => { + const req = createMockRequest('POST', { + code: 'return {{special.chars+*?}}', + envVars: { + 'special.chars+*?': 'escaped-value', + }, + }) + + const response = await POST(req) + + expect(response.status).toBe(200) + }) + + it.concurrent('should handle JSON serialization edge cases', async () => { + const complexData = { + special: 'chars"with\'quotes', + unicode: '🎉 Unicode content', + nested: { + deep: { + value: 'test', + }, + }, + } + + const req = createMockRequest('POST', { + code: 'return ', + blockData: { + 'block-complex': complexData, + }, + blockNameMapping: { + complexdata: 'block-complex', + }, + }) + + const response = await POST(req) + + expect(response.status).toBe(200) + }) + }) +}) diff --git a/apps/sim/lib/function-execution/execute-request.ts b/apps/sim/lib/function-execution/execute-request.ts index cb3904b0804..443a3fdc0e1 100644 --- a/apps/sim/lib/function-execution/execute-request.ts +++ b/apps/sim/lib/function-execution/execute-request.ts @@ -1,9 +1,9 @@ -import type { Principal } from '@sim/auth/principal' +import type { DelegatedPrincipal, Principal } from '@sim/auth/principal' import { createLogger } from '@sim/logger' import { sha256Hex } from '@sim/security/hash' import { getErrorMessage } from '@sim/utils/errors' import { toRecord } from '@sim/utils/object' -import { type NextRequest, NextResponse } from 'next/server' +import { NextResponse } from 'next/server' import type { ParsedFunctionExecuteBody } from '@/lib/api/contracts' import { FORMAT_TO_CONTENT_TYPE, @@ -22,6 +22,7 @@ import { isTimeoutAbortReason, type TimeoutAbortController, } from '@/lib/core/execution-limits' +import { asOrchestrationError } from '@/lib/core/orchestration/types' import { encryptSecret } from '@/lib/core/security/encryption' import { setRecordValue } from '@/lib/core/utils/records' import { generateRequestId } from '@/lib/core/utils/request' @@ -84,15 +85,10 @@ import { type WorkspaceFileSecretProvenance, } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' import { getWorkflowById } from '@/lib/workflows/utils' -import { createWorkspaceFileDelegatedPrincipal } from '@/lib/workspace-files/application/delegated-principal' +import { rebindWorkspaceFileDelegatedPrincipal } from '@/lib/workspace-files/application/delegated-principal' import { fileOperations } from '@/lib/workspace-files/application/operations' import { readWorkspaceFileContent } from '@/lib/workspace-files/application/read-workspace-file-content' import { resolveWorkspaceFileReference } from '@/lib/workspace-files/application/resolve-workspace-file-reference' -import { - checkWorkspaceAccess, - resolveWorkspaceAccess, - type WorkspaceAccess, -} from '@/lib/workspaces/permissions/utils' import { escapeRegExp, normalizeName, REFERENCE } from '@/executor/constants' import { type OutputSchema, resolveBlockReference } from '@/executor/utils/block-reference' import { @@ -970,6 +966,7 @@ function serializeForShellEnv(value: unknown, nullValue = ''): string { } interface FunctionRouteExecutionContext { + principal: DelegatedPrincipal workflowId?: string workspaceId?: string executionId?: string @@ -977,7 +974,8 @@ interface FunctionRouteExecutionContext { largeValueKeys?: string[] fileKeys?: string[] allowLargeValueWorkflowScope?: boolean - userId?: string + attributedUserId: string + fileAccessUserId?: string requestId: string resolvedSecretNames: Set includePrivateResolvedSecretNames: boolean @@ -1078,6 +1076,7 @@ function createFunctionRuntimeBrokers( const largeValueKeys = context.largeValueKeys const fileKeys = context.fileKeys const base = { + principal: context.principal, requestId: context.requestId, workflowId: context.workflowId, workspaceId: context.workspaceId, @@ -1086,7 +1085,7 @@ function createFunctionRuntimeBrokers( largeValueKeys, fileKeys, allowLargeValueWorkflowScope: context.allowLargeValueWorkflowScope, - userId: context.userId, + userId: context.fileAccessUserId, logger, } @@ -1158,7 +1157,7 @@ async function compactFunctionRouteBody( workflowId: context.workflowId, workspaceId: context.workspaceId, executionId: context.executionId, - userId: context.userId, + userId: context.attributedUserId, preserveRoot: true, requireDurable: Boolean(context.workspaceId && context.workflowId && context.executionId), }) @@ -1326,8 +1325,13 @@ async function appendPrivateResolvedSecretNames( ) } +export interface FunctionExecutionRequestContext { + headers: Headers + signal: AbortSignal +} + export function projectFunctionValidationResponse( - req: NextRequest, + req: Pick, response: NextResponse ): Promise { const metadataType = getRequestedResolvedSecretNamesMetadataType(req.headers) @@ -1406,21 +1410,8 @@ function exportFailure( ) } -/** - * Both `workspaceId` and `workflowId` arrive in the request body, so the workspace an export - * resolves to is caller-controlled either way. Returns null when the acting user cannot write to - * it, gating the secret-provenance scan and overwrite probe that run before the write itself. - */ -async function authorizeExportWorkspace( - workspaceId: string, - authUserId: string, - provided?: WorkspaceAccess -): Promise { - const access = await resolveWorkspaceAccess(workspaceId, authUserId, provided) - if (access.exists && access.canWrite) return access - - logger.warn('Sandbox file export denied for workspace', { workspaceId, userId: authUserId }) - return null +function workspaceFileExportErrorStatus(error: unknown): number { + return asOrchestrationError(error)?.code === 'forbidden' ? 403 : 400 } async function maybeExportSandboxFileToWorkspace(args: { @@ -1428,7 +1419,6 @@ async function maybeExportSandboxFileToWorkspace(args: { authUserId: string workflowId?: string workspaceId?: string - workspaceAccess?: WorkspaceAccess outputPath?: string outputFormat?: string outputMimeType?: string @@ -1444,7 +1434,6 @@ async function maybeExportSandboxFileToWorkspace(args: { authUserId, workflowId, workspaceId, - workspaceAccess, outputPath, outputFormat, outputMimeType, @@ -1479,9 +1468,6 @@ async function maybeExportSandboxFileToWorkspace(args: { ) } - const access = await authorizeExportWorkspace(resolvedWorkspaceId, authUserId, workspaceAccess) - if (!access) return exportFailure('Workspace access denied', 403, stdout, executionTime) - if (exportedFileContent === undefined) { return exportFailure( `Sandbox file "${outputSandboxPath}" was not found or could not be read`, @@ -1518,9 +1504,8 @@ async function maybeExportSandboxFileToWorkspace(args: { const mode = outputMode ?? (overwriteFileId ? 'overwrite' : 'create') const targetPath = mode === 'create' ? outputPath : overwriteFileId || outputPath - const principal = createWorkspaceFileDelegatedPrincipal({ - serviceId: 'executor', - subjectUserId: authUserId, + const principal = rebindWorkspaceFileDelegatedPrincipal({ + principal: routeContext.principal, workspaceId: resolvedWorkspaceId, delegationId: `function-execute:${routeContext.requestId}`, executionId: routeContext.executionId, @@ -1586,7 +1571,7 @@ async function maybeExportSandboxFileToWorkspace(args: { } catch (error) { return exportFailure( getErrorMessage(error, 'Failed to export sandbox file'), - 400, + workspaceFileExportErrorStatus(error), stdout, executionTime ) @@ -1598,7 +1583,6 @@ async function maybeExportSandboxFilesToWorkspace(args: { authUserId: string workflowId?: string workspaceId?: string - workspaceAccess?: WorkspaceAccess outputFiles: OutputFileDeclaration[] exportedFiles?: Record exportedFileContent?: string @@ -1623,7 +1607,6 @@ async function maybeExportSandboxFilesToWorkspace(args: { authUserId: args.authUserId, workflowId: args.workflowId, workspaceId: args.workspaceId, - workspaceAccess: args.workspaceAccess, outputPath: file.formatPath ?? file.path, outputFormat: file.format, outputMimeType: file.mimeType, @@ -1649,15 +1632,6 @@ async function maybeExportSandboxFilesToWorkspace(args: { ) } - const access = await authorizeExportWorkspace( - resolvedWorkspaceId, - args.authUserId, - args.workspaceAccess - ) - if (!access) { - return exportFailure('Workspace access denied', 403, args.stdout, args.executionTime) - } - const preparedFiles = [] let totalOutputBytes = 0 for (const file of sandboxFiles) { @@ -1711,9 +1685,8 @@ async function maybeExportSandboxFilesToWorkspace(args: { }) } - const principal = createWorkspaceFileDelegatedPrincipal({ - serviceId: 'executor', - subjectUserId: args.authUserId, + const principal = rebindWorkspaceFileDelegatedPrincipal({ + principal: args.routeContext.principal, workspaceId: resolvedWorkspaceId, delegationId: `function-execute:${args.routeContext.requestId}`, executionId: args.routeContext.executionId, @@ -1733,7 +1706,7 @@ async function maybeExportSandboxFilesToWorkspace(args: { } catch (error) { return exportFailure( getErrorMessage(error, 'Invalid sandbox output destination'), - 400, + workspaceFileExportErrorStatus(error), args.stdout, args.executionTime ) @@ -1800,7 +1773,7 @@ async function maybeExportSandboxFilesToWorkspace(args: { } catch (error) { return exportFailure( getErrorMessage(error, 'Failed to export sandbox files'), - 400, + workspaceFileExportErrorStatus(error), args.stdout, args.executionTime ) @@ -1852,19 +1825,15 @@ async function maybeExportSandboxFilesToWorkspace(args: { } export interface TrustedFunctionExecutionAuth { - userId: string + attributedUserId: string + fileAccessUserId?: string + principal: DelegatedPrincipal sandboxProfile?: 'mothership' } -/** - * Executes the Function protocol after the caller has authenticated the human subject. - * - * The public route uses the legacy internal-token adapter below. Trusted in-process callers use - * the authorized Function application operation, which supplies this subject without creating a - * second Sim-to-Sim HTTP request. - */ +/** Executes the Function protocol after the application operation authorizes its principal. */ export async function executeFunctionRequest( - req: NextRequest, + req: FunctionExecutionRequestContext, body: ParsedFunctionExecuteBody, auth: TrustedFunctionExecutionAuth ): Promise { @@ -1958,24 +1927,6 @@ export async function executeFunctionRequest( _sandboxFiles, } = body - // The internal JWT carries no workspace scope, so a body-supplied workspaceId would - // otherwise be the sole authorization input for sandbox selection and file exports. - // Denial is returned rather than thrown: this handler's catch-all would turn a thrown - // WorkspaceAccessDeniedError into a 500 before withRouteHandler could map it. - const workspaceAccess = workspaceId - ? await checkWorkspaceAccess(workspaceId, auth.userId) - : undefined - if (workspaceAccess && (!workspaceAccess.exists || !workspaceAccess.hasAccess)) { - logger.warn(`[${requestId}] Function execution denied for workspace`, { - workspaceId, - userId: auth.userId, - }) - return NextResponse.json( - { success: false, error: 'Workspace access denied' }, - { status: 403 } - ) - } - if (selectedSandboxId && !isRemoteSandboxEnabled) { return NextResponse.json( { success: false, error: 'The Function code sandbox is not configured' }, @@ -2048,6 +1999,7 @@ export async function executeFunctionRequest( }) routeContext = { + principal: auth.principal, workflowId, workspaceId, executionId, @@ -2055,7 +2007,8 @@ export async function executeFunctionRequest( largeValueKeys, fileKeys, allowLargeValueWorkflowScope, - userId: auth.userId, + attributedUserId: auth.attributedUserId, + fileAccessUserId: auth.fileAccessUserId, requestId, resolvedSecretNames: new Set(), includePrivateResolvedSecretNames, @@ -2218,10 +2171,9 @@ export async function executeFunctionRequest( if (outputSandboxPaths.length > 0 || outputSandboxPath) { const fileExportResponse = await maybeExportSandboxFilesToWorkspace({ routeContext, - authUserId: auth.userId, + authUserId: auth.attributedUserId, workflowId, workspaceId, - workspaceAccess, outputFiles, exportedFiles, exportedFileContent, @@ -2398,10 +2350,9 @@ export async function executeFunctionRequest( if (outputSandboxPaths.length > 0 || outputSandboxPath) { const fileExportResponse = await maybeExportSandboxFilesToWorkspace({ routeContext, - authUserId: auth.userId, + authUserId: auth.attributedUserId, workflowId, workspaceId, - workspaceAccess, outputFiles, exportedFiles, exportedFileContent, @@ -2489,10 +2440,9 @@ export async function executeFunctionRequest( if (outputSandboxPaths.length > 0 || outputSandboxPath) { const fileExportResponse = await maybeExportSandboxFilesToWorkspace({ routeContext, - authUserId: auth.userId, + authUserId: auth.attributedUserId, workflowId, workspaceId, - workspaceAccess, outputFiles, exportedFiles, exportedFileContent, @@ -2545,7 +2495,7 @@ export async function executeFunctionRequest( runtimeBindings: compilerRuntimeBindings, timeoutMs: timeout, requestId, - ownerKey: `user:${auth.userId}`, + ownerKey: `user:${auth.attributedUserId}`, ownerWeight: 1, }, { brokers: createFunctionRuntimeBrokers(routeContext), signal: executionSignal } diff --git a/apps/sim/lib/guardrails/validate_hallucination.test.ts b/apps/sim/lib/guardrails/validate_hallucination.test.ts index 7d77a607a7b..50c731d8715 100644 --- a/apps/sim/lib/guardrails/validate_hallucination.test.ts +++ b/apps/sim/lib/guardrails/validate_hallucination.test.ts @@ -1,28 +1,19 @@ /** * @vitest-environment node */ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { - BILLING_ATTRIBUTION_HEADER, - type BillingAttributionSnapshot, - serializeBillingAttributionHeader, -} from '@/lib/billing/core/billing-attribution' -import { - PRIVATE_TOOL_METADATA_REQUEST_HEADER, - PRIVATE_TOOL_METADATA_RESPONSE_HEADER, - RESOLVED_SECRET_PROVENANCE_FIELD, - RESOLVED_SECRET_PROVENANCE_METADATA_V1, -} from '@/lib/execution/private-tool-metadata' - -const { mockDecryptSecret, mockExecuteProviderRequest, mockGenerateInternalDelegationToken } = - vi.hoisted(() => ({ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' + +const { mockDecryptSecret, mockExecuteProviderRequest, mockSearchKnowledgeAsExecutor } = vi.hoisted( + () => ({ mockDecryptSecret: vi.fn(), mockExecuteProviderRequest: vi.fn(), - mockGenerateInternalDelegationToken: vi.fn(), - })) + mockSearchKnowledgeAsExecutor: vi.fn(), + }) +) -vi.mock('@/lib/auth/internal', () => ({ - generateInternalDelegationToken: mockGenerateInternalDelegationToken, +vi.mock('@/lib/internal/knowledge/search', () => ({ + searchKnowledgeAsExecutor: mockSearchKnowledgeAsExecutor, })) vi.mock('@/lib/core/security/encryption', () => ({ @@ -53,21 +44,18 @@ const BILLING_ATTRIBUTION: BillingAttributionSnapshot = { payerSubscription: null, } -function createPrivateKnowledgeResponse( - functionalBody: Record, - provenance: Record = { version: 1, complete: true, entries: [] } -): Response { - return Response.json( - { ...functionalBody, [RESOLVED_SECRET_PROVENANCE_FIELD]: provenance }, - { - headers: { - [PRIVATE_TOOL_METADATA_RESPONSE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1, - }, - } - ) -} - function createInput(registry: ResolvedSecretTraceRegistry) { + const executionContext = { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionId: 'execution-1', + userId: 'user-1', + executorDelegationOrigin: { + subjectUserId: 'user-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }, + } return { userInput: 'secret-value __var_FOREIGN', knowledgeBaseId: 'kb-1', @@ -77,6 +65,7 @@ function createInput(registry: ResolvedSecretTraceRegistry) { workflowId: 'workflow-1', workspaceId: 'workspace-1', actorUserId: 'user-1', + executionContext, billingAttribution: BILLING_ATTRIBUTION, requestId: 'request-1', resolvedSecretTraceRegistry: registry, @@ -86,11 +75,16 @@ function createInput(registry: ResolvedSecretTraceRegistry) { describe('validateHallucination', () => { beforeEach(() => { vi.clearAllMocks() - mockGenerateInternalDelegationToken.mockResolvedValue('minted-internal-token') mockDecryptSecret.mockImplementation(async (encryptedValue: string) => ({ decrypted: encryptedValue === 'encrypted-reference-secret' ? 'reference-secret' : encryptedValue, })) + mockSearchKnowledgeAsExecutor.mockImplementation( + async ({ resolvedSecretTraceRegistry, modelInputPaths }) => ({ + results: [{ content: 'public context' }], + registry: resolvedSecretTraceRegistry.forkForInputPaths(modelInputPaths), + }) + ) mockExecuteProviderRequest.mockResolvedValue({ content: JSON.stringify({ score: 8, reasoning: 'supported' }), model: 'test-model', @@ -98,10 +92,6 @@ describe('validateHallucination', () => { }) }) - afterEach(() => { - vi.unstubAllGlobals() - }) - it('carries exact query and result provenance across both model boundaries', async () => { const registry = new ResolvedSecretTraceRegistry([ { name: 'TOKEN', plaintext: 'secret-value', encryptedValue: 'ciphertext' }, @@ -117,50 +107,39 @@ describe('validateHallucination', () => { 'secret-value __var_FOREIGN', '{{TOKEN}} __var_FOREIGN' ) - const knowledgeBody = { - data: { results: [{ content: 'Box reference-secret' }] }, - } - const fetchMock = vi.fn(async () => - createPrivateKnowledgeResponse(knowledgeBody, { - version: 1, - complete: true, - entries: [{ name: 'KB_TOKEN', encryptedValue: 'encrypted-reference-secret' }], - }) - ) - vi.stubGlobal('fetch', fetchMock) + mockSearchKnowledgeAsExecutor.mockImplementation(async (input) => { + const resultRegistry = input.resolvedSecretTraceRegistry.forkForInputPaths( + input.modelInputPaths + ) + await resultRegistry.importProvenance( + { + version: 1, + complete: true, + entries: [{ name: 'KB_TOKEN', encryptedValue: 'encrypted-reference-secret' }], + }, + { origin: 'test.knowledgeResult', trusted: true } + ) + return { results: [{ content: 'Box reference-secret' }], registry: resultRegistry } + }) const result = await validateHallucination(createInput(registry)) expect(result).toMatchObject({ passed: true, score: 8 }) - expect(mockGenerateInternalDelegationToken).toHaveBeenCalledWith({ - subjectUserId: 'user-1', - workflowId: 'workflow-1', + expect(mockSearchKnowledgeAsExecutor).toHaveBeenCalledWith({ + knowledgeBaseIds: ['kb-1'], + query: 'secret-value __var_FOREIGN', + topK: 10, + workspaceId: 'workspace-1', + context: expect.objectContaining({ + workflowId: 'workflow-1', + executorDelegationOrigin: expect.objectContaining({ workflowId: 'workflow-1' }), + }), + billingAttribution: BILLING_ATTRIBUTION, + resolvedSecretTraceRegistry: registry, + modelInputPaths: [['input']], + signal: undefined, }) - const [, searchOptions] = fetchMock.mock.calls[0] - const searchBody = JSON.parse(String(searchOptions?.body)) as { - query: string - __resolvedSecretTraceProvenance: unknown - } - const searchHeaders = new Headers(searchOptions?.headers) - expect(searchHeaders.get('authorization')).toBe('Bearer minted-internal-token') - expect(searchHeaders.get(BILLING_ATTRIBUTION_HEADER)).toBe( - serializeBillingAttributionHeader(BILLING_ATTRIBUTION) - ) - expect(searchHeaders.get(PRIVATE_TOOL_METADATA_REQUEST_HEADER)).toBe( - RESOLVED_SECRET_PROVENANCE_METADATA_V1 - ) - expect(searchHeaders.get('x-sim-private-model-input-provenance')).toBe( - RESOLVED_SECRET_PROVENANCE_METADATA_V1 - ) - expect(searchBody.query).toBe('secret-value __var_FOREIGN') - expect(searchBody.__resolvedSecretTraceProvenance).toEqual({ - version: 1, - complete: true, - entries: [{ encryptedValue: 'ciphertext', name: 'TOKEN' }], - }) - expect(JSON.stringify(searchBody)).toContain('__var_FOREIGN') - const providerCall = mockExecuteProviderRequest.mock.calls[0] const providerRequest = providerCall[1] as { messages: Array<{ content: string }> } const providerContext = providerCall[2] as { @@ -171,7 +150,6 @@ describe('validateHallucination', () => { expect(providerRequest.messages[0].content).not.toContain('{{UNUSED}}') expect(providerRequest.messages[0].content).not.toContain('secret-value') expect(providerRequest.messages[0].content).not.toContain('reference-secret') - expect(providerRequest.messages[0].content).not.toContain(RESOLVED_SECRET_PROVENANCE_FIELD) expect(providerContext.resolvedSecretTraceRegistry).not.toBe(registry) expect(providerContext.resolvedSecretTraceRegistry.getModelEgressSnapshot()).toMatchObject({ complete: true, @@ -182,12 +160,8 @@ describe('validateHallucination', () => { }) }) - it('accepts a successful legacy Knowledge response without private provenance', async () => { + it('accepts public Knowledge context without output secret provenance', async () => { const registry = new ResolvedSecretTraceRegistry() - vi.stubGlobal( - 'fetch', - vi.fn(async () => Response.json({ data: { results: [{ content: 'public context' }] } })) - ) const result = await validateHallucination(createInput(registry)) @@ -199,19 +173,15 @@ describe('validateHallucination', () => { expect(registry.isComplete()).toBe(true) }) - it('fails validation when the delegated Knowledge query is rejected', async () => { + it('fails validation when the authorized Knowledge operation is rejected', async () => { const registry = new ResolvedSecretTraceRegistry() - vi.stubGlobal( - 'fetch', - vi.fn(async () => new Response(null, { status: 401 })) - ) + mockSearchKnowledgeAsExecutor.mockRejectedValue(new Error('Unauthorized')) const result = await validateHallucination(createInput(registry)) expect(result).toEqual({ passed: false, - error: - 'Validation error: Failed to query knowledge base: Knowledge base query failed with status 401', + error: 'Validation error: Failed to query knowledge base: Unauthorized', }) expect(mockExecuteProviderRequest).not.toHaveBeenCalled() }) @@ -224,10 +194,12 @@ describe('validateHallucination', () => { */ it('surfaces a cancelled run as cancellation, not as a failed guardrail', async () => { const registry = new ResolvedSecretTraceRegistry() - const fetchMock = vi.fn(async () => - createPrivateKnowledgeResponse({ data: { results: [{ content: 'reference' }] } }) + mockSearchKnowledgeAsExecutor.mockImplementation( + async ({ resolvedSecretTraceRegistry, modelInputPaths }) => ({ + results: [{ content: 'reference' }], + registry: resolvedSecretTraceRegistry.forkForInputPaths(modelInputPaths), + }) ) - vi.stubGlobal('fetch', fetchMock) const abort = Object.assign(new Error('The operation was aborted.'), { name: 'AbortError' }) mockExecuteProviderRequest.mockRejectedValueOnce(abort) diff --git a/apps/sim/lib/guardrails/validate_hallucination.ts b/apps/sim/lib/guardrails/validate_hallucination.ts index cfa1f07e73d..0e25c657a5b 100644 --- a/apps/sim/lib/guardrails/validate_hallucination.ts +++ b/apps/sim/lib/guardrails/validate_hallucination.ts @@ -4,23 +4,9 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isPlainRecord } from '@sim/utils/object' import { eq } from 'drizzle-orm' -import { generateInternalDelegationToken } from '@/lib/auth/internal' -import { - BILLING_ATTRIBUTION_HEADER, - type BillingAttributionSnapshot, - serializeBillingAttributionHeader, -} from '@/lib/billing/core/billing-attribution' -import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' -import { - addModelInputProvenanceToRequest, - createModelInputProvenanceRequestMetadata, -} from '@/lib/execution/model-input-provenance' -import { - inspectPrivateToolMetadataEnvelope, - PRIVATE_TOOL_METADATA_REQUEST_HEADER, - RESOLVED_SECRET_PROVENANCE_FIELD, - RESOLVED_SECRET_PROVENANCE_METADATA_V1, -} from '@/lib/execution/private-tool-metadata' +import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' +import { searchKnowledgeAsExecutor } from '@/lib/internal/knowledge/search' +import type { InternalToolOperationContext } from '@/lib/internal/tool-operations/types' import { refreshTokenIfNeeded } from '@/lib/oauth/credential-service' import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection' import { refuseResolvedSecretProjection } from '@/executor/utils/resolved-secret-projection-refusal' @@ -30,16 +16,8 @@ import { isAbortError } from '@/providers/streaming-tool-loop-shared' import { getProviderFromModel } from '@/providers/utils' const logger = createLogger('HallucinationValidator') -const KNOWLEDGE_PROVENANCE_ERROR = 'Knowledge result secret provenance is unavailable' const HALLUCINATION_INPUT_PATHS = [['input']] as const -class KnowledgeProvenanceError extends Error { - constructor() { - super(KNOWLEDGE_PROVENANCE_ERROR) - this.name = 'KnowledgeProvenanceError' - } -} - export interface HallucinationValidationResult { passed: boolean error?: string @@ -69,6 +47,7 @@ export interface HallucinationValidationInput { workflowId?: string workspaceId?: string actorUserId: string + executionContext: InternalToolOperationContext billingAttribution: BillingAttributionSnapshot requestId: string resolvedSecretTraceRegistry: ResolvedSecretTraceRegistry @@ -81,97 +60,46 @@ export interface HallucinationValidationInput { } /** - * Query knowledge base to get relevant context chunks using the search API + * Queries the authorized Knowledge application operation for relevant chunks. */ async function queryKnowledgeBase( knowledgeBaseId: string, query: string, topK: number, requestId: string, - actorUserId: string, + executionContext: InternalToolOperationContext, billingAttribution: BillingAttributionSnapshot, workflowId: string | undefined, - resolvedSecretTraceRegistry: ResolvedSecretTraceRegistry + workspaceId: string | undefined, + resolvedSecretTraceRegistry: ResolvedSecretTraceRegistry, + abortSignal: AbortSignal | undefined ): Promise<{ context: string[]; registry: ResolvedSecretTraceRegistry }> { - const resultRegistry = resolvedSecretTraceRegistry.forkForInputPaths([]) if (!workflowId) throw new Error('Hallucination validation requires a workflow ID') + if (!workspaceId) throw new Error('Hallucination validation requires a workspace ID') try { - const searchUrl = `${getInternalApiBaseUrl()}/api/knowledge/search` - const internalToken = await generateInternalDelegationToken({ - subjectUserId: actorUserId, - workflowId, - }) - const headers = new Headers({ - 'Content-Type': 'application/json', - Authorization: `Bearer ${internalToken}`, - [BILLING_ATTRIBUTION_HEADER]: serializeBillingAttributionHeader(billingAttribution), - [PRIVATE_TOOL_METADATA_REQUEST_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1, - }) - const requestBody = { + const result = await searchKnowledgeAsExecutor({ knowledgeBaseIds: [knowledgeBaseId], query, topK, - workflowId, - } - const modelInputMetadata = createModelInputProvenanceRequestMetadata( + workspaceId, + context: executionContext, + billingAttribution, resolvedSecretTraceRegistry, - HALLUCINATION_INPUT_PATHS - ) - if (!modelInputMetadata) throw new KnowledgeProvenanceError() - const body = addModelInputProvenanceToRequest(requestBody, headers, modelInputMetadata) - // boundary-raw-fetch: authenticated internal Knowledge call with private provenance envelopes - const response = await fetch(searchUrl, { - method: 'POST', - headers, - body: JSON.stringify(body), + modelInputPaths: HALLUCINATION_INPUT_PATHS, + signal: abortSignal, }) - if (!response.ok) { - throw new Error(`Knowledge base query failed with status ${response.status}`) - } - - const payload: unknown = await response.json() - if (!isPlainRecord(payload)) throw new KnowledgeProvenanceError() - - const inspection = inspectPrivateToolMetadataEnvelope( - response.headers, - payload, - RESOLVED_SECRET_PROVENANCE_METADATA_V1 - ) - if (inspection.status === 'invalid') throw new KnowledgeProvenanceError() - - let functionalResponse = payload - if (inspection.status === 'verified') { - functionalResponse = { ...payload } - delete functionalResponse[RESOLVED_SECRET_PROVENANCE_FIELD] - const imported = await resultRegistry.importProvenance(inspection.value, { - origin: 'guardrails.hallucinationResult', - trusted: true, - }) - if (!imported || !resultRegistry.isComplete()) { - throw new KnowledgeProvenanceError() - } - } - - const data = isPlainRecord(functionalResponse.data) ? functionalResponse.data : undefined - const results = Array.isArray(data?.results) ? data.results : [] - return { - context: results.flatMap((result) => { - if ( - !isPlainRecord(result) || - typeof result.content !== 'string' || - result.content.length === 0 - ) { + context: result.results.flatMap((item) => { + if (!isPlainRecord(item) || typeof item.content !== 'string' || item.content.length === 0) { return [] } - return [result.content] + return [item.content] }), - registry: resultRegistry, + registry: result.registry, } } catch (error) { - if (error instanceof KnowledgeProvenanceError) throw error const message = getErrorMessage(error, 'Unknown Knowledge query error') logger.error(`[${requestId}] Error querying knowledge base`, { error: message, @@ -339,6 +267,7 @@ export async function validateHallucination( workflowId, workspaceId, actorUserId, + executionContext, billingAttribution, requestId, resolvedSecretTraceRegistry, @@ -365,10 +294,12 @@ export async function validateHallucination( userInput, topK, requestId, - actorUserId, + executionContext, billingAttribution, workflowId, - resolvedSecretTraceRegistry + workspaceId, + resolvedSecretTraceRegistry, + abortSignal ) const ragContext = knowledgeResult.context diff --git a/apps/sim/lib/guardrails/validate_pii.test.ts b/apps/sim/lib/guardrails/validate_pii.test.ts index 158f5be29af..d427951bd43 100644 --- a/apps/sim/lib/guardrails/validate_pii.test.ts +++ b/apps/sim/lib/guardrails/validate_pii.test.ts @@ -125,6 +125,25 @@ describe('validate_pii (Presidio service)', () => { }) describe('validatePII', () => { + it('forwards cancellation to Presidio and does not reshape it into a verdict', async () => { + const controller = new AbortController() + fetchMock.mockImplementationOnce(async (_url: string, init: RequestInit) => { + expect(init.signal).toBe(controller.signal) + controller.abort(new DOMException('cancelled', 'AbortError')) + throw controller.signal.reason + }) + + await expect( + validatePII({ + text: 'claim', + entityTypes: [], + mode: 'block', + requestId: 'cancelled-request', + abortSignal: controller.signal, + }) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) + it('block mode fails with a summary when PII is detected', async () => { const res = await validatePII({ text: 'reach me at a@b.com', diff --git a/apps/sim/lib/guardrails/validate_pii.ts b/apps/sim/lib/guardrails/validate_pii.ts index 44087d79c60..8b77841dabd 100644 --- a/apps/sim/lib/guardrails/validate_pii.ts +++ b/apps/sim/lib/guardrails/validate_pii.ts @@ -4,6 +4,7 @@ import { env } from '@/lib/core/config/env' import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import { chunkIndicesByBudget } from '@/lib/guardrails/pii-batching' import type { CustomPiiPattern } from '@/lib/guardrails/pii-entities' +import { isAbortError } from '@/providers/streaming-tool-loop-shared' const logger = createLogger('PIIValidator') @@ -53,6 +54,7 @@ export interface PIIValidationInput { /** User-supplied custom regex patterns applied alongside `entityTypes`. */ customPatterns?: CustomPiiPattern[] requestId: string + abortSignal?: AbortSignal } interface DetectedPIIEntity { @@ -85,7 +87,8 @@ async function analyze( text: string, entityTypes: string[], language: string, - patterns?: CustomPiiPattern[] + patterns?: CustomPiiPattern[], + signal?: AbortSignal ): Promise { // Guardrails convention: an empty selection means "detect all". Sending no // `entities` keeps that, and the server still runs the custom recognizers under @@ -103,6 +106,7 @@ async function analyze( ...(entities ? { entities } : {}), ...(patterns?.length ? { patterns } : {}), }), + signal, }) if (!response.ok) { const detail = await response.text().catch(() => '') @@ -230,7 +234,8 @@ async function redactBatch( async function anonymize( text: string, spans: AnalyzerSpan[], - patterns?: CustomPiiPattern[] + patterns?: CustomPiiPattern[], + signal?: AbortSignal ): Promise { if (spans.length === 0) return text @@ -243,6 +248,7 @@ async function anonymize( analyzer_results: spans, ...(patterns?.length ? { patterns } : {}), }), + signal, }) if (!response.ok) { const detail = await response.text().catch(() => '') @@ -259,7 +265,7 @@ async function anonymize( * - mask: passes and returns masked text with PII replaced by `` */ export async function validatePII(input: PIIValidationInput): Promise { - const { text, entityTypes, mode, language = 'en', customPatterns, requestId } = input + const { text, entityTypes, mode, language = 'en', customPatterns, requestId, abortSignal } = input logger.info(`[${requestId}] Starting PII validation`, { textLength: text.length, @@ -270,7 +276,9 @@ export async function validatePII(input: PIIValidationInput): Promise ({ type: displayEntityType(s.entity_type, customPatterns), @@ -300,7 +308,8 @@ export async function validatePII(input: PIIValidationInput): Promise` (or the // pattern's `replacement` for custom-pattern spans). - const maskedText = await anonymize(text, spans, customPatterns) + const maskedText = await anonymize(text, spans, customPatterns, abortSignal) + abortSignal?.throwIfAborted() logger.info(`[${requestId}] PII validation completed`, { passed: true, detectedCount: detectedEntities.length, @@ -308,6 +317,7 @@ export async function validatePII(input: PIIValidationInput): Promise ({ + cancelA2ATask: vi.fn(), + enforceUserRateLimit: vi.fn(), + getA2AAgentCard: vi.fn(), + getA2ATask: vi.fn(), + sendA2AMessage: vi.fn(), +})) + +vi.mock('@/lib/core/rate-limiter', () => ({ + enforceUserRateLimit: mocks.enforceUserRateLimit, +})) + +vi.mock('@/lib/internal/a2a/operations', () => ({ + cancelA2ATask: mocks.cancelA2ATask, + getA2AAgentCard: mocks.getA2AAgentCard, + getA2ATask: mocks.getA2ATask, + sendA2AMessage: mocks.sendA2AMessage, +})) + +import { A2AOperationError } from '@/lib/internal/a2a/errors' +import { executeA2ATool } from '@/lib/internal/a2a/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +function request(overrides: Partial = {}): InternalToolOperationCall { + return { + toolId: 'a2a_get_agent_card', + input: { agentUrl: 'https://agent.example' }, + headers: new Headers(), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + userId: 'user-1', + workspaceId: 'workspace-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +const CASES = [ + ['a2a_get_agent_card', { agentUrl: 'https://agent.example' }, mocks.getA2AAgentCard], + [ + 'a2a_send_message', + { agentUrl: 'https://agent.example', message: 'Hello' }, + mocks.sendA2AMessage, + ], + ['a2a_get_task', { agentUrl: 'https://agent.example', taskId: 'task-1' }, mocks.getA2ATask], + ['a2a_cancel_task', { agentUrl: 'https://agent.example', taskId: 'task-1' }, mocks.cancelA2ATask], +] as const + +describe('executeA2ATool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.enforceUserRateLimit.mockResolvedValue(null) + for (const operation of [ + mocks.cancelA2ATask, + mocks.getA2AAgentCard, + mocks.getA2ATask, + mocks.sendA2AMessage, + ]) { + operation.mockResolvedValue({ success: true, output: {} }) + } + }) + + it.each(CASES)('dispatches %s with trusted context', async (toolId, input, operation) => { + const controller = new AbortController() + const headers = new Headers({ 'x-sim-private-model-input-provenance': 'resolved-values-v1' }) + + const response = await executeA2ATool( + request({ toolId, input, headers, signal: controller.signal }) + ) + + expect(response.status).toBe(200) + expect(operation).toHaveBeenCalledWith(input, { + headers, + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + }) + }) + + it('keeps the existing per-user rate limit ahead of validation', async () => { + mocks.enforceUserRateLimit.mockResolvedValue( + Response.json({ error: 'Rate limit exceeded' }, { status: 429 }) + ) + + const response = await executeA2ATool(request({ input: null })) + + expect(response.status).toBe(429) + expect(mocks.getA2AAgentCard).not.toHaveBeenCalled() + }) + + it.each([false, 0, null, ['structured', 'data']])( + 'preserves non-object structured JSON data: %j', + async (data) => { + const response = await executeA2ATool( + request({ + toolId: 'a2a_send_message', + input: { agentUrl: 'https://agent.example', message: 'Hello', data }, + }) + ) + + expect(response.status).toBe(200) + expect(mocks.sendA2AMessage).toHaveBeenCalledWith( + { agentUrl: 'https://agent.example', message: 'Hello', data }, + expect.any(Object) + ) + } + ) + + it('preserves operation error statuses', async () => { + mocks.getA2AAgentCard.mockRejectedValue(new A2AOperationError('unsafe input', 400)) + + const response = await executeA2ATool(request()) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ success: false, error: 'unsafe input' }) + }) + + it('propagates cancellation before rate-limit or provider work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect(executeA2ATool(request({ signal: controller.signal }))).rejects.toMatchObject({ + name: 'AbortError', + }) + expect(mocks.enforceUserRateLimit).not.toHaveBeenCalled() + expect(mocks.getA2AAgentCard).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/a2a/execute-tool.ts b/apps/sim/lib/internal/a2a/execute-tool.ts new file mode 100644 index 00000000000..c7ef748e8b3 --- /dev/null +++ b/apps/sim/lib/internal/a2a/execute-tool.ts @@ -0,0 +1,126 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { getValidationErrorMessage } from '@/lib/api/server' +import { enforceUserRateLimit } from '@/lib/core/rate-limiter' +import { A2AOperationError } from '@/lib/internal/a2a/errors' +import { + a2aCancelTaskInputSchema, + a2aGetAgentCardInputSchema, + a2aGetTaskInputSchema, + a2aSendMessageInputSchema, +} from '@/lib/internal/a2a/input' +import { + cancelA2ATask, + getA2AAgentCard, + getA2ATask, + sendA2AMessage, +} from '@/lib/internal/a2a/operations' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' + +const logger = createLogger('A2AToolExecution') + +const RATE_LIMIT_BUCKETS = { + a2a_cancel_task: 'a2a-cancel-task', + a2a_get_agent_card: 'a2a-get-agent-card', + a2a_get_task: 'a2a-get-task', + a2a_send_message: 'a2a-send-message', +} as const + +export const executeA2ATool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + const userId = request.context.userId + if (!userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + if (!Object.hasOwn(RATE_LIMIT_BUCKETS, request.toolId)) { + return Response.json( + { success: false, error: `Unsupported A2A tool: ${request.toolId}` }, + { status: 500 } + ) + } + const bucket = RATE_LIMIT_BUCKETS[request.toolId as keyof typeof RATE_LIMIT_BUCKETS] + const rateLimited = await enforceUserRateLimit(bucket, userId) + request.signal?.throwIfAborted() + if (rateLimited) return rateLimited + + const context = { + headers: request.headers, + requestId: request.requestId, + signal: request.signal, + userId, + } + + try { + switch (request.toolId) { + case 'a2a_cancel_task': { + const parsed = a2aCancelTaskInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { + success: false, + error: getValidationErrorMessage(parsed.error, 'Invalid request data'), + }, + { status: 400 } + ) + } + return Response.json(await cancelA2ATask(parsed.data, context)) + } + case 'a2a_get_agent_card': { + const parsed = a2aGetAgentCardInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { + success: false, + error: getValidationErrorMessage(parsed.error, 'Invalid request data'), + }, + { status: 400 } + ) + } + return Response.json(await getA2AAgentCard(parsed.data, context)) + } + case 'a2a_get_task': { + const parsed = a2aGetTaskInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { + success: false, + error: getValidationErrorMessage(parsed.error, 'Invalid request data'), + }, + { status: 400 } + ) + } + return Response.json(await getA2ATask(parsed.data, context)) + } + case 'a2a_send_message': { + const parsed = a2aSendMessageInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { + success: false, + error: getValidationErrorMessage(parsed.error, 'Invalid request data'), + }, + { status: 400 } + ) + } + return Response.json(await sendA2AMessage(parsed.data, context)) + } + } + return Response.json( + { success: false, error: `Unsupported A2A tool: ${request.toolId}` }, + { status: 500 } + ) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof A2AOperationError) { + return Response.json({ success: false, error: error.message }, { status: error.status }) + } + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${request.requestId}] A2A operation failed`, { + error: getErrorMessage(error), + toolId: request.toolId, + }) + return Response.json({ success: false, error: getErrorMessage(error) }, { status: 502 }) + } +} diff --git a/apps/sim/lib/internal/a2a/input.ts b/apps/sim/lib/internal/a2a/input.ts new file mode 100644 index 00000000000..aa8849701af --- /dev/null +++ b/apps/sim/lib/internal/a2a/input.ts @@ -0,0 +1,39 @@ +import { z } from 'zod' +import { resolvedSecretTraceProvenanceSchema } from '@/lib/api/contracts/primitives' +import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' + +const baseInputSchema = z.object({ + agentUrl: z.string().url('Agent URL must be a valid URL').max(2048), + apiKey: z.string().optional(), +}) + +export const a2aSendMessageInputSchema = baseInputSchema.extend({ + message: z.string().min(1, 'Message is required'), + data: z.unknown().optional(), + files: z.array(RawFileInputSchema).max(20).optional(), + taskId: z.string().optional(), + contextId: z.string().optional(), + [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), +}) + +export const a2aGetTaskInputSchema = baseInputSchema.extend({ + taskId: z.string().min(1, 'Task ID is required'), + historyLength: z + .number() + .int() + .positive() + .max(1000, 'History length cannot exceed 1000') + .optional(), +}) + +export const a2aCancelTaskInputSchema = baseInputSchema.extend({ + taskId: z.string().min(1, 'Task ID is required'), +}) + +export const a2aGetAgentCardInputSchema = baseInputSchema + +export type A2ASendMessageInput = z.output +export type A2AGetTaskInput = z.output +export type A2ACancelTaskInput = z.output +export type A2AGetAgentCardInput = z.output diff --git a/apps/sim/lib/internal/a2a/operations.test.ts b/apps/sim/lib/internal/a2a/operations.test.ts new file mode 100644 index 00000000000..12e28b5d854 --- /dev/null +++ b/apps/sim/lib/internal/a2a/operations.test.ts @@ -0,0 +1,139 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + assertToolFileAccess: vi.fn(), + buildUserMessage: vi.fn(), + createA2AClient: vi.fn(), + downloadServableFileFromStorage: vi.fn(), + isModelSafeWorkspaceFileKey: vi.fn(), + isTaskResult: vi.fn(), + messageOutput: vi.fn(), + processFilesToUserFiles: vi.fn(), + validateOpaqueModelInputProvenance: vi.fn(), +})) + +vi.mock('@/lib/a2a/client', () => ({ + buildUserMessage: mocks.buildUserMessage, + createA2AClient: mocks.createA2AClient, + isTaskResult: mocks.isTaskResult, + messageOutput: mocks.messageOutput, + taskErrored: vi.fn(), + taskOutput: vi.fn(), + agentCardOutput: vi.fn(), +})) + +vi.mock('@/lib/execution/model-input-provenance', () => ({ + validateOpaqueModelInputProvenance: mocks.validateOpaqueModelInputProvenance, +})) + +vi.mock('@/lib/uploads/shared/types', () => ({ MAX_BUFFERED_TRANSFER_BYTES: 5 })) + +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + processFilesToUserFiles: mocks.processFilesToUserFiles, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mocks.downloadServableFileFromStorage, +})) + +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({ + isModelSafeWorkspaceFileKey: mocks.isModelSafeWorkspaceFileKey, + MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE: 'unsafe file', +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertToolFileAccess, +})) + +import { sendA2AMessage } from '@/lib/internal/a2a/operations' + +describe('sendA2AMessage', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.validateOpaqueModelInputProvenance.mockReturnValue({ success: true }) + mocks.assertToolFileAccess.mockResolvedValue(null) + mocks.isModelSafeWorkspaceFileKey.mockResolvedValue(true) + mocks.buildUserMessage.mockReturnValue({ messageId: 'message-1' }) + mocks.isTaskResult.mockReturnValue(false) + mocks.messageOutput.mockReturnValue({ content: 'done' }) + mocks.createA2AClient.mockResolvedValue({ + sendMessage: vi.fn().mockResolvedValue({ messageId: 'response-1' }), + }) + }) + + it('validates private model-input provenance before any file or provider work', async () => { + mocks.validateOpaqueModelInputProvenance.mockReturnValue({ + success: false, + error: 'Model input contains a resolved secret', + status: 400, + }) + + await expect( + sendA2AMessage( + { agentUrl: 'https://agent.example', message: 'Hello' }, + { + headers: new Headers(), + requestId: 'request-1', + userId: 'user-1', + } + ) + ).rejects.toMatchObject({ status: 400 }) + expect(mocks.processFilesToUserFiles).not.toHaveBeenCalled() + expect(mocks.createA2AClient).not.toHaveBeenCalled() + }) + + it('resolves attachments sequentially and enforces a cumulative byte budget', async () => { + const files = [ + { key: 'workspace/ws/file-1', name: 'one.txt', size: 3, type: 'text/plain' }, + { key: 'workspace/ws/file-2', name: 'two.txt', size: 3, type: 'text/plain' }, + ] + mocks.processFilesToUserFiles.mockReturnValue(files) + mocks.downloadServableFileFromStorage + .mockResolvedValueOnce({ buffer: Buffer.from('one'), contentType: 'text/plain' }) + .mockResolvedValueOnce({ buffer: Buffer.from('two'), contentType: 'text/plain' }) + + await expect( + sendA2AMessage( + { + agentUrl: 'https://agent.example', + message: 'Hello', + files: [{ key: files[0].key }, { key: files[1].key }], + }, + { + headers: new Headers(), + requestId: 'request-1', + userId: 'user-1', + } + ) + ).rejects.toMatchObject({ name: 'PayloadSizeLimitError' }) + expect(mocks.downloadServableFileFromStorage).toHaveBeenCalledTimes(2) + expect(mocks.createA2AClient).not.toHaveBeenCalled() + }) + + it('parses structured data and returns a direct A2A message', async () => { + const result = await sendA2AMessage( + { + agentUrl: 'https://agent.example', + message: 'Hello', + data: '{"kind":"probe"}', + }, + { + headers: new Headers(), + requestId: 'request-1', + userId: 'user-1', + } + ) + + expect(mocks.buildUserMessage).toHaveBeenCalledWith({ + text: 'Hello', + data: { kind: 'probe' }, + files: undefined, + taskId: undefined, + contextId: undefined, + }) + expect(result).toEqual({ success: true, output: { content: 'done' } }) + }) +}) diff --git a/apps/sim/lib/internal/a2a/operations.ts b/apps/sim/lib/internal/a2a/operations.ts new file mode 100644 index 00000000000..e05d8a0d920 --- /dev/null +++ b/apps/sim/lib/internal/a2a/operations.ts @@ -0,0 +1,170 @@ +import { createLogger } from '@sim/logger' +import { + type A2AFileInput, + agentCardOutput, + buildUserMessage, + createA2AClient, + isTaskResult, + messageOutput, + taskErrored, + taskOutput, +} from '@/lib/a2a/client' +import { assertKnownSizeWithinLimit } from '@/lib/core/utils/stream-limits' +import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' +import { A2AOperationError } from '@/lib/internal/a2a/errors' +import type { + A2ACancelTaskInput, + A2AGetAgentCardInput, + A2AGetTaskInput, + A2ASendMessageInput, +} from '@/lib/internal/a2a/input' +import { + isModelSafeWorkspaceFileKey, + MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, +} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +const logger = createLogger('A2AOperations') +const A2A_MAX_FILE_BYTES = 10 * 1024 * 1024 + +export interface A2AOperationContext { + headers: Headers + requestId: string + signal?: AbortSignal + userId: string +} + +async function resolveA2AFiles( + input: A2ASendMessageInput, + context: A2AOperationContext +): Promise { + if (!input.files?.length) return undefined + const userFiles = processFilesToUserFiles(input.files, context.requestId, logger) + const files: A2AFileInput[] = [] + let totalBytes = 0 + + for (const userFile of userFiles) { + context.signal?.throwIfAborted() + const denied = await assertToolFileAccess( + userFile.key, + context.userId, + context.requestId, + logger + ) + context.signal?.throwIfAborted() + if (denied) { + let message = 'File not found' + try { + const body = (await denied.json()) as { error?: unknown } + if (typeof body.error === 'string') message = body.error + } catch {} + throw new A2AOperationError(message, denied.status) + } + if (!(await isModelSafeWorkspaceFileKey(userFile.key))) { + throw new A2AOperationError(MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, 400) + } + + assertKnownSizeWithinLimit(userFile.size, A2A_MAX_FILE_BYTES, 'A2A attachment') + const { buffer, contentType } = await downloadServableFileFromStorage( + userFile, + context.requestId, + logger, + { maxBytes: A2A_MAX_FILE_BYTES, signal: context.signal } + ) + totalBytes += buffer.length + assertKnownSizeWithinLimit(totalBytes, MAX_BUFFERED_TRANSFER_BYTES, 'Total A2A attachment size') + files.push({ + bytes: buffer, + name: userFile.name, + mediaType: contentType || userFile.type || 'application/octet-stream', + }) + } + return files +} + +export async function sendA2AMessage(input: A2ASendMessageInput, context: A2AOperationContext) { + const provenance = validateOpaqueModelInputProvenance({ + headers: context.headers, + payload: input, + isInternalRequest: true, + }) + if (!provenance.success) throw new A2AOperationError(provenance.error, provenance.status) + + let data: unknown + if (input.data !== undefined) { + if (typeof input.data === 'string') { + try { + data = JSON.parse(input.data) + } catch { + throw new A2AOperationError('Data must be valid JSON', 400) + } + } else { + data = input.data + } + } + + const files = await resolveA2AFiles(input, context) + const client = await createA2AClient(input.agentUrl, input.apiKey, { signal: context.signal }) + const message = buildUserMessage({ + text: input.message, + data, + files, + taskId: input.taskId, + contextId: input.contextId, + }) + const result = await client.sendMessage({ + tenant: '', + message, + configuration: undefined, + metadata: undefined, + }) + context.signal?.throwIfAborted() + + if (!isTaskResult(result)) { + logger.info(`[${context.requestId}] A2A send returned a direct message`) + return { success: true as const, output: messageOutput(result) } + } + const output = taskOutput(result) + const errored = taskErrored(result) + logger.info(`[${context.requestId}] A2A send produced task ${result.id} (${output.state})`) + return { + success: !errored, + ...(errored ? { error: output.content || `Agent task ${output.state}` } : {}), + output, + } +} + +export async function getA2ATask(input: A2AGetTaskInput, context: A2AOperationContext) { + const client = await createA2AClient(input.agentUrl, input.apiKey, { signal: context.signal }) + const task = await client.getTask({ + tenant: '', + id: input.taskId, + historyLength: input.historyLength, + }) + context.signal?.throwIfAborted() + logger.info(`[${context.requestId}] Retrieved A2A task ${task.id}`) + return { success: true as const, output: taskOutput(task) } +} + +export async function cancelA2ATask(input: A2ACancelTaskInput, context: A2AOperationContext) { + const client = await createA2AClient(input.agentUrl, input.apiKey, { signal: context.signal }) + const task = await client.cancelTask({ tenant: '', id: input.taskId, metadata: undefined }) + context.signal?.throwIfAborted() + const output = taskOutput(task) + logger.info(`[${context.requestId}] Cancel requested for A2A task ${task.id}`) + return { + success: true as const, + output: { taskId: output.taskId, state: output.state, canceled: output.state === 'canceled' }, + } +} + +export async function getA2AAgentCard(input: A2AGetAgentCardInput, context: A2AOperationContext) { + const client = await createA2AClient(input.agentUrl, input.apiKey, { signal: context.signal }) + const card = await client.getAgentCard() + context.signal?.throwIfAborted() + logger.info(`[${context.requestId}] Fetched agent card for ${card.name}`) + return { success: true as const, output: agentCardOutput(card, input.agentUrl) } +} diff --git a/apps/sim/lib/internal/agiloft/client.test.ts b/apps/sim/lib/internal/agiloft/client.test.ts new file mode 100644 index 00000000000..b20c68312ef --- /dev/null +++ b/apps/sim/lib/internal/agiloft/client.test.ts @@ -0,0 +1,196 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +/** Obvious non-secret so credential scanners do not flag these fixtures. */ +const PLACEHOLDER_PASSWORD = 'not-a-real-password' + +const { mockValidateUrlWithDNS, mockSecureFetch } = vi.hoisted(() => ({ + mockValidateUrlWithDNS: vi.fn(), + mockSecureFetch: vi.fn(), +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + MAX_JSON_API_RESPONSE_BYTES: 10 * 1024 * 1024, + validateUrlWithDNS: mockValidateUrlWithDNS, + secureFetchWithPinnedIP: mockSecureFetch, +})) + +import { + AgiloftAlrestError, + executeAgiloftRequest, + readAlrestJson, +} from '@/lib/internal/agiloft/client' + +const baseParams = { + instanceUrl: 'https://example.agiloft.com', + knowledgeBase: 'demo', + login: 'admin', + password: PLACEHOLDER_PASSWORD, + table: 'contracts', +} + +function mockResponse(body: { ok?: boolean; status?: number; json?: unknown; text?: string }) { + return { + ok: body.ok ?? true, + status: body.status ?? 200, + statusText: '', + headers: { get: () => null, getSetCookie: () => [], toRecord: () => ({}) }, + body: null, + text: async () => body.text ?? JSON.stringify(body.json ?? {}), + json: async () => body.json ?? {}, + arrayBuffer: async () => new ArrayBuffer(0), + } +} + +beforeEach(() => { + mockValidateUrlWithDNS.mockReset() + mockSecureFetch.mockReset() + mockValidateUrlWithDNS.mockResolvedValue({ isValid: true, resolvedIP: '203.0.113.10' }) +}) + +describe('executeAgiloftRequest', () => { + it('resolves DNS once, logs in, runs the operation with the bearer token, then logs out — all pinned', async () => { + const controller = new AbortController() + mockSecureFetch + .mockResolvedValueOnce(mockResponse({ json: { access_token: 'tok-1' } })) + .mockResolvedValueOnce(mockResponse({ json: { id: 42, fields: { name: 'foo' } } })) + .mockResolvedValueOnce(mockResponse({})) + + const result = await executeAgiloftRequest( + baseParams, + (base) => ({ + url: `${base}/ewws/REST/demo/contracts/42`, + method: 'GET', + headers: { Accept: 'application/json' }, + }), + async (response) => { + const data = (await response.json()) as { id: number; fields: Record } + return { + success: response.ok, + output: { id: String(data.id), fields: data.fields }, + } + }, + controller.signal + ) + + expect(result).toEqual({ success: true, output: { id: '42', fields: { name: 'foo' } } }) + + expect(mockValidateUrlWithDNS).toHaveBeenCalledWith( + 'https://example.agiloft.com', + 'instanceUrl' + ) + + const calls = mockSecureFetch.mock.calls + expect(calls).toHaveLength(3) + /** Credentials go in a form body, never the URL. */ + expect(calls[0][0]).toBe('https://example.agiloft.com/ewws/EWLogin') + expect(calls[0][2]).toMatchObject({ + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + maxResponseBytes: 10 * 1024 * 1024, + signal: controller.signal, + }) + const sent = new URLSearchParams(calls[0][2].body as string) + expect(sent.get('$KB')).toBe('demo') + expect(sent.get('$table')).toBe('contracts') + expect(sent.get('$lang')).toBe('en') + expect(calls[1][0]).toBe('https://example.agiloft.com/ewws/REST/demo/contracts/42') + expect(calls[2][0]).toBe('https://example.agiloft.com/ewws/EWLogout?$KB=demo&$lang=en') + + for (const call of calls) { + expect(call[1]).toBe('203.0.113.10') + } + expect(calls[1][2]).toMatchObject({ + method: 'GET', + headers: { Accept: 'application/json', Authorization: 'Bearer tok-1' }, + maxResponseBytes: 10 * 1024 * 1024, + signal: controller.signal, + }) + expect(calls[2][2]).toMatchObject({ + maxResponseBytes: 10 * 1024 * 1024, + }) + expect(calls[2][2].signal).toBeUndefined() + }) + + it('still logs out when the operation throws', async () => { + mockSecureFetch + .mockResolvedValueOnce(mockResponse({ json: { access_token: 'tok-2' } })) + .mockResolvedValueOnce(mockResponse({ ok: false, status: 500 })) + .mockResolvedValueOnce(mockResponse({})) + + await expect( + executeAgiloftRequest( + baseParams, + (base) => ({ url: `${base}/ewws/REST/demo/contracts/42`, method: 'GET' }), + async (response) => { + if (!response.ok) throw new Error('operation failed') + return { success: true, output: {} } + } + ) + ).rejects.toThrow('operation failed') + + expect(mockSecureFetch).toHaveBeenCalledTimes(3) + expect(mockSecureFetch.mock.calls[2][0]).toContain('/ewws/EWLogout') + }) + + it('swallows logout failures (best-effort)', async () => { + mockSecureFetch + .mockResolvedValueOnce(mockResponse({ json: { access_token: 'tok-3' } })) + .mockResolvedValueOnce(mockResponse({ json: { ok: true } })) + .mockRejectedValueOnce(new Error('logout network error')) + + const result = await executeAgiloftRequest( + baseParams, + (base) => ({ url: `${base}/ewws/REST/demo/contracts/42`, method: 'GET' }), + async () => ({ success: true, output: {} }) + ) + + expect(result.success).toBe(true) + }) + + it('throws when login does not return an access token', async () => { + mockSecureFetch.mockResolvedValueOnce(mockResponse({ json: {} })) + + await expect( + executeAgiloftRequest( + baseParams, + (base) => ({ url: `${base}/ewws/REST/demo/contracts/42`, method: 'GET' }), + async () => ({ success: true, output: {} }) + ) + ).rejects.toThrow('Agiloft login did not return an access token') + + expect(mockSecureFetch).toHaveBeenCalledTimes(1) + }) + + it('rejects an instance URL that resolves to a blocked IP without issuing any request', async () => { + mockValidateUrlWithDNS.mockResolvedValue({ + isValid: false, + error: 'instanceUrl resolves to a blocked IP address', + }) + + await expect( + executeAgiloftRequest( + { ...baseParams, instanceUrl: 'https://internal.attacker.com' }, + (base) => ({ url: `${base}/ewws/REST/demo/contracts/42`, method: 'GET' }), + async () => ({ success: true, output: {} }) + ) + ).rejects.toThrow(/blocked IP address/) + + expect(mockSecureFetch).not.toHaveBeenCalled() + }) +}) + +describe('readAlrestJson', () => { + it('treats an HTTP-200 text refusal as an upstream refusal, not success', async () => { + const error = await readAlrestJson( + mockResponse({ status: 200, text: 'Permission denied by Agiloft' }) + ).catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(AgiloftAlrestError) + expect(error).toMatchObject({ + message: expect.stringContaining('Permission denied by Agiloft'), + }) + }) +}) diff --git a/apps/sim/lib/internal/agiloft/client.ts b/apps/sim/lib/internal/agiloft/client.ts new file mode 100644 index 00000000000..a837c890f38 --- /dev/null +++ b/apps/sim/lib/internal/agiloft/client.ts @@ -0,0 +1,325 @@ +import { createLogger } from '@sim/logger' +import { filterUndefined } from '@sim/utils/object' +import { truncate } from '@sim/utils/string' +import { + MAX_JSON_API_RESPONSE_BYTES, + type SecureFetchResponse, + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { AGILOFT_LANG, agiloftAlrestBase, describeAgiloftError } from '@/lib/internal/agiloft/urls' +import type { AgiloftBaseParams, AgiloftCredentials } from '@/tools/agiloft/types' +import type { HttpMethod, ToolResponse } from '@/tools/types' + +const logger = createLogger('AgiloftAuthServer') + +export interface AgiloftRequestConfig { + url: string + method: HttpMethod + headers?: Record + body?: string +} + +/** + * Validates the Agiloft instance URL and resolves its DNS once, returning the + * resolved IP so subsequent requests can pin to it. This prevents DNS-rebinding + * (TOCTOU) SSRF where the hostname could resolve to a private IP on a later + * lookup. Server-only — uses node:dns/promises. + */ +export async function resolveAgiloftInstance( + instanceUrl: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const validation = await validateUrlWithDNS(instanceUrl, 'instanceUrl') + signal?.throwIfAborted() + if (!validation.isValid || !validation.resolvedIP) { + throw new Error(validation.error || 'Invalid Agiloft instance URL') + } + return validation.resolvedIP +} + +/** + * Serializes credentials the way EWLogin expects them. + * + * The parameters go in a form-encoded request body rather than the query + * string: Agiloft's own documentation notes they "can be filled to request + * body", and it keeps the password out of URLs, access logs, and proxy traces. + */ +function formEncode(fields: Record): string { + return Object.entries(fields) + .map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(value)}`) + .join('&') +} + +export interface AgiloftSession { + /** Ready-to-send Authorization header value, e.g. `Bearer eyJ...`. */ + authorization: string + token: string +} + +/** + * DNS-pinned login. Requires a pre-resolved IP so the connection cannot be + * steered to a different host between validation and the actual TCP connect. + * + * `$table` and `$lang` are as mandatory as `$KB` here even though only `$KB`, + * `$login`, `$password`, and `$lang` are documented — a live instance rejects + * the call with: + * + * EWWrongDataException ... One has to specify $table, $KB, $lang parameters + * + * The scheme is read back from `authentication_scheme` rather than hardcoded, + * because Agiloft returns it with a trailing space ("Bearer ") and naively + * concatenating it yields a malformed `Bearer ` header. + */ +export async function agiloftLoginPinned( + params: AgiloftCredentials, + resolvedIP: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const base = params.instanceUrl.replace(/\/$/, '') + + const response = await secureFetchWithPinnedIP(`${base}/ewws/EWLogin`, resolvedIP, { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: formEncode( + filterUndefined({ + $KB: params.knowledgeBase, + $login: params.login, + $password: params.password, + /** + * Undocumented on EWLogin, but a live instance rejects the call without + * it. Omitted for KB-scoped operations that have no table. + */ + $table: params.table || undefined, + $lang: AGILOFT_LANG, + }) + ), + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + signal, + }) + + const text = await response.text() + signal?.throwIfAborted() + + if (!response.ok) { + throw new Error(`Agiloft login failed (${response.status}): ${describeAgiloftError(text)}`) + } + + let data: { access_token?: string; authentication_scheme?: string } + try { + data = JSON.parse(text) + } catch { + throw new Error(`Agiloft login returned a non-JSON response: ${truncate(text, 200)}`) + } + + if (!data.access_token) { + throw new Error(`Agiloft login did not return an access token: ${truncate(text, 200)}`) + } + + const scheme = (data.authentication_scheme || 'Bearer').trim() || 'Bearer' + return { authorization: `${scheme} ${data.access_token}`, token: data.access_token } +} + +/** + * DNS-pinned variant of agiloftLogout. Best-effort — failures are logged but + * not thrown. + */ +export async function agiloftLogoutPinned( + instanceUrl: string, + knowledgeBase: string, + authorization: string, + resolvedIP: string +): Promise { + try { + const base = instanceUrl.replace(/\/$/, '') + const kb = encodeURIComponent(knowledgeBase) + await secureFetchWithPinnedIP( + `${base}/ewws/EWLogout?$KB=${kb}&$lang=${AGILOFT_LANG}`, + resolvedIP, + { + method: 'POST', + headers: { Authorization: authorization }, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + } + ) + } catch (error) { + logger.warn('Agiloft logout failed (best-effort)', { error }) + } +} + +/** + * Shared wrapper that handles the full Agiloft auth lifecycle behind the + * codebase's SSRF-safe fetch path. The instance URL is validated and resolved + * to a concrete IP once via `validateUrlWithDNS` (which rejects hostnames that + * resolve to private/reserved addresses), and every hop — login, the operation + * request, and logout — is issued through `secureFetchWithPinnedIP` so the + * connection is pinned to that validated IP. This defeats DNS-rebinding (TOCTOU) + * SSRF where a hostname could resolve to an internal address on a later lookup. + * + * 1. Validate + resolve the instance URL once. + * 2. Login to obtain a Bearer token. + * 3. Execute the operation request with the token. + * 4. Logout to clean up the session (best-effort). + * + * The `buildRequest` callback receives the base URL and returns the request + * config. The `transformResponse` callback converts the raw response into the + * tool's output format. + * + * Server-only — uses node:dns/promises and node:http(s) via the pinned fetch. + */ +export async function executeAgiloftRequest( + params: AgiloftCredentials, + buildRequest: (base: string) => AgiloftRequestConfig, + transformResponse: (response: SecureFetchResponse) => Promise, + signal?: AbortSignal +): Promise { + const resolvedIP = await resolveAgiloftInstance(params.instanceUrl, signal) + const session = await agiloftLoginPinned(params, resolvedIP, signal) + const base = params.instanceUrl.replace(/\/$/, '') + + try { + const req = buildRequest(base) + const response = await secureFetchWithPinnedIP(req.url, resolvedIP, { + method: req.method, + headers: { + ...req.headers, + Authorization: session.authorization, + }, + body: req.body, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + signal, + }) + const result = await transformResponse(response) + signal?.throwIfAborted() + return result + } finally { + await agiloftLogoutPinned( + params.instanceUrl, + params.knowledgeBase, + session.authorization, + resolvedIP + ) + } +} + +export type { SecureFetchResponse } + +/** + * Shape every `/ewws/alrest` endpoint answers with. + * + * The surface reports failures as HTTP 200 with `success: false`, so checking + * `response.ok` alone silently turns an upstream refusal into a successful + * empty result. `readAlrestJson` is the only sanctioned way to read one. + */ +export interface AlrestEnvelope { + success?: boolean + message?: string + errors?: Array<{ message?: string }> + result?: T +} + +export class AgiloftAlrestError extends Error {} + +/** + * True for an upstream refusal Agiloft already decided on — a validation error, + * a permission denial, a conflicting match. + * + * These must not surface as HTTP 500: the tool runner treats 500 as retryable, + * and retrying a refused create can duplicate a record rather than converge. + */ +export function isAgiloftRefusal(error: unknown): error is AgiloftAlrestError { + return error instanceof AgiloftAlrestError +} + +/** + * Parses an alrest envelope, throwing `AgiloftAlrestError` when the call failed + * — whether it failed by status code, by `success: false`, or by returning + * something that is not JSON at all. + */ +export async function readAlrestJson(response: SecureFetchResponse): Promise { + const text = await response.text() + + let envelope: AlrestEnvelope + try { + envelope = JSON.parse(text) + } catch { + throw new AgiloftAlrestError( + `Agiloft returned a non-JSON response (${response.status}): ${truncate(text, 300)}` + ) + } + + if (!response.ok || envelope.success === false) { + const detail = + envelope.errors + ?.map((entry) => entry?.message) + .filter(Boolean) + .join('; ') || + envelope.message || + describeAgiloftError(truncate(text, 300)) + throw new AgiloftAlrestError(`Agiloft error: ${detail}`) + } + + return envelope.result +} + +/** + * Runs a single authenticated `/ewws/alrest/{KB}` call. `buildRequest` receives + * the KB-scoped base URL, so callers compose `${base}/{table}/...` paths. + */ +export async function executeAlrestRequest( + params: AgiloftBaseParams, + buildRequest: (base: string) => AgiloftRequestConfig, + transformResponse: (response: SecureFetchResponse) => Promise, + signal?: AbortSignal +): Promise { + const resolvedIP = await resolveAgiloftInstance(params.instanceUrl, signal) + const session = await agiloftLoginPinned(params, resolvedIP, signal) + + try { + const req = buildRequest(agiloftAlrestBase(params.instanceUrl, params.knowledgeBase)) + const response = await secureFetchWithPinnedIP(req.url, resolvedIP, { + method: req.method, + headers: { ...req.headers, Authorization: session.authorization }, + body: req.body, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + signal, + }) + const result = await transformResponse(response) + signal?.throwIfAborted() + return result + } finally { + await agiloftLogoutPinned( + params.instanceUrl, + params.knowledgeBase, + session.authorization, + resolvedIP + ) + } +} + +/** + * Runs a single `/ewws/EW*` call. No login round-trip: that surface rejects the + * bearer token and authenticates from the inline `$login`/`$password` already + * present in the URL built by the caller. + */ +export async function executeEwRequest( + params: AgiloftCredentials, + buildRequest: (base: string) => AgiloftRequestConfig, + transformResponse: (response: SecureFetchResponse) => Promise, + signal?: AbortSignal +): Promise { + const resolvedIP = await resolveAgiloftInstance(params.instanceUrl, signal) + const req = buildRequest(params.instanceUrl.replace(/\/$/, '')) + const response = await secureFetchWithPinnedIP(req.url, resolvedIP, { + method: req.method, + headers: req.headers, + body: req.body, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + signal, + }) + const result = await transformResponse(response) + signal?.throwIfAborted() + return result +} diff --git a/apps/sim/lib/internal/agiloft/errors.ts b/apps/sim/lib/internal/agiloft/errors.ts new file mode 100644 index 00000000000..00548f23d7b --- /dev/null +++ b/apps/sim/lib/internal/agiloft/errors.ts @@ -0,0 +1,13 @@ +export class AgiloftOperationError extends Error { + constructor( + readonly status: number, + readonly body: unknown + ) { + super( + body && typeof body === 'object' && 'error' in body && typeof body.error === 'string' + ? body.error + : 'Agiloft operation failed' + ) + this.name = 'AgiloftOperationError' + } +} diff --git a/apps/sim/lib/internal/agiloft/execute-tool.test.ts b/apps/sim/lib/internal/agiloft/execute-tool.test.ts new file mode 100644 index 00000000000..d0c98cc55f8 --- /dev/null +++ b/apps/sim/lib/internal/agiloft/execute-tool.test.ts @@ -0,0 +1,246 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operationMocks = vi.hoisted(() => ({ + executeAgiloftAsyncStatus: vi.fn(), + executeAgiloftAttachFile: vi.fn(), + executeAgiloftAttachmentInfo: vi.fn(), + executeAgiloftCreateRecord: vi.fn(), + executeAgiloftDeleteRecord: vi.fn(), + executeAgiloftGetChoiceLineId: vi.fn(), + executeAgiloftListTables: vi.fn(), + executeAgiloftLockRecord: vi.fn(), + executeAgiloftNlpSearch: vi.fn(), + executeAgiloftReadRecord: vi.fn(), + executeAgiloftRemoveAttachment: vi.fn(), + executeAgiloftRetrieveAttachment: vi.fn(), + executeAgiloftRunActionButton: vi.fn(), + executeAgiloftSavedSearch: vi.fn(), + executeAgiloftSearchRecords: vi.fn(), + executeAgiloftSelectRecords: vi.fn(), + executeAgiloftUpdateRecord: vi.fn(), + executeAgiloftUpsertRecord: vi.fn(), +})) + +vi.mock('@/lib/internal/agiloft/operations', () => operationMocks) + +import { AgiloftOperationError } from '@/lib/internal/agiloft/errors' +import { executeAgiloftTool } from '@/lib/internal/agiloft/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const CREDENTIALS = { + instanceUrl: 'https://example.agiloft.com', + knowledgeBase: 'demo', + login: 'user', + password: 'not-a-real-password', +} + +const BASE = { + ...CREDENTIALS, + table: 'contracts', +} + +const FILE = { + id: 'file-1', + name: 'evidence.txt', + key: 'workspace-1/file-1', + size: 5, + type: 'text/plain', +} + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'agiloft_create_record', + input: { ...BASE, data: '{"name":"Contract"}' }, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + userId: 'user-current', + }, + requestId: 'request-1', + ...overrides, + } +} + +const TOOL_CASES = [ + [ + 'agiloft_async_status', + { ...BASE, callbackId: 'callback-1' }, + operationMocks.executeAgiloftAsyncStatus, + ], + [ + 'agiloft_attach_file', + { ...BASE, recordId: '1', fieldName: 'files', file: FILE }, + operationMocks.executeAgiloftAttachFile, + ], + [ + 'agiloft_attachment_info', + { ...BASE, recordId: '1', fieldName: 'files' }, + operationMocks.executeAgiloftAttachmentInfo, + ], + [ + 'agiloft_create_record', + { ...BASE, data: '{"name":"Contract"}' }, + operationMocks.executeAgiloftCreateRecord, + ], + ['agiloft_delete_record', { ...BASE, recordId: '1' }, operationMocks.executeAgiloftDeleteRecord], + [ + 'agiloft_get_choice_line_id', + { ...BASE, fieldName: 'status', value: 'Open' }, + operationMocks.executeAgiloftGetChoiceLineId, + ], + ['agiloft_list_tables', BASE, operationMocks.executeAgiloftListTables], + [ + 'agiloft_lock_record', + { ...BASE, recordId: '1', lockAction: 'check' }, + operationMocks.executeAgiloftLockRecord, + ], + [ + 'agiloft_nlp_search', + { ...CREDENTIALS, nlpQuery: 'open contracts', fields: 'id,summary' }, + operationMocks.executeAgiloftNlpSearch, + ], + ['agiloft_read_record', { ...BASE, recordId: '1' }, operationMocks.executeAgiloftReadRecord], + [ + 'agiloft_remove_attachment', + { ...BASE, recordId: '1', fieldName: 'files', position: '0' }, + operationMocks.executeAgiloftRemoveAttachment, + ], + [ + 'agiloft_retrieve_attachment', + { ...BASE, recordId: '1', fieldName: 'files', position: '0' }, + operationMocks.executeAgiloftRetrieveAttachment, + ], + [ + 'agiloft_run_action_button', + { ...BASE, recordId: '1', actionButtonField: 'approve' }, + operationMocks.executeAgiloftRunActionButton, + ], + ['agiloft_saved_search', BASE, operationMocks.executeAgiloftSavedSearch], + [ + 'agiloft_search_records', + { ...BASE, query: 'status=Open' }, + operationMocks.executeAgiloftSearchRecords, + ], + [ + 'agiloft_select_records', + { ...BASE, where: 'status=Open' }, + operationMocks.executeAgiloftSelectRecords, + ], + [ + 'agiloft_update_record', + { ...BASE, recordId: '1', data: '{"name":"Updated"}' }, + operationMocks.executeAgiloftUpdateRecord, + ], + [ + 'agiloft_upsert_record', + { ...BASE, match: 'external_id', data: '{"external_id":"1"}' }, + operationMocks.executeAgiloftUpsertRecord, + ], +] as const + +describe('executeAgiloftTool', () => { + beforeEach(() => { + vi.clearAllMocks() + for (const operation of Object.values(operationMocks)) { + operation.mockResolvedValue({ success: true, output: { handled: true } }) + } + }) + + it.each(TOOL_CASES)('validates and dispatches %s', async (toolId, input, operation) => { + const response = await executeAgiloftTool(createRequest({ toolId, input })) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ success: true, output: { handled: true } }) + expect(operation).toHaveBeenCalledWith( + expect.objectContaining(input), + expect.objectContaining({ requestId: 'request-1', userId: 'user-current' }) + ) + }) + + it('uses the trusted delegation origin and forwards cancellation', async () => { + const controller = new AbortController() + const input = { ...BASE, data: '{"name":"Contract"}' } + + await executeAgiloftTool( + createRequest({ + input, + signal: controller.signal, + context: { + ...createExecutionContext({ workflowId: 'workflow-current' }), + workspaceId: 'workspace-1', + userId: 'user-current', + executorDelegationOrigin: { + subjectUserId: 'user-origin', + workflowId: 'workflow-origin', + executionId: 'execution-origin', + }, + }, + }) + ) + + expect(operationMocks.executeAgiloftCreateRecord).toHaveBeenCalledWith(input, { + requestId: 'request-1', + userId: 'user-origin', + signal: controller.signal, + }) + }) + + it('preserves non-object input and canonical validation envelopes', async () => { + const invalidInput = await executeAgiloftTool(createRequest({ input: '{' })) + expect(invalidInput.status).toBe(400) + await expect(invalidInput.json()).resolves.toMatchObject({ + success: false, + error: 'Invalid input: expected object, received string', + details: expect.any(Array), + }) + + const invalidBody = await executeAgiloftTool(createRequest({ input: { ...BASE, data: '' } })) + expect(invalidBody.status).toBe(400) + await expect(invalidBody.json()).resolves.toMatchObject({ + success: false, + error: 'Data is required', + details: expect.any(Array), + }) + expect(operationMocks.executeAgiloftCreateRecord).not.toHaveBeenCalled() + }) + + it('preserves explicit operation and generic provider errors', async () => { + operationMocks.executeAgiloftCreateRecord.mockRejectedValueOnce( + new AgiloftOperationError(429, { success: false, error: 'rate limited' }) + ) + const provider = await executeAgiloftTool(createRequest()) + expect(provider.status).toBe(429) + await expect(provider.json()).resolves.toEqual({ success: false, error: 'rate limited' }) + + operationMocks.executeAgiloftCreateRecord.mockRejectedValueOnce(new Error('network down')) + const generic = await executeAgiloftTool(createRequest()) + expect(generic.status).toBe(500) + await expect(generic.json()).resolves.toEqual({ success: false, error: 'network down' }) + }) + + it('propagates cancellation before provider work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeAgiloftTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(operationMocks.executeAgiloftCreateRecord).not.toHaveBeenCalled() + }) + + it('returns a deterministic error for unsupported IDs', async () => { + const response = await executeAgiloftTool(createRequest({ toolId: 'agiloft_unknown' })) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'Unsupported Agiloft tool: agiloft_unknown', + }) + }) +}) diff --git a/apps/sim/lib/internal/agiloft/execute-tool.ts b/apps/sim/lib/internal/agiloft/execute-tool.ts new file mode 100644 index 00000000000..e59c8ba22ba --- /dev/null +++ b/apps/sim/lib/internal/agiloft/execute-tool.ts @@ -0,0 +1,152 @@ +import { getErrorMessage } from '@sim/utils/errors' +import type { AnyApiRouteContract, ContractBody } from '@/lib/api/contracts' +import { + agiloftAsyncStatusContract, + agiloftAttachContract, + agiloftAttachmentInfoContract, + agiloftCreateRecordContract, + agiloftDeleteRecordContract, + agiloftGetChoiceLineIdContract, + agiloftListTablesContract, + agiloftLockRecordContract, + agiloftNlpSearchContract, + agiloftReadRecordContract, + agiloftRemoveAttachmentContract, + agiloftRetrieveContract, + agiloftRunActionButtonContract, + agiloftSavedSearchContract, + agiloftSearchRecordsContract, + agiloftSelectRecordsContract, + agiloftUpdateRecordContract, + agiloftUpsertRecordContract, +} from '@/lib/api/contracts/tools/agiloft' +import { AgiloftOperationError } from '@/lib/internal/agiloft/errors' +import { + type AgiloftOperationContext, + executeAgiloftAsyncStatus, + executeAgiloftAttachFile, + executeAgiloftAttachmentInfo, + executeAgiloftCreateRecord, + executeAgiloftDeleteRecord, + executeAgiloftGetChoiceLineId, + executeAgiloftListTables, + executeAgiloftLockRecord, + executeAgiloftNlpSearch, + executeAgiloftReadRecord, + executeAgiloftRemoveAttachment, + executeAgiloftRetrieveAttachment, + executeAgiloftRunActionButton, + executeAgiloftSavedSearch, + executeAgiloftSearchRecords, + executeAgiloftSelectRecords, + executeAgiloftUpdateRecord, + executeAgiloftUpsertRecord, +} from '@/lib/internal/agiloft/operations' +import type { + InternalToolOperationCall, + InternalToolOperationHandler, +} from '@/lib/internal/tool-operations/types' + +function parseInput(contract: C, input: unknown) { + const parsed = contract.body?.safeParse(input) + if (!parsed?.success) { + return { + success: false as const, + response: Response.json( + { + success: false, + error: parsed?.error.issues[0]?.message || 'Invalid request data', + details: parsed?.error.issues ?? [], + }, + { status: 400 } + ), + } + } + return { success: true as const, data: parsed.data as ContractBody } +} + +async function executeOperation( + contract: C, + request: InternalToolOperationCall, + operation: (input: ContractBody, context: AgiloftOperationContext) => Promise +): Promise { + request.signal?.throwIfAborted() + const parsed = parseInput(contract, request.input) + if (!parsed.success) return parsed.response + try { + const result = await operation(parsed.data, { + requestId: request.requestId, + userId: request.context.executorDelegationOrigin?.subjectUserId ?? request.context.userId, + signal: request.signal, + }) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof AgiloftOperationError) { + return Response.json(error.body, { status: error.status }) + } + return Response.json( + { success: false, error: getErrorMessage(error, 'Agiloft request failed') }, + { status: 500 } + ) + } +} + +export const executeAgiloftTool: InternalToolOperationHandler = async (request) => { + switch (request.toolId) { + case 'agiloft_async_status': + return executeOperation(agiloftAsyncStatusContract, request, executeAgiloftAsyncStatus) + case 'agiloft_attach_file': + return executeOperation(agiloftAttachContract, request, executeAgiloftAttachFile) + case 'agiloft_attachment_info': + return executeOperation(agiloftAttachmentInfoContract, request, executeAgiloftAttachmentInfo) + case 'agiloft_create_record': + return executeOperation(agiloftCreateRecordContract, request, executeAgiloftCreateRecord) + case 'agiloft_delete_record': + return executeOperation(agiloftDeleteRecordContract, request, executeAgiloftDeleteRecord) + case 'agiloft_get_choice_line_id': + return executeOperation( + agiloftGetChoiceLineIdContract, + request, + executeAgiloftGetChoiceLineId + ) + case 'agiloft_list_tables': + return executeOperation(agiloftListTablesContract, request, executeAgiloftListTables) + case 'agiloft_lock_record': + return executeOperation(agiloftLockRecordContract, request, executeAgiloftLockRecord) + case 'agiloft_nlp_search': + return executeOperation(agiloftNlpSearchContract, request, executeAgiloftNlpSearch) + case 'agiloft_read_record': + return executeOperation(agiloftReadRecordContract, request, executeAgiloftReadRecord) + case 'agiloft_remove_attachment': + return executeOperation( + agiloftRemoveAttachmentContract, + request, + executeAgiloftRemoveAttachment + ) + case 'agiloft_retrieve_attachment': + return executeOperation(agiloftRetrieveContract, request, executeAgiloftRetrieveAttachment) + case 'agiloft_run_action_button': + return executeOperation( + agiloftRunActionButtonContract, + request, + executeAgiloftRunActionButton + ) + case 'agiloft_saved_search': + return executeOperation(agiloftSavedSearchContract, request, executeAgiloftSavedSearch) + case 'agiloft_search_records': + return executeOperation(agiloftSearchRecordsContract, request, executeAgiloftSearchRecords) + case 'agiloft_select_records': + return executeOperation(agiloftSelectRecordsContract, request, executeAgiloftSelectRecords) + case 'agiloft_update_record': + return executeOperation(agiloftUpdateRecordContract, request, executeAgiloftUpdateRecord) + case 'agiloft_upsert_record': + return executeOperation(agiloftUpsertRecordContract, request, executeAgiloftUpsertRecord) + default: + return Response.json( + { error: `Unsupported Agiloft tool: ${request.toolId}` }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/agiloft/file-input.test.ts b/apps/sim/lib/internal/agiloft/file-input.test.ts new file mode 100644 index 00000000000..2c9be28225f --- /dev/null +++ b/apps/sim/lib/internal/agiloft/file-input.test.ts @@ -0,0 +1,198 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const fileMocks = vi.hoisted(() => ({ + assertToolFileAccess: vi.fn(), + docNotReadyResponse: vi.fn(), + downloadServableFileFromStorage: vi.fn(), + isPayloadSizeLimitError: vi.fn(), + processFilesToUserFiles: vi.fn(), +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: fileMocks.assertToolFileAccess, +})) +vi.mock('@/lib/core/utils/stream-limits', () => ({ + isPayloadSizeLimitError: fileMocks.isPayloadSizeLimitError, +})) +vi.mock('@/lib/uploads/shared/types', () => ({ + MAX_BUFFERED_TRANSFER_BYTES: 50 * 1024 * 1024, +})) +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + processFilesToUserFiles: fileMocks.processFilesToUserFiles, +})) +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: fileMocks.downloadServableFileFromStorage, +})) +vi.mock('@/lib/uploads/utils/servable-file-response', () => ({ + docNotReadyResponse: fileMocks.docNotReadyResponse, +})) + +import { AgiloftOperationError } from '@/lib/internal/agiloft/errors' +import { resolveAgiloftAttachmentFile } from '@/lib/internal/agiloft/file-input' + +const RAW_FILE = { + id: 'file-1', + name: 'evidence.txt', + url: '/api/files/serve/file-1', + size: 5, + type: 'text/plain', + key: 'workspace-1/file-1', +} + +const USER_FILE = { + ...RAW_FILE, + context: 'workspace', +} + +async function expectOperationError( + promise: Promise, + expected: { status: number; body: unknown } +) { + try { + await promise + throw new Error('Expected AgiloftOperationError') + } catch (error) { + expect(error).toBeInstanceOf(AgiloftOperationError) + expect(error).toMatchObject(expected) + } +} + +describe('Agiloft attachment file resolution', () => { + beforeEach(() => { + vi.clearAllMocks() + fileMocks.processFilesToUserFiles.mockReturnValue([USER_FILE]) + fileMocks.assertToolFileAccess.mockResolvedValue(null) + fileMocks.downloadServableFileFromStorage.mockResolvedValue({ + buffer: Buffer.from('hello'), + contentType: 'text/plain', + }) + fileMocks.docNotReadyResponse.mockReturnValue(null) + fileMocks.isPayloadSizeLimitError.mockReturnValue(false) + }) + + it.each(['workspace', 'mothership', 'execution', 'copilot', 'knowledge-base', 'chat', 'general'])( + 'preserves fail-closed authorization for %s file inputs', + async (context) => { + const file = { ...RAW_FILE, context, key: `${context}/file-1` } + const userFile = { ...USER_FILE, ...file } + fileMocks.processFilesToUserFiles.mockReturnValueOnce([userFile]) + + const result = await resolveAgiloftAttachmentFile(file, { + userId: 'user-1', + requestId: 'request-1', + }) + + expect(result).toEqual({ userFile, buffer: Buffer.from('hello') }) + expect(fileMocks.assertToolFileAccess).toHaveBeenCalledWith( + userFile.key, + 'user-1', + 'request-1', + expect.anything() + ) + } + ) + + it('forwards cancellation and the bounded-transfer limit to storage', async () => { + const controller = new AbortController() + + await resolveAgiloftAttachmentFile(RAW_FILE, { + userId: 'user-1', + requestId: 'request-1', + signal: controller.signal, + }) + + expect(fileMocks.downloadServableFileFromStorage).toHaveBeenCalledWith( + USER_FILE, + 'request-1', + expect.anything(), + { maxBytes: 50 * 1024 * 1024, signal: controller.signal } + ) + }) + + it('propagates cancellation before and after file authorization', async () => { + const before = new AbortController() + before.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + resolveAgiloftAttachmentFile(RAW_FILE, { + userId: 'user-1', + requestId: 'request-1', + signal: before.signal, + }) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(fileMocks.assertToolFileAccess).not.toHaveBeenCalled() + + const after = new AbortController() + fileMocks.assertToolFileAccess.mockImplementationOnce(async () => { + after.abort(new DOMException('cancelled', 'AbortError')) + return null + }) + await expect( + resolveAgiloftAttachmentFile(RAW_FILE, { + userId: 'user-1', + requestId: 'request-1', + signal: after.signal, + }) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(fileMocks.downloadServableFileFromStorage).not.toHaveBeenCalled() + }) + + it('accepts the serialized file shape produced by advanced-mode inputs', async () => { + const result = await resolveAgiloftAttachmentFile(JSON.stringify(RAW_FILE), { + userId: 'user-1', + requestId: 'request-1', + }) + + expect(result).toEqual({ userFile: USER_FILE, buffer: Buffer.from('hello') }) + expect(fileMocks.processFilesToUserFiles).toHaveBeenCalledWith( + [RAW_FILE], + 'request-1', + expect.anything() + ) + }) + + it('preserves authorization denials and rejects missing or invalid files', async () => { + fileMocks.assertToolFileAccess.mockResolvedValueOnce( + Response.json({ success: false, error: 'File not found' }, { status: 404 }) + ) + await expectOperationError( + resolveAgiloftAttachmentFile(RAW_FILE, { + userId: 'user-1', + requestId: 'request-1', + }), + { status: 404, body: { success: false, error: 'File not found' } } + ) + expect(fileMocks.downloadServableFileFromStorage).not.toHaveBeenCalled() + + await expectOperationError( + resolveAgiloftAttachmentFile(undefined, { + userId: 'user-1', + requestId: 'request-1', + }), + { status: 400, body: { success: false, error: 'File is required' } } + ) + await expectOperationError( + resolveAgiloftAttachmentFile('/api/files/serve/file-1', { + userId: 'user-1', + requestId: 'request-1', + }), + { status: 400, body: { success: false, error: 'Invalid file input' } } + ) + }) + + it('preserves payload-too-large responses', async () => { + fileMocks.downloadServableFileFromStorage.mockRejectedValueOnce(new Error('file too large')) + fileMocks.isPayloadSizeLimitError.mockReturnValueOnce(true) + + await expectOperationError( + resolveAgiloftAttachmentFile(RAW_FILE, { + userId: 'user-1', + requestId: 'request-1', + }), + { status: 413, body: { success: false, error: 'file too large' } } + ) + }) +}) diff --git a/apps/sim/lib/internal/agiloft/file-input.ts b/apps/sim/lib/internal/agiloft/file-input.ts new file mode 100644 index 00000000000..b3b71d13ee1 --- /dev/null +++ b/apps/sim/lib/internal/agiloft/file-input.ts @@ -0,0 +1,72 @@ +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { AgiloftOperationError } from '@/lib/internal/agiloft/errors' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { parseRawFileInput, type RawFileInput } from '@/lib/uploads/utils/file-schemas' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +const logger = createLogger('AgiloftFileInput') + +async function bodyFromResponse(response: Response): Promise { + try { + return await response.json() + } catch { + return { success: false, error: response.statusText || 'File operation failed' } + } +} + +export async function resolveAgiloftAttachmentFile( + file: RawFileInput | string | undefined, + context: { userId?: string; requestId: string; signal?: AbortSignal } +) { + context.signal?.throwIfAborted() + if (!context.userId) { + throw new AgiloftOperationError(401, { + success: false, + error: 'Authentication required', + }) + } + if (!file) { + throw new AgiloftOperationError(400, { success: false, error: 'File is required' }) + } + + const parsedFile = parseRawFileInput(file) + if (!parsedFile) { + throw new AgiloftOperationError(400, { success: false, error: 'Invalid file input' }) + } + + const userFiles = processFilesToUserFiles([parsedFile], context.requestId, logger) + if (userFiles.length === 0) { + throw new AgiloftOperationError(400, { success: false, error: 'Invalid file input' }) + } + + const userFile = userFiles[0]! + const denied = await assertToolFileAccess(userFile.key, context.userId, context.requestId, logger) + if (denied) { + throw new AgiloftOperationError(denied.status, await bodyFromResponse(denied)) + } + + context.signal?.throwIfAborted() + try { + const servable = await downloadServableFileFromStorage(userFile, context.requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + signal: context.signal, + }) + context.signal?.throwIfAborted() + return { userFile, buffer: servable.buffer } + } catch (error) { + context.signal?.throwIfAborted() + const notReady = docNotReadyResponse(error) + if (notReady) { + throw new AgiloftOperationError(notReady.status, await bodyFromResponse(notReady)) + } + throw new AgiloftOperationError(isPayloadSizeLimitError(error) ? 413 : 500, { + success: false, + error: toError(error).message, + }) + } +} diff --git a/apps/sim/lib/internal/agiloft/operations.test.ts b/apps/sim/lib/internal/agiloft/operations.test.ts new file mode 100644 index 00000000000..922e1b33a57 --- /dev/null +++ b/apps/sim/lib/internal/agiloft/operations.test.ts @@ -0,0 +1,187 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { SecureFetchResponse } from '@/lib/core/security/input-validation.server' +import type { ToolResponse } from '@/tools/types' + +const clientMocks = vi.hoisted(() => ({ + executeAgiloftRequest: vi.fn(), + executeAlrestRequest: vi.fn(), + executeEwRequest: vi.fn(), + isAgiloftRefusal: vi.fn(), + readAlrestJson: vi.fn(), + resolveAgiloftInstance: vi.fn(), +})) + +const providerMocks = vi.hoisted(() => ({ + secureFetchWithPinnedIP: vi.fn(), +})) + +const fileMocks = vi.hoisted(() => ({ + resolveAgiloftAttachmentFile: vi.fn(), +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + MAX_JSON_API_RESPONSE_BYTES: 10 * 1024 * 1024, + secureFetchWithPinnedIP: providerMocks.secureFetchWithPinnedIP, +})) +vi.mock('@/lib/internal/agiloft/client', () => clientMocks) +vi.mock('@/lib/internal/agiloft/file-input', () => fileMocks) + +import { + executeAgiloftCreateRecord, + executeAgiloftRetrieveAttachment, + executeAgiloftSearchRecords, + executeAgiloftSelectRecords, +} from '@/lib/internal/agiloft/operations' + +const BASE = { + instanceUrl: 'https://example.agiloft.com', + knowledgeBase: 'demo', + login: 'user', + password: 'not-a-real-password', + table: 'contracts', +} + +function createResponse( + options: { + status?: number + text?: string + bytes?: Uint8Array + headers?: Record + } = {} +): SecureFetchResponse { + const status = options.status ?? 200 + const bytes = options.bytes ?? new TextEncoder().encode(options.text ?? '') + const headers = new Map( + Object.entries(options.headers ?? {}).map(([key, value]) => [key.toLowerCase(), value]) + ) + return { + ok: status >= 200 && status < 300, + status, + statusText: '', + headers: { + get: (name: string) => headers.get(name.toLowerCase()) ?? null, + getSetCookie: () => [], + toRecord: () => Object.fromEntries(headers), + [Symbol.iterator]: () => headers.entries(), + }, + body: null, + text: async () => new TextDecoder().decode(bytes), + json: async () => JSON.parse(new TextDecoder().decode(bytes)), + arrayBuffer: async () => + bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength), + } +} + +type ResponseTransform = (response: SecureFetchResponse) => Promise + +describe('Agiloft operations', () => { + beforeEach(() => { + vi.clearAllMocks() + clientMocks.isAgiloftRefusal.mockReturnValue(false) + clientMocks.resolveAgiloftInstance.mockResolvedValue('203.0.113.10') + }) + + it('rejects invalid record JSON before opening an Agiloft session', async () => { + const result = await executeAgiloftCreateRecord( + { ...BASE, data: '[]' }, + { requestId: 'request-1' } + ) + + expect(result).toEqual({ + success: false, + output: { id: null, fields: {} }, + error: 'The data parameter must be a JSON object of field names to values', + }) + expect(clientMocks.executeAlrestRequest).not.toHaveBeenCalled() + }) + + it('caps search results and forwards cancellation through the authenticated operation', async () => { + const controller = new AbortController() + const records = Array.from({ length: 205 }, (_, id) => ({ id })) + clientMocks.readAlrestJson.mockResolvedValue(records) + clientMocks.executeAlrestRequest.mockImplementation( + async ( + _params: unknown, + _buildRequest: unknown, + transformResponse: ResponseTransform, + _signal?: AbortSignal + ) => transformResponse(createResponse()) + ) + + const result = await executeAgiloftSearchRecords( + { ...BASE, query: 'status=Open' }, + { requestId: 'request-1', signal: controller.signal } + ) + + expect(result.output).toMatchObject({ + records: records.slice(0, 200), + totalCount: 200, + truncated: true, + }) + expect(clientMocks.executeAlrestRequest.mock.calls[0]?.[3]).toBe(controller.signal) + }) + + it('caps legacy select IDs without materializing them into an unbounded result', async () => { + const assignments = Array.from( + { length: 1005 }, + (_, index) => `EWREST_id_${index} = '${index}';` + ).join('\n') + clientMocks.executeEwRequest.mockImplementation( + async (_params: unknown, _buildRequest: unknown, transformResponse: ResponseTransform) => + transformResponse(createResponse({ text: assignments })) + ) + + const result = await executeAgiloftSelectRecords( + { ...BASE, where: 'status=Open' }, + { requestId: 'request-1' } + ) + + expect(result.output).toMatchObject({ + totalCount: 1000, + truncated: true, + }) + expect(result.output?.recordIds).toHaveLength(1000) + }) + + it('bounds attachment downloads and preserves binary metadata', async () => { + const controller = new AbortController() + providerMocks.secureFetchWithPinnedIP.mockResolvedValue( + createResponse({ + bytes: new TextEncoder().encode('hello'), + headers: { + 'content-type': 'text/plain', + 'content-disposition': 'attachment; filename="evidence.txt"', + }, + }) + ) + + const result = await executeAgiloftRetrieveAttachment( + { ...BASE, recordId: '1', fieldName: 'files', position: '0' }, + { requestId: 'request-1', signal: controller.signal } + ) + + expect(result).toEqual({ + success: true, + output: { + file: { + name: 'evidence.txt', + mimeType: 'text/plain', + data: Buffer.from('hello').toString('base64'), + size: 5, + }, + }, + }) + expect(providerMocks.secureFetchWithPinnedIP).toHaveBeenCalledWith( + expect.stringContaining('/ewws/EWRetrieve'), + '203.0.113.10', + { + method: 'GET', + maxResponseBytes: 25 * 1024 * 1024, + signal: controller.signal, + } + ) + }) +}) diff --git a/apps/sim/lib/internal/agiloft/operations.ts b/apps/sim/lib/internal/agiloft/operations.ts new file mode 100644 index 00000000000..91c983b44b5 --- /dev/null +++ b/apps/sim/lib/internal/agiloft/operations.ts @@ -0,0 +1,938 @@ +import { toError } from '@sim/utils/errors' +import { filterUndefined } from '@sim/utils/object' +import type { + AgiloftAsyncStatusBody, + AgiloftAttachBody, + AgiloftAttachmentInfoBody, + AgiloftCreateRecordBody, + AgiloftDeleteRecordBody, + AgiloftGetChoiceLineIdBody, + AgiloftListTablesBody, + AgiloftLockRecordBody, + AgiloftNlpSearchBody, + AgiloftReadRecordBody, + AgiloftRemoveAttachmentBody, + AgiloftRetrieveBody, + AgiloftRunActionButtonBody, + AgiloftSavedSearchBody, + AgiloftSearchRecordsBody, + AgiloftSelectRecordsBody, + AgiloftUpdateRecordBody, + AgiloftUpsertRecordBody, +} from '@/lib/api/contracts/tools/agiloft' +import { + MAX_JSON_API_RESPONSE_BYTES, + secureFetchWithPinnedIP, +} from '@/lib/core/security/input-validation.server' +import { + type AgiloftRequestConfig, + executeAgiloftRequest, + executeAlrestRequest, + executeEwRequest, + isAgiloftRefusal, + readAlrestJson, + resolveAgiloftInstance, +} from '@/lib/internal/agiloft/client' +import { AgiloftOperationError } from '@/lib/internal/agiloft/errors' +import { resolveAgiloftAttachmentFile } from '@/lib/internal/agiloft/file-input' +import { isEwRestBody, parseEwRest, toRecordIds } from '@/lib/internal/agiloft/protocol' +import { + AGILOFT_ASYNC_STATUS, + AGILOFT_LANG, + AGILOFT_MAX_ATTACHMENT_BYTES, + AGILOFT_MAX_SEARCH_RECORDS, + AGILOFT_MAX_SELECT_IDS, + alrestDeleteRecordUrl, + alrestRecordCollectionUrl, + alrestRecordUrl, + alrestSearchUrl, + buildAsyncStatusUrl, + buildAttachFileUrl, + buildAttachmentInfoUrl, + buildGetChoiceLineIdUrl, + buildListTablesUrl, + buildLockRecordUrl, + buildNlpSearchUrl, + buildRemoveAttachmentUrl, + buildRetrieveAttachmentUrl, + buildRunActionButtonUrl, + buildSavedSearchUrl, + buildSelectRecordsUrl, + buildUpsertRecordBody, + buildUpsertRecordUrl, + describeAgiloftError, + ewCredentialBody, + getLockHttpMethod, + parseFieldList, +} from '@/lib/internal/agiloft/urls' +import { resolveEffectiveMimeType } from '@/lib/uploads/utils/file-utils' +import type { + AgiloftAsyncStatusResponse, + AgiloftAttachmentInfoResponse, + AgiloftDeleteResponse, + AgiloftGetChoiceLineIdResponse, + AgiloftListTablesResponse, + AgiloftLockResponse, + AgiloftNlpSearchResponse, + AgiloftRecordResponse, + AgiloftRemoveAttachmentResponse, + AgiloftRunActionButtonResponse, + AgiloftSavedSearchResponse, + AgiloftSearchResponse, + AgiloftSelectResponse, + AgiloftTableField, + AgiloftUpsertRecordResponse, +} from '@/tools/agiloft/types' +import type { ToolResponse } from '@/tools/types' + +export interface AgiloftOperationContext { + requestId: string + userId?: string + signal?: AbortSignal +} + +function parseRecordData(data: string): Record | null { + try { + const parsed = JSON.parse(data) + return typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed) + ? (parsed as Record) + : null + } catch { + return null + } +} + +async function preserveRefusal( + run: () => Promise, + output: R['output'] +): Promise { + try { + return await run() + } catch (error) { + if (!isAgiloftRefusal(error)) throw error + return { success: false, output, error: error.message } as R + } +} + +export async function executeAgiloftCreateRecord( + input: AgiloftCreateRecordBody, + context: AgiloftOperationContext +): Promise { + const fields = parseRecordData(input.data) + if (!fields) { + return { + success: false, + output: { id: null, fields: {} }, + error: 'The data parameter must be a JSON object of field names to values', + } + } + return preserveRefusal( + () => + executeAlrestRequest( + input, + (base) => ({ + url: alrestRecordCollectionUrl(base, input.table), + method: 'POST', + headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, + body: JSON.stringify(fields), + }), + async (response) => { + const record = await readAlrestJson>(response) + const id = record?.id + return id == null + ? { + success: false, + output: { id: null, fields: record ?? {} }, + error: 'Agiloft did not return an ID for the created record', + } + : { success: true, output: { id: String(id), fields: record ?? {} } } + }, + context.signal + ), + { id: null, fields: {} } + ) +} + +export async function executeAgiloftReadRecord( + input: AgiloftReadRecordBody, + context: AgiloftOperationContext +): Promise { + const requestedFields = parseFieldList(input.fields) + const recordId = input.recordId.trim() + if (requestedFields && !/^\d+$/.test(recordId)) { + return { + success: false, + output: { id: null, fields: {} }, + error: `Record ID must be numeric to read specific fields, got "${recordId}"`, + } + } + return preserveRefusal( + () => + executeAlrestRequest( + input, + (base): AgiloftRequestConfig => + requestedFields + ? { + url: alrestSearchUrl(base, input.table), + method: 'POST', + headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, + body: JSON.stringify({ + field: requestedFields.includes('id') + ? requestedFields + : ['id', ...requestedFields], + query: `id=${recordId}`, + }), + } + : { + url: alrestRecordUrl(base, input.table, input.recordId), + method: 'GET', + headers: { Accept: 'application/json' }, + }, + async (response) => { + const payload = await readAlrestJson | Record[]>( + response + ) + const record = Array.isArray(payload) + ? payload.find((row) => { + const rowId = String(row?.id ?? '') + return /^\d+$/.test(rowId) && BigInt(rowId) === BigInt(recordId) + }) + : payload + if (!record) { + return { + success: false, + output: { id: null, fields: {} }, + error: `Agiloft returned no record for ID ${recordId}`, + } + } + return { + success: true, + output: { id: record.id == null ? null : String(record.id), fields: record }, + } + }, + context.signal + ), + { id: null, fields: {} } + ) +} + +export async function executeAgiloftUpdateRecord( + input: AgiloftUpdateRecordBody, + context: AgiloftOperationContext +): Promise { + const fields = parseRecordData(input.data) + if (!fields) { + return { + success: false, + output: { id: null, fields: {} }, + error: 'The data parameter must be a JSON object of field names to values', + } + } + return preserveRefusal( + () => + executeAlrestRequest( + input, + (base) => ({ + url: alrestRecordUrl(base, input.table, input.recordId), + method: 'PUT', + headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, + body: JSON.stringify(fields), + }), + async (response) => { + const record = await readAlrestJson>(response) + return { + success: true, + output: { + id: String(record?.id ?? input.recordId.trim()), + fields: record ?? {}, + }, + } + }, + context.signal + ), + { id: null, fields: {} } + ) +} + +export async function executeAgiloftDeleteRecord( + input: AgiloftDeleteRecordBody, + context: AgiloftOperationContext +): Promise { + return preserveRefusal( + () => + executeAlrestRequest( + input, + (base) => ({ + url: alrestDeleteRecordUrl( + base, + input.table, + input.recordId, + input.deleteRule, + input.substituteIds + ), + method: 'DELETE', + headers: { Accept: 'application/json' }, + }), + async (response) => { + await readAlrestJson(response) + return { + success: true, + output: { id: input.recordId.trim(), deleted: true }, + } + }, + context.signal + ), + { id: '', deleted: false } + ) +} + +export async function executeAgiloftSearchRecords( + input: AgiloftSearchRecordsBody, + context: AgiloftOperationContext +): Promise { + const page = input.page ? Number(input.page) : 0 + const limit = input.limit ? Number(input.limit) : 0 + return preserveRefusal( + () => + executeAlrestRequest( + input, + (base) => ({ + url: alrestSearchUrl(base, input.table), + method: 'POST', + headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, + body: JSON.stringify( + filterUndefined({ + search: input.search?.trim() || undefined, + query: input.query?.trim() || undefined, + field: parseFieldList(input.fields), + page: input.page ? page : undefined, + limit: input.limit ? limit : undefined, + }) + ), + }), + async (response) => { + const returned = (await readAlrestJson[]>(response)) ?? [] + const records = returned.slice(0, AGILOFT_MAX_SEARCH_RECORDS) + return { + success: true, + output: { + records, + totalCount: records.length, + page, + limit, + truncated: returned.length > records.length, + }, + } + }, + context.signal + ), + { records: [], totalCount: 0, page: 0, limit: 0, truncated: false } + ) +} + +export async function executeAgiloftSelectRecords( + input: AgiloftSelectRecordsBody, + context: AgiloftOperationContext +): Promise { + return executeEwRequest( + input, + (base) => ({ + url: buildSelectRecordsUrl(base, input), + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: ewCredentialBody(input), + }), + async (response) => { + const body = await response.text() + if (!response.ok) { + return { + success: false, + output: { recordIds: [], totalCount: 0, truncated: false }, + error: `Agiloft error ${response.status}: ${describeAgiloftError(body)}`, + } + } + const values = parseEwRest(body) + if (values.size === 0) { + return { + success: false, + output: { recordIds: [], totalCount: 0, truncated: false }, + error: `Agiloft did not return a result set: ${body.trim() || '(empty response)'}`, + } + } + const recordIds = toRecordIds(values).recordIds + const capped = recordIds.slice(0, AGILOFT_MAX_SELECT_IDS) + return { + success: true, + output: { + recordIds: capped, + totalCount: capped.length, + truncated: recordIds.length > capped.length, + }, + } + }, + context.signal + ) +} + +function emptyLock(recordId: string) { + return { + id: recordId.trim(), + tableId: null, + lockStatus: '', + lockedBy: null, + lockExpiresInMinutes: null, + } +} + +export async function executeAgiloftAttachmentInfo( + input: AgiloftAttachmentInfoBody, + context: AgiloftOperationContext +): Promise { + return executeEwRequest( + input, + (base) => ({ url: buildAttachmentInfoUrl(base, input), method: 'GET' }), + async (response) => { + if (!response.ok) { + const text = await response.text() + return { + success: false, + output: { attachments: [], totalCount: 0 }, + error: `Agiloft error ${response.status}: ${describeAgiloftError(text)}`, + } + } + const data = (await response.json()) as Record + const payload = (data.result ?? data) as Record + const attachments: Array<{ position: number; name: string; size: number }> = [] + if (Array.isArray(payload)) { + for (let index = 0; index < payload.length; index++) { + const item = payload[index] as Record + attachments.push({ + position: (item.filePosition as number) ?? (item.position as number) ?? index, + name: + (item.fileName as string) ?? (item.name as string) ?? (item.filename as string) ?? '', + size: (item.size as number) ?? (item.fileSize as number) ?? 0, + }) + } + } + return { + success: data.success !== false, + output: { attachments, totalCount: attachments.length }, + } + }, + context.signal + ) +} + +export async function executeAgiloftLockRecord( + input: AgiloftLockRecordBody, + context: AgiloftOperationContext +): Promise { + return executeEwRequest( + input, + (base) => ({ + url: buildLockRecordUrl(base, input), + method: getLockHttpMethod(input.lockAction), + }), + async (response) => { + if (!response.ok) { + const text = await response.text() + return { + success: false, + output: emptyLock(input.recordId), + error: `Agiloft error ${response.status}: ${describeAgiloftError(text)}`, + } + } + const data = (await response.json()) as Record + if (typeof data.lock_status !== 'string') { + const code = typeof data.error === 'string' ? data.error : 'UNKNOWN' + const detail = + typeof data.error_description === 'string' ? data.error_description : JSON.stringify(data) + return { + success: false, + output: emptyLock(input.recordId), + error: `Agiloft lock error (${code}): ${detail}`, + } + } + return { + success: true, + output: { + id: String(data.id ?? input.recordId.trim()), + tableId: typeof data.table_id === 'number' ? data.table_id : null, + lockStatus: data.lock_status, + lockedBy: typeof data.locked_by === 'string' ? data.locked_by : null, + lockExpiresInMinutes: + typeof data.lock_expires_in_minutes === 'number' ? data.lock_expires_in_minutes : null, + }, + } + }, + context.signal + ) +} + +export async function executeAgiloftRemoveAttachment( + input: AgiloftRemoveAttachmentBody, + context: AgiloftOperationContext +): Promise { + return executeEwRequest( + input, + (base) => ({ url: buildRemoveAttachmentUrl(base, input), method: 'GET' }), + async (response) => { + const text = await response.text() + const fieldName = input.fieldName.trim() + if (!response.ok) { + return { + success: false, + output: { recordId: input.recordId.trim(), fieldName, remainingAttachments: 0 }, + error: `Agiloft error ${response.status}: ${describeAgiloftError(text)}`, + } + } + const values = parseEwRest(text) + const remainingAttachments = Number( + values.get(`${fieldName}.length`) ?? [...values.values()][0] + ) + if (!Number.isFinite(remainingAttachments)) { + return { + success: false, + output: { recordId: input.recordId.trim(), fieldName, remainingAttachments: 0 }, + error: `Agiloft did not report the remaining attachment count: ${text.trim() || '(empty response)'}`, + } + } + return { + success: true, + output: { recordId: input.recordId.trim(), fieldName, remainingAttachments }, + } + }, + context.signal + ) +} + +export async function executeAgiloftGetChoiceLineId( + input: AgiloftGetChoiceLineIdBody, + context: AgiloftOperationContext +): Promise { + return executeEwRequest( + input, + (base) => ({ url: buildGetChoiceLineIdUrl(base, input), method: 'GET' }), + async (response) => { + const body = await response.text() + if (!response.ok) { + return { + success: false, + output: { choiceLineId: null }, + error: `Agiloft error ${response.status}: ${describeAgiloftError(body)}`, + } + } + const raw = parseEwRest(body).get('choiceLineId') + const id = Number(raw) + if (raw === undefined) { + return { + success: false, + output: { choiceLineId: null }, + error: `Agiloft did not return a choice line ID for "${input.value}" in field "${input.fieldName}": ${body.trim() || '(empty response)'}`, + } + } + if (raw.trim() === '' || !Number.isFinite(id)) { + return { + success: false, + output: { choiceLineId: null }, + error: `Agiloft returned a non-numeric choice line ID for "${input.value}" in field "${input.fieldName}": "${raw}"`, + } + } + return { success: true, output: { choiceLineId: id } } + }, + context.signal + ) +} + +export async function executeAgiloftRunActionButton( + input: AgiloftRunActionButtonBody, + context: AgiloftOperationContext +): Promise { + return executeEwRequest( + input, + (base) => ({ + url: buildRunActionButtonUrl(base, input), + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + }), + async (response) => { + const body = await response.text() + const recordId = input.recordId.trim() + if (!response.ok) { + return { + success: false, + output: { recordId, callbackId: null }, + error: `Agiloft error ${response.status}: ${describeAgiloftError(body)}`, + } + } + const values = parseEwRest(body) + if (values.size === 0) { + return { + success: false, + output: { recordId, callbackId: null }, + error: `Agiloft did not acknowledge the action button: ${body.trim() || '(empty response)'}`, + } + } + return { + success: true, + output: { + recordId: values.get('id') ?? recordId, + callbackId: values.get('EWCALLBACK_ID') ?? null, + }, + } + }, + context.signal + ) +} + +export async function executeAgiloftAsyncStatus( + input: AgiloftAsyncStatusBody, + context: AgiloftOperationContext +): Promise { + return executeEwRequest( + input, + (base) => ({ url: buildAsyncStatusUrl(base, input), method: 'GET' }), + async (response) => { + const known = AGILOFT_ASYNC_STATUS[response.status] + if (!known) { + const body = await response.text() + return { + success: false, + output: { + callbackId: input.callbackId.trim(), + statusCode: response.status, + status: 'unrecognized', + complete: false, + }, + error: `Agiloft returned an unrecognized async status ${response.status}: ${body.trim() || '(empty response)'}`, + } + } + return { + success: true, + output: { + callbackId: input.callbackId.trim(), + statusCode: response.status, + status: known.status, + complete: known.complete, + }, + } + }, + context.signal + ) +} + +export async function executeAgiloftSavedSearch( + input: AgiloftSavedSearchBody, + context: AgiloftOperationContext +): Promise { + return preserveRefusal( + () => + executeAgiloftRequest( + input, + (base) => ({ + url: buildSavedSearchUrl(base, input), + method: 'GET', + headers: { Accept: 'application/json' }, + }), + async (response) => { + const rows = + (await readAlrestJson< + Array<{ name?: string; label?: string; id?: number; description?: string }> + >(response)) ?? [] + const searches = rows.map((row) => ({ + name: row.name ?? '', + label: row.label ?? row.name ?? '', + id: row.id ?? null, + description: row.description ?? null, + })) + return { success: true, output: { searches, totalCount: searches.length } } + }, + context.signal + ), + { searches: [], totalCount: 0 } + ) +} + +interface EwTableResult { + tables?: Array<{ + label?: string + logicalName?: string + fields?: Array<{ + columnName?: string + columnLabel?: string + columnType?: string + columnTypeDomain?: string + required?: boolean + isLinked?: boolean + linkedInfo?: Array<{ linkedTable?: string; linkedColumn?: string }> + textFieldType?: string + }> + }> +} + +export async function executeAgiloftListTables( + input: AgiloftListTablesBody, + context: AgiloftOperationContext +): Promise { + try { + return await executeAgiloftRequest( + input, + (base) => ({ + url: buildListTablesUrl(base, input), + method: 'GET', + headers: { Accept: 'application/json' }, + }), + async (response) => { + const payload = await readAlrestJson(response) + const tables = (payload?.tables ?? []).map((table) => ({ + label: table.label ?? '', + logicalName: table.logicalName ?? '', + fields: (table.fields ?? []).map( + (field): AgiloftTableField => ({ + columnName: field.columnName ?? '', + columnLabel: field.columnLabel ?? '', + columnType: field.columnType ?? '', + columnTypeDomain: field.columnTypeDomain ?? '', + required: field.required === true, + isLinked: field.isLinked === true, + linkedInfo: (field.linkedInfo ?? []).map((link) => ({ + linkedTable: link.linkedTable ?? '', + linkedColumn: link.linkedColumn ?? '', + })), + textFieldType: field.textFieldType ?? null, + }) + ), + })) + return { success: true, output: { tables, totalCount: tables.length } } + }, + context.signal + ) + } catch (error) { + if (!input.table && /\$table/.test(toError(error).message)) { + return { + success: false, + output: { tables: [], totalCount: 0 }, + error: + 'This Agiloft instance requires a table name to authenticate. Put any known table in the Table field — it also narrows the result to that table.', + } + } + if (!isAgiloftRefusal(error)) throw error + return { + success: false, + output: { tables: [], totalCount: 0 }, + error: error.message, + } + } +} + +export async function executeAgiloftNlpSearch( + input: AgiloftNlpSearchBody, + context: AgiloftOperationContext +): Promise { + return executeEwRequest( + input, + (base) => ({ + url: buildNlpSearchUrl(base), + method: 'POST', + headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, + body: JSON.stringify( + filterUndefined({ + $KB: input.knowledgeBase, + $login: input.login, + $password: input.password, + $lang: AGILOFT_LANG, + field: parseFieldList(input.fields), + nlp_query: input.nlpQuery.trim(), + page: input.page ? Number(input.page) : undefined, + limit: input.limit ? Number(input.limit) : undefined, + }) + ), + }), + async (response) => { + const returned = (await readAlrestJson[]>(response)) ?? [] + const records = returned.slice(0, AGILOFT_MAX_SEARCH_RECORDS) + return { + success: true, + output: { + records, + totalCount: records.length, + truncated: returned.length > records.length, + }, + } + }, + context.signal + ) +} + +export async function executeAgiloftUpsertRecord( + input: AgiloftUpsertRecordBody, + context: AgiloftOperationContext +): Promise { + const fields = parseRecordData(input.data) + if (!fields) { + return { + success: false, + output: { id: null, created: false, callbackId: null }, + error: 'The data parameter must be a JSON object of field names to values', + } + } + let encoded: string + try { + encoded = buildUpsertRecordBody(input, fields) + } catch (error) { + return { + success: false, + output: { id: null, created: false, callbackId: null }, + error: toError(error).message, + } + } + return executeEwRequest( + input, + (base) => ({ + url: buildUpsertRecordUrl(base), + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: encoded, + }), + async (response) => { + const body = await response.text() + if (response.status === 409) { + return { + success: false, + output: { id: null, created: false, callbackId: null }, + error: `Agiloft found more than one record matching "${input.match}", so it did not write: ${body.trim()}`, + } + } + if (!response.ok) { + return { + success: false, + output: { id: null, created: false, callbackId: null }, + error: `Agiloft error ${response.status}: ${describeAgiloftError(body)}`, + } + } + if (response.status === 202) { + return { + success: true, + output: { + id: null, + created: false, + callbackId: parseEwRest(body).get('EWCALLBACK_ID') ?? null, + }, + } + } + const id = parseEwRest(body).get('id') + return id === undefined + ? { + success: false, + output: { id: null, created: false, callbackId: null }, + error: `Agiloft did not return a record ID: ${body.trim() || '(empty response)'}`, + } + : { + success: true, + output: { id, created: response.status === 201, callbackId: null }, + } + }, + context.signal + ) +} + +export async function executeAgiloftAttachFile( + input: AgiloftAttachBody, + context: AgiloftOperationContext +): Promise { + const { userFile, buffer } = await resolveAgiloftAttachmentFile(input.file, context) + const fileName = input.fileName || userFile.name || 'attachment' + let resolvedIP: string + try { + resolvedIP = await resolveAgiloftInstance(input.instanceUrl, context.signal) + } catch (error) { + context.signal?.throwIfAborted() + throw new AgiloftOperationError(400, { success: false, error: toError(error).message }) + } + const response = await secureFetchWithPinnedIP( + buildAttachFileUrl(input.instanceUrl.replace(/\/$/, ''), input, fileName), + resolvedIP, + { + method: 'PUT', + headers: { 'Content-Type': 'application/octet-stream' }, + body: buffer, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + signal: context.signal, + } + ) + const text = await response.text() + context.signal?.throwIfAborted() + if (!response.ok) { + throw new AgiloftOperationError(response.status, { + success: false, + error: `Agiloft error ${response.status}: ${describeAgiloftError(text)}`, + }) + } + const values = parseEwRest(text) + const totalAttachments = Number( + values.get(`${input.fieldName.trim()}.length`) ?? [...values.values()][0] + ) + if (!Number.isFinite(totalAttachments)) { + throw new AgiloftOperationError(502, { + success: false, + error: `Agiloft did not confirm the attachment: ${describeAgiloftError(text) || '(empty response)'}`, + }) + } + return { + success: true, + output: { + recordId: input.recordId.trim(), + fieldName: input.fieldName.trim(), + fileName, + totalAttachments, + }, + } +} + +export async function executeAgiloftRetrieveAttachment( + input: AgiloftRetrieveBody, + context: AgiloftOperationContext +): Promise { + let resolvedIP: string + try { + resolvedIP = await resolveAgiloftInstance(input.instanceUrl, context.signal) + } catch (error) { + context.signal?.throwIfAborted() + throw new AgiloftOperationError(400, { success: false, error: toError(error).message }) + } + const response = await secureFetchWithPinnedIP( + buildRetrieveAttachmentUrl(input.instanceUrl.replace(/\/$/, ''), input), + resolvedIP, + { method: 'GET', maxResponseBytes: AGILOFT_MAX_ATTACHMENT_BYTES, signal: context.signal } + ) + if (!response.ok) { + const text = await response.text() + throw new AgiloftOperationError(response.status, { + success: false, + error: `Agiloft error ${response.status}: ${describeAgiloftError(text)}`, + }) + } + const contentType = response.headers.get('content-type') || 'application/octet-stream' + const disposition = response.headers.get('content-disposition') + const match = disposition?.match(/filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/) + const fileName = match?.[1] ? match[1].replace(/['"]/g, '') : 'attachment' + const buffer = Buffer.from(await response.arrayBuffer()) + context.signal?.throwIfAborted() + if (isEwRestBody(buffer.subarray(0, 512).toString('utf8'))) { + throw new AgiloftOperationError(502, { + success: false, + error: `Agiloft error: ${buffer.toString('utf8').slice(0, 300)}`, + }) + } + return { + success: true, + output: { + file: { + name: fileName, + mimeType: resolveEffectiveMimeType(contentType, fileName), + data: buffer.toString('base64'), + size: buffer.length, + }, + }, + } +} diff --git a/apps/sim/lib/internal/agiloft/protocol.test.ts b/apps/sim/lib/internal/agiloft/protocol.test.ts new file mode 100644 index 00000000000..3491fad926d --- /dev/null +++ b/apps/sim/lib/internal/agiloft/protocol.test.ts @@ -0,0 +1,77 @@ +/** + * @vitest-environment node + * + * Every fixture below is quoted from Agiloft's own REST documentation so the + * parser is pinned to the published response format rather than to a shape we + * assumed. + */ +import { describe, expect, it } from 'vitest' +import { parseEwRest, toRecordIds } from '@/lib/internal/agiloft/protocol' + +/** REST - Create: "A result similar to the following will be returned". */ +const CREATE_BODY = "EWREST_id='353';" + +/** REST - Read: the documented result for record 358 of contacts.employees. */ +const READ_BODY = `EWREST_full_name='John Doe'; +EWREST_first_name='John'; +EWREST__1576_company_name0='IBM'; +EWREST_f_group_0='Service Manager'; +EWREST_id='358'; +EWREST__login='jdoe'; +EWREST_date_updated='Dec 27 2017 04:40:24'; +EWREST_last_name='Doe';` + +/** REST - Select, Example 1: three matching records. */ +const SELECT_BODY = `EWREST_id_length = '3'; +EWREST_id_0 = '150'; +EWREST_id_1 = '169'; +EWREST_id_2 = '325';` + +/** REST - Select: the documented empty result. */ +const SELECT_EMPTY_BODY = "EWREST_id_length = '0';" + +/** REST - Search, Example 2: two requested fields across four records. */ +const SEARCH_BODY = `EWREST_length = '4'; +EWREST_summary_0='Here is a new service request with some tasks'; +EWREST_priority_0='High'; +EWREST_summary_1='New Employee Setup for Patricia Smith'; +EWREST_priority_1='High'; +EWREST_summary_2='Upgrading Our Software'; +EWREST_priority_2='High'; +EWREST_summary_3='Need New Wireless Card for Laptop'; +EWREST_priority_3='High';` + +describe('parseEwRest', () => { + it('reads the field assignments EWRead returns', () => { + const values = parseEwRest(READ_BODY) + + expect(values.get('id')).toBe('358') + expect(values.get('full_name')).toBe('John Doe') + expect(values.get('date_updated')).toBe('Dec 27 2017 04:40:24') + expect(values.get('_login')).toBe('jdoe') + }) + + it('tolerates the spaces around = that Select and Search use', () => { + expect(parseEwRest(SELECT_BODY).get('id_length')).toBe('3') + }) + + it('ignores blank lines and non-assignment noise instead of aborting', () => { + const values = parseEwRest(`\n${CREATE_BODY}\nnot an assignment\n\n`) + + expect(values.size).toBe(1) + expect(values.get('id')).toBe('353') + }) +}) + +describe('toRecordIds', () => { + it('reads the documented EWSelect result', () => { + expect(toRecordIds(parseEwRest(SELECT_BODY))).toEqual({ + recordIds: ['150', '169', '325'], + count: 3, + }) + }) + + it('reads the documented empty EWSelect result', () => { + expect(toRecordIds(parseEwRest(SELECT_EMPTY_BODY))).toEqual({ recordIds: [], count: 0 }) + }) +}) diff --git a/apps/sim/lib/internal/agiloft/protocol.ts b/apps/sim/lib/internal/agiloft/protocol.ts new file mode 100644 index 00000000000..fb354a1e845 --- /dev/null +++ b/apps/sim/lib/internal/agiloft/protocol.ts @@ -0,0 +1,78 @@ +/** + * Parses Agiloft's `EWREST_` response format. + * + * The legacy `/ewws/EW*` operations answer with a body of JavaScript assignments + * rather than JSON — the interface was designed to be `eval`-ed by a browser + * client. The documented shapes are: + * + * EWCreate -> EWREST_id='353'; + * EWRead -> EWREST_full_name='John Doe'; + * EWREST_id='358'; + * EWSelect -> EWREST_id_length = '3'; + * EWREST_id_0 = '150'; + * EWSearch -> EWREST_length = '4'; + * EWREST_summary_0='Upgrading Our Software'; + * + * Note the inconsistent spacing around `=`: the record-field forms have none, + * the `_length` and indexed-id forms in the Select/Search examples do. Both are + * accepted here. + */ + +/** + * Matches one `EWREST_key='value';` assignment anywhere in the body. Most + * responses put one per line, but EWActionButton documents both of its + * assignments on a single line, so the scan is not line-anchored. The value is + * non-greedy up to the closing `';` for the same reason. + */ +const ASSIGNMENT = /EWREST_(?[^=\s']+)\s*=\s*'(?[\s\S]*?)'\s*;/g + +/** + * Parses a body into its raw `EWREST_` key/value pairs, preserving document + * order. Lines that are blank or do not match the assignment form are skipped, + * so a trailing newline or an incidental banner does not abort the parse. + */ +export function parseEwRest(body: string): Map { + const values = new Map() + + for (const match of body.matchAll(ASSIGNMENT)) { + const key = match.groups?.key + if (!key) continue + values.set(key, match.groups?.value ?? '') + } + + return values +} + +/** + * Reads the `EWREST_id_length` / `EWREST_id_` pairs EWSelect returns. A + * result of zero records is reported as `EWREST_id_length = '0';` with no + * indexed entries. + */ +/** + * True when the body carries at least one `EWREST_` assignment. Used where a + * binary payload is expected, since a refusal arrives as an assignment or + * plain text instead of file bytes. + */ +export function isEwRestBody(body: string): boolean { + return parseEwRest(body).size > 0 +} + +export function toRecordIds(values: Map): { + recordIds: string[] + count: number +} { + const recordIds: string[] = [] + for (let index = 0; ; index++) { + const id = values.get(`id_${index}`) + if (id === undefined) break + recordIds.push(id) + } + + /** + * Report what was actually parsed rather than the declared length. A + * declared count that disagrees with the rows present means the body was + * truncated, and returning the larger number would hide that from callers + * who compare `totalCount` against `recordIds.length`. + */ + return { recordIds, count: recordIds.length } +} diff --git a/apps/sim/lib/internal/agiloft/urls.test.ts b/apps/sim/lib/internal/agiloft/urls.test.ts new file mode 100644 index 00000000000..5b48a039f08 --- /dev/null +++ b/apps/sim/lib/internal/agiloft/urls.test.ts @@ -0,0 +1,209 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + agiloftAlrestBase, + alrestDeleteRecordUrl, + alrestRecordCollectionUrl, + alrestRecordUrl, + alrestSearchUrl, + buildAttachFileUrl, + buildLockRecordUrl, + buildRetrieveAttachmentUrl, + buildSavedSearchUrl, + buildSelectRecordsUrl, + describeAgiloftError, + ewCredentialBody, + parseFieldList, +} from '@/lib/internal/agiloft/urls' + +/** Obvious non-secret so credential scanners do not flag these fixtures. */ +const PLACEHOLDER_PASSWORD = 'not-a-real-password' + +const INSTANCE = 'https://example.agiloft.com' + +const baseParams = { + instanceUrl: INSTANCE, + knowledgeBase: 'Russell Investments', + login: 'svc.user', + password: PLACEHOLDER_PASSWORD, + table: 'contract', +} + +const BASE = agiloftAlrestBase(INSTANCE, baseParams.knowledgeBase) + +describe('agiloftAlrestBase', () => { + it('targets the alrest surface that accepts the EWLogin token', () => { + expect(BASE).toBe('https://example.agiloft.com/ewws/alrest/Russell%20Investments') + }) + + it('encodes the KB name and tolerates a trailing slash on the instance URL', () => { + expect(agiloftAlrestBase('https://example.agiloft.com/', 'A B&C')).toBe( + 'https://example.agiloft.com/ewws/alrest/A%20B%26C' + ) + }) +}) + +describe('alrest record routes', () => { + it('builds collection, item, and search paths under the KB base', () => { + expect(alrestRecordCollectionUrl(BASE, 'contract')).toBe(`${BASE}/contract?lang=en`) + expect(alrestRecordUrl(BASE, 'contract', ' 6342 ')).toBe(`${BASE}/contract/6342?lang=en`) + expect(alrestSearchUrl(BASE, 'contract')).toBe(`${BASE}/contract/search?lang=en`) + }) + + it('retrieves attachments through the documented EWRetrieve endpoint', () => { + const url = buildRetrieveAttachmentUrl(INSTANCE, { + ...baseParams, + recordId: '1234', + fieldName: 'someField', + position: '1', + }) + + expect(url).toContain('/ewws/EWRetrieve?') + expect(url).toContain('&id=1234') + expect(url).toContain('&field=someField') + /** Documented parameter name is filePosition, not position. */ + expect(url).toContain('&filePosition=1') + }) + + it('always carries a delete rule so linked-record behavior is explicit', () => { + expect(alrestDeleteRecordUrl(BASE, 'contract', '6342', 'ERROR_IF_DEPENDANTS')).toBe( + `${BASE}/contract/6342?lang=en&deleteRule=ERROR_IF_DEPENDANTS` + ) + }) +}) + +describe('parseFieldList', () => { + it('splits and trims a comma-separated projection', () => { + expect(parseFieldList(' id , contract_title1 ,, ')).toEqual(['id', 'contract_title1']) + }) + + it('returns undefined when nothing usable was given, so no projection is sent', () => { + expect(parseFieldList(undefined)).toBeUndefined() + expect(parseFieldList(' ')).toBeUndefined() + expect(parseFieldList(',,')).toBeUndefined() + }) +}) + +describe('legacy EW* endpoints', () => { + it('keeps credentials out of the EWSelect URL, which supports a POST body', () => { + const url = buildSelectRecordsUrl(INSTANCE, { ...baseParams, where: "id='1'" }) + + expect(url).toContain('/ewws/EWSelect?') + expect(url).toContain('&$lang=en') + expect(url).not.toContain('$login') + expect(url).not.toContain('$password') + }) + + it('form-encodes credentials for the operations that accept a body', () => { + const body = ewCredentialBody(baseParams) + + const sent = new URLSearchParams(body) + expect(sent.get('$login')).toBe('svc.user') + expect(sent.get('$password')).toBe(PLACEHOLDER_PASSWORD) + }) + + it('percent-encodes credentials in the URL for operations with no body option', () => { + const url = buildLockRecordUrl(INSTANCE, { + ...baseParams, + login: 'a&b=c', + password: 'placeholder&pass=word', + recordId: '18', + lockAction: 'check', + }) + + expect(url).toContain('&$login=a%26b%3Dc') + expect(url).toContain('&$password=placeholder%26pass%3Dword') + }) + + it('adds force only when unlocking', () => { + expect( + buildLockRecordUrl(INSTANCE, { + ...baseParams, + recordId: '18', + lockAction: 'unlock', + force: true, + }) + ).toContain('&force=true') + + expect( + buildLockRecordUrl(INSTANCE, { + ...baseParams, + recordId: '18', + lockAction: 'lock', + force: true, + }) + ).not.toContain('force=true') + }) +}) + +describe('documented response keys', () => { + it('builds the EWSavedSearch URL with the mandatory .json decorator and no credentials', () => { + const url = buildSavedSearchUrl(INSTANCE, baseParams) + + expect(url).toContain('/ewws/EWSavedSearch/.json?') + expect(url).toContain('$table=contract') + /** Must run under EWLogin/OAuth, so inline credentials are not appended. */ + expect(url).not.toContain('$login') + expect(url).not.toContain('$password') + }) +}) + +describe('describeAgiloftError', () => { + it('reduces the HTML-wrapped exception to its message', () => { + const body = + 'ErrorEWWrongDataException has occurred: ' + + '[default task-70331][1786479740423] One has to specify $table, $KB, $lang parameters' + + '' + + expect(describeAgiloftError(body)).toBe( + 'EWWrongDataException: One has to specify $table, $KB, $lang parameters' + ) + }) + + it('passes through a body that is not a typed exception', () => { + expect(describeAgiloftError('Error executing query, please consult logs')).toBe( + 'Error executing query, please consult logs' + ) + }) +}) + +describe('documented optional parameters', () => { + it('adds subs only for the delete rule that reads it', () => { + const withReplace = alrestDeleteRecordUrl( + BASE, + 'contract', + '6342', + 'REPLACE_WITH_ANOTHER', + '7,8' + ) + expect(withReplace).toContain('&subs=7') + expect(withReplace).toContain('&subs=8') + + const withoutReplace = alrestDeleteRecordUrl(BASE, 'contract', '6342', 'APPLY_UNLINK', '7,8') + expect(withoutReplace).not.toContain('subs') + }) + + it('asks the JSON decorator for real status codes', () => { + expect(buildSavedSearchUrl(INSTANCE, baseParams)).toContain('err_code_resp=1') + }) +}) + +describe('attach overwrite', () => { + it('sends the documented fieldName$overwrite marker only when requested', () => { + const on = buildAttachFileUrl( + INSTANCE, + { ...baseParams, recordId: '1', fieldName: 'docs', overwrite: true }, + 'a.pdf' + ) + expect(on).toContain('docs%24overwrite=true') + + const off = buildAttachFileUrl( + INSTANCE, + { ...baseParams, recordId: '1', fieldName: 'docs' }, + 'a.pdf' + ) + expect(off).not.toContain('overwrite') + }) +}) diff --git a/apps/sim/lib/internal/agiloft/urls.ts b/apps/sim/lib/internal/agiloft/urls.ts new file mode 100644 index 00000000000..cd39fb645ac --- /dev/null +++ b/apps/sim/lib/internal/agiloft/urls.ts @@ -0,0 +1,394 @@ +import type { + AgiloftAsyncStatusParams, + AgiloftAttachmentInfoParams, + AgiloftBaseParams, + AgiloftCredentials, + AgiloftGetChoiceLineIdParams, + AgiloftListTablesParams, + AgiloftLockRecordParams, + AgiloftRemoveAttachmentParams, + AgiloftRetrieveAttachmentParams, + AgiloftRunActionButtonParams, + AgiloftSelectRecordsParams, + AgiloftUpsertRecordParams, +} from '@/tools/agiloft/types' +import type { HttpMethod } from '@/tools/types' + +/** + * Requests real HTTP status codes from Agiloft's JSON decorator. Without it Agiloft + * answers 200 even on failure, which is what forces callers to infer errors + * from the body shape. + */ +export const AGILOFT_JSON_ERROR_CODES = 'err_code_resp=1' + +/** + * Reduces an Agiloft error body to its message. + * + * Failures arrive as an HTML document wrapping a typed exception and an + * internal task id — `…EWWrongDataException has occurred: + * [default task-70331][1786479740423] One has to specify $table…` — + * none of which helps the person reading the workflow log. + */ +export function describeAgiloftError(body: string): string { + const text = body + .replace(/<[^>]*>/g, ' ') + .replace(/\s+/g, ' ') + .trim() + + const typed = /(EW[A-Za-z]*Exception)\s*(?:has occurred)?\s*:?\s*(.*)/.exec(text) + if (!typed) return text + + const detail = typed[2].replace(/^(\[[^\]]*\]\s*)+/, '').trim() + return detail ? `${typed[1]}: ${detail}` : typed[1] +} + +/** Language sent on every Agiloft call; EWLogin rejects the request without it. */ +export const AGILOFT_LANG = 'en' + +/** + * Base URL for the `/ewws/alrest/{KB}` REST surface, which is the one that + * accepts the token EWLogin issues. The legacy `/ewws/EW*` endpoints expect + * inline `$login`/`$password` credentials instead and reject a bearer token. + */ +export function agiloftAlrestBase(instanceUrl: string, knowledgeBase: string): string { + return `${instanceUrl.replace(/\/$/, '')}/ewws/alrest/${encodeURIComponent(knowledgeBase)}` +} + +/** Table segment of an alrest path. */ +function tableSegment(table: string): string { + return encodeURIComponent(table.trim()) +} + +export function alrestRecordCollectionUrl(base: string, table: string): string { + return `${base}/${tableSegment(table)}?lang=${AGILOFT_LANG}` +} + +export function alrestRecordUrl(base: string, table: string, recordId: string): string { + return `${base}/${tableSegment(table)}/${encodeURIComponent(recordId.trim())}?lang=${AGILOFT_LANG}` +} + +/** + * EWDelete's dependent-record strategy carries over to alrest as a query + * parameter; omitting it leaves the behavior for linked records unspecified. + */ +export function alrestDeleteRecordUrl( + base: string, + table: string, + recordId: string, + deleteRule: string, + substituteIds?: string +): string { + let url = `${alrestRecordUrl(base, table, recordId)}&deleteRule=${encodeURIComponent(deleteRule)}` + + /** + * `subs` is read only under REPLACE_WITH_ANOTHER, and names records from the + * same table that adopt the dependants of the one being deleted. + */ + if (deleteRule === 'REPLACE_WITH_ANOTHER' && substituteIds) { + for (const id of substituteIds + .split(',') + .map((value) => value.trim()) + .filter(Boolean)) { + url += `&subs=${encodeURIComponent(id)}` + } + } + + return url +} + +export function alrestSearchUrl(base: string, table: string): string { + return `${base}/${tableSegment(table)}/search?lang=${AGILOFT_LANG}` +} + +/** + * Hard ceiling on records returned from a search. + * + * Whether alrest honours `page`/`limit` in the request body is unverified — the + * names carry over from the legacy EWSearch query string. If it ignores them a + * broad query returns the whole table, and an unfiltered contract record runs + * to roughly 184 KB, so the result is capped here rather than trusting the + * server to bound it. + */ +export const AGILOFT_MAX_SEARCH_RECORDS = 200 + +/** + * Byte ceiling for a single attachment download. The route base64-encodes the + * body into a JSON response, so peak memory is several times the file size; + * without a cap it inherits the shared 100 MiB default. + */ +export const AGILOFT_MAX_ATTACHMENT_BYTES = 25 * 1024 * 1024 + +/** + * Ceiling on record IDs returned by EWSelect. The operation has no page size + * of its own — the documented way to bound it is a database `limit` inside the + * WHERE clause — so an unqualified clause returns every matching ID. + */ +export const AGILOFT_MAX_SELECT_IDS = 1000 + +/** + * Splits a comma-separated field list into the `field` array alrest search + * accepts. Field selection is the only way to keep a response small — a single + * contract record runs to roughly 184 KB unfiltered. + */ +export function parseFieldList(fields?: string): string[] | undefined { + const list = fields + ?.split(',') + .map((field) => field.trim()) + .filter(Boolean) + return list?.length ? list : undefined +} + +/** URL builders for the legacy `/ewws/EW*` surface (inline-credential auth) */ + +function encodeTable(params: AgiloftBaseParams) { + return { + kb: encodeURIComponent(params.knowledgeBase), + table: encodeURIComponent(params.table), + } +} + +/** Non-secret query prefix shared by the legacy `/ewws/EW*` endpoints. */ +function buildEwBaseQuery(params: AgiloftBaseParams): string { + const { kb, table } = encodeTable(params) + return `$KB=${kb}&$table=${table}&$lang=${AGILOFT_LANG}` +} + +/** + * Credentials appended to an EW* URL. + * + * "Every REST call should contain the user's credentials in the form + * login={login}&password={password}" — the legacy surface authenticates this + * way rather than from the bearer token EWLogin issues. Used only for the + * operations that cannot carry them in a body instead; see + * `ewCredentialBody`. + */ +function ewCredentialQuery(params: AgiloftCredentials): string { + const login = encodeURIComponent(params.login) + const password = encodeURIComponent(params.password) + return `&$login=${login}&$password=${password}` +} + +/** + * Credentials as a form-encoded POST body, which is how Agiloft recommends + * production systems pass them: "you can avoid passing the login or password + * in REST calls by using POST instead of GET to pass the parameters in the + * request body." + * + * Only EWRead, EWSelect, EWCreate, EWUpdate, and EWDelete accept credentials + * this way. Every other EW* operation has to keep them in the query string. + */ +export function ewCredentialBody(params: AgiloftCredentials): string { + return `$login=${encodeURIComponent(params.login)}&$password=${encodeURIComponent(params.password)}` +} + +/** + * EWSelect is one of the five operations that accept credentials in a POST + * body, so the URL deliberately carries no `$login`/`$password`. + */ +/** + * EWSavedSearch answers JSON only, so the `.json` decorator is mandatory. It + * also has to run under EWLogin or OAuth authorization rather than inline + * credentials, so no `$login`/`$password` are appended here. + */ +/** + * EWTable is knowledge-base scoped, so the query carries `$KB` and `$lang` but + * no `$table`. Narrowing to one table uses the plain `table` parameter, and + * JSON is the only supported output, hence the mandatory `.json` decorator. + * Runs under EWLogin or OAuth authorization, so no inline credentials. + */ +export function buildListTablesUrl(base: string, params: AgiloftListTablesParams): string { + const kb = encodeURIComponent(params.knowledgeBase) + let url = `${base}/ewws/EWTable/.json?${AGILOFT_JSON_ERROR_CODES}&$KB=${kb}&$lang=${AGILOFT_LANG}` + + const table = params.table?.trim() + if (table) url += `&table=${encodeURIComponent(table)}` + if (params.includeLinkedInfo) url += '&includelinkedinfo=true' + if (params.skipColumnsInfo) url += '&skipColumnsInfo=true' + + return url +} + +/** + * EWUpsert takes every parameter — credentials included — in a form-encoded + * body, so nothing sensitive reaches the URL and there is no request-line + * length ceiling on the record data. + */ +export function buildUpsertRecordUrl(base: string): string { + return `${base}/ewws/EWUpsert` +} + +export function buildUpsertRecordBody( + params: AgiloftUpsertRecordParams, + data: Record +): string { + const fields: Array<[string, string]> = [ + ['$KB', params.knowledgeBase], + ['$table', params.table], + ['$login', params.login], + ['$password', params.password], + ['$lang', AGILOFT_LANG], + ['$match', params.match.trim()], + ] + + if (params.async) fields.push(['$async', 'true']) + + for (const [field, value] of Object.entries(data)) { + if (value === undefined || value === null) continue + + /** + * Multi-value fields are encoded as repeated key/value pairs, not as a + * joined string. Objects have no documented encoding at all, and + * String()-ing one silently writes "[object Object]" into the record. + */ + if (Array.isArray(value)) { + for (const entry of value) { + if (entry === undefined || entry === null) continue + fields.push([field, String(entry)]) + } + continue + } + + if (typeof value === 'object') { + throw new TypeError( + `Field "${field}" is an object, which Agiloft has no encoding for. Use a string, a number, or an array of values.` + ) + } + + fields.push([field, String(value)]) + } + + return fields + .map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(value)}`) + .join('&') +} + +export function buildSavedSearchUrl(base: string, params: AgiloftBaseParams): string { + return `${base}/ewws/EWSavedSearch/.json?${AGILOFT_JSON_ERROR_CODES}&${buildEwBaseQuery(params)}` +} + +export function buildSelectRecordsUrl(base: string, params: AgiloftSelectRecordsParams): string { + const where = encodeURIComponent(params.where) + return `${base}/ewws/EWSelect?${buildEwBaseQuery(params)}&where=${where}` +} + +export function buildRetrieveAttachmentUrl( + base: string, + params: AgiloftRetrieveAttachmentParams +): string { + const id = encodeURIComponent(params.recordId.trim()) + const field = encodeURIComponent(params.fieldName.trim()) + const position = encodeURIComponent(params.position.trim()) + return `${base}/ewws/EWRetrieve?${buildEwBaseQuery(params)}${ewCredentialQuery(params)}&id=${id}&field=${field}&filePosition=${position}` +} + +export function buildRemoveAttachmentUrl( + base: string, + params: AgiloftRemoveAttachmentParams +): string { + const id = encodeURIComponent(params.recordId.trim()) + const field = encodeURIComponent(params.fieldName.trim()) + const position = encodeURIComponent(params.position) + return `${base}/ewws/EWRemoveAttachment?${buildEwBaseQuery(params)}${ewCredentialQuery(params)}&id=${id}&field=${field}&filePosition=${position}` +} + +export function buildAttachmentInfoUrl(base: string, params: AgiloftAttachmentInfoParams): string { + const id = encodeURIComponent(params.recordId.trim()) + const fieldName = encodeURIComponent(params.fieldName.trim()) + return `${base}/ewws/EWAttachInfo/.json?${AGILOFT_JSON_ERROR_CODES}&${buildEwBaseQuery(params)}${ewCredentialQuery(params)}&id=${id}&field=${fieldName}` +} + +/** + * `force` is only meaningful on the DELETE (unlock) variant, where it lets an + * admin release a lock held by another user. + */ +export function buildLockRecordUrl(base: string, params: AgiloftLockRecordParams): string { + const id = encodeURIComponent(params.recordId.trim()) + let url = `${base}/ewws/EWLock?${buildEwBaseQuery(params)}${ewCredentialQuery(params)}&id=${id}` + if (params.lockAction === 'unlock' && params.force) { + url += '&force=true' + } + return url +} + +/** + * EWAttach carries the file as the raw request body, so its credentials have to + * travel in the query string like the rest of the EW* surface — there is no + * room for a form-encoded credential body here. + */ +export function buildAttachFileUrl( + base: string, + params: AgiloftBaseParams & { recordId: string; fieldName: string; overwrite?: boolean }, + fileName: string +): string { + const recordId = encodeURIComponent(params.recordId.trim()) + const fieldName = encodeURIComponent(params.fieldName.trim()) + const encodedFileName = encodeURIComponent(fileName) + let url = `${base}/ewws/EWAttach?${buildEwBaseQuery(params)}${ewCredentialQuery(params)}&id=${recordId}&field=${fieldName}&fileName=${encodedFileName}` + + /** `$overwrite` replaces the field's contents instead of appending. */ + if (params.overwrite) { + url += `&${fieldName}%24overwrite=true` + } + + return url +} + +export function buildGetChoiceLineIdUrl( + base: string, + params: AgiloftGetChoiceLineIdParams +): string { + const field = encodeURIComponent(params.fieldName.trim()) + const value = encodeURIComponent(params.value.trim()) + return `${base}/ewws/EWGetChoiceLineId?${buildEwBaseQuery(params)}${ewCredentialQuery(params)}&field=${field}&value=${value}` +} + +/** + * EWAsyncStatus reports the outcome of a call made through the `/ewws/async` + * prefix — including EWActionButton, whose callback ID is otherwise unusable. + */ +export function buildAsyncStatusUrl(base: string, params: AgiloftAsyncStatusParams): string { + const callbackId = encodeURIComponent(params.callbackId.trim()) + return `${base}/ewws/EWAsyncStatus?${buildEwBaseQuery(params)}${ewCredentialQuery(params)}&callback_id=${callbackId}` +} + +/** Documented EWAsyncStatus response codes. */ +export const AGILOFT_ASYNC_STATUS: Record = { + 200: { status: 'completed', complete: true }, + 201: { status: 'queued', complete: false }, + 202: { status: 'in_progress', complete: false }, + 501: { status: 'failed', complete: true }, + 523: { status: 'unknown_callback', complete: true }, +} + +/** + * EWNLPSearch answers semantic queries and returns records in the same shape as + * search. It is KB-scoped — the table is chosen by the KB's chat-search + * configuration rather than by the caller. + */ +export function buildNlpSearchUrl(base: string): string { + return `${base}/ewws/EWNLPSearch` +} + +export function getLockHttpMethod(lockAction: string): HttpMethod { + switch (lockAction) { + case 'lock': + return 'PUT' + case 'unlock': + return 'DELETE' + default: + return 'GET' + } +} + +/** + * EWActionButton runs asynchronously and therefore lives under the `/ewws/async` + * prefix rather than `/ewws` directly. + */ +export function buildRunActionButtonUrl( + base: string, + params: AgiloftRunActionButtonParams +): string { + const id = encodeURIComponent(params.recordId.trim()) + const name = encodeURIComponent(params.actionButtonField.trim()) + return `${base}/ewws/async/EWActionButton?${buildEwBaseQuery(params)}${ewCredentialQuery(params)}&name=${name}&id=${id}` +} diff --git a/apps/sim/lib/internal/appconfig/client.ts b/apps/sim/lib/internal/appconfig/client.ts new file mode 100644 index 00000000000..38a5e285031 --- /dev/null +++ b/apps/sim/lib/internal/appconfig/client.ts @@ -0,0 +1,740 @@ +import { + AppConfigClient, + CreateApplicationCommand, + CreateConfigurationProfileCommand, + CreateEnvironmentCommand, + CreateHostedConfigurationVersionCommand, + DeleteApplicationCommand, + DeleteConfigurationProfileCommand, + DeleteEnvironmentCommand, + DeleteHostedConfigurationVersionCommand, + GetApplicationCommand, + GetConfigurationProfileCommand, + GetDeploymentCommand, + GetEnvironmentCommand, + GetHostedConfigurationVersionCommand, + ListApplicationsCommand, + ListConfigurationProfilesCommand, + ListDeploymentStrategiesCommand, + ListDeploymentsCommand, + ListEnvironmentsCommand, + ListHostedConfigurationVersionsCommand, + StartDeploymentCommand, + StopDeploymentCommand, + UpdateApplicationCommand, + UpdateConfigurationProfileCommand, + UpdateEnvironmentCommand, +} from '@aws-sdk/client-appconfig' +import { + AppConfigDataClient, + GetLatestConfigurationCommand, + StartConfigurationSessionCommand, +} from '@aws-sdk/client-appconfigdata' +import type { AppConfigConnectionConfig } from '@/tools/appconfig/types' + +export function createAppConfigClient(config: AppConfigConnectionConfig): AppConfigClient { + return new AppConfigClient({ + region: config.region, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + }) +} + +export function createAppConfigDataClient(config: AppConfigConnectionConfig): AppConfigDataClient { + return new AppConfigDataClient({ + region: config.region, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + }) +} + +const textDecoder = new TextDecoder() + +function decodeContent(content?: Uint8Array): string { + if (!content || content.length === 0) return '' + return textDecoder.decode(content) +} + +export async function listApplications( + client: AppConfigClient, + signal: AbortSignal | undefined, + maxResults?: number | null, + nextToken?: string | null +) { + const response = await client.send( + new ListApplicationsCommand({ + ...(maxResults ? { MaxResults: maxResults } : {}), + ...(nextToken ? { NextToken: nextToken } : {}), + }), + { abortSignal: signal } + ) + + const applications = (response.Items ?? []).map((item) => ({ + id: item.Id ?? '', + name: item.Name ?? '', + description: item.Description ?? null, + })) + + return { + applications, + nextToken: response.NextToken ?? null, + count: applications.length, + } +} + +export async function createApplication( + client: AppConfigClient, + signal: AbortSignal | undefined, + name: string, + description?: string | null +) { + const response = await client.send( + new CreateApplicationCommand({ + Name: name, + ...(description ? { Description: description } : {}), + }), + { abortSignal: signal } + ) + + return { + message: `Application "${response.Name ?? name}" created`, + id: response.Id ?? '', + name: response.Name ?? '', + description: response.Description ?? null, + } +} + +export async function listEnvironments( + client: AppConfigClient, + signal: AbortSignal | undefined, + applicationId: string, + maxResults?: number | null, + nextToken?: string | null +) { + const response = await client.send( + new ListEnvironmentsCommand({ + ApplicationId: applicationId, + ...(maxResults ? { MaxResults: maxResults } : {}), + ...(nextToken ? { NextToken: nextToken } : {}), + }), + { abortSignal: signal } + ) + + const environments = (response.Items ?? []).map((item) => ({ + applicationId: item.ApplicationId ?? '', + id: item.Id ?? '', + name: item.Name ?? '', + description: item.Description ?? null, + state: item.State ?? null, + })) + + return { + environments, + nextToken: response.NextToken ?? null, + count: environments.length, + } +} + +export async function createEnvironment( + client: AppConfigClient, + signal: AbortSignal | undefined, + applicationId: string, + name: string, + description?: string | null +) { + const response = await client.send( + new CreateEnvironmentCommand({ + ApplicationId: applicationId, + Name: name, + ...(description ? { Description: description } : {}), + }), + { abortSignal: signal } + ) + + return { + message: `Environment "${response.Name ?? name}" created`, + applicationId: response.ApplicationId ?? applicationId, + id: response.Id ?? '', + name: response.Name ?? '', + state: response.State ?? null, + } +} + +export async function listConfigurationProfiles( + client: AppConfigClient, + signal: AbortSignal | undefined, + applicationId: string, + maxResults?: number | null, + nextToken?: string | null +) { + const response = await client.send( + new ListConfigurationProfilesCommand({ + ApplicationId: applicationId, + ...(maxResults ? { MaxResults: maxResults } : {}), + ...(nextToken ? { NextToken: nextToken } : {}), + }), + { abortSignal: signal } + ) + + const configurationProfiles = (response.Items ?? []).map((item) => ({ + applicationId: item.ApplicationId ?? '', + id: item.Id ?? '', + name: item.Name ?? '', + description: null, + locationUri: item.LocationUri ?? null, + retrievalRoleArn: null, + type: item.Type ?? null, + validatorTypes: item.ValidatorTypes ?? [], + })) + + return { + configurationProfiles, + nextToken: response.NextToken ?? null, + count: configurationProfiles.length, + } +} + +export async function createConfigurationProfile( + client: AppConfigClient, + signal: AbortSignal | undefined, + applicationId: string, + name: string, + locationUri: string, + description?: string | null, + retrievalRoleArn?: string | null, + type?: string | null +) { + const response = await client.send( + new CreateConfigurationProfileCommand({ + ApplicationId: applicationId, + Name: name, + LocationUri: locationUri, + ...(description ? { Description: description } : {}), + ...(retrievalRoleArn ? { RetrievalRoleArn: retrievalRoleArn } : {}), + ...(type ? { Type: type } : {}), + }), + { abortSignal: signal } + ) + + return { + message: `Configuration profile "${response.Name ?? name}" created`, + applicationId: response.ApplicationId ?? applicationId, + id: response.Id ?? '', + name: response.Name ?? '', + locationUri: response.LocationUri ?? null, + type: response.Type ?? null, + } +} + +export async function createHostedConfigurationVersion( + client: AppConfigClient, + signal: AbortSignal | undefined, + applicationId: string, + configurationProfileId: string, + content: string, + contentType: string, + description?: string | null, + latestVersionNumber?: number | null, + versionLabel?: string | null +) { + const response = await client.send( + new CreateHostedConfigurationVersionCommand({ + ApplicationId: applicationId, + ConfigurationProfileId: configurationProfileId, + Content: new TextEncoder().encode(content), + ContentType: contentType, + ...(description ? { Description: description } : {}), + ...(latestVersionNumber != null ? { LatestVersionNumber: latestVersionNumber } : {}), + ...(versionLabel ? { VersionLabel: versionLabel } : {}), + }), + { abortSignal: signal } + ) + + return { + message: `Hosted configuration version ${response.VersionNumber ?? ''} created`, + applicationId: response.ApplicationId ?? applicationId, + configurationProfileId: response.ConfigurationProfileId ?? configurationProfileId, + versionNumber: response.VersionNumber ?? null, + contentType: response.ContentType ?? null, + versionLabel: response.VersionLabel ?? null, + } +} + +export async function getHostedConfigurationVersion( + client: AppConfigClient, + signal: AbortSignal | undefined, + applicationId: string, + configurationProfileId: string, + versionNumber: number +) { + const response = await client.send( + new GetHostedConfigurationVersionCommand({ + ApplicationId: applicationId, + ConfigurationProfileId: configurationProfileId, + VersionNumber: versionNumber, + }), + { abortSignal: signal } + ) + + return { + applicationId: response.ApplicationId ?? applicationId, + configurationProfileId: response.ConfigurationProfileId ?? configurationProfileId, + versionNumber: response.VersionNumber ?? null, + description: response.Description ?? null, + content: decodeContent(response.Content), + contentType: response.ContentType ?? null, + versionLabel: response.VersionLabel ?? null, + } +} + +export async function listHostedConfigurationVersions( + client: AppConfigClient, + signal: AbortSignal | undefined, + applicationId: string, + configurationProfileId: string, + maxResults?: number | null, + nextToken?: string | null +) { + const response = await client.send( + new ListHostedConfigurationVersionsCommand({ + ApplicationId: applicationId, + ConfigurationProfileId: configurationProfileId, + ...(maxResults ? { MaxResults: maxResults } : {}), + ...(nextToken ? { NextToken: nextToken } : {}), + }), + { abortSignal: signal } + ) + + const versions = (response.Items ?? []).map((item) => ({ + applicationId: item.ApplicationId ?? null, + configurationProfileId: item.ConfigurationProfileId ?? null, + versionNumber: item.VersionNumber ?? null, + description: item.Description ?? null, + contentType: item.ContentType ?? null, + versionLabel: item.VersionLabel ?? null, + })) + + return { + versions, + nextToken: response.NextToken ?? null, + count: versions.length, + } +} + +export async function listDeploymentStrategies( + client: AppConfigClient, + signal: AbortSignal | undefined, + maxResults?: number | null, + nextToken?: string | null +) { + const response = await client.send( + new ListDeploymentStrategiesCommand({ + ...(maxResults ? { MaxResults: maxResults } : {}), + ...(nextToken ? { NextToken: nextToken } : {}), + }), + { abortSignal: signal } + ) + + const deploymentStrategies = (response.Items ?? []).map((item) => ({ + id: item.Id ?? '', + name: item.Name ?? '', + description: item.Description ?? null, + deploymentDurationInMinutes: item.DeploymentDurationInMinutes ?? null, + growthType: item.GrowthType ?? null, + growthFactor: item.GrowthFactor ?? null, + finalBakeTimeInMinutes: item.FinalBakeTimeInMinutes ?? null, + replicateTo: item.ReplicateTo ?? null, + })) + + return { + deploymentStrategies, + nextToken: response.NextToken ?? null, + count: deploymentStrategies.length, + } +} + +export async function startDeployment( + client: AppConfigClient, + signal: AbortSignal | undefined, + applicationId: string, + environmentId: string, + deploymentStrategyId: string, + configurationProfileId: string, + configurationVersion: string, + description?: string | null +) { + const response = await client.send( + new StartDeploymentCommand({ + ApplicationId: applicationId, + EnvironmentId: environmentId, + DeploymentStrategyId: deploymentStrategyId, + ConfigurationProfileId: configurationProfileId, + ConfigurationVersion: configurationVersion, + ...(description ? { Description: description } : {}), + }), + { abortSignal: signal } + ) + + return { + message: `Deployment ${response.DeploymentNumber ?? ''} started`, + deploymentNumber: response.DeploymentNumber ?? null, + state: response.State ?? null, + percentageComplete: response.PercentageComplete ?? null, + } +} + +export async function getDeployment( + client: AppConfigClient, + signal: AbortSignal | undefined, + applicationId: string, + environmentId: string, + deploymentNumber: number +) { + const response = await client.send( + new GetDeploymentCommand({ + ApplicationId: applicationId, + EnvironmentId: environmentId, + DeploymentNumber: deploymentNumber, + }), + { abortSignal: signal } + ) + + return { + applicationId: response.ApplicationId ?? applicationId, + environmentId: response.EnvironmentId ?? environmentId, + deploymentStrategyId: response.DeploymentStrategyId ?? '', + configurationProfileId: response.ConfigurationProfileId ?? '', + deploymentNumber: response.DeploymentNumber ?? null, + configurationName: response.ConfigurationName ?? null, + configurationVersion: response.ConfigurationVersion ?? null, + description: response.Description ?? null, + state: response.State ?? null, + percentageComplete: response.PercentageComplete ?? null, + startedAt: response.StartedAt?.toISOString() ?? null, + completedAt: response.CompletedAt?.toISOString() ?? null, + } +} + +export async function listDeployments( + client: AppConfigClient, + signal: AbortSignal | undefined, + applicationId: string, + environmentId: string, + maxResults?: number | null, + nextToken?: string | null +) { + const response = await client.send( + new ListDeploymentsCommand({ + ApplicationId: applicationId, + EnvironmentId: environmentId, + ...(maxResults ? { MaxResults: maxResults } : {}), + ...(nextToken ? { NextToken: nextToken } : {}), + }), + { abortSignal: signal } + ) + + const deployments = (response.Items ?? []).map((item) => ({ + deploymentNumber: item.DeploymentNumber ?? null, + configurationName: item.ConfigurationName ?? null, + configurationVersion: item.ConfigurationVersion ?? null, + deploymentDurationInMinutes: item.DeploymentDurationInMinutes ?? null, + growthType: item.GrowthType ?? null, + growthFactor: item.GrowthFactor ?? null, + finalBakeTimeInMinutes: item.FinalBakeTimeInMinutes ?? null, + state: item.State ?? null, + percentageComplete: item.PercentageComplete ?? null, + startedAt: item.StartedAt?.toISOString() ?? null, + completedAt: item.CompletedAt?.toISOString() ?? null, + versionLabel: item.VersionLabel ?? null, + })) + + return { + deployments, + nextToken: response.NextToken ?? null, + count: deployments.length, + } +} + +export async function stopDeployment( + client: AppConfigClient, + signal: AbortSignal | undefined, + applicationId: string, + environmentId: string, + deploymentNumber: number +) { + const response = await client.send( + new StopDeploymentCommand({ + ApplicationId: applicationId, + EnvironmentId: environmentId, + DeploymentNumber: deploymentNumber, + }), + { abortSignal: signal } + ) + + return { + message: `Deployment ${response.DeploymentNumber ?? deploymentNumber} stopped`, + deploymentNumber: response.DeploymentNumber ?? null, + state: response.State ?? null, + } +} + +export async function getConfiguration( + client: AppConfigDataClient, + signal: AbortSignal | undefined, + applicationId: string, + environmentId: string, + configurationProfileId: string +) { + const session = await client.send( + new StartConfigurationSessionCommand({ + ApplicationIdentifier: applicationId, + EnvironmentIdentifier: environmentId, + ConfigurationProfileIdentifier: configurationProfileId, + }), + { abortSignal: signal } + ) + + const response = await client.send( + new GetLatestConfigurationCommand({ + ConfigurationToken: session.InitialConfigurationToken, + }), + { abortSignal: signal } + ) + + return { + configuration: decodeContent(response.Configuration), + contentType: response.ContentType ?? null, + versionLabel: response.VersionLabel ?? null, + } +} + +export async function getApplication( + client: AppConfigClient, + signal: AbortSignal | undefined, + applicationId: string +) { + const response = await client.send(new GetApplicationCommand({ ApplicationId: applicationId }), { + abortSignal: signal, + }) + + return { + id: response.Id ?? '', + name: response.Name ?? '', + description: response.Description ?? null, + } +} + +export async function updateApplication( + client: AppConfigClient, + signal: AbortSignal | undefined, + applicationId: string, + name?: string | null, + description?: string | null +) { + const response = await client.send( + new UpdateApplicationCommand({ + ApplicationId: applicationId, + ...(name ? { Name: name } : {}), + ...(description != null ? { Description: description } : {}), + }), + { abortSignal: signal } + ) + + return { + message: `Application "${response.Name ?? applicationId}" updated`, + id: response.Id ?? '', + name: response.Name ?? '', + description: response.Description ?? null, + } +} + +export async function deleteApplication( + client: AppConfigClient, + signal: AbortSignal | undefined, + applicationId: string +) { + await client.send(new DeleteApplicationCommand({ ApplicationId: applicationId }), { + abortSignal: signal, + }) + + return { + message: `Application ${applicationId} deleted`, + id: applicationId, + } +} + +export async function getEnvironment( + client: AppConfigClient, + signal: AbortSignal | undefined, + applicationId: string, + environmentId: string +) { + const response = await client.send( + new GetEnvironmentCommand({ ApplicationId: applicationId, EnvironmentId: environmentId }), + { abortSignal: signal } + ) + + return { + applicationId: response.ApplicationId ?? applicationId, + id: response.Id ?? '', + name: response.Name ?? '', + description: response.Description ?? null, + state: response.State ?? null, + monitors: (response.Monitors ?? []).map((monitor) => ({ + alarmArn: monitor.AlarmArn ?? '', + alarmRoleArn: monitor.AlarmRoleArn ?? null, + })), + } +} + +export async function updateEnvironment( + client: AppConfigClient, + signal: AbortSignal | undefined, + applicationId: string, + environmentId: string, + name?: string | null, + description?: string | null +) { + const response = await client.send( + new UpdateEnvironmentCommand({ + ApplicationId: applicationId, + EnvironmentId: environmentId, + ...(name ? { Name: name } : {}), + ...(description != null ? { Description: description } : {}), + }), + { abortSignal: signal } + ) + + return { + message: `Environment "${response.Name ?? environmentId}" updated`, + applicationId: response.ApplicationId ?? applicationId, + id: response.Id ?? '', + name: response.Name ?? '', + state: response.State ?? null, + } +} + +export async function deleteEnvironment( + client: AppConfigClient, + signal: AbortSignal | undefined, + applicationId: string, + environmentId: string +) { + await client.send( + new DeleteEnvironmentCommand({ ApplicationId: applicationId, EnvironmentId: environmentId }), + { abortSignal: signal } + ) + + return { + message: `Environment ${environmentId} deleted`, + applicationId, + id: environmentId, + } +} + +export async function getConfigurationProfile( + client: AppConfigClient, + signal: AbortSignal | undefined, + applicationId: string, + configurationProfileId: string +) { + const response = await client.send( + new GetConfigurationProfileCommand({ + ApplicationId: applicationId, + ConfigurationProfileId: configurationProfileId, + }), + { abortSignal: signal } + ) + + return { + applicationId: response.ApplicationId ?? applicationId, + id: response.Id ?? '', + name: response.Name ?? '', + description: response.Description ?? null, + locationUri: response.LocationUri ?? null, + retrievalRoleArn: response.RetrievalRoleArn ?? null, + type: response.Type ?? null, + validators: (response.Validators ?? []).map((validator) => ({ + type: validator.Type ?? '', + })), + } +} + +export async function updateConfigurationProfile( + client: AppConfigClient, + signal: AbortSignal | undefined, + applicationId: string, + configurationProfileId: string, + name?: string | null, + description?: string | null, + retrievalRoleArn?: string | null +) { + const response = await client.send( + new UpdateConfigurationProfileCommand({ + ApplicationId: applicationId, + ConfigurationProfileId: configurationProfileId, + ...(name ? { Name: name } : {}), + ...(description != null ? { Description: description } : {}), + ...(retrievalRoleArn != null ? { RetrievalRoleArn: retrievalRoleArn } : {}), + }), + { abortSignal: signal } + ) + + return { + message: `Configuration profile "${response.Name ?? configurationProfileId}" updated`, + applicationId: response.ApplicationId ?? applicationId, + id: response.Id ?? '', + name: response.Name ?? '', + description: response.Description ?? null, + type: response.Type ?? null, + } +} + +export async function deleteConfigurationProfile( + client: AppConfigClient, + signal: AbortSignal | undefined, + applicationId: string, + configurationProfileId: string +) { + await client.send( + new DeleteConfigurationProfileCommand({ + ApplicationId: applicationId, + ConfigurationProfileId: configurationProfileId, + }), + { abortSignal: signal } + ) + + return { + message: `Configuration profile ${configurationProfileId} deleted`, + applicationId, + id: configurationProfileId, + } +} + +export async function deleteHostedConfigurationVersion( + client: AppConfigClient, + signal: AbortSignal | undefined, + applicationId: string, + configurationProfileId: string, + versionNumber: number +) { + await client.send( + new DeleteHostedConfigurationVersionCommand({ + ApplicationId: applicationId, + ConfigurationProfileId: configurationProfileId, + VersionNumber: versionNumber, + }), + { abortSignal: signal } + ) + + return { + message: `Hosted configuration version ${versionNumber} deleted`, + applicationId, + configurationProfileId, + versionNumber, + } +} diff --git a/apps/sim/lib/internal/appconfig/execute-tool.test.ts b/apps/sim/lib/internal/appconfig/execute-tool.test.ts new file mode 100644 index 00000000000..f8c224d1df4 --- /dev/null +++ b/apps/sim/lib/internal/appconfig/execute-tool.test.ts @@ -0,0 +1,160 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operationMocks = vi.hoisted(() => ({ + executeAppConfigCreateApplication: vi.fn(), + executeAppConfigCreateConfigurationProfile: vi.fn(), + executeAppConfigCreateEnvironment: vi.fn(), + executeAppConfigCreateHostedConfigurationVersion: vi.fn(), + executeAppConfigDeleteApplication: vi.fn(), + executeAppConfigDeleteConfigurationProfile: vi.fn(), + executeAppConfigDeleteEnvironment: vi.fn(), + executeAppConfigDeleteHostedConfigurationVersion: vi.fn(), + executeAppConfigGetApplication: vi.fn(), + executeAppConfigGetConfiguration: vi.fn(), + executeAppConfigGetConfigurationProfile: vi.fn(), + executeAppConfigGetDeployment: vi.fn(), + executeAppConfigGetEnvironment: vi.fn(), + executeAppConfigGetHostedConfigurationVersion: vi.fn(), + executeAppConfigListApplications: vi.fn(), + executeAppConfigListConfigurationProfiles: vi.fn(), + executeAppConfigListDeployments: vi.fn(), + executeAppConfigListDeploymentStrategies: vi.fn(), + executeAppConfigListEnvironments: vi.fn(), + executeAppConfigListHostedConfigurationVersions: vi.fn(), + executeAppConfigStartDeployment: vi.fn(), + executeAppConfigStopDeployment: vi.fn(), + executeAppConfigUpdateApplication: vi.fn(), + executeAppConfigUpdateConfigurationProfile: vi.fn(), + executeAppConfigUpdateEnvironment: vi.fn(), +})) + +vi.mock('@/lib/internal/appconfig/operations', () => operationMocks) + +import { executeAppConfigTool } from '@/lib/internal/appconfig/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const VALID_BODY = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', + maxResults: 25, + nextToken: 'next-token', +} as const + +const SUPPORTED_TOOL_IDS = [ + 'appconfig_create_application', + 'appconfig_create_configuration_profile', + 'appconfig_create_environment', + 'appconfig_create_hosted_configuration_version', + 'appconfig_delete_application', + 'appconfig_delete_configuration_profile', + 'appconfig_delete_environment', + 'appconfig_delete_hosted_configuration_version', + 'appconfig_get_application', + 'appconfig_get_configuration', + 'appconfig_get_configuration_profile', + 'appconfig_get_deployment', + 'appconfig_get_environment', + 'appconfig_get_hosted_configuration_version', + 'appconfig_list_applications', + 'appconfig_list_configuration_profiles', + 'appconfig_list_deployment_strategies', + 'appconfig_list_deployments', + 'appconfig_list_environments', + 'appconfig_list_hosted_configuration_versions', + 'appconfig_start_deployment', + 'appconfig_stop_deployment', + 'appconfig_update_application', + 'appconfig_update_configuration_profile', + 'appconfig_update_environment', +] as const + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'appconfig_list_applications', + input: VALID_BODY, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeAppConfigTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('validates and executes the matching AppConfig operation with cancellation', async () => { + const controller = new AbortController() + operationMocks.executeAppConfigListApplications.mockResolvedValue({ + applications: [], + nextToken: null, + count: 0, + }) + + const response = await executeAppConfigTool(createRequest({ signal: controller.signal })) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + applications: [], + nextToken: null, + count: 0, + }) + expect(operationMocks.executeAppConfigListApplications).toHaveBeenCalledWith( + VALID_BODY, + controller.signal + ) + }) + + it('returns the route-compatible validation envelope before provider work', async () => { + const response = await executeAppConfigTool(createRequest({ input: { region: 'us-east-1' } })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(operationMocks.executeAppConfigListApplications).not.toHaveBeenCalled() + }) + + it.each(SUPPORTED_TOOL_IDS)('recognizes the canonical tool ID %s', async (toolId) => { + const response = await executeAppConfigTool(createRequest({ toolId, input: {} })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ error: 'Invalid request data' }) + }) + + it('preserves the provider error envelope', async () => { + operationMocks.executeAppConfigListApplications.mockRejectedValue( + new Error('AWS rejected credentials') + ) + + const response = await executeAppConfigTool(createRequest()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'Failed to list applications: AWS rejected credentials', + }) + }) + + it('propagates cancellation without converting it into a provider failure', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeAppConfigTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(operationMocks.executeAppConfigListApplications).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/appconfig/execute-tool.ts b/apps/sim/lib/internal/appconfig/execute-tool.ts new file mode 100644 index 00000000000..8ce1fd00fca --- /dev/null +++ b/apps/sim/lib/internal/appconfig/execute-tool.ts @@ -0,0 +1,292 @@ +import { getErrorMessage } from '@sim/utils/errors' +import type { AnyApiRouteContract, ContractBody } from '@/lib/api/contracts' +import { awsAppConfigCreateApplicationContract } from '@/lib/api/contracts/tools/aws/appconfig-create-application' +import { awsAppConfigCreateConfigurationProfileContract } from '@/lib/api/contracts/tools/aws/appconfig-create-configuration-profile' +import { awsAppConfigCreateEnvironmentContract } from '@/lib/api/contracts/tools/aws/appconfig-create-environment' +import { awsAppConfigCreateHostedConfigurationVersionContract } from '@/lib/api/contracts/tools/aws/appconfig-create-hosted-configuration-version' +import { awsAppConfigDeleteApplicationContract } from '@/lib/api/contracts/tools/aws/appconfig-delete-application' +import { awsAppConfigDeleteConfigurationProfileContract } from '@/lib/api/contracts/tools/aws/appconfig-delete-configuration-profile' +import { awsAppConfigDeleteEnvironmentContract } from '@/lib/api/contracts/tools/aws/appconfig-delete-environment' +import { awsAppConfigDeleteHostedConfigurationVersionContract } from '@/lib/api/contracts/tools/aws/appconfig-delete-hosted-configuration-version' +import { awsAppConfigGetApplicationContract } from '@/lib/api/contracts/tools/aws/appconfig-get-application' +import { awsAppConfigGetConfigurationContract } from '@/lib/api/contracts/tools/aws/appconfig-get-configuration' +import { awsAppConfigGetConfigurationProfileContract } from '@/lib/api/contracts/tools/aws/appconfig-get-configuration-profile' +import { awsAppConfigGetDeploymentContract } from '@/lib/api/contracts/tools/aws/appconfig-get-deployment' +import { awsAppConfigGetEnvironmentContract } from '@/lib/api/contracts/tools/aws/appconfig-get-environment' +import { awsAppConfigGetHostedConfigurationVersionContract } from '@/lib/api/contracts/tools/aws/appconfig-get-hosted-configuration-version' +import { awsAppConfigListApplicationsContract } from '@/lib/api/contracts/tools/aws/appconfig-list-applications' +import { awsAppConfigListConfigurationProfilesContract } from '@/lib/api/contracts/tools/aws/appconfig-list-configuration-profiles' +import { awsAppConfigListDeploymentStrategiesContract } from '@/lib/api/contracts/tools/aws/appconfig-list-deployment-strategies' +import { awsAppConfigListDeploymentsContract } from '@/lib/api/contracts/tools/aws/appconfig-list-deployments' +import { awsAppConfigListEnvironmentsContract } from '@/lib/api/contracts/tools/aws/appconfig-list-environments' +import { awsAppConfigListHostedConfigurationVersionsContract } from '@/lib/api/contracts/tools/aws/appconfig-list-hosted-configuration-versions' +import { awsAppConfigStartDeploymentContract } from '@/lib/api/contracts/tools/aws/appconfig-start-deployment' +import { awsAppConfigStopDeploymentContract } from '@/lib/api/contracts/tools/aws/appconfig-stop-deployment' +import { awsAppConfigUpdateApplicationContract } from '@/lib/api/contracts/tools/aws/appconfig-update-application' +import { awsAppConfigUpdateConfigurationProfileContract } from '@/lib/api/contracts/tools/aws/appconfig-update-configuration-profile' +import { awsAppConfigUpdateEnvironmentContract } from '@/lib/api/contracts/tools/aws/appconfig-update-environment' +import { + executeAppConfigCreateApplication, + executeAppConfigCreateConfigurationProfile, + executeAppConfigCreateEnvironment, + executeAppConfigCreateHostedConfigurationVersion, + executeAppConfigDeleteApplication, + executeAppConfigDeleteConfigurationProfile, + executeAppConfigDeleteEnvironment, + executeAppConfigDeleteHostedConfigurationVersion, + executeAppConfigGetApplication, + executeAppConfigGetConfiguration, + executeAppConfigGetConfigurationProfile, + executeAppConfigGetDeployment, + executeAppConfigGetEnvironment, + executeAppConfigGetHostedConfigurationVersion, + executeAppConfigListApplications, + executeAppConfigListConfigurationProfiles, + executeAppConfigListDeploymentStrategies, + executeAppConfigListDeployments, + executeAppConfigListEnvironments, + executeAppConfigListHostedConfigurationVersions, + executeAppConfigStartDeployment, + executeAppConfigStopDeployment, + executeAppConfigUpdateApplication, + executeAppConfigUpdateConfigurationProfile, + executeAppConfigUpdateEnvironment, +} from '@/lib/internal/appconfig/operations' +import { parseInternalToolInput } from '@/lib/internal/tool-operations/parse-input' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +async function executeOperation( + contract: C, + input: unknown, + execute: (input: ContractBody, signal?: AbortSignal) => Promise, + errorMessage: string, + signal?: AbortSignal +): Promise { + const parsed = parseInternalToolInput(contract, input) + if (!parsed.success) return parsed.response + + try { + const result = await execute(parsed.data, signal) + signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + signal?.throwIfAborted() + return Response.json( + { error: `${errorMessage}: ${getErrorMessage(error, 'Unknown error occurred')}` }, + { status: 500 } + ) + } +} + +export const executeAppConfigTool: InternalToolOperationHandler = async ({ + toolId, + input, + signal, +}) => { + signal?.throwIfAborted() + + switch (toolId) { + case 'appconfig_create_application': + return executeOperation( + awsAppConfigCreateApplicationContract, + input, + executeAppConfigCreateApplication, + 'Failed to create application', + signal + ) + case 'appconfig_create_configuration_profile': + return executeOperation( + awsAppConfigCreateConfigurationProfileContract, + input, + executeAppConfigCreateConfigurationProfile, + 'Failed to create configuration profile', + signal + ) + case 'appconfig_create_environment': + return executeOperation( + awsAppConfigCreateEnvironmentContract, + input, + executeAppConfigCreateEnvironment, + 'Failed to create environment', + signal + ) + case 'appconfig_create_hosted_configuration_version': + return executeOperation( + awsAppConfigCreateHostedConfigurationVersionContract, + input, + executeAppConfigCreateHostedConfigurationVersion, + 'Failed to create hosted configuration version', + signal + ) + case 'appconfig_delete_application': + return executeOperation( + awsAppConfigDeleteApplicationContract, + input, + executeAppConfigDeleteApplication, + 'Failed to delete application', + signal + ) + case 'appconfig_delete_configuration_profile': + return executeOperation( + awsAppConfigDeleteConfigurationProfileContract, + input, + executeAppConfigDeleteConfigurationProfile, + 'Failed to delete configuration profile', + signal + ) + case 'appconfig_delete_environment': + return executeOperation( + awsAppConfigDeleteEnvironmentContract, + input, + executeAppConfigDeleteEnvironment, + 'Failed to delete environment', + signal + ) + case 'appconfig_delete_hosted_configuration_version': + return executeOperation( + awsAppConfigDeleteHostedConfigurationVersionContract, + input, + executeAppConfigDeleteHostedConfigurationVersion, + 'Failed to delete hosted configuration version', + signal + ) + case 'appconfig_get_application': + return executeOperation( + awsAppConfigGetApplicationContract, + input, + executeAppConfigGetApplication, + 'Failed to get application', + signal + ) + case 'appconfig_get_configuration': + return executeOperation( + awsAppConfigGetConfigurationContract, + input, + executeAppConfigGetConfiguration, + 'Failed to retrieve configuration', + signal + ) + case 'appconfig_get_configuration_profile': + return executeOperation( + awsAppConfigGetConfigurationProfileContract, + input, + executeAppConfigGetConfigurationProfile, + 'Failed to get configuration profile', + signal + ) + case 'appconfig_get_deployment': + return executeOperation( + awsAppConfigGetDeploymentContract, + input, + executeAppConfigGetDeployment, + 'Failed to get deployment', + signal + ) + case 'appconfig_get_environment': + return executeOperation( + awsAppConfigGetEnvironmentContract, + input, + executeAppConfigGetEnvironment, + 'Failed to get environment', + signal + ) + case 'appconfig_get_hosted_configuration_version': + return executeOperation( + awsAppConfigGetHostedConfigurationVersionContract, + input, + executeAppConfigGetHostedConfigurationVersion, + 'Failed to get hosted configuration version', + signal + ) + case 'appconfig_list_applications': + return executeOperation( + awsAppConfigListApplicationsContract, + input, + executeAppConfigListApplications, + 'Failed to list applications', + signal + ) + case 'appconfig_list_configuration_profiles': + return executeOperation( + awsAppConfigListConfigurationProfilesContract, + input, + executeAppConfigListConfigurationProfiles, + 'Failed to list configuration profiles', + signal + ) + case 'appconfig_list_deployment_strategies': + return executeOperation( + awsAppConfigListDeploymentStrategiesContract, + input, + executeAppConfigListDeploymentStrategies, + 'Failed to list deployment strategies', + signal + ) + case 'appconfig_list_deployments': + return executeOperation( + awsAppConfigListDeploymentsContract, + input, + executeAppConfigListDeployments, + 'Failed to list deployments', + signal + ) + case 'appconfig_list_environments': + return executeOperation( + awsAppConfigListEnvironmentsContract, + input, + executeAppConfigListEnvironments, + 'Failed to list environments', + signal + ) + case 'appconfig_list_hosted_configuration_versions': + return executeOperation( + awsAppConfigListHostedConfigurationVersionsContract, + input, + executeAppConfigListHostedConfigurationVersions, + 'Failed to list hosted configuration versions', + signal + ) + case 'appconfig_start_deployment': + return executeOperation( + awsAppConfigStartDeploymentContract, + input, + executeAppConfigStartDeployment, + 'Failed to start deployment', + signal + ) + case 'appconfig_stop_deployment': + return executeOperation( + awsAppConfigStopDeploymentContract, + input, + executeAppConfigStopDeployment, + 'Failed to stop deployment', + signal + ) + case 'appconfig_update_application': + return executeOperation( + awsAppConfigUpdateApplicationContract, + input, + executeAppConfigUpdateApplication, + 'Failed to update application', + signal + ) + case 'appconfig_update_configuration_profile': + return executeOperation( + awsAppConfigUpdateConfigurationProfileContract, + input, + executeAppConfigUpdateConfigurationProfile, + 'Failed to update configuration profile', + signal + ) + case 'appconfig_update_environment': + return executeOperation( + awsAppConfigUpdateEnvironmentContract, + input, + executeAppConfigUpdateEnvironment, + 'Failed to update environment', + signal + ) + default: + return Response.json({ error: `Unsupported AppConfig tool: ${toolId}` }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/appconfig/operations.test.ts b/apps/sim/lib/internal/appconfig/operations.test.ts new file mode 100644 index 00000000000..13c044227bd --- /dev/null +++ b/apps/sim/lib/internal/appconfig/operations.test.ts @@ -0,0 +1,123 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const clientMocks = vi.hoisted(() => ({ + createAppConfigClient: vi.fn(), + createAppConfigDataClient: vi.fn(), + createApplication: vi.fn(), + createConfigurationProfile: vi.fn(), + createEnvironment: vi.fn(), + createHostedConfigurationVersion: vi.fn(), + deleteApplication: vi.fn(), + deleteConfigurationProfile: vi.fn(), + deleteEnvironment: vi.fn(), + deleteHostedConfigurationVersion: vi.fn(), + getApplication: vi.fn(), + getConfiguration: vi.fn(), + getConfigurationProfile: vi.fn(), + getDeployment: vi.fn(), + getEnvironment: vi.fn(), + getHostedConfigurationVersion: vi.fn(), + listApplications: vi.fn(), + listConfigurationProfiles: vi.fn(), + listDeployments: vi.fn(), + listDeploymentStrategies: vi.fn(), + listEnvironments: vi.fn(), + listHostedConfigurationVersions: vi.fn(), + startDeployment: vi.fn(), + stopDeployment: vi.fn(), + updateApplication: vi.fn(), + updateConfigurationProfile: vi.fn(), + updateEnvironment: vi.fn(), +})) + +vi.mock('@/lib/internal/appconfig/client', () => clientMocks) + +import { + executeAppConfigGetConfiguration, + executeAppConfigListApplications, +} from '@/lib/internal/appconfig/operations' + +const CONNECTION = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', +} as const + +describe('AppConfig operations', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('passes the abort signal to AppConfig and destroys the client after success', async () => { + const controller = new AbortController() + const client = { destroy: vi.fn() } + clientMocks.createAppConfigClient.mockReturnValue(client) + clientMocks.listApplications.mockResolvedValue({ + applications: [], + nextToken: null, + count: 0, + }) + + await expect( + executeAppConfigListApplications( + { ...CONNECTION, maxResults: 25, nextToken: 'next-token' }, + controller.signal + ) + ).resolves.toEqual({ applications: [], nextToken: null, count: 0 }) + + expect(clientMocks.createAppConfigClient).toHaveBeenCalledWith({ + ...CONNECTION, + maxResults: 25, + nextToken: 'next-token', + }) + expect(clientMocks.listApplications).toHaveBeenCalledWith( + client, + controller.signal, + 25, + 'next-token' + ) + expect(client.destroy).toHaveBeenCalledOnce() + }) + + it('destroys the AppConfig client when the provider rejects', async () => { + const client = { destroy: vi.fn() } + clientMocks.createAppConfigClient.mockReturnValue(client) + clientMocks.listApplications.mockRejectedValue(new Error('provider failed')) + + await expect(executeAppConfigListApplications(CONNECTION)).rejects.toThrow('provider failed') + expect(client.destroy).toHaveBeenCalledOnce() + }) + + it('passes cancellation through both AppConfig Data requests and destroys that client', async () => { + const controller = new AbortController() + const client = { destroy: vi.fn() } + clientMocks.createAppConfigDataClient.mockReturnValue(client) + clientMocks.getConfiguration.mockResolvedValue({ + configuration: '{}', + contentType: 'application/json', + versionLabel: null, + }) + + await executeAppConfigGetConfiguration( + { + ...CONNECTION, + applicationId: 'application-1', + environmentId: 'environment-1', + configurationProfileId: 'profile-1', + }, + controller.signal + ) + + expect(clientMocks.getConfiguration).toHaveBeenCalledWith( + client, + controller.signal, + 'application-1', + 'environment-1', + 'profile-1' + ) + expect(client.destroy).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/internal/appconfig/operations.ts b/apps/sim/lib/internal/appconfig/operations.ts new file mode 100644 index 00000000000..0d3f6ceddb1 --- /dev/null +++ b/apps/sim/lib/internal/appconfig/operations.ts @@ -0,0 +1,385 @@ +import type { AppConfigClient } from '@aws-sdk/client-appconfig' +import type { AppConfigDataClient } from '@aws-sdk/client-appconfigdata' +import type { AwsAppConfigCreateApplicationBody } from '@/lib/api/contracts/tools/aws/appconfig-create-application' +import type { AwsAppConfigCreateConfigurationProfileBody } from '@/lib/api/contracts/tools/aws/appconfig-create-configuration-profile' +import type { AwsAppConfigCreateEnvironmentBody } from '@/lib/api/contracts/tools/aws/appconfig-create-environment' +import type { AwsAppConfigCreateHostedConfigurationVersionBody } from '@/lib/api/contracts/tools/aws/appconfig-create-hosted-configuration-version' +import type { AwsAppConfigDeleteApplicationBody } from '@/lib/api/contracts/tools/aws/appconfig-delete-application' +import type { AwsAppConfigDeleteConfigurationProfileBody } from '@/lib/api/contracts/tools/aws/appconfig-delete-configuration-profile' +import type { AwsAppConfigDeleteEnvironmentBody } from '@/lib/api/contracts/tools/aws/appconfig-delete-environment' +import type { AwsAppConfigDeleteHostedConfigurationVersionBody } from '@/lib/api/contracts/tools/aws/appconfig-delete-hosted-configuration-version' +import type { AwsAppConfigGetApplicationBody } from '@/lib/api/contracts/tools/aws/appconfig-get-application' +import type { AwsAppConfigGetConfigurationBody } from '@/lib/api/contracts/tools/aws/appconfig-get-configuration' +import type { AwsAppConfigGetConfigurationProfileBody } from '@/lib/api/contracts/tools/aws/appconfig-get-configuration-profile' +import type { AwsAppConfigGetDeploymentBody } from '@/lib/api/contracts/tools/aws/appconfig-get-deployment' +import type { AwsAppConfigGetEnvironmentBody } from '@/lib/api/contracts/tools/aws/appconfig-get-environment' +import type { AwsAppConfigGetHostedConfigurationVersionBody } from '@/lib/api/contracts/tools/aws/appconfig-get-hosted-configuration-version' +import type { AwsAppConfigListApplicationsBody } from '@/lib/api/contracts/tools/aws/appconfig-list-applications' +import type { AwsAppConfigListConfigurationProfilesBody } from '@/lib/api/contracts/tools/aws/appconfig-list-configuration-profiles' +import type { AwsAppConfigListDeploymentStrategiesBody } from '@/lib/api/contracts/tools/aws/appconfig-list-deployment-strategies' +import type { AwsAppConfigListDeploymentsBody } from '@/lib/api/contracts/tools/aws/appconfig-list-deployments' +import type { AwsAppConfigListEnvironmentsBody } from '@/lib/api/contracts/tools/aws/appconfig-list-environments' +import type { AwsAppConfigListHostedConfigurationVersionsBody } from '@/lib/api/contracts/tools/aws/appconfig-list-hosted-configuration-versions' +import type { AwsAppConfigStartDeploymentBody } from '@/lib/api/contracts/tools/aws/appconfig-start-deployment' +import type { AwsAppConfigStopDeploymentBody } from '@/lib/api/contracts/tools/aws/appconfig-stop-deployment' +import type { AwsAppConfigUpdateApplicationBody } from '@/lib/api/contracts/tools/aws/appconfig-update-application' +import type { AwsAppConfigUpdateConfigurationProfileBody } from '@/lib/api/contracts/tools/aws/appconfig-update-configuration-profile' +import type { AwsAppConfigUpdateEnvironmentBody } from '@/lib/api/contracts/tools/aws/appconfig-update-environment' +import { + createAppConfigClient, + createAppConfigDataClient, + createApplication, + createConfigurationProfile, + createEnvironment, + createHostedConfigurationVersion, + deleteApplication, + deleteConfigurationProfile, + deleteEnvironment, + deleteHostedConfigurationVersion, + getApplication, + getConfiguration, + getConfigurationProfile, + getDeployment, + getEnvironment, + getHostedConfigurationVersion, + listApplications, + listConfigurationProfiles, + listDeploymentStrategies, + listDeployments, + listEnvironments, + listHostedConfigurationVersions, + startDeployment, + stopDeployment, + updateApplication, + updateConfigurationProfile, + updateEnvironment, +} from '@/lib/internal/appconfig/client' +import type { AppConfigConnectionConfig } from '@/tools/appconfig/types' + +async function withAppConfigClient( + input: AppConfigConnectionConfig, + execute: (client: AppConfigClient) => Promise +): Promise { + const client = createAppConfigClient(input) + try { + return await execute(client) + } finally { + client.destroy() + } +} + +async function withAppConfigDataClient( + input: AppConfigConnectionConfig, + execute: (client: AppConfigDataClient) => Promise +): Promise { + const client = createAppConfigDataClient(input) + try { + return await execute(client) + } finally { + client.destroy() + } +} + +export function executeAppConfigListApplications( + input: AwsAppConfigListApplicationsBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => + listApplications(client, signal, input.maxResults, input.nextToken) + ) +} + +export function executeAppConfigCreateApplication( + input: AwsAppConfigCreateApplicationBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => + createApplication(client, signal, input.name, input.description) + ) +} + +export function executeAppConfigListEnvironments( + input: AwsAppConfigListEnvironmentsBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => + listEnvironments(client, signal, input.applicationId, input.maxResults, input.nextToken) + ) +} + +export function executeAppConfigCreateEnvironment( + input: AwsAppConfigCreateEnvironmentBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => + createEnvironment(client, signal, input.applicationId, input.name, input.description) + ) +} + +export function executeAppConfigListConfigurationProfiles( + input: AwsAppConfigListConfigurationProfilesBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => + listConfigurationProfiles( + client, + signal, + input.applicationId, + input.maxResults, + input.nextToken + ) + ) +} + +export function executeAppConfigCreateConfigurationProfile( + input: AwsAppConfigCreateConfigurationProfileBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => + createConfigurationProfile( + client, + signal, + input.applicationId, + input.name, + input.locationUri, + input.description, + input.retrievalRoleArn, + input.type + ) + ) +} + +export function executeAppConfigCreateHostedConfigurationVersion( + input: AwsAppConfigCreateHostedConfigurationVersionBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => + createHostedConfigurationVersion( + client, + signal, + input.applicationId, + input.configurationProfileId, + input.content, + input.contentType, + input.description, + input.latestVersionNumber, + input.versionLabel + ) + ) +} + +export function executeAppConfigGetHostedConfigurationVersion( + input: AwsAppConfigGetHostedConfigurationVersionBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => + getHostedConfigurationVersion( + client, + signal, + input.applicationId, + input.configurationProfileId, + input.versionNumber + ) + ) +} + +export function executeAppConfigListHostedConfigurationVersions( + input: AwsAppConfigListHostedConfigurationVersionsBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => + listHostedConfigurationVersions( + client, + signal, + input.applicationId, + input.configurationProfileId, + input.maxResults, + input.nextToken + ) + ) +} + +export function executeAppConfigListDeploymentStrategies( + input: AwsAppConfigListDeploymentStrategiesBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => + listDeploymentStrategies(client, signal, input.maxResults, input.nextToken) + ) +} + +export function executeAppConfigStartDeployment( + input: AwsAppConfigStartDeploymentBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => + startDeployment( + client, + signal, + input.applicationId, + input.environmentId, + input.deploymentStrategyId, + input.configurationProfileId, + input.configurationVersion, + input.description + ) + ) +} + +export function executeAppConfigGetDeployment( + input: AwsAppConfigGetDeploymentBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => + getDeployment(client, signal, input.applicationId, input.environmentId, input.deploymentNumber) + ) +} + +export function executeAppConfigListDeployments( + input: AwsAppConfigListDeploymentsBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => + listDeployments( + client, + signal, + input.applicationId, + input.environmentId, + input.maxResults, + input.nextToken + ) + ) +} + +export function executeAppConfigStopDeployment( + input: AwsAppConfigStopDeploymentBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => + stopDeployment(client, signal, input.applicationId, input.environmentId, input.deploymentNumber) + ) +} + +export function executeAppConfigGetConfiguration( + input: AwsAppConfigGetConfigurationBody, + signal?: AbortSignal +) { + return withAppConfigDataClient(input, (client) => + getConfiguration( + client, + signal, + input.applicationId, + input.environmentId, + input.configurationProfileId + ) + ) +} + +export function executeAppConfigGetApplication( + input: AwsAppConfigGetApplicationBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => getApplication(client, signal, input.applicationId)) +} + +export function executeAppConfigUpdateApplication( + input: AwsAppConfigUpdateApplicationBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => + updateApplication(client, signal, input.applicationId, input.name, input.description) + ) +} + +export function executeAppConfigDeleteApplication( + input: AwsAppConfigDeleteApplicationBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => + deleteApplication(client, signal, input.applicationId) + ) +} + +export function executeAppConfigGetEnvironment( + input: AwsAppConfigGetEnvironmentBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => + getEnvironment(client, signal, input.applicationId, input.environmentId) + ) +} + +export function executeAppConfigUpdateEnvironment( + input: AwsAppConfigUpdateEnvironmentBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => + updateEnvironment( + client, + signal, + input.applicationId, + input.environmentId, + input.name, + input.description + ) + ) +} + +export function executeAppConfigDeleteEnvironment( + input: AwsAppConfigDeleteEnvironmentBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => + deleteEnvironment(client, signal, input.applicationId, input.environmentId) + ) +} + +export function executeAppConfigGetConfigurationProfile( + input: AwsAppConfigGetConfigurationProfileBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => + getConfigurationProfile(client, signal, input.applicationId, input.configurationProfileId) + ) +} + +export function executeAppConfigUpdateConfigurationProfile( + input: AwsAppConfigUpdateConfigurationProfileBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => + updateConfigurationProfile( + client, + signal, + input.applicationId, + input.configurationProfileId, + input.name, + input.description, + input.retrievalRoleArn + ) + ) +} + +export function executeAppConfigDeleteConfigurationProfile( + input: AwsAppConfigDeleteConfigurationProfileBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => + deleteConfigurationProfile(client, signal, input.applicationId, input.configurationProfileId) + ) +} + +export function executeAppConfigDeleteHostedConfigurationVersion( + input: AwsAppConfigDeleteHostedConfigurationVersionBody, + signal?: AbortSignal +) { + return withAppConfigClient(input, (client) => + deleteHostedConfigurationVersion( + client, + signal, + input.applicationId, + input.configurationProfileId, + input.versionNumber + ) + ) +} diff --git a/apps/sim/lib/internal/asana/client.test.ts b/apps/sim/lib/internal/asana/client.test.ts new file mode 100644 index 00000000000..790ebd8bebe --- /dev/null +++ b/apps/sim/lib/internal/asana/client.test.ts @@ -0,0 +1,118 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { AsanaClient } from '@/lib/internal/asana/client' +import { AsanaOperationError } from '@/lib/internal/asana/errors' + +describe('AsanaClient', () => { + const fetchMock = vi.fn() + + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', fetchMock) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('sends OAuth credentials and cancellation to the provider', async () => { + fetchMock.mockResolvedValue(new Response('{"data":{"gid":"task1"}}')) + const controller = new AbortController() + + await expect( + new AsanaClient('access-token').json('/tasks/task1', { method: 'GET' }, controller.signal) + ).resolves.toEqual({ data: { gid: 'task1' } }) + + expect(fetchMock).toHaveBeenCalledWith('https://app.asana.com/api/1.0/tasks/task1', { + method: 'GET', + headers: { + Authorization: 'Bearer access-token', + Accept: 'application/json', + }, + signal: controller.signal, + }) + }) + + it('preserves structured provider status, message, help, and raw details', async () => { + const details = JSON.stringify({ + errors: [{ message: 'Rate limited', help: 'Retry after the reset time' }], + }) + fetchMock.mockResolvedValue( + new Response(details, { status: 429, statusText: 'Too Many Requests' }) + ) + + await expect(new AsanaClient('access-token').json('/tasks')).rejects.toEqual( + new AsanaOperationError('Rate limited (Retry after the reset time)', 429, { + success: false, + error: 'Rate limited (Retry after the reset time)', + details, + }) + ) + }) + + it('uses the route-compatible fallback for unstructured provider errors', async () => { + fetchMock.mockResolvedValue( + new Response('upstream unavailable', { status: 502, statusText: 'Bad Gateway' }) + ) + + await expect(new AsanaClient('access-token').json('/tasks')).rejects.toMatchObject({ + status: 502, + body: { + success: false, + error: 'Asana API error: 502 Bad Gateway', + details: 'upstream unavailable', + }, + }) + }) + + it('caps provider responses before materializing oversized bodies', async () => { + let cancelled = false + const stream = new ReadableStream({ + cancel: () => { + cancelled = true + }, + }) + fetchMock.mockResolvedValue( + new Response(stream, { + headers: { 'Content-Length': String(10 * 1024 * 1024 + 1) }, + }) + ) + + await expect(new AsanaClient('access-token').json('/tasks')).rejects.toEqual( + new PayloadSizeLimitError({ + label: 'Asana API response', + maxBytes: 10 * 1024 * 1024, + observedBytes: 10 * 1024 * 1024 + 1, + }) + ) + expect(cancelled).toBe(true) + }) + + it('cancels successful empty responses for delete operations', async () => { + let cancelled = false + const stream = new ReadableStream({ + cancel: () => { + cancelled = true + }, + }) + fetchMock.mockResolvedValue(new Response(stream, { status: 200 })) + + await expect( + new AsanaClient('access-token').empty('/tasks/task1', { method: 'DELETE' }) + ).resolves.toBeUndefined() + expect(cancelled).toBe(true) + }) + + it('does not start provider work after cancellation', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + new AsanaClient('access-token').json('/tasks', {}, controller.signal) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(fetchMock).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/asana/client.ts b/apps/sim/lib/internal/asana/client.ts new file mode 100644 index 00000000000..1ea4e1b025d --- /dev/null +++ b/apps/sim/lib/internal/asana/client.ts @@ -0,0 +1,94 @@ +import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' +import { AsanaOperationError } from '@/lib/internal/asana/errors' + +const ASANA_API_BASE_URL = 'https://app.asana.com/api/1.0' +const ASANA_RESPONSE_MAX_BYTES = 10 * 1024 * 1024 + +export type AsanaJsonObject = Record + +export function asObject(value: unknown): AsanaJsonObject { + return value && typeof value === 'object' && !Array.isArray(value) + ? (value as AsanaJsonObject) + : {} +} + +export function asArray(value: unknown): unknown[] { + return Array.isArray(value) ? value : [] +} + +function providerErrorMessage(response: Response, text: string): string { + let message = `Asana API error: ${response.status} ${response.statusText}` + try { + const data = asObject(JSON.parse(text)) + const firstError = asObject(asArray(data.errors)[0]) + if (Object.keys(firstError).length > 0) { + const providerMessage = + typeof firstError.message === 'string' && firstError.message ? firstError.message : message + const help = typeof firstError.help === 'string' ? firstError.help : '' + message = `${providerMessage} (${help})` + } + } catch { + return message + } + return message +} + +export class AsanaClient { + constructor(private readonly accessToken: string) {} + + private url(path: string): string { + return `${ASANA_API_BASE_URL}${path}` + } + + private async fetch(path: string, init: RequestInit, signal?: AbortSignal): Promise { + signal?.throwIfAborted() + return fetch(this.url(path), { + ...init, + headers: { + Authorization: `Bearer ${this.accessToken}`, + Accept: 'application/json', + ...init.headers, + }, + signal, + }) + } + + private async read(response: Response, signal?: AbortSignal): Promise { + const text = await readResponseTextWithLimit(response, { + maxBytes: ASANA_RESPONSE_MAX_BYTES, + label: 'Asana API response', + signal, + }) + signal?.throwIfAborted() + return text + } + + async json(path: string, init: RequestInit = {}, signal?: AbortSignal): Promise { + const response = await this.fetch(path, init, signal) + const text = await this.read(response, signal) + if (!response.ok) { + const error = providerErrorMessage(response, text) + throw new AsanaOperationError(error, response.status, { + success: false, + error, + details: text, + }) + } + return asObject(JSON.parse(text)) + } + + async empty(path: string, init: RequestInit, signal?: AbortSignal): Promise { + const response = await this.fetch(path, init, signal) + if (!response.ok) { + const text = await this.read(response, signal) + const error = providerErrorMessage(response, text) + throw new AsanaOperationError(error, response.status, { + success: false, + error, + details: text, + }) + } + await response.body?.cancel() + signal?.throwIfAborted() + } +} diff --git a/apps/sim/lib/internal/asana/errors.ts b/apps/sim/lib/internal/asana/errors.ts new file mode 100644 index 00000000000..4333d68b31c --- /dev/null +++ b/apps/sim/lib/internal/asana/errors.ts @@ -0,0 +1,12 @@ +export type AsanaErrorBody = Record & { error: string } + +export class AsanaOperationError extends Error { + constructor( + message: string, + readonly status: number, + readonly body: AsanaErrorBody + ) { + super(message) + this.name = 'AsanaOperationError' + } +} diff --git a/apps/sim/lib/internal/asana/execute-tool.test.ts b/apps/sim/lib/internal/asana/execute-tool.test.ts new file mode 100644 index 00000000000..e578c8c60bc --- /dev/null +++ b/apps/sim/lib/internal/asana/execute-tool.test.ts @@ -0,0 +1,231 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operationMocks = vi.hoisted(() => ({ + executeAsanaAddComment: vi.fn(), + executeAsanaAddFollowers: vi.fn(), + executeAsanaCreateProject: vi.fn(), + executeAsanaCreateSection: vi.fn(), + executeAsanaCreateSubtask: vi.fn(), + executeAsanaCreateTask: vi.fn(), + executeAsanaDeleteTask: vi.fn(), + executeAsanaGetProject: vi.fn(), + executeAsanaGetProjects: vi.fn(), + executeAsanaGetTask: vi.fn(), + executeAsanaListSections: vi.fn(), + executeAsanaListWorkspaces: vi.fn(), + executeAsanaSearchTasks: vi.fn(), + executeAsanaUpdateTask: vi.fn(), +})) + +vi.mock('@/lib/internal/asana/operations', () => operationMocks) + +import { AsanaOperationError } from '@/lib/internal/asana/errors' +import { executeAsanaTool } from '@/lib/internal/asana/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const ACCESS_TOKEN = { accessToken: 'access-token' } + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'asana_list_workspaces', + input: ACCESS_TOKEN, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } as InternalToolOperationCall +} + +const TOOL_CASES = [ + [ + 'asana_add_comment', + { ...ACCESS_TOKEN, taskGid: 'task1', text: 'Comment' }, + operationMocks.executeAsanaAddComment, + ], + [ + 'asana_add_followers', + { ...ACCESS_TOKEN, taskGid: 'task1', followers: ['user1'] }, + operationMocks.executeAsanaAddFollowers, + ], + [ + 'asana_create_project', + { ...ACCESS_TOKEN, workspace: 'workspace1', name: 'Project' }, + operationMocks.executeAsanaCreateProject, + ], + [ + 'asana_create_section', + { ...ACCESS_TOKEN, projectGid: 'project1', name: 'Section' }, + operationMocks.executeAsanaCreateSection, + ], + [ + 'asana_create_subtask', + { ...ACCESS_TOKEN, taskGid: 'task1', name: 'Subtask' }, + operationMocks.executeAsanaCreateSubtask, + ], + [ + 'asana_create_task', + { ...ACCESS_TOKEN, workspace: 'workspace1', name: 'Task' }, + operationMocks.executeAsanaCreateTask, + ], + [ + 'asana_delete_task', + { ...ACCESS_TOKEN, taskGid: 'task1' }, + operationMocks.executeAsanaDeleteTask, + ], + [ + 'asana_get_project', + { ...ACCESS_TOKEN, projectGid: 'project1' }, + operationMocks.executeAsanaGetProject, + ], + [ + 'asana_get_projects', + { ...ACCESS_TOKEN, workspace: 'workspace1' }, + operationMocks.executeAsanaGetProjects, + ], + ['asana_get_task', { ...ACCESS_TOKEN, taskGid: 'task1' }, operationMocks.executeAsanaGetTask], + [ + 'asana_list_sections', + { ...ACCESS_TOKEN, projectGid: 'project1' }, + operationMocks.executeAsanaListSections, + ], + ['asana_list_workspaces', ACCESS_TOKEN, operationMocks.executeAsanaListWorkspaces], + [ + 'asana_search_tasks', + { ...ACCESS_TOKEN, workspace: 'workspace1' }, + operationMocks.executeAsanaSearchTasks, + ], + [ + 'asana_update_task', + { ...ACCESS_TOKEN, taskGid: 'task1', completed: true }, + operationMocks.executeAsanaUpdateTask, + ], +] as const + +describe('executeAsanaTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it.each(TOOL_CASES)('validates and dispatches %s', async (toolId, input, operation) => { + const controller = new AbortController() + operation.mockResolvedValue({ toolId }) + + const response = await executeAsanaTool( + createRequest({ toolId, input, signal: controller.signal }) + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ toolId }) + expect(operation).toHaveBeenCalledWith(input, controller.signal) + }) + + it('requires an authenticated user before parsing or provider work', async () => { + const response = await executeAsanaTool( + createRequest({ input: '{', context: createExecutionContext({ workflowId: 'workflow-1' }) }) + ) + + expect(response.status).toBe(401) + await expect(response.json()).resolves.toEqual({ error: 'Unauthorized' }) + expect(operationMocks.executeAsanaListWorkspaces).not.toHaveBeenCalled() + }) + + it('returns the canonical validation envelope before provider work', async () => { + const response = await executeAsanaTool(createRequest({ input: { accessToken: '' } })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(operationMocks.executeAsanaListWorkspaces).not.toHaveBeenCalled() + }) + + it('rejects non-object operation input before provider work', async () => { + const response = await executeAsanaTool(createRequest({ input: '{' })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(operationMocks.executeAsanaListWorkspaces).not.toHaveBeenCalled() + }) + + it('preserves provider status and error envelopes', async () => { + operationMocks.executeAsanaListWorkspaces.mockRejectedValue( + new AsanaOperationError('Asana API error: 429 Too Many Requests', 429, { + success: false, + error: 'Asana API error: 429 Too Many Requests', + details: '{"errors":[]}', + }) + ) + + const response = await executeAsanaTool(createRequest()) + + expect(response.status).toBe(429) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Asana API error: 429 Too Many Requests', + details: '{"errors":[]}', + }) + }) + + it('preserves fixed unexpected error envelopes', async () => { + operationMocks.executeAsanaListWorkspaces.mockRejectedValue(new Error('Asana unavailable')) + + const response = await executeAsanaTool(createRequest()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'Failed to retrieve Asana workspaces', + details: 'Asana unavailable', + }) + }) + + it('preserves dynamic unexpected error envelopes', async () => { + operationMocks.executeAsanaCreateTask.mockRejectedValue(new Error('Asana unavailable')) + + const response = await executeAsanaTool( + createRequest({ + toolId: 'asana_create_task', + input: { ...ACCESS_TOKEN, workspace: 'workspace1', name: 'Task' }, + }) + ) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'Asana unavailable', + success: false, + }) + }) + + it('rejects unsupported Asana IDs without provider work', async () => { + const response = await executeAsanaTool(createRequest({ toolId: 'asana_unknown' })) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'Unsupported Asana tool: asana_unknown', + }) + expect(operationMocks.executeAsanaListWorkspaces).not.toHaveBeenCalled() + }) + + it('propagates cancellation without starting provider work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeAsanaTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(operationMocks.executeAsanaListWorkspaces).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/asana/execute-tool.ts b/apps/sim/lib/internal/asana/execute-tool.ts new file mode 100644 index 00000000000..f0212500706 --- /dev/null +++ b/apps/sim/lib/internal/asana/execute-tool.ts @@ -0,0 +1,175 @@ +import { getErrorMessage } from '@sim/utils/errors' +import type { AnyApiRouteContract, ContractBody } from '@/lib/api/contracts' +import { + asanaAddCommentContract, + asanaAddFollowersContract, + asanaCreateProjectContract, + asanaCreateSectionContract, + asanaCreateSubtaskContract, + asanaCreateTaskContract, + asanaDeleteTaskContract, + asanaGetProjectContract, + asanaGetProjectsContract, + asanaGetTaskContract, + asanaListSectionsContract, + asanaListWorkspacesContract, + asanaSearchTasksContract, + asanaUpdateTaskContract, +} from '@/lib/api/contracts/tools/asana' +import { AsanaOperationError } from '@/lib/internal/asana/errors' +import { + executeAsanaAddComment, + executeAsanaAddFollowers, + executeAsanaCreateProject, + executeAsanaCreateSection, + executeAsanaCreateSubtask, + executeAsanaCreateTask, + executeAsanaDeleteTask, + executeAsanaGetProject, + executeAsanaGetProjects, + executeAsanaGetTask, + executeAsanaListSections, + executeAsanaListWorkspaces, + executeAsanaSearchTasks, + executeAsanaUpdateTask, +} from '@/lib/internal/asana/operations' +import type { + InternalToolOperationCall, + InternalToolOperationHandler, +} from '@/lib/internal/tool-operations/types' + +type UnexpectedErrorPolicy = { kind: 'dynamic' } | { kind: 'fixed'; message: string } + +function unexpectedErrorBody( + error: unknown, + policy: UnexpectedErrorPolicy +): Record { + if (policy.kind === 'dynamic') { + return { error: getErrorMessage(error, 'Internal server error'), success: false } + } + return { + error: policy.message, + details: getErrorMessage(error, '') || undefined, + } +} + +async function executeOperation( + contract: C, + request: InternalToolOperationCall, + execute: (input: ContractBody, signal?: AbortSignal) => Promise, + unexpectedErrorPolicy: UnexpectedErrorPolicy +): Promise { + request.signal?.throwIfAborted() + if (!contract.body) throw new Error(`Asana contract ${contract.path} has no operation input`) + const parsed = contract.body.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { error: 'Invalid request data', details: parsed.error.issues }, + { status: 400 } + ) + } + + try { + const result = await execute(parsed.data as ContractBody, request.signal) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof AsanaOperationError) { + return Response.json(error.body, { status: error.status }) + } + return Response.json(unexpectedErrorBody(error, unexpectedErrorPolicy), { status: 500 }) + } +} + +const DYNAMIC_ERROR = { kind: 'dynamic' } as const + +export const executeAsanaTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!request.context.userId) return Response.json({ error: 'Unauthorized' }, { status: 401 }) + + switch (request.toolId) { + case 'asana_add_comment': + return executeOperation(asanaAddCommentContract, request, executeAsanaAddComment, { + kind: 'fixed', + message: 'Failed to add comment to Asana task', + }) + case 'asana_add_followers': + return executeOperation(asanaAddFollowersContract, request, executeAsanaAddFollowers, { + kind: 'fixed', + message: 'Failed to add followers to Asana task', + }) + case 'asana_create_project': + return executeOperation( + asanaCreateProjectContract, + request, + executeAsanaCreateProject, + DYNAMIC_ERROR + ) + case 'asana_create_section': + return executeOperation( + asanaCreateSectionContract, + request, + executeAsanaCreateSection, + DYNAMIC_ERROR + ) + case 'asana_create_subtask': + return executeOperation( + asanaCreateSubtaskContract, + request, + executeAsanaCreateSubtask, + DYNAMIC_ERROR + ) + case 'asana_create_task': + return executeOperation( + asanaCreateTaskContract, + request, + executeAsanaCreateTask, + DYNAMIC_ERROR + ) + case 'asana_delete_task': + return executeOperation(asanaDeleteTaskContract, request, executeAsanaDeleteTask, { + kind: 'fixed', + message: 'Failed to delete Asana task', + }) + case 'asana_get_project': + return executeOperation(asanaGetProjectContract, request, executeAsanaGetProject, { + kind: 'fixed', + message: 'Failed to retrieve Asana project', + }) + case 'asana_get_projects': + return executeOperation(asanaGetProjectsContract, request, executeAsanaGetProjects, { + kind: 'fixed', + message: 'Failed to retrieve Asana projects', + }) + case 'asana_get_task': + return executeOperation(asanaGetTaskContract, request, executeAsanaGetTask, { + kind: 'fixed', + message: 'Failed to retrieve Asana task(s)', + }) + case 'asana_list_sections': + return executeOperation(asanaListSectionsContract, request, executeAsanaListSections, { + kind: 'fixed', + message: 'Failed to retrieve Asana sections', + }) + case 'asana_list_workspaces': + return executeOperation(asanaListWorkspacesContract, request, executeAsanaListWorkspaces, { + kind: 'fixed', + message: 'Failed to retrieve Asana workspaces', + }) + case 'asana_search_tasks': + return executeOperation(asanaSearchTasksContract, request, executeAsanaSearchTasks, { + kind: 'fixed', + message: 'Failed to search Asana tasks', + }) + case 'asana_update_task': + return executeOperation( + asanaUpdateTaskContract, + request, + executeAsanaUpdateTask, + DYNAMIC_ERROR + ) + default: + return Response.json({ error: `Unsupported Asana tool: ${request.toolId}` }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/asana/operations.test.ts b/apps/sim/lib/internal/asana/operations.test.ts new file mode 100644 index 00000000000..af0dd3f50f9 --- /dev/null +++ b/apps/sim/lib/internal/asana/operations.test.ts @@ -0,0 +1,266 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { AsanaOperationError } from '@/lib/internal/asana/errors' +import { + executeAsanaAddComment, + executeAsanaAddFollowers, + executeAsanaCreateProject, + executeAsanaCreateSection, + executeAsanaCreateSubtask, + executeAsanaCreateTask, + executeAsanaDeleteTask, + executeAsanaGetProject, + executeAsanaGetProjects, + executeAsanaGetTask, + executeAsanaListSections, + executeAsanaListWorkspaces, + executeAsanaSearchTasks, + executeAsanaUpdateTask, +} from '@/lib/internal/asana/operations' + +const API_BASE_URL = 'https://app.asana.com/api/1.0' +const TASK_OPT_FIELDS = + 'gid,name,notes,completed,assignee,assignee.name,due_on,created_at,modified_at,created_by,created_by.name,resource_type,resource_subtype' +const PROJECT_OPT_FIELDS = 'name,notes,archived,color,created_at,modified_at,permalink_url' +const AUTH = { accessToken: 'access-token' } + +describe('Asana operations', () => { + const fetchMock = vi.fn() + + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', fetchMock) + fetchMock.mockImplementation( + async () => new Response(JSON.stringify({ data: {}, next_page: { offset: 'next' } })) + ) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + const searchParams = new URLSearchParams({ opt_fields: TASK_OPT_FIELDS }) + const operationCases = [ + { + name: 'add comment', + run: (signal: AbortSignal) => + executeAsanaAddComment({ ...AUTH, taskGid: 'task1', text: 'Comment' }, signal), + url: `${API_BASE_URL}/tasks/task1/stories`, + method: 'POST', + }, + { + name: 'add followers', + run: (signal: AbortSignal) => + executeAsanaAddFollowers({ ...AUTH, taskGid: 'task1', followers: ['user1'] }, signal), + url: `${API_BASE_URL}/tasks/task1/addFollowers?opt_fields=name,followers.name`, + method: 'POST', + }, + { + name: 'create project', + run: (signal: AbortSignal) => + executeAsanaCreateProject({ ...AUTH, workspace: 'workspace1', name: 'Project' }, signal), + url: `${API_BASE_URL}/projects?opt_fields=${PROJECT_OPT_FIELDS}`, + method: 'POST', + }, + { + name: 'create section', + run: (signal: AbortSignal) => + executeAsanaCreateSection({ ...AUTH, projectGid: 'project1', name: 'Section' }, signal), + url: `${API_BASE_URL}/projects/project1/sections`, + method: 'POST', + }, + { + name: 'create subtask', + run: (signal: AbortSignal) => + executeAsanaCreateSubtask({ ...AUTH, taskGid: 'task1', name: 'Subtask' }, signal), + url: `${API_BASE_URL}/tasks/task1/subtasks?opt_fields=name,notes,completed,created_at,permalink_url`, + method: 'POST', + }, + { + name: 'create task', + run: (signal: AbortSignal) => + executeAsanaCreateTask({ ...AUTH, workspace: 'workspace1', name: 'Task' }, signal), + url: `${API_BASE_URL}/tasks?opt_fields=name,notes,completed,created_at,permalink_url`, + method: 'POST', + }, + { + name: 'delete task', + run: (signal: AbortSignal) => executeAsanaDeleteTask({ ...AUTH, taskGid: 'task1' }, signal), + url: `${API_BASE_URL}/tasks/task1`, + method: 'DELETE', + }, + { + name: 'get project', + run: (signal: AbortSignal) => + executeAsanaGetProject({ ...AUTH, projectGid: 'project1' }, signal), + url: `${API_BASE_URL}/projects/project1?opt_fields=${PROJECT_OPT_FIELDS}`, + method: 'GET', + }, + { + name: 'get projects', + run: (signal: AbortSignal) => + executeAsanaGetProjects({ ...AUTH, workspace: 'workspace1' }, signal), + url: `${API_BASE_URL}/projects?workspace=workspace1`, + method: 'GET', + }, + { + name: 'get task', + run: (signal: AbortSignal) => executeAsanaGetTask({ ...AUTH, taskGid: 'task1' }, signal), + url: `${API_BASE_URL}/tasks/task1?opt_fields=${TASK_OPT_FIELDS}`, + method: 'GET', + }, + { + name: 'list sections', + run: (signal: AbortSignal) => + executeAsanaListSections({ ...AUTH, projectGid: 'project1' }, signal), + url: `${API_BASE_URL}/projects/project1/sections`, + method: 'GET', + }, + { + name: 'list workspaces', + run: (signal: AbortSignal) => executeAsanaListWorkspaces(AUTH, signal), + url: `${API_BASE_URL}/workspaces?limit=100`, + method: 'GET', + }, + { + name: 'search tasks', + run: (signal: AbortSignal) => + executeAsanaSearchTasks({ ...AUTH, workspace: 'workspace1' }, signal), + url: `${API_BASE_URL}/workspaces/workspace1/tasks/search?${searchParams.toString()}`, + method: 'GET', + }, + { + name: 'update task', + run: (signal: AbortSignal) => + executeAsanaUpdateTask({ ...AUTH, taskGid: 'task1', completed: true }, signal), + url: `${API_BASE_URL}/tasks/task1`, + method: 'PUT', + }, + ] + + it.each(operationCases)('executes $name with cancellation', async ({ run, url, method }) => { + const controller = new AbortController() + + await run(controller.signal) + + expect(fetchMock).toHaveBeenCalledOnce() + expect(fetchMock).toHaveBeenCalledWith( + url, + expect.objectContaining({ method, signal: controller.signal }) + ) + }) + + it('preserves task-list pagination, project precedence, and the default limit', async () => { + fetchMock.mockResolvedValueOnce( + new Response( + JSON.stringify({ + data: [ + { + gid: 'task1', + name: 'Task', + notes: '', + completed: false, + assignee: { gid: 'user1', name: 'Person' }, + }, + ], + next_page: { offset: 'next' }, + }) + ) + ) + + const result = await executeAsanaGetTask({ + ...AUTH, + workspace: 'workspace1', + project: 'project1', + }) + + const [url] = fetchMock.mock.calls[0] + const parsedUrl = new URL(String(url)) + expect(parsedUrl.searchParams.get('project')).toBe('project1') + expect(parsedUrl.searchParams.has('workspace')).toBe(false) + expect(parsedUrl.searchParams.get('limit')).toBe('50') + expect(result).toMatchObject({ + success: true, + tasks: [ + { + gid: 'task1', + name: 'Task', + completed: false, + assignee: { gid: 'user1', name: 'Person' }, + }, + ], + next_page: { offset: 'next' }, + }) + }) + + it('preserves nullable search filters in the provider query', async () => { + await executeAsanaSearchTasks({ + ...AUTH, + workspace: 'workspace1', + text: 'urgent', + assignee: 'user1', + projects: ['project1', 'project2'], + completed: null, + }) + + const [url] = fetchMock.mock.calls[0] + const parsedUrl = new URL(String(url)) + expect(parsedUrl.searchParams.get('text')).toBe('urgent') + expect(parsedUrl.searchParams.get('assignee.any')).toBe('user1') + expect(parsedUrl.searchParams.get('projects.any')).toBe('project1,project2') + expect(parsedUrl.searchParams.get('completed')).toBe('null') + }) + + it('preserves optional create and nullable update payloads', async () => { + await executeAsanaCreateTask({ + ...AUTH, + workspace: 'workspace1', + name: 'Task', + notes: '', + assignee: 'user1', + due_on: '2026-09-01', + }) + await executeAsanaUpdateTask({ + ...AUTH, + taskGid: 'task1', + name: null, + notes: null, + assignee: null, + completed: null, + due_on: null, + }) + + const createInit = fetchMock.mock.calls[0]?.[1] + expect(JSON.parse(String(createInit?.body))).toEqual({ + data: { + name: 'Task', + workspace: 'workspace1', + assignee: 'user1', + due_on: '2026-09-01', + }, + }) + const updateInit = fetchMock.mock.calls[1]?.[1] + expect(JSON.parse(String(updateInit?.body))).toEqual({ + data: { + name: null, + notes: null, + assignee: null, + completed: null, + due_on: null, + }, + }) + }) + + it('rejects invalid identifiers and missing task selectors before provider work', async () => { + await expect( + executeAsanaAddFollowers({ ...AUTH, taskGid: 'task1', followers: ['../user'] }) + ).rejects.toBeInstanceOf(AsanaOperationError) + await expect(executeAsanaGetTask(AUTH)).rejects.toMatchObject({ + status: 400, + body: { error: 'Either taskGid or workspace/project must be provided' }, + }) + expect(fetchMock).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/asana/operations.ts b/apps/sim/lib/internal/asana/operations.ts new file mode 100644 index 00000000000..617f0fec877 --- /dev/null +++ b/apps/sim/lib/internal/asana/operations.ts @@ -0,0 +1,389 @@ +import type { + AsanaAddCommentBody, + AsanaAddFollowersBody, + AsanaCreateProjectBody, + AsanaCreateSectionBody, + AsanaCreateSubtaskBody, + AsanaCreateTaskBody, + AsanaDeleteTaskBody, + AsanaGetProjectBody, + AsanaGetProjectsBody, + AsanaGetTaskBody, + AsanaListSectionsBody, + AsanaListWorkspacesBody, + AsanaSearchTasksBody, + AsanaUpdateTaskBody, +} from '@/lib/api/contracts/tools/asana' +import { validateAlphanumericId } from '@/lib/core/security/input-validation' +import { AsanaClient, type AsanaJsonObject, asArray, asObject } from '@/lib/internal/asana/client' +import { AsanaOperationError } from '@/lib/internal/asana/errors' + +const TASK_OPT_FIELDS = + 'gid,name,notes,completed,assignee,assignee.name,due_on,created_at,modified_at,created_by,created_by.name,resource_type,resource_subtype' +const PROJECT_OPT_FIELDS = 'name,notes,archived,color,created_at,modified_at,permalink_url' + +function timestamp(): string { + return new Date().toISOString() +} + +function validateId(value: string, name: string): void { + const validation = validateAlphanumericId(value, name, 100) + if (!validation.isValid) { + const error = validation.error || `Invalid ${name}` + throw new AsanaOperationError(error, 400, { error }) + } +} + +function dataObject(result: AsanaJsonObject): AsanaJsonObject { + return asObject(result.data) +} + +function optionalString(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined +} + +function requiredString(value: unknown): string { + return typeof value === 'string' ? value : '' +} + +function optionalBoolean(value: unknown): boolean | undefined { + return typeof value === 'boolean' ? value : undefined +} + +function taskSummary(value: unknown) { + const task = asObject(value) + const assignee = asObject(task.assignee) + const createdBy = asObject(task.created_by) + return { + gid: requiredString(task.gid), + resource_type: optionalString(task.resource_type), + resource_subtype: optionalString(task.resource_subtype), + name: requiredString(task.name), + notes: requiredString(task.notes), + completed: task.completed === true, + assignee: + Object.keys(assignee).length > 0 + ? { gid: requiredString(assignee.gid), name: requiredString(assignee.name) } + : undefined, + created_by: + Object.keys(createdBy).length > 0 + ? { + gid: requiredString(createdBy.gid), + resource_type: optionalString(createdBy.resource_type), + name: requiredString(createdBy.name), + } + : undefined, + due_on: optionalString(task.due_on) || undefined, + created_at: optionalString(task.created_at), + modified_at: optionalString(task.modified_at), + } +} + +function taskMutation(task: AsanaJsonObject) { + return { + success: true as const, + ts: timestamp(), + gid: requiredString(task.gid), + name: requiredString(task.name), + notes: requiredString(task.notes), + completed: task.completed === true, + created_at: optionalString(task.created_at), + modified_at: optionalString(task.modified_at), + permalink_url: optionalString(task.permalink_url), + } +} + +function projectRecord(project: AsanaJsonObject) { + return { + success: true as const, + ts: timestamp(), + gid: requiredString(project.gid), + name: requiredString(project.name), + notes: requiredString(project.notes), + archived: optionalBoolean(project.archived) ?? false, + color: typeof project.color === 'string' ? project.color : null, + created_at: optionalString(project.created_at), + modified_at: optionalString(project.modified_at), + permalink_url: optionalString(project.permalink_url), + } +} + +function jsonBody(data: Record): RequestInit { + return { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ data }), + } +} + +export async function executeAsanaAddComment(input: AsanaAddCommentBody, signal?: AbortSignal) { + validateId(input.taskGid, 'taskGid') + const result = await new AsanaClient(input.accessToken).json( + `/tasks/${input.taskGid}/stories`, + jsonBody({ text: input.text }), + signal + ) + const story = dataObject(result) + const createdBy = asObject(story.created_by) + return { + success: true as const, + ts: timestamp(), + gid: requiredString(story.gid), + text: requiredString(story.text), + created_at: optionalString(story.created_at), + created_by: + Object.keys(createdBy).length > 0 + ? { gid: requiredString(createdBy.gid), name: requiredString(createdBy.name) } + : undefined, + } +} + +export async function executeAsanaAddFollowers(input: AsanaAddFollowersBody, signal?: AbortSignal) { + validateId(input.taskGid, 'taskGid') + for (const follower of input.followers) validateId(follower, 'follower') + const result = await new AsanaClient(input.accessToken).json( + `/tasks/${input.taskGid}/addFollowers?opt_fields=name,followers.name`, + jsonBody({ followers: input.followers }), + signal + ) + const task = dataObject(result) + return { + success: true as const, + ts: timestamp(), + gid: requiredString(task.gid), + name: requiredString(task.name), + followers: asArray(task.followers).map((value) => { + const follower = asObject(value) + return { gid: requiredString(follower.gid), name: requiredString(follower.name) } + }), + } +} + +export async function executeAsanaCreateProject( + input: AsanaCreateProjectBody, + signal?: AbortSignal +) { + validateId(input.workspace, 'workspace') + const data: Record = { name: input.name, workspace: input.workspace } + if (input.notes) data.notes = input.notes + const result = await new AsanaClient(input.accessToken).json( + `/projects?opt_fields=${PROJECT_OPT_FIELDS}`, + jsonBody(data), + signal + ) + return projectRecord(dataObject(result)) +} + +export async function executeAsanaCreateSection( + input: AsanaCreateSectionBody, + signal?: AbortSignal +) { + validateId(input.projectGid, 'projectGid') + const result = await new AsanaClient(input.accessToken).json( + `/projects/${input.projectGid}/sections`, + jsonBody({ name: input.name }), + signal + ) + const section = dataObject(result) + return { + success: true as const, + ts: timestamp(), + gid: requiredString(section.gid), + name: requiredString(section.name), + created_at: optionalString(section.created_at), + } +} + +export async function executeAsanaCreateSubtask( + input: AsanaCreateSubtaskBody, + signal?: AbortSignal +) { + validateId(input.taskGid, 'taskGid') + const data: Record = { name: input.name } + if (input.notes) data.notes = input.notes + if (input.assignee) data.assignee = input.assignee + if (input.due_on) data.due_on = input.due_on + const result = await new AsanaClient(input.accessToken).json( + `/tasks/${input.taskGid}/subtasks?opt_fields=name,notes,completed,created_at,permalink_url`, + jsonBody(data), + signal + ) + return taskMutation(dataObject(result)) +} + +export async function executeAsanaCreateTask(input: AsanaCreateTaskBody, signal?: AbortSignal) { + validateId(input.workspace, 'workspace') + const data: Record = { name: input.name, workspace: input.workspace } + if (input.notes) data.notes = input.notes + if (input.assignee) data.assignee = input.assignee + if (input.due_on) data.due_on = input.due_on + const result = await new AsanaClient(input.accessToken).json( + '/tasks?opt_fields=name,notes,completed,created_at,permalink_url', + jsonBody(data), + signal + ) + return taskMutation(dataObject(result)) +} + +export async function executeAsanaDeleteTask(input: AsanaDeleteTaskBody, signal?: AbortSignal) { + validateId(input.taskGid, 'taskGid') + await new AsanaClient(input.accessToken).empty( + `/tasks/${input.taskGid}`, + { method: 'DELETE' }, + signal + ) + return { success: true as const, ts: timestamp(), gid: input.taskGid, deleted: true as const } +} + +export async function executeAsanaGetProject(input: AsanaGetProjectBody, signal?: AbortSignal) { + validateId(input.projectGid, 'projectGid') + const result = await new AsanaClient(input.accessToken).json( + `/projects/${input.projectGid}?opt_fields=${PROJECT_OPT_FIELDS}`, + { method: 'GET' }, + signal + ) + return projectRecord(dataObject(result)) +} + +export async function executeAsanaGetProjects(input: AsanaGetProjectsBody, signal?: AbortSignal) { + validateId(input.workspace, 'workspace') + const result = await new AsanaClient(input.accessToken).json( + `/projects?workspace=${input.workspace}`, + { method: 'GET' }, + signal + ) + return { + success: true as const, + ts: timestamp(), + projects: asArray(result.data).map((value) => { + const project = asObject(value) + return { + gid: requiredString(project.gid), + name: requiredString(project.name), + resource_type: requiredString(project.resource_type), + } + }), + } +} + +export async function executeAsanaGetTask(input: AsanaGetTaskBody, signal?: AbortSignal) { + const client = new AsanaClient(input.accessToken) + if (input.taskGid) { + validateId(input.taskGid, 'taskGid') + const result = await client.json( + `/tasks/${input.taskGid}?opt_fields=${TASK_OPT_FIELDS}`, + { method: 'GET' }, + signal + ) + return { success: true as const, ts: timestamp(), ...taskSummary(result.data) } + } + + if (!input.workspace && !input.project) { + const error = 'Either taskGid or workspace/project must be provided' + throw new AsanaOperationError(error, 400, { error }) + } + const params = new URLSearchParams() + if (input.project) { + validateId(input.project, 'project') + params.append('project', input.project) + } else if (input.workspace) { + validateId(input.workspace, 'workspace') + params.append('workspace', input.workspace) + } + params.append('limit', input.limit ? String(input.limit) : '50') + params.append('opt_fields', TASK_OPT_FIELDS) + const result = await client.json(`/tasks?${params.toString()}`, { method: 'GET' }, signal) + return { + success: true as const, + ts: timestamp(), + tasks: asArray(result.data).map(taskSummary), + next_page: result.next_page, + } +} + +export async function executeAsanaListSections(input: AsanaListSectionsBody, signal?: AbortSignal) { + validateId(input.projectGid, 'projectGid') + const result = await new AsanaClient(input.accessToken).json( + `/projects/${input.projectGid}/sections`, + { method: 'GET' }, + signal + ) + return { + success: true as const, + ts: timestamp(), + sections: asArray(result.data).map((value) => { + const section = asObject(value) + return { + gid: requiredString(section.gid), + name: requiredString(section.name), + resource_type: optionalString(section.resource_type), + } + }), + } +} + +export async function executeAsanaListWorkspaces( + input: AsanaListWorkspacesBody, + signal?: AbortSignal +) { + const result = await new AsanaClient(input.accessToken).json( + '/workspaces?limit=100', + { method: 'GET' }, + signal + ) + return { + success: true as const, + ts: timestamp(), + workspaces: asArray(result.data).map((value) => { + const workspace = asObject(value) + return { + gid: requiredString(workspace.gid), + name: requiredString(workspace.name), + resource_type: optionalString(workspace.resource_type), + } + }), + } +} + +export async function executeAsanaSearchTasks(input: AsanaSearchTasksBody, signal?: AbortSignal) { + validateId(input.workspace, 'workspace') + const params = new URLSearchParams() + if (input.text) params.append('text', input.text) + if (input.assignee) params.append('assignee.any', input.assignee) + if (input.projects?.length) params.append('projects.any', input.projects.join(',')) + if (input.completed !== undefined) { + params.append('completed', String(input.completed)) + } + params.append('opt_fields', TASK_OPT_FIELDS) + const result = await new AsanaClient(input.accessToken).json( + `/workspaces/${input.workspace}/tasks/search?${params.toString()}`, + { method: 'GET' }, + signal + ) + return { + success: true as const, + ts: timestamp(), + tasks: asArray(result.data).map(taskSummary), + next_page: result.next_page, + } +} + +export async function executeAsanaUpdateTask(input: AsanaUpdateTaskBody, signal?: AbortSignal) { + validateId(input.taskGid, 'taskGid') + const data: Record = {} + if (input.name !== undefined) data.name = input.name + if (input.notes !== undefined) data.notes = input.notes + if (input.assignee !== undefined) data.assignee = input.assignee + if (input.completed !== undefined) data.completed = input.completed + if (input.due_on !== undefined) data.due_on = input.due_on + const result = await new AsanaClient(input.accessToken).json( + `/tasks/${input.taskGid}`, + { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ data }), + }, + signal + ) + return taskMutation(dataObject(result)) +} diff --git a/apps/sim/lib/internal/athena/client.ts b/apps/sim/lib/internal/athena/client.ts new file mode 100644 index 00000000000..c584509c627 --- /dev/null +++ b/apps/sim/lib/internal/athena/client.ts @@ -0,0 +1,17 @@ +import { AthenaClient } from '@aws-sdk/client-athena' + +export interface AthenaConnectionConfig { + region: string + accessKeyId: string + secretAccessKey: string +} + +export function createAthenaClient(config: AthenaConnectionConfig): AthenaClient { + return new AthenaClient({ + region: config.region, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + }) +} diff --git a/apps/sim/lib/internal/athena/execute-tool.test.ts b/apps/sim/lib/internal/athena/execute-tool.test.ts new file mode 100644 index 00000000000..7cd48124e11 --- /dev/null +++ b/apps/sim/lib/internal/athena/execute-tool.test.ts @@ -0,0 +1,135 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mockOperations = vi.hoisted(() => ({ + executeAthenaBatchGetQueryExecution: vi.fn(), + executeAthenaCreateNamedQuery: vi.fn(), + executeAthenaDeleteNamedQuery: vi.fn(), + executeAthenaGetNamedQuery: vi.fn(), + executeAthenaGetQueryExecution: vi.fn(), + executeAthenaGetQueryResults: vi.fn(), + executeAthenaListDatabases: vi.fn(), + executeAthenaListNamedQueries: vi.fn(), + executeAthenaListQueryExecutions: vi.fn(), + executeAthenaListTableMetadata: vi.fn(), + executeAthenaStartQuery: vi.fn(), + executeAthenaStopQuery: vi.fn(), +})) + +vi.mock('@/lib/internal/athena/operations', () => mockOperations) + +import { executeAthenaTool } from '@/lib/internal/athena/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const CONNECTION = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', +} + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'athena_list_named_queries', + input: CONNECTION, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + userId: 'user-1', + metadata: {}, + }, + requestId: 'request-1', + ...overrides, + } +} + +const NAMED_QUERY = { ...CONNECTION, namedQueryId: 'named-query-id' } +const QUERY_EXECUTION = { ...CONNECTION, queryExecutionId: 'query-execution-id' } + +const TOOL_CASES = [ + [ + 'athena_batch_get_query_execution', + { ...CONNECTION, queryExecutionIds: ['query-execution-id'] }, + mockOperations.executeAthenaBatchGetQueryExecution, + ], + [ + 'athena_create_named_query', + { ...CONNECTION, name: 'query', database: 'analytics', queryString: 'SELECT 1' }, + mockOperations.executeAthenaCreateNamedQuery, + ], + ['athena_delete_named_query', NAMED_QUERY, mockOperations.executeAthenaDeleteNamedQuery], + ['athena_get_named_query', NAMED_QUERY, mockOperations.executeAthenaGetNamedQuery], + ['athena_get_query_execution', QUERY_EXECUTION, mockOperations.executeAthenaGetQueryExecution], + ['athena_get_query_results', QUERY_EXECUTION, mockOperations.executeAthenaGetQueryResults], + [ + 'athena_list_databases', + { ...CONNECTION, catalogName: 'AwsDataCatalog' }, + mockOperations.executeAthenaListDatabases, + ], + ['athena_list_named_queries', CONNECTION, mockOperations.executeAthenaListNamedQueries], + ['athena_list_query_executions', CONNECTION, mockOperations.executeAthenaListQueryExecutions], + [ + 'athena_list_table_metadata', + { ...CONNECTION, catalogName: 'AwsDataCatalog', databaseName: 'analytics' }, + mockOperations.executeAthenaListTableMetadata, + ], + [ + 'athena_start_query', + { ...CONNECTION, queryString: 'SELECT 1' }, + mockOperations.executeAthenaStartQuery, + ], + ['athena_stop_query', QUERY_EXECUTION, mockOperations.executeAthenaStopQuery], +] as const + +describe('executeAthenaTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it.each(TOOL_CASES)('validates and dispatches %s', async (toolId, input, operation) => { + const controller = new AbortController() + operation.mockResolvedValue({ toolId }) + + const response = await executeAthenaTool( + createRequest({ toolId, input, signal: controller.signal }) + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ toolId }) + expect(operation).toHaveBeenCalledWith(input, controller.signal) + }) + + it('returns the canonical validation envelope before provider work', async () => { + const response = await executeAthenaTool(createRequest({ input: { region: '' } })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(mockOperations.executeAthenaListNamedQueries).not.toHaveBeenCalled() + }) + + it('preserves the provider error envelope', async () => { + mockOperations.executeAthenaListNamedQueries.mockRejectedValue(new Error('Athena rejected')) + + const response = await executeAthenaTool(createRequest()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ error: 'Athena rejected' }) + }) + + it('propagates cancellation without starting provider work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeAthenaTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mockOperations.executeAthenaListNamedQueries).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/athena/execute-tool.ts b/apps/sim/lib/internal/athena/execute-tool.ts new file mode 100644 index 00000000000..f604f524b24 --- /dev/null +++ b/apps/sim/lib/internal/athena/execute-tool.ts @@ -0,0 +1,159 @@ +import { getErrorMessage } from '@sim/utils/errors' +import type { AnyApiRouteContract, ContractBody } from '@/lib/api/contracts' +import { awsAthenaBatchGetQueryExecutionContract } from '@/lib/api/contracts/tools/aws/athena-batch-get-query-execution' +import { awsAthenaCreateNamedQueryContract } from '@/lib/api/contracts/tools/aws/athena-create-named-query' +import { awsAthenaDeleteNamedQueryContract } from '@/lib/api/contracts/tools/aws/athena-delete-named-query' +import { awsAthenaGetNamedQueryContract } from '@/lib/api/contracts/tools/aws/athena-get-named-query' +import { awsAthenaGetQueryExecutionContract } from '@/lib/api/contracts/tools/aws/athena-get-query-execution' +import { awsAthenaGetQueryResultsContract } from '@/lib/api/contracts/tools/aws/athena-get-query-results' +import { awsAthenaListDatabasesContract } from '@/lib/api/contracts/tools/aws/athena-list-databases' +import { awsAthenaListNamedQueriesContract } from '@/lib/api/contracts/tools/aws/athena-list-named-queries' +import { awsAthenaListQueryExecutionsContract } from '@/lib/api/contracts/tools/aws/athena-list-query-executions' +import { awsAthenaListTableMetadataContract } from '@/lib/api/contracts/tools/aws/athena-list-table-metadata' +import { awsAthenaStartQueryContract } from '@/lib/api/contracts/tools/aws/athena-start-query' +import { awsAthenaStopQueryContract } from '@/lib/api/contracts/tools/aws/athena-stop-query' +import { + executeAthenaBatchGetQueryExecution, + executeAthenaCreateNamedQuery, + executeAthenaDeleteNamedQuery, + executeAthenaGetNamedQuery, + executeAthenaGetQueryExecution, + executeAthenaGetQueryResults, + executeAthenaListDatabases, + executeAthenaListNamedQueries, + executeAthenaListQueryExecutions, + executeAthenaListTableMetadata, + executeAthenaStartQuery, + executeAthenaStopQuery, +} from '@/lib/internal/athena/operations' +import { parseInternalToolInput } from '@/lib/internal/tool-operations/parse-input' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +async function executeOperation( + contract: C, + input: unknown, + execute: (input: ContractBody, signal?: AbortSignal) => Promise, + fallbackError: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const parsed = parseInternalToolInput(contract, input) + if (!parsed.success) return parsed.response + + try { + const result = await execute(parsed.data, signal) + signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + signal?.throwIfAborted() + return Response.json({ error: getErrorMessage(error, fallbackError) }, { status: 500 }) + } +} + +export const executeAthenaTool: InternalToolOperationHandler = async ({ + toolId, + input, + signal, +}) => { + signal?.throwIfAborted() + switch (toolId) { + case 'athena_batch_get_query_execution': + return executeOperation( + awsAthenaBatchGetQueryExecutionContract, + input, + executeAthenaBatchGetQueryExecution, + 'Failed to batch get Athena query executions', + signal + ) + case 'athena_create_named_query': + return executeOperation( + awsAthenaCreateNamedQueryContract, + input, + executeAthenaCreateNamedQuery, + 'Failed to create Athena named query', + signal + ) + case 'athena_delete_named_query': + return executeOperation( + awsAthenaDeleteNamedQueryContract, + input, + executeAthenaDeleteNamedQuery, + 'Failed to delete Athena named query', + signal + ) + case 'athena_get_named_query': + return executeOperation( + awsAthenaGetNamedQueryContract, + input, + executeAthenaGetNamedQuery, + 'Failed to get Athena named query', + signal + ) + case 'athena_get_query_execution': + return executeOperation( + awsAthenaGetQueryExecutionContract, + input, + executeAthenaGetQueryExecution, + 'Failed to get Athena query execution', + signal + ) + case 'athena_get_query_results': + return executeOperation( + awsAthenaGetQueryResultsContract, + input, + executeAthenaGetQueryResults, + 'Failed to get Athena query results', + signal + ) + case 'athena_list_databases': + return executeOperation( + awsAthenaListDatabasesContract, + input, + executeAthenaListDatabases, + 'Failed to list Athena databases', + signal + ) + case 'athena_list_named_queries': + return executeOperation( + awsAthenaListNamedQueriesContract, + input, + executeAthenaListNamedQueries, + 'Failed to list Athena named queries', + signal + ) + case 'athena_list_query_executions': + return executeOperation( + awsAthenaListQueryExecutionsContract, + input, + executeAthenaListQueryExecutions, + 'Failed to list Athena query executions', + signal + ) + case 'athena_list_table_metadata': + return executeOperation( + awsAthenaListTableMetadataContract, + input, + executeAthenaListTableMetadata, + 'Failed to list Athena table metadata', + signal + ) + case 'athena_start_query': + return executeOperation( + awsAthenaStartQueryContract, + input, + executeAthenaStartQuery, + 'Failed to start Athena query', + signal + ) + case 'athena_stop_query': + return executeOperation( + awsAthenaStopQueryContract, + input, + executeAthenaStopQuery, + 'Failed to stop Athena query', + signal + ) + default: + return Response.json({ error: `Unsupported Athena tool: ${toolId}` }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/athena/operations.test.ts b/apps/sim/lib/internal/athena/operations.test.ts new file mode 100644 index 00000000000..0f2f924a79b --- /dev/null +++ b/apps/sim/lib/internal/athena/operations.test.ts @@ -0,0 +1,107 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + createAthenaClient: vi.fn(), + destroy: vi.fn(), + send: vi.fn(), +})) + +vi.mock('@/lib/internal/athena/client', () => ({ + createAthenaClient: mocks.createAthenaClient, +})) + +import { + executeAthenaGetQueryResults, + executeAthenaListNamedQueries, +} from '@/lib/internal/athena/operations' + +const CONNECTION = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', +} + +describe('Athena operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.createAthenaClient.mockReturnValue({ send: mocks.send, destroy: mocks.destroy }) + }) + + it('preserves first-page header handling and forwards cancellation', async () => { + const controller = new AbortController() + mocks.send.mockResolvedValue({ + ResultSet: { + ResultSetMetadata: { + ColumnInfo: [ + { Name: 'name', Type: 'varchar' }, + { Name: 'count', Type: 'bigint' }, + ], + }, + Rows: [ + { Data: [{ VarCharValue: 'name' }, { VarCharValue: 'count' }] }, + { Data: [{ VarCharValue: 'sim' }, { VarCharValue: '3' }] }, + ], + }, + NextToken: 'next-page', + UpdateCount: 1, + }) + + await expect( + executeAthenaGetQueryResults( + { ...CONNECTION, queryExecutionId: 'query-id', maxResults: 10 }, + controller.signal + ) + ).resolves.toEqual({ + success: true, + output: { + columns: [ + { name: 'name', type: 'varchar' }, + { name: 'count', type: 'bigint' }, + ], + rows: [{ name: 'sim', count: '3' }], + nextToken: 'next-page', + updateCount: 1, + }, + }) + expect(mocks.send.mock.calls[0]?.[0].input).toEqual({ + QueryExecutionId: 'query-id', + MaxResults: 11, + }) + expect(mocks.send.mock.calls[0]?.[1]).toEqual({ abortSignal: controller.signal }) + expect(mocks.destroy).toHaveBeenCalledOnce() + }) + + it('does not strip a row or increase the page size on continuation pages', async () => { + mocks.send.mockResolvedValue({ + ResultSet: { + ResultSetMetadata: { ColumnInfo: [{ Name: 'name', Type: 'varchar' }] }, + Rows: [{ Data: [{ VarCharValue: 'continued' }] }], + }, + }) + + await expect( + executeAthenaGetQueryResults({ + ...CONNECTION, + queryExecutionId: 'query-id', + maxResults: 10, + nextToken: 'current-page', + }) + ).resolves.toMatchObject({ output: { rows: [{ name: 'continued' }] } }) + expect(mocks.send.mock.calls[0]?.[0].input).toEqual({ + QueryExecutionId: 'query-id', + MaxResults: 10, + NextToken: 'current-page', + }) + expect(mocks.destroy).toHaveBeenCalledOnce() + }) + + it('destroys the client when provider execution fails', async () => { + mocks.send.mockRejectedValue(new Error('provider failure')) + + await expect(executeAthenaListNamedQueries(CONNECTION)).rejects.toThrow('provider failure') + expect(mocks.destroy).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/internal/athena/operations.ts b/apps/sim/lib/internal/athena/operations.ts new file mode 100644 index 00000000000..8bc9ca16086 --- /dev/null +++ b/apps/sim/lib/internal/athena/operations.ts @@ -0,0 +1,367 @@ +import { + type AthenaClient, + BatchGetQueryExecutionCommand, + CreateNamedQueryCommand, + DeleteNamedQueryCommand, + GetNamedQueryCommand, + GetQueryExecutionCommand, + GetQueryResultsCommand, + ListDatabasesCommand, + ListNamedQueriesCommand, + ListQueryExecutionsCommand, + ListTableMetadataCommand, + StartQueryExecutionCommand, + StopQueryExecutionCommand, +} from '@aws-sdk/client-athena' +import type { AwsAthenaBatchGetQueryExecutionBody } from '@/lib/api/contracts/tools/aws/athena-batch-get-query-execution' +import type { AwsAthenaCreateNamedQueryBody } from '@/lib/api/contracts/tools/aws/athena-create-named-query' +import type { AwsAthenaDeleteNamedQueryBody } from '@/lib/api/contracts/tools/aws/athena-delete-named-query' +import type { AwsAthenaGetNamedQueryBody } from '@/lib/api/contracts/tools/aws/athena-get-named-query' +import type { AwsAthenaGetQueryExecutionBody } from '@/lib/api/contracts/tools/aws/athena-get-query-execution' +import type { AwsAthenaGetQueryResultsBody } from '@/lib/api/contracts/tools/aws/athena-get-query-results' +import type { AwsAthenaListDatabasesBody } from '@/lib/api/contracts/tools/aws/athena-list-databases' +import type { AwsAthenaListNamedQueriesBody } from '@/lib/api/contracts/tools/aws/athena-list-named-queries' +import type { AwsAthenaListQueryExecutionsBody } from '@/lib/api/contracts/tools/aws/athena-list-query-executions' +import type { AwsAthenaListTableMetadataBody } from '@/lib/api/contracts/tools/aws/athena-list-table-metadata' +import type { AwsAthenaStartQueryBody } from '@/lib/api/contracts/tools/aws/athena-start-query' +import type { AwsAthenaStopQueryBody } from '@/lib/api/contracts/tools/aws/athena-stop-query' +import { type AthenaConnectionConfig, createAthenaClient } from '@/lib/internal/athena/client' + +async function withAthenaClient( + input: AthenaConnectionConfig, + execute: (client: AthenaClient) => Promise +): Promise { + const client = createAthenaClient(input) + try { + return await execute(client) + } finally { + client.destroy() + } +} + +export async function executeAthenaBatchGetQueryExecution( + input: AwsAthenaBatchGetQueryExecutionBody, + signal?: AbortSignal +) { + return withAthenaClient(input, async (client) => { + const response = await client.send( + new BatchGetQueryExecutionCommand({ QueryExecutionIds: input.queryExecutionIds }), + { abortSignal: signal } + ) + return { + success: true, + output: { + queryExecutions: (response.QueryExecutions ?? []).map((execution) => ({ + queryExecutionId: execution.QueryExecutionId ?? '', + query: execution.Query ?? null, + state: execution.Status?.State ?? null, + stateChangeReason: execution.Status?.StateChangeReason ?? null, + statementType: execution.StatementType ?? null, + database: execution.QueryExecutionContext?.Database ?? null, + catalog: execution.QueryExecutionContext?.Catalog ?? null, + workGroup: execution.WorkGroup ?? null, + submissionDateTime: execution.Status?.SubmissionDateTime?.getTime() ?? null, + completionDateTime: execution.Status?.CompletionDateTime?.getTime() ?? null, + dataScannedInBytes: execution.Statistics?.DataScannedInBytes ?? null, + engineExecutionTimeInMillis: execution.Statistics?.EngineExecutionTimeInMillis ?? null, + queryPlanningTimeInMillis: execution.Statistics?.QueryPlanningTimeInMillis ?? null, + queryQueueTimeInMillis: execution.Statistics?.QueryQueueTimeInMillis ?? null, + totalExecutionTimeInMillis: execution.Statistics?.TotalExecutionTimeInMillis ?? null, + outputLocation: execution.ResultConfiguration?.OutputLocation ?? null, + })), + unprocessedQueryExecutionIds: (response.UnprocessedQueryExecutionIds ?? []).map((item) => ({ + queryExecutionId: item.QueryExecutionId ?? null, + errorCode: item.ErrorCode ?? null, + errorMessage: item.ErrorMessage ?? null, + })), + }, + } + }) +} + +export async function executeAthenaCreateNamedQuery( + input: AwsAthenaCreateNamedQueryBody, + signal?: AbortSignal +) { + return withAthenaClient(input, async (client) => { + const response = await client.send( + new CreateNamedQueryCommand({ + Name: input.name, + Database: input.database, + QueryString: input.queryString, + ...(input.description ? { Description: input.description } : {}), + ...(input.workGroup ? { WorkGroup: input.workGroup } : {}), + }), + { abortSignal: signal } + ) + if (!response.NamedQueryId) throw new Error('No named query ID returned') + return { success: true, output: { namedQueryId: response.NamedQueryId } } + }) +} + +export async function executeAthenaDeleteNamedQuery( + input: AwsAthenaDeleteNamedQueryBody, + signal?: AbortSignal +) { + return withAthenaClient(input, async (client) => { + await client.send(new DeleteNamedQueryCommand({ NamedQueryId: input.namedQueryId }), { + abortSignal: signal, + }) + return { success: true, output: { success: true } } + }) +} + +export async function executeAthenaGetNamedQuery( + input: AwsAthenaGetNamedQueryBody, + signal?: AbortSignal +) { + return withAthenaClient(input, async (client) => { + const response = await client.send( + new GetNamedQueryCommand({ NamedQueryId: input.namedQueryId }), + { + abortSignal: signal, + } + ) + const namedQuery = response.NamedQuery + if (!namedQuery) throw new Error('No named query data returned') + return { + success: true, + output: { + namedQueryId: namedQuery.NamedQueryId ?? input.namedQueryId, + name: namedQuery.Name ?? '', + description: namedQuery.Description ?? null, + database: namedQuery.Database ?? '', + queryString: namedQuery.QueryString ?? '', + workGroup: namedQuery.WorkGroup ?? null, + }, + } + }) +} + +export async function executeAthenaGetQueryExecution( + input: AwsAthenaGetQueryExecutionBody, + signal?: AbortSignal +) { + return withAthenaClient(input, async (client) => { + const response = await client.send( + new GetQueryExecutionCommand({ QueryExecutionId: input.queryExecutionId }), + { abortSignal: signal } + ) + const execution = response.QueryExecution + if (!execution) throw new Error('No query execution data returned') + return { + success: true, + output: { + queryExecutionId: execution.QueryExecutionId ?? input.queryExecutionId, + query: execution.Query ?? '', + state: execution.Status?.State ?? 'UNKNOWN', + stateChangeReason: execution.Status?.StateChangeReason ?? null, + statementType: execution.StatementType ?? null, + database: execution.QueryExecutionContext?.Database ?? null, + catalog: execution.QueryExecutionContext?.Catalog ?? null, + workGroup: execution.WorkGroup ?? null, + submissionDateTime: execution.Status?.SubmissionDateTime?.getTime() ?? null, + completionDateTime: execution.Status?.CompletionDateTime?.getTime() ?? null, + dataScannedInBytes: execution.Statistics?.DataScannedInBytes ?? null, + engineExecutionTimeInMillis: execution.Statistics?.EngineExecutionTimeInMillis ?? null, + queryPlanningTimeInMillis: execution.Statistics?.QueryPlanningTimeInMillis ?? null, + queryQueueTimeInMillis: execution.Statistics?.QueryQueueTimeInMillis ?? null, + totalExecutionTimeInMillis: execution.Statistics?.TotalExecutionTimeInMillis ?? null, + outputLocation: execution.ResultConfiguration?.OutputLocation ?? null, + }, + } + }) +} + +export async function executeAthenaGetQueryResults( + input: AwsAthenaGetQueryResultsBody, + signal?: AbortSignal +) { + return withAthenaClient(input, async (client) => { + const isFirstPage = !input.nextToken + const adjustedMaxResults = + input.maxResults !== undefined && isFirstPage ? input.maxResults + 1 : input.maxResults + const response = await client.send( + new GetQueryResultsCommand({ + QueryExecutionId: input.queryExecutionId, + ...(adjustedMaxResults !== undefined ? { MaxResults: adjustedMaxResults } : {}), + ...(input.nextToken ? { NextToken: input.nextToken } : {}), + }), + { abortSignal: signal } + ) + const columns = (response.ResultSet?.ResultSetMetadata?.ColumnInfo ?? []).map((column) => ({ + name: column.Name ?? '', + type: column.Type ?? 'varchar', + })) + const rawRows = response.ResultSet?.Rows ?? [] + const dataRows = input.nextToken ? rawRows : rawRows.slice(1) + const rows = dataRows.map((row) => { + const record: Record = {} + const rowData = row.Data ?? [] + for (let index = 0; index < columns.length; index++) { + record[columns[index].name] = rowData[index]?.VarCharValue ?? '' + } + return record + }) + return { + success: true, + output: { + columns, + rows, + nextToken: response.NextToken ?? null, + updateCount: response.UpdateCount ?? null, + }, + } + }) +} + +export async function executeAthenaListDatabases( + input: AwsAthenaListDatabasesBody, + signal?: AbortSignal +) { + return withAthenaClient(input, async (client) => { + const response = await client.send( + new ListDatabasesCommand({ + CatalogName: input.catalogName, + ...(input.workGroup ? { WorkGroup: input.workGroup } : {}), + ...(input.maxResults !== undefined ? { MaxResults: input.maxResults } : {}), + ...(input.nextToken ? { NextToken: input.nextToken } : {}), + }), + { abortSignal: signal } + ) + return { + success: true, + output: { + databases: (response.DatabaseList ?? []).map((database) => ({ + name: database.Name ?? '', + description: database.Description ?? null, + })), + nextToken: response.NextToken ?? null, + }, + } + }) +} + +export async function executeAthenaListNamedQueries( + input: AwsAthenaListNamedQueriesBody, + signal?: AbortSignal +) { + return withAthenaClient(input, async (client) => { + const response = await client.send( + new ListNamedQueriesCommand({ + ...(input.workGroup ? { WorkGroup: input.workGroup } : {}), + ...(input.maxResults !== undefined ? { MaxResults: input.maxResults } : {}), + ...(input.nextToken ? { NextToken: input.nextToken } : {}), + }), + { abortSignal: signal } + ) + return { + success: true, + output: { + namedQueryIds: response.NamedQueryIds ?? [], + nextToken: response.NextToken ?? null, + }, + } + }) +} + +export async function executeAthenaListQueryExecutions( + input: AwsAthenaListQueryExecutionsBody, + signal?: AbortSignal +) { + return withAthenaClient(input, async (client) => { + const response = await client.send( + new ListQueryExecutionsCommand({ + ...(input.workGroup ? { WorkGroup: input.workGroup } : {}), + ...(input.maxResults !== undefined ? { MaxResults: input.maxResults } : {}), + ...(input.nextToken ? { NextToken: input.nextToken } : {}), + }), + { abortSignal: signal } + ) + return { + success: true, + output: { + queryExecutionIds: response.QueryExecutionIds ?? [], + nextToken: response.NextToken ?? null, + }, + } + }) +} + +export async function executeAthenaListTableMetadata( + input: AwsAthenaListTableMetadataBody, + signal?: AbortSignal +) { + return withAthenaClient(input, async (client) => { + const response = await client.send( + new ListTableMetadataCommand({ + CatalogName: input.catalogName, + DatabaseName: input.databaseName, + ...(input.expression ? { Expression: input.expression } : {}), + ...(input.workGroup ? { WorkGroup: input.workGroup } : {}), + ...(input.maxResults !== undefined ? { MaxResults: input.maxResults } : {}), + ...(input.nextToken ? { NextToken: input.nextToken } : {}), + }), + { abortSignal: signal } + ) + return { + success: true, + output: { + tables: (response.TableMetadataList ?? []).map((table) => ({ + name: table.Name ?? '', + tableType: table.TableType ?? null, + createTime: table.CreateTime?.getTime() ?? null, + lastAccessTime: table.LastAccessTime?.getTime() ?? null, + columns: (table.Columns ?? []).map((column) => ({ + name: column.Name ?? '', + type: column.Type ?? null, + comment: column.Comment ?? null, + })), + partitionKeys: (table.PartitionKeys ?? []).map((column) => ({ + name: column.Name ?? '', + type: column.Type ?? null, + comment: column.Comment ?? null, + })), + })), + nextToken: response.NextToken ?? null, + }, + } + }) +} + +export async function executeAthenaStartQuery( + input: AwsAthenaStartQueryBody, + signal?: AbortSignal +) { + return withAthenaClient(input, async (client) => { + const response = await client.send( + new StartQueryExecutionCommand({ + QueryString: input.queryString, + ...(input.database || input.catalog + ? { + QueryExecutionContext: { + ...(input.database ? { Database: input.database } : {}), + ...(input.catalog ? { Catalog: input.catalog } : {}), + }, + } + : {}), + ...(input.outputLocation + ? { ResultConfiguration: { OutputLocation: input.outputLocation } } + : {}), + ...(input.workGroup ? { WorkGroup: input.workGroup } : {}), + }), + { abortSignal: signal } + ) + if (!response.QueryExecutionId) throw new Error('No query execution ID returned') + return { success: true, output: { queryExecutionId: response.QueryExecutionId } } + }) +} + +export async function executeAthenaStopQuery(input: AwsAthenaStopQueryBody, signal?: AbortSignal) { + return withAthenaClient(input, async (client) => { + await client.send(new StopQueryExecutionCommand({ QueryExecutionId: input.queryExecutionId }), { + abortSignal: signal, + }) + return { success: true, output: { success: true } } + }) +} diff --git a/apps/sim/lib/internal/azure-data-explorer/client.test.ts b/apps/sim/lib/internal/azure-data-explorer/client.test.ts new file mode 100644 index 00000000000..5100e54da21 --- /dev/null +++ b/apps/sim/lib/internal/azure-data-explorer/client.test.ts @@ -0,0 +1,147 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { secureFetch } = vi.hoisted(() => ({ secureFetch: vi.fn() })) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + MAX_JSON_API_RESPONSE_BYTES: 10 * 1024 * 1024, + secureFetchWithValidation: secureFetch, +})) + +import { + AzureDataExplorerOperationError, + requestAzureDataExplorer, +} from '@/lib/internal/azure-data-explorer/client' + +const BASE_INPUT = { + clusterUri: 'https://mycluster.eastus.kusto.windows.net', + tenantId: 'tenant-1', + clientId: 'client-1', + clientSecret: 'secret-1', + endpoint: 'query' as const, + database: 'Samples', + csl: 'print Test="Hello, World!"', +} + +function jsonResponse(body: unknown, status = 200): Response { + return { + ok: status >= 200 && status < 300, + status, + headers: new Headers(), + json: async () => body, + text: async () => JSON.stringify(body), + } as Response +} + +function queryResponse(severity = 4) { + return { + Tables: [ + { + TableName: 'Table_0', + Columns: [{ ColumnName: 'Value', DataType: 'String', ColumnType: 'string' }], + Rows: [['metadata']], + }, + { + TableName: 'Table_1', + Columns: [{ ColumnName: 'Test', DataType: 'String', ColumnType: 'string' }], + Rows: [['Hello, World!']], + }, + { + TableName: 'Table_2', + Columns: [ + { ColumnName: 'Severity', DataType: 'Int32', ColumnType: 'int' }, + { ColumnName: 'StatusDescription', DataType: 'String', ColumnType: 'string' }, + ], + Rows: [[severity, severity <= 2 ? 'Query failed' : 'Query completed']], + }, + { + TableName: 'Table_3', + Columns: [{ ColumnName: 'Ordinal' }, { ColumnName: 'Kind' }], + Rows: [ + [0, 'QueryProperties'], + [1, 'QueryResult'], + [2, 'QueryStatus'], + ], + }, + ], + } +} + +describe('requestAzureDataExplorer', () => { + beforeEach(() => { + vi.clearAllMocks() + secureFetch + .mockResolvedValueOnce(jsonResponse({ access_token: 'token-1', expires_in: 3600 })) + .mockResolvedValueOnce(jsonResponse(queryResponse())) + }) + + it('preserves the token audience, read-only header, and primary-table projection', async () => { + const output = await requestAzureDataExplorer( + { ...BASE_INPUT, clientSecret: 'unique-1', readOnly: true }, + 'request-1' + ) + + expect(secureFetch.mock.calls[0][0]).toBe( + 'https://login.microsoftonline.com/tenant-1/oauth2/token' + ) + expect( + Object.fromEntries(new URLSearchParams(secureFetch.mock.calls[0][1].body)) + ).toMatchObject({ + client_id: 'client-1', + resource: 'https://mycluster.eastus.kusto.windows.net', + }) + expect(secureFetch.mock.calls[1][0]).toBe( + 'https://mycluster.eastus.kusto.windows.net/v1/rest/query' + ) + expect(secureFetch.mock.calls[1][1].headers['x-ms-readonly']).toBe('true') + expect(output).toMatchObject({ + tableName: 'Table_1', + records: [{ Test: 'Hello, World!' }], + rowCount: 1, + totalRowCount: 1, + truncated: false, + }) + }) + + it('preserves partial Kusto failures reported inside an HTTP 200 response', async () => { + secureFetch.mockReset() + secureFetch + .mockResolvedValueOnce(jsonResponse({ access_token: 'token-1', expires_in: 3600 })) + .mockResolvedValueOnce(jsonResponse(queryResponse(2))) + + await expect( + requestAzureDataExplorer({ ...BASE_INPUT, clientSecret: 'unique-2' }, 'request-2') + ).rejects.toEqual(new AzureDataExplorerOperationError('Query failed', 400, 200)) + }) + + it('caps projected rows while retaining the provider row count', async () => { + secureFetch.mockReset() + const response = queryResponse() + response.Tables[1].Rows = Array.from({ length: 10_050 }, (_, index) => [`row-${index}`]) + secureFetch + .mockResolvedValueOnce(jsonResponse({ access_token: 'token-1', expires_in: 3600 })) + .mockResolvedValueOnce(jsonResponse(response)) + + const output = await requestAzureDataExplorer( + { ...BASE_INPUT, clientSecret: 'unique-3' }, + 'request-3' + ) + + expect(output.rows).toHaveLength(10_000) + expect(output.records).toHaveLength(10_000) + expect(output).toMatchObject({ rowCount: 10_000, totalRowCount: 10_050, truncated: true }) + }) + + it('propagates cancellation before any provider request', async () => { + secureFetch.mockReset() + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + requestAzureDataExplorer(BASE_INPUT, 'request-4', controller.signal) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(secureFetch).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/azure-data-explorer/client.ts b/apps/sim/lib/internal/azure-data-explorer/client.ts new file mode 100644 index 00000000000..eaca58026b7 --- /dev/null +++ b/apps/sim/lib/internal/azure-data-explorer/client.ts @@ -0,0 +1,321 @@ +import { createHash } from 'node:crypto' +import { createLogger } from '@sim/logger' +import { truncate } from '@sim/utils/string' +import { + MAX_JSON_API_RESPONSE_BYTES, + secureFetchWithValidation, +} from '@/lib/core/security/input-validation.server' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { + type AzureDataExplorerInput, + assertSafeAzureDataExplorerClusterUri, + resolveEntraAuthority, +} from '@/lib/internal/azure-data-explorer/schema' +import type { AzureDataExplorerTable } from '@/tools/azure_data_explorer/types' + +const logger = createLogger('AzureDataExplorerClient') + +const OUTBOUND_FETCH_TIMEOUT_MS = 120_000 +const TOKEN_FETCH_TIMEOUT_MS = 30_000 +const TOKEN_CACHE_MAX_ENTRIES = 500 +const TOKEN_SAFETY_WINDOW_MS = 60_000 +const MAX_ERROR_MESSAGE_LENGTH = 2000 +const MAX_TOKEN_RESPONSE_BYTES = 256 * 1024 +const MAX_PROJECTED_ROWS = 10_000 + +interface CachedToken { + accessToken: string + expiresAt: number +} + +const TOKEN_CACHE = new Map() + +export class AzureDataExplorerOperationError extends Error { + constructor( + message: string, + readonly status: number, + readonly providerStatus?: number + ) { + super(message) + this.name = 'AzureDataExplorerOperationError' + } +} + +function resolveResource(input: AzureDataExplorerInput, clusterUrl: URL): string { + return (input.resource || clusterUrl.origin).replace(/\/+$/, '') +} + +function tokenCacheKey(input: AzureDataExplorerInput, authority: string, resource: string): string { + const secretHash = createHash('sha256').update(input.clientSecret).digest('hex').slice(0, 16) + return `${authority}::${input.tenantId}::${input.clientId}::${secretHash}::${resource}` +} + +function rememberToken(key: string, token: CachedToken): void { + if (TOKEN_CACHE.has(key)) TOKEN_CACHE.delete(key) + TOKEN_CACHE.set(key, token) + while (TOKEN_CACHE.size > TOKEN_CACHE_MAX_ENTRIES) { + const oldestKey = TOKEN_CACHE.keys().next().value + if (oldestKey === undefined) break + TOKEN_CACHE.delete(oldestKey) + } +} + +async function fetchAccessToken( + input: AzureDataExplorerInput, + authority: string, + resource: string, + requestId: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const cacheKey = tokenCacheKey(input, authority, resource) + const cached = TOKEN_CACHE.get(cacheKey) + if (cached && cached.expiresAt - TOKEN_SAFETY_WINDOW_MS > Date.now()) { + return cached.accessToken + } + + const body = new URLSearchParams({ + grant_type: 'client_credentials', + client_id: input.clientId, + client_secret: input.clientSecret, + resource, + }) + + const response = await secureFetchWithValidation( + `${authority}/${encodeURIComponent(input.tenantId)}/oauth2/token`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + Accept: 'application/json', + }, + body: body.toString(), + timeout: TOKEN_FETCH_TIMEOUT_MS, + maxResponseBytes: MAX_TOKEN_RESPONSE_BYTES, + signal, + }, + 'tokenUrl' + ) + signal?.throwIfAborted() + + if (!response.ok) { + const text = await response.text().catch(() => '') + logger.warn('Entra token fetch failed', { requestId, status: response.status, error: text }) + throw new AzureDataExplorerOperationError( + `Microsoft Entra token request failed: HTTP ${response.status}. Verify tenantId, clientId, clientSecret, and that the app has access to the cluster.`, + 500 + ) + } + + const data = (await response.json()) as { access_token?: string; expires_in?: string | number } + if (!data.access_token) { + throw new AzureDataExplorerOperationError( + 'Microsoft Entra token response did not include an access token', + 500 + ) + } + + const expiresInSeconds = Number(data.expires_in) + const expiresInMs = (Number.isFinite(expiresInSeconds) ? expiresInSeconds : 3600) * 1000 + rememberToken(cacheKey, { + accessToken: data.access_token, + expiresAt: Date.now() + expiresInMs, + }) + return data.access_token +} + +interface KustoColumn { + ColumnName?: string + DataType?: string + ColumnType?: string +} + +interface KustoTable { + TableName?: string + Columns?: KustoColumn[] + Rows?: unknown[][] +} + +function columnNames(table: KustoTable): string[] { + return (table.Columns ?? []).map((column) => column.ColumnName ?? '') +} + +interface TableOfContents { + primaryOrdinal: number | null + statusOrdinal: number | null +} + +function readTableOfContents(tables: KustoTable[]): TableOfContents | null { + if (tables.length === 0) return null + const contents = tables[tables.length - 1] + const names = columnNames(contents) + const ordinalIndex = names.indexOf('Ordinal') + const kindIndex = names.indexOf('Kind') + if (ordinalIndex < 0 || kindIndex < 0) return null + + let primaryOrdinal: number | null = null + let statusOrdinal: number | null = null + for (const row of contents.Rows ?? []) { + const ordinal = Number(row[ordinalIndex]) + if (!Number.isInteger(ordinal) || !tables[ordinal]) continue + if (row[kindIndex] === 'QueryResult' && primaryOrdinal === null) primaryOrdinal = ordinal + if (row[kindIndex] === 'QueryStatus' && statusOrdinal === null) statusOrdinal = ordinal + } + return { primaryOrdinal, statusOrdinal } +} + +function selectPrimaryTable( + tables: KustoTable[], + contents: TableOfContents | null +): KustoTable | null { + if (tables.length === 0) return null + if (contents?.primaryOrdinal != null) return tables[contents.primaryOrdinal] ?? tables[0] + return tables[0] +} + +function findQueryFailure(tables: KustoTable[], contents: TableOfContents | null): string | null { + if (contents?.statusOrdinal == null) return null + const table = tables[contents.statusOrdinal] + if (!table) return null + const names = columnNames(table) + const severityIndex = names.indexOf('Severity') + const descriptionIndex = names.indexOf('StatusDescription') + if (severityIndex < 0 || descriptionIndex < 0) return null + + for (const row of table.Rows ?? []) { + const severity = Number(row[severityIndex]) + if (!Number.isFinite(severity) || severity > 2) continue + const description = row[descriptionIndex] + return typeof description === 'string' && description.length > 0 + ? description + : 'Kusto reported a query failure' + } + return null +} + +const EMPTY_PROJECTION: AzureDataExplorerTable = { + tableName: null, + columns: [], + rows: [], + records: [], + rowCount: 0, + totalRowCount: 0, + truncated: false, +} + +function projectTable(table: KustoTable | null): AzureDataExplorerTable { + if (!table) return EMPTY_PROJECTION + const columns = (table.Columns ?? []).map((column) => ({ + name: column.ColumnName ?? '', + type: column.ColumnType ?? null, + dataType: column.DataType ?? null, + })) + const allRows = table.Rows ?? [] + const rows = allRows.length > MAX_PROJECTED_ROWS ? allRows.slice(0, MAX_PROJECTED_ROWS) : allRows + const records = rows.map((row) => { + const record: Record = {} + columns.forEach((column, index) => { + if (column.name) record[column.name] = row[index] ?? null + }) + return record + }) + return { + tableName: table.TableName ?? null, + columns, + rows, + records, + rowCount: rows.length, + totalRowCount: allRows.length, + truncated: allRows.length > rows.length, + } +} + +function extractKustoError(body: unknown, status: number): string { + if (body && typeof body === 'object') { + const error = (body as { error?: { code?: unknown; message?: unknown } }).error + if (error && typeof error === 'object') { + const message = typeof error.message === 'string' ? error.message : '' + const code = typeof error.code === 'string' ? error.code : '' + if (message) return code ? `[${code}] ${message}` : message + if (code) return code + } + } + if (typeof body === 'string' && body.length > 0) { + return truncate(body, MAX_ERROR_MESSAGE_LENGTH) + } + return `Azure Data Explorer request failed with HTTP ${status}` +} + +export async function requestAzureDataExplorer( + input: AzureDataExplorerInput, + requestId: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const clusterUrl = assertSafeAzureDataExplorerClusterUri(input.clusterUri) + const resource = resolveResource(input, clusterUrl) + const authority = resolveEntraAuthority(clusterUrl.hostname) + const accessToken = await fetchAccessToken(input, authority, resource, requestId, signal) + + const headers: Record = { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + 'Content-Type': 'application/json; charset=utf-8', + 'x-ms-client-request-id': `Sim.Workflow;${requestId}`, + 'x-ms-app': 'Sim', + } + if (input.readOnly) headers['x-ms-readonly'] = 'true' + + const response = await secureFetchWithValidation( + `${clusterUrl.origin}/v1/rest/${input.endpoint}`, + { + method: 'POST', + headers, + body: JSON.stringify({ + ...(input.database ? { db: input.database } : {}), + csl: input.csl, + ...(input.properties ? { properties: input.properties } : {}), + }), + timeout: OUTBOUND_FETCH_TIMEOUT_MS, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + signal, + }, + 'clusterUri' + ).catch((error: unknown) => { + if (isPayloadSizeLimitError(error)) { + throw new AzureDataExplorerOperationError( + 'The Azure Data Explorer response was too large to return. Narrow the query — add a `where` filter, aggregate with `summarize`, or bound it with `take` or `top N by`.', + 413 + ) + } + throw error + }) + signal?.throwIfAborted() + + const raw = await response.text() + let body: unknown = null + if (raw.length > 0) { + try { + body = JSON.parse(raw) + } catch { + body = raw + } + } + if (!response.ok) { + throw new AzureDataExplorerOperationError( + extractKustoError(body, response.status), + response.status, + response.status + ) + } + + const tables = Array.isArray((body as { Tables?: KustoTable[] } | null)?.Tables) + ? ((body as { Tables: KustoTable[] }).Tables ?? []) + : [] + const contents = readTableOfContents(tables) + const failure = findQueryFailure(tables, contents) + if (failure) { + throw new AzureDataExplorerOperationError(truncate(failure, MAX_ERROR_MESSAGE_LENGTH), 400, 200) + } + return projectTable(selectPrimaryTable(tables, contents)) +} diff --git a/apps/sim/lib/internal/azure-data-explorer/execute-tool.ts b/apps/sim/lib/internal/azure-data-explorer/execute-tool.ts new file mode 100644 index 00000000000..9ad5bfdea50 --- /dev/null +++ b/apps/sim/lib/internal/azure-data-explorer/execute-tool.ts @@ -0,0 +1,96 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { AzureDataExplorerOperationError } from '@/lib/internal/azure-data-explorer/client' +import { executeAzureDataExplorerOperation } from '@/lib/internal/azure-data-explorer/operations' +import { azureDataExplorerInputSchema } from '@/lib/internal/azure-data-explorer/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const logger = createLogger('AzureDataExplorerToolExecution') + +const TOOL_IDS = new Set([ + 'azure_data_explorer_create_table', + 'azure_data_explorer_drop_table', + 'azure_data_explorer_ingest_from_query', + 'azure_data_explorer_ingest_inline', + 'azure_data_explorer_list_databases', + 'azure_data_explorer_list_functions', + 'azure_data_explorer_list_tables', + 'azure_data_explorer_management', + 'azure_data_explorer_query', + 'azure_data_explorer_show_database_schema', + 'azure_data_explorer_show_ingestion_failures', + 'azure_data_explorer_show_operations', + 'azure_data_explorer_show_table_details', + 'azure_data_explorer_show_table_schema', +]) + +function exceedsInputCap(input: unknown): boolean { + try { + return Buffer.byteLength(JSON.stringify(input) ?? '') > DEFAULT_MAX_JSON_BODY_BYTES + } catch { + return true + } +} + +export const executeAzureDataExplorerTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!TOOL_IDS.has(request.toolId)) { + return Response.json( + { success: false, error: `Unsupported Azure Data Explorer tool: ${request.toolId}` }, + { status: 500 } + ) + } + if (!request.context.userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + if (exceedsInputCap(request.input)) { + return Response.json( + { + success: false, + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + + const parsed = azureDataExplorerInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { + success: false, + error: getValidationErrorMessage(parsed.error, 'Validation failed'), + }, + { status: 400 } + ) + } + + try { + const output = await executeAzureDataExplorerOperation( + parsed.data, + request.requestId, + request.signal + ) + request.signal?.throwIfAborted() + return Response.json({ success: true, output }) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof AzureDataExplorerOperationError) { + return Response.json( + { + success: false, + error: error.message, + ...(error.providerStatus === undefined ? {} : { status: error.providerStatus }), + }, + { status: error.status } + ) + } + const message = getErrorMessage(error, 'Unknown error occurred') + logger.error('Azure Data Explorer operation failed', { + error: message, + requestId: request.requestId, + }) + return Response.json({ success: false, error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/azure-data-explorer/operations.ts b/apps/sim/lib/internal/azure-data-explorer/operations.ts new file mode 100644 index 00000000000..964d367f47b --- /dev/null +++ b/apps/sim/lib/internal/azure-data-explorer/operations.ts @@ -0,0 +1,10 @@ +import { requestAzureDataExplorer } from '@/lib/internal/azure-data-explorer/client' +import type { AzureDataExplorerInput } from '@/lib/internal/azure-data-explorer/schema' + +export async function executeAzureDataExplorerOperation( + input: AzureDataExplorerInput, + requestId: string, + signal?: AbortSignal +) { + return requestAzureDataExplorer(input, requestId, signal) +} diff --git a/apps/sim/lib/internal/azure-data-explorer/schema.ts b/apps/sim/lib/internal/azure-data-explorer/schema.ts new file mode 100644 index 00000000000..6e99c7df1a4 --- /dev/null +++ b/apps/sim/lib/internal/azure-data-explorer/schema.ts @@ -0,0 +1,117 @@ +import { isPrivateIpHost } from '@sim/security/ssrf' +import { z } from 'zod' + +const KUSTO_CLOUDS = [ + { hostSuffix: 'kusto.windows.net', authority: 'https://login.microsoftonline.com' }, + { hostSuffix: 'kusto.fabric.microsoft.com', authority: 'https://login.microsoftonline.com' }, + { hostSuffix: 'kusto.usgovcloudapi.net', authority: 'https://login.microsoftonline.us' }, + { hostSuffix: 'kusto.chinacloudapi.cn', authority: 'https://login.partner.microsoftonline.cn' }, +] as const + +const ALLOWED_CLUSTER_HOSTS = KUSTO_CLOUDS.map((cloud) => cloud.hostSuffix).join(', ') + +function matchKustoCloud(host: string): (typeof KUSTO_CLOUDS)[number] | null { + return ( + KUSTO_CLOUDS.find( + (cloud) => host === cloud.hostSuffix || host.endsWith(`.${cloud.hostSuffix}`) + ) ?? null + ) +} + +export function resolveEntraAuthority(clusterHost: string): string { + const cloud = matchKustoCloud(clusterHost.toLowerCase()) + if (!cloud) { + throw new Error(`No Microsoft Entra authority is configured for cluster host ${clusterHost}`) + } + return cloud.authority +} + +export function checkAzureDataExplorerClusterUri( + rawUrl: string, + label = 'clusterUri' +): { ok: true; url: URL } | { ok: false; message: string } { + let parsed: URL + try { + parsed = new URL(rawUrl) + } catch { + return { + ok: false, + message: `${label} must be a full URL (e.g., https://mycluster.eastus.kusto.windows.net)`, + } + } + if (parsed.protocol !== 'https:') { + return { ok: false, message: `${label} must use https://` } + } + const host = parsed.hostname.toLowerCase() + if (isPrivateIpHost(host)) { + return { ok: false, message: `${label} host is not allowed (private/loopback range)` } + } + if (!matchKustoCloud(host)) { + return { + ok: false, + message: `${label} host must be an Azure Data Explorer or Fabric Eventhouse endpoint (${ALLOWED_CLUSTER_HOSTS})`, + } + } + return { ok: true, url: parsed } +} + +export function assertSafeAzureDataExplorerClusterUri(rawUrl: string, label?: string): URL { + const result = checkAzureDataExplorerClusterUri(rawUrl, label) + if (!result.ok) throw new Error(result.message) + return result.url +} + +const entityNameSchema = z + .string() + .trim() + .min(1, 'name is required') + .max(1024, 'name must be at most 1024 characters') + .regex( + /^[\p{L}\p{N}_ .-]+$/u, + 'name may contain only letters, digits, underscores, spaces, dots, and dashes' + ) + +const tenantIdSchema = z + .string() + .trim() + .min(1, 'tenantId is required') + .max(253, 'tenantId is too long') + .regex( + /^[A-Za-z0-9][A-Za-z0-9.-]*$/, + 'tenantId must be a GUID or a domain name (e.g., contoso.onmicrosoft.com)' + ) + +export const azureDataExplorerInputSchema = z + .object({ + clusterUri: z.string().min(1, 'clusterUri is required'), + tenantId: tenantIdSchema, + clientId: z.string().min(1, 'clientId is required'), + clientSecret: z.string().min(1, 'clientSecret is required'), + resource: z.string().optional(), + endpoint: z.enum(['query', 'mgmt']), + database: entityNameSchema.optional(), + csl: z.string().min(1, 'csl is required').max(1_000_000, 'csl is too long'), + properties: z.record(z.string(), z.unknown()).optional(), + readOnly: z.boolean().optional(), + }) + .superRefine((input, context) => { + const clusterCheck = checkAzureDataExplorerClusterUri(input.clusterUri) + if (!clusterCheck.ok) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ['clusterUri'], + message: clusterCheck.message, + }) + } + if (input.resource === undefined) return + const resourceCheck = checkAzureDataExplorerClusterUri(input.resource, 'resource') + if (!resourceCheck.ok) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ['resource'], + message: resourceCheck.message, + }) + } + }) + +export type AzureDataExplorerInput = z.output diff --git a/apps/sim/lib/internal/box/client.test.ts b/apps/sim/lib/internal/box/client.test.ts new file mode 100644 index 00000000000..290c27abc4f --- /dev/null +++ b/apps/sim/lib/internal/box/client.test.ts @@ -0,0 +1,71 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ fetch: vi.fn() })) + +import { BoxClient, BoxUploadError } from '@/lib/internal/box/client' + +describe('BoxClient', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mocks.fetch) + }) + + it('uses Box multipart semantics and projects the exact output contract', async () => { + const controller = new AbortController() + mocks.fetch.mockResolvedValue( + Response.json({ + entries: [ + { + id: 'box-1', + name: 'file.pdf', + size: 4, + sha1: 'sha', + created_at: 'created', + modified_at: 'modified', + parent: { id: '0', name: 'All Files' }, + }, + ], + }) + ) + const output = await new BoxClient('token', controller.signal).upload( + '0', + 'file.pdf', + Buffer.from('file') + ) + + const [url, init] = mocks.fetch.mock.calls[0] as [string, RequestInit] + expect(url).toBe('https://upload.box.com/api/2.0/files/content') + expect(init.method).toBe('POST') + expect(init.headers).toEqual({ Authorization: 'Bearer token' }) + expect(init.signal).toBe(controller.signal) + const form = init.body as FormData + expect(JSON.parse(String(form.get('attributes')))).toEqual({ + name: 'file.pdf', + parent: { id: '0' }, + }) + expect((form.get('file') as File).name).toBe('file.pdf') + expect(output).toEqual({ + id: 'box-1', + name: 'file.pdf', + size: 4, + sha1: 'sha', + createdAt: 'created', + modifiedAt: 'modified', + parentId: '0', + parentName: 'All Files', + }) + }) + + it('preserves Box provider status and error message', async () => { + mocks.fetch.mockResolvedValue(Response.json({ message: 'Folder not found' }, { status: 404 })) + const error = await new BoxClient('token') + .upload('0', 'file.pdf', Buffer.from('file')) + .catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(BoxUploadError) + expect(error).toMatchObject({ message: 'Folder not found', status: 404 }) + }) +}) diff --git a/apps/sim/lib/internal/box/client.ts b/apps/sim/lib/internal/box/client.ts new file mode 100644 index 00000000000..be7cc2150f1 --- /dev/null +++ b/apps/sim/lib/internal/box/client.ts @@ -0,0 +1,76 @@ +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + readResponseJsonWithLimit, +} from '@/lib/core/utils/stream-limits' + +interface BoxUploadEntry { + id?: string + name?: string + size?: number + sha1?: string | null + created_at?: string | null + modified_at?: string | null + parent?: { id?: string; name?: string } +} + +interface BoxUploadResponseBody { + entries?: BoxUploadEntry[] + message?: string +} + +export class BoxUploadError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'BoxUploadError' + } +} + +export class BoxClient { + constructor( + private readonly accessToken: string, + private readonly signal?: AbortSignal + ) {} + + async upload(parentFolderId: string, fileName: string, buffer: Buffer) { + this.signal?.throwIfAborted() + const formData = new FormData() + formData.append( + 'attributes', + JSON.stringify({ name: fileName, parent: { id: parentFolderId } }) + ) + formData.append( + 'file', + new Blob([new Uint8Array(buffer)], { type: 'application/octet-stream' }), + fileName + ) + const response = await fetch('https://upload.box.com/api/2.0/files/content', { + method: 'POST', + headers: { Authorization: `Bearer ${this.accessToken}` }, + body: formData, + signal: this.signal, + }) + const data = await readResponseJsonWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Box file upload response', + signal: this.signal, + }) + if (!response.ok) { + throw new BoxUploadError(data.message || 'Failed to upload file', response.status) + } + const file = data.entries?.[0] + if (!file) throw new BoxUploadError('No file returned in upload response', 500) + return { + id: file.id ?? '', + name: file.name ?? '', + size: file.size ?? 0, + sha1: file.sha1 ?? null, + createdAt: file.created_at ?? null, + modifiedAt: file.modified_at ?? null, + parentId: file.parent?.id ?? null, + parentName: file.parent?.name ?? null, + } + } +} diff --git a/apps/sim/lib/internal/box/execute-tool.test.ts b/apps/sim/lib/internal/box/execute-tool.test.ts new file mode 100644 index 00000000000..66b95c8a9db --- /dev/null +++ b/apps/sim/lib/internal/box/execute-tool.test.ts @@ -0,0 +1,69 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ upload: vi.fn() })) + +vi.mock('@/lib/internal/box/operations', () => ({ executeBoxUploadFile: mocks.upload })) + +import { executeBoxTool } from '@/lib/internal/box/execute-tool' +import { boxUploadFileTool } from '@/tools/box/upload_file' + +const file = { key: 'uploads/file.pdf', name: 'file.pdf', size: 5 } + +describe('executeBoxTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.upload.mockResolvedValue(Response.json({ success: true, output: {} })) + }) + + it('dispatches the typed upload with trusted execution context', async () => { + const controller = new AbortController() + const input = { accessToken: 'token', parentFolderId: '0', file } + await executeBoxTool({ + toolId: 'box_upload_file', + input, + headers: new Headers(), + context: { userId: 'user-1' }, + requestId: 'request-1', + signal: controller.signal, + }) + + expect(mocks.upload).toHaveBeenCalledWith(input, { + userId: 'user-1', + requestId: 'request-1', + signal: controller.signal, + }) + }) + + it('requires trusted identity before parsing', async () => { + const response = await executeBoxTool({ + toolId: 'box_upload_file', + input: {}, + headers: new Headers(), + context: {}, + requestId: 'request-1', + }) + + expect(response.status).toBe(401) + expect(mocks.upload).not.toHaveBeenCalled() + }) + + it('uses only typed operation metadata and keeps OAuth and legacy content private', () => { + expect(boxUploadFileTool).not.toHaveProperty('request') + const params = { + accessToken: 'private-token', + parentFolderId: '0', + file, + fileContent: 'private-base64', + fileName: 'override.pdf', + } + expect(boxUploadFileTool.operation.modelInput?.select?.(params)).toEqual({ + parentFolderId: '0', + file, + fileName: 'override.pdf', + }) + expect(boxUploadFileTool.operation.input(params)).toEqual(params) + }) +}) diff --git a/apps/sim/lib/internal/box/execute-tool.ts b/apps/sim/lib/internal/box/execute-tool.ts new file mode 100644 index 00000000000..04775c4e257 --- /dev/null +++ b/apps/sim/lib/internal/box/execute-tool.ts @@ -0,0 +1,55 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { executeBoxUploadFile } from '@/lib/internal/box/operations' +import { boxUploadFileInputSchema } from '@/lib/internal/box/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +export const executeBoxTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + const userId = request.context.userId + if (!userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + if (request.toolId !== 'box_upload_file') { + return Response.json( + { success: false, error: `Unsupported Box tool: ${request.toolId}` }, + { status: 500 } + ) + } + let serializedInput: string + try { + serializedInput = JSON.stringify(request.input) ?? '' + } catch { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + if (Buffer.byteLength(serializedInput, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + success: false, + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + const parsed = boxUploadFileInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { success: false, error: getValidationErrorMessage(parsed.error, 'Invalid request data') }, + { status: 400 } + ) + } + try { + return await executeBoxUploadFile(parsed.data, { + userId, + requestId: request.requestId, + signal: request.signal, + }) + } catch (error) { + request.signal?.throwIfAborted() + return Response.json( + { success: false, error: getErrorMessage(error, 'Unknown error') }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/box/operations.test.ts b/apps/sim/lib/internal/box/operations.test.ts new file mode 100644 index 00000000000..16464784d38 --- /dev/null +++ b/apps/sim/lib/internal/box/operations.test.ts @@ -0,0 +1,116 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + clientConstructed: vi.fn(), + upload: vi.fn(), + processFiles: vi.fn(), + downloadStorage: vi.fn(), + assertAccess: vi.fn(), +})) + +vi.mock('@/lib/internal/box/client', () => { + class BoxUploadError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + } + } + class BoxClient { + constructor(token: string, signal?: AbortSignal) { + mocks.clientConstructed(token, signal) + } + + upload = mocks.upload + } + return { BoxClient, BoxUploadError } +}) + +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + processFilesToUserFiles: mocks.processFiles, +})) +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mocks.downloadStorage, +})) +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertAccess, +})) + +import { executeBoxUploadFile } from '@/lib/internal/box/operations' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' + +const rawFile = { key: 'uploads/file.pdf', name: 'file.pdf', size: 4 } +const userFile = { ...rawFile, type: 'application/pdf' } + +describe('executeBoxUploadFile', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.processFiles.mockReturnValue([userFile]) + mocks.assertAccess.mockResolvedValue(null) + mocks.downloadStorage.mockResolvedValue({ buffer: Buffer.from('file') }) + mocks.upload.mockResolvedValue({ id: 'box-1', name: 'override.pdf', size: 4 }) + }) + + it('authorizes provenance and propagates cancellation through storage and Box', async () => { + const controller = new AbortController() + const response = await executeBoxUploadFile( + { + accessToken: 'token', + parentFolderId: '0', + file: rawFile, + fileName: 'override.pdf', + }, + { userId: 'user-1', requestId: 'request-1', signal: controller.signal } + ) + + expect(mocks.assertAccess).toHaveBeenCalledWith( + userFile.key, + 'user-1', + 'request-1', + expect.anything() + ) + expect(mocks.downloadStorage).toHaveBeenCalledWith(userFile, 'request-1', expect.anything(), { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + signal: controller.signal, + }) + expect(mocks.clientConstructed).toHaveBeenCalledWith('token', controller.signal) + expect(mocks.upload).toHaveBeenCalledWith('0', 'override.pdf', Buffer.from('file')) + expect(await response.json()).toEqual({ + success: true, + output: { id: 'box-1', name: 'override.pdf', size: 4 }, + }) + }) + + it('never materializes an unauthorized file', async () => { + mocks.assertAccess.mockResolvedValue( + Response.json({ success: false, error: 'File not found' }, { status: 404 }) + ) + const response = await executeBoxUploadFile( + { accessToken: 'token', parentFolderId: '0', file: rawFile }, + { userId: 'user-1', requestId: 'request-1' } + ) + + expect(response.status).toBe(404) + expect(mocks.downloadStorage).not.toHaveBeenCalled() + expect(mocks.upload).not.toHaveBeenCalled() + }) + + it('preserves legacy base64 uploads without invoking file authorization', async () => { + await executeBoxUploadFile( + { + accessToken: 'token', + parentFolderId: 'folder-1', + fileContent: Buffer.from('legacy').toString('base64'), + fileName: 'legacy.txt', + }, + { userId: 'user-1', requestId: 'request-1' } + ) + + expect(mocks.assertAccess).not.toHaveBeenCalled() + expect(mocks.upload).toHaveBeenCalledWith('folder-1', 'legacy.txt', Buffer.from('legacy')) + }) +}) diff --git a/apps/sim/lib/internal/box/operations.ts b/apps/sim/lib/internal/box/operations.ts new file mode 100644 index 00000000000..773f49623d7 --- /dev/null +++ b/apps/sim/lib/internal/box/operations.ts @@ -0,0 +1,92 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { assertKnownSizeWithinLimit, isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { BoxClient, BoxUploadError } from '@/lib/internal/box/client' +import type { BoxUploadFileInput } from '@/lib/internal/box/schema' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +const logger = createLogger('BoxOperations') + +export interface BoxOperationContext { + userId: string + requestId: string + signal?: AbortSignal +} + +function failureResponse(error: string, status: number): Response { + return Response.json({ success: false, error }, { status }) +} + +export async function executeBoxUploadFile( + input: BoxUploadFileInput, + context: BoxOperationContext +): Promise { + context.signal?.throwIfAborted() + let buffer: Buffer + let fileName: string + + if (input.file) { + if (typeof input.file === 'string') return failureResponse('Invalid file input', 400) + const userFile = processFilesToUserFiles([input.file], context.requestId, logger)[0] + if (!userFile) return failureResponse('Invalid file input', 400) + const denied = await assertToolFileAccess( + userFile.key, + context.userId, + context.requestId, + logger + ) + context.signal?.throwIfAborted() + if (denied) return denied + try { + const resolved = await downloadServableFileFromStorage(userFile, context.requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + signal: context.signal, + }) + buffer = resolved.buffer + } catch (error) { + context.signal?.throwIfAborted() + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + return failureResponse( + getErrorMessage(error, 'Failed to download file'), + isPayloadSizeLimitError(error) ? 413 : 500 + ) + } + fileName = input.fileName || userFile.name + } else if (input.fileContent) { + buffer = Buffer.from(input.fileContent, 'base64') + try { + assertKnownSizeWithinLimit(buffer.length, MAX_BUFFERED_TRANSFER_BYTES, 'Box upload file') + } catch (error) { + return failureResponse( + getErrorMessage(error, 'Failed to decode file'), + isPayloadSizeLimitError(error) ? 413 : 500 + ) + } + fileName = input.fileName || 'file' + } else { + return failureResponse('File is required', 400) + } + + try { + const output = await new BoxClient(input.accessToken, context.signal).upload( + input.parentFolderId, + fileName, + buffer + ) + context.signal?.throwIfAborted() + return Response.json({ success: true, output }) + } catch (error) { + context.signal?.throwIfAborted() + if (error instanceof BoxUploadError) return failureResponse(error.message, error.status) + logger.error('Unexpected Box upload error', { + error: getErrorMessage(error), + requestId: context.requestId, + }) + return failureResponse(getErrorMessage(error, 'Unknown error'), 500) + } +} diff --git a/apps/sim/lib/internal/box/schema.ts b/apps/sim/lib/internal/box/schema.ts new file mode 100644 index 00000000000..420ac45fcfd --- /dev/null +++ b/apps/sim/lib/internal/box/schema.ts @@ -0,0 +1,12 @@ +import { z } from 'zod' +import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' + +export const boxUploadFileInputSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + parentFolderId: z.string().min(1, 'Parent folder ID is required'), + file: FileInputSchema.optional().nullable(), + fileContent: z.string().optional().nullable(), + fileName: z.string().optional().nullable(), +}) + +export type BoxUploadFileInput = z.output diff --git a/apps/sim/lib/internal/brex/client.test.ts b/apps/sim/lib/internal/brex/client.test.ts new file mode 100644 index 00000000000..1c07cfa4b7d --- /dev/null +++ b/apps/sim/lib/internal/brex/client.test.ts @@ -0,0 +1,93 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { DEFAULT_MAX_ERROR_BODY_BYTES } from '@/lib/core/utils/stream-limits' + +const mocks = vi.hoisted(() => ({ + validateUrl: vi.fn(), + pinnedFetch: vi.fn(), + fetch: vi.fn(), +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + validateUrlWithDNS: mocks.validateUrl, + secureFetchWithPinnedIP: mocks.pinnedFetch, +})) + +import { BrexReceiptClient, BrexReceiptError } from '@/lib/internal/brex/client' + +describe('BrexReceiptClient', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mocks.fetch) + mocks.validateUrl.mockResolvedValue({ isValid: true, resolvedIP: '52.216.0.1' }) + }) + + it('uses the exact expense upload contract and forwards cancellation', async () => { + const controller = new AbortController() + mocks.fetch.mockResolvedValue( + Response.json({ id: 'receipt-1', uri: 'https://upload.example/file' }) + ) + const target = await new BrexReceiptClient('token', controller.signal).createUploadTarget( + 'dinner.pdf', + 'expense/id' + ) + + expect(target).toEqual({ id: 'receipt-1', uri: 'https://upload.example/file' }) + expect(mocks.fetch).toHaveBeenCalledWith( + 'https://api.brex.com/v1/expenses/card/expense%2Fid/receipt_upload', + { + method: 'POST', + headers: { + Authorization: 'Bearer token', + Accept: 'application/json', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ receipt_name: 'dinner.pdf' }), + signal: controller.signal, + } + ) + }) + + it('preserves Brex provider status and message', async () => { + mocks.fetch.mockResolvedValue(Response.json({ message: 'Expense not found' }, { status: 404 })) + const error = await new BrexReceiptClient('token') + .createUploadTarget('receipt.pdf', 'expense-1') + .catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(BrexReceiptError) + expect(error).toMatchObject({ + message: 'Brex API error (404): Expense not found', + status: 404, + }) + }) + + it('pins the pre-signed upload and caps its response', async () => { + const controller = new AbortController() + mocks.pinnedFetch.mockResolvedValue(new Response(null, { status: 200 })) + const buffer = Buffer.from('receipt') + await new BrexReceiptClient('token', controller.signal).uploadReceipt( + 'https://upload.example/file', + buffer + ) + + expect(mocks.pinnedFetch).toHaveBeenCalledWith('https://upload.example/file', '52.216.0.1', { + method: 'PUT', + headers: { 'Content-Length': String(buffer.byteLength) }, + body: new Uint8Array(buffer), + maxResponseBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + signal: controller.signal, + }) + }) + + it('rejects a provider upload URL that fails SSRF validation', async () => { + mocks.validateUrl.mockResolvedValue({ isValid: false, error: 'blocked' }) + const error = await new BrexReceiptClient('token') + .uploadReceipt('https://169.254.169.254/latest', Buffer.from('receipt')) + .catch((caught: unknown) => caught) + + expect(error).toMatchObject({ message: 'Brex returned an invalid upload URL', status: 502 }) + expect(mocks.pinnedFetch).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/brex/client.ts b/apps/sim/lib/internal/brex/client.ts new file mode 100644 index 00000000000..eab3a5b0edf --- /dev/null +++ b/apps/sim/lib/internal/brex/client.ts @@ -0,0 +1,98 @@ +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + readResponseTextWithLimit, + readStreamToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' +import { BREX_API_BASE, buildBrexHeaders } from '@/tools/brex/utils' + +interface BrexReceiptUploadTarget { + id?: string + uri?: string +} + +export class BrexReceiptError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'BrexReceiptError' + } +} + +function parseBrexError(errorText: string): string { + try { + const parsed: unknown = JSON.parse(errorText) + if (parsed && typeof parsed === 'object' && 'message' in parsed) { + const message = (parsed as { message?: unknown }).message + if (typeof message === 'string') return message + } + } catch {} + return errorText +} + +export class BrexReceiptClient { + constructor( + private readonly apiKey: string, + private readonly signal?: AbortSignal + ) {} + + async createUploadTarget( + receiptName: string, + expenseId?: string + ): Promise { + this.signal?.throwIfAborted() + const endpoint = expenseId + ? `${BREX_API_BASE}/v1/expenses/card/${encodeURIComponent(expenseId)}/receipt_upload` + : `${BREX_API_BASE}/v1/expenses/card/receipt_match` + const response = await fetch(endpoint, { + method: 'POST', + headers: buildBrexHeaders(this.apiKey), + body: JSON.stringify({ receipt_name: receiptName }), + signal: this.signal, + }) + const body = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Brex receipt upload target response', + signal: this.signal, + }) + if (!response.ok) { + throw new BrexReceiptError( + `Brex API error (${response.status}): ${parseBrexError(body)}`, + response.status + ) + } + return JSON.parse(body) as BrexReceiptUploadTarget + } + + async uploadReceipt(uri: string, buffer: Buffer): Promise { + this.signal?.throwIfAborted() + const validation = await validateUrlWithDNS(uri, 'uri') + this.signal?.throwIfAborted() + if (!validation.isValid || !validation.resolvedIP) { + throw new BrexReceiptError('Brex returned an invalid upload URL', 502) + } + const response = await secureFetchWithPinnedIP(uri, validation.resolvedIP, { + method: 'PUT', + headers: { 'Content-Length': String(buffer.byteLength) }, + body: new Uint8Array(buffer), + maxResponseBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + signal: this.signal, + }) + if (response.body) { + await readStreamToBufferWithLimit(response.body, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Brex pre-signed upload response', + signal: this.signal, + }) + } + this.signal?.throwIfAborted() + if (!response.ok) { + throw new BrexReceiptError(`Failed to upload receipt file (${response.status})`, 502) + } + } +} diff --git a/apps/sim/lib/internal/brex/execute-tool.test.ts b/apps/sim/lib/internal/brex/execute-tool.test.ts new file mode 100644 index 00000000000..ed7240ed975 --- /dev/null +++ b/apps/sim/lib/internal/brex/execute-tool.test.ts @@ -0,0 +1,116 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + match: vi.fn(), + upload: vi.fn(), +})) + +vi.mock('@/lib/internal/brex/operations', () => ({ + executeBrexMatchReceipt: mocks.match, + executeBrexUploadReceipt: mocks.upload, +})) + +import { executeBrexTool } from '@/lib/internal/brex/execute-tool' +import { brexMatchReceiptTool } from '@/tools/brex/match_receipt' +import { brexUploadReceiptTool } from '@/tools/brex/upload_receipt' + +const file = { key: 'uploads/receipt.pdf', name: 'receipt.pdf', size: 5 } +const context = { userId: 'user-1' } + +describe('executeBrexTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.match.mockResolvedValue(Response.json({ success: true, output: {} })) + mocks.upload.mockResolvedValue(Response.json({ success: true, output: {} })) + }) + + it.each([ + ['brex_match_receipt', mocks.match, { apiKey: 'token', file }], + ['brex_upload_receipt', mocks.upload, { apiKey: 'token', expenseId: 'expense-1', file }], + ])('dispatches %s through its typed operation', async (toolId, operation, input) => { + await executeBrexTool({ + toolId, + input, + headers: new Headers(), + context, + requestId: 'request-1', + }) + + expect(operation).toHaveBeenCalledWith( + input, + expect.objectContaining({ userId: 'user-1', requestId: 'request-1' }) + ) + }) + + it('validates upload-only expense IDs without falling back to receipt matching', async () => { + const response = await executeBrexTool({ + toolId: 'brex_upload_receipt', + input: { apiKey: 'token', expenseId: ' ', file }, + headers: new Headers(), + context, + requestId: 'request-1', + }) + + expect(response.status).toBe(400) + expect(mocks.upload).not.toHaveBeenCalled() + expect(mocks.match).not.toHaveBeenCalled() + }) + + it('requires trusted execution identity before parsing', async () => { + const response = await executeBrexTool({ + toolId: 'brex_match_receipt', + input: {}, + headers: new Headers(), + context: {}, + requestId: 'request-1', + }) + + expect(response.status).toBe(401) + expect(await response.json()).toEqual({ success: false, error: 'Authentication required' }) + }) + + it('awaits operation failures so the handler preserves its error envelope', async () => { + mocks.match.mockRejectedValueOnce(new Error('provider unavailable')) + + const response = await executeBrexTool({ + toolId: 'brex_match_receipt', + input: { apiKey: 'token', file }, + headers: new Headers(), + context, + requestId: 'request-1', + }) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'provider unavailable', + }) + }) +}) + +describe('Brex receipt internal tool declarations', () => { + it('keep provider credentials private while projecting the model-visible file reference', () => { + expect(brexMatchReceiptTool).not.toHaveProperty('request') + expect(brexUploadReceiptTool).not.toHaveProperty('request') + + const params = { + apiKey: 'private-token', + expenseId: 'expense-1', + file, + receiptName: 'dinner.pdf', + } + expect(brexUploadReceiptTool.operation.modelInput?.select?.(params)).toEqual({ + expenseId: 'expense-1', + file, + receiptName: 'dinner.pdf', + }) + expect(brexUploadReceiptTool.operation.input(params)).toEqual(params) + expect(brexMatchReceiptTool.operation.modelInput?.select?.(params)).toEqual({ + file, + receiptName: 'dinner.pdf', + }) + }) +}) diff --git a/apps/sim/lib/internal/brex/execute-tool.ts b/apps/sim/lib/internal/brex/execute-tool.ts new file mode 100644 index 00000000000..146c6272910 --- /dev/null +++ b/apps/sim/lib/internal/brex/execute-tool.ts @@ -0,0 +1,83 @@ +import { getErrorMessage } from '@sim/utils/errors' +import type { z } from 'zod' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { + type BrexReceiptOperationContext, + executeBrexMatchReceipt, + executeBrexUploadReceipt, +} from '@/lib/internal/brex/operations' +import { + brexMatchReceiptInputSchema, + brexUploadReceiptInputSchema, +} from '@/lib/internal/brex/schema' +import type { + InternalToolOperationCall, + InternalToolOperationHandler, +} from '@/lib/internal/tool-operations/types' + +async function executeParsed( + request: InternalToolOperationCall, + schema: S, + execute: (input: z.output, context: BrexReceiptOperationContext) => Promise +): Promise { + let serializedInput: string + try { + serializedInput = JSON.stringify(request.input) ?? '' + } catch { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + if (Buffer.byteLength(serializedInput, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + success: false, + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + const parsed = schema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { success: false, error: getValidationErrorMessage(parsed.error, 'Invalid request data') }, + { status: 400 } + ) + } + const userId = request.context.userId + if (!userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + const response = await execute(parsed.data, { + userId, + requestId: request.requestId, + signal: request.signal, + }) + request.signal?.throwIfAborted() + return response +} + +export const executeBrexTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!request.context.userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + try { + switch (request.toolId) { + case 'brex_match_receipt': + return await executeParsed(request, brexMatchReceiptInputSchema, executeBrexMatchReceipt) + case 'brex_upload_receipt': + return await executeParsed(request, brexUploadReceiptInputSchema, executeBrexUploadReceipt) + default: + return Response.json( + { success: false, error: `Unsupported Brex tool: ${request.toolId}` }, + { status: 500 } + ) + } + } catch (error) { + request.signal?.throwIfAborted() + return Response.json( + { success: false, error: getErrorMessage(error, 'Unknown error') }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/brex/operations.test.ts b/apps/sim/lib/internal/brex/operations.test.ts new file mode 100644 index 00000000000..bf087b344ce --- /dev/null +++ b/apps/sim/lib/internal/brex/operations.test.ts @@ -0,0 +1,146 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + clientConstructed: vi.fn(), + createTarget: vi.fn(), + uploadReceipt: vi.fn(), + processFiles: vi.fn(), + downloadStorage: vi.fn(), + assertAccess: vi.fn(), +})) + +vi.mock('@/lib/internal/brex/client', () => { + class BrexReceiptError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + } + } + class BrexReceiptClient { + constructor(apiKey: string, signal?: AbortSignal) { + mocks.clientConstructed(apiKey, signal) + } + + createUploadTarget = mocks.createTarget + uploadReceipt = mocks.uploadReceipt + } + return { BrexReceiptClient, BrexReceiptError } +}) + +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + processFilesToUserFiles: mocks.processFiles, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mocks.downloadStorage, +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertAccess, +})) + +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { + executeBrexMatchReceipt, + executeBrexUploadReceipt, + MAX_BREX_RECEIPT_BYTES, +} from '@/lib/internal/brex/operations' + +const rawFile = { key: 'uploads/receipt.pdf', name: 'receipt.pdf', size: 5 } +const userFile = { ...rawFile, type: 'application/pdf' } + +describe('Brex receipt operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.processFiles.mockReturnValue([userFile]) + mocks.assertAccess.mockResolvedValue(null) + mocks.downloadStorage.mockResolvedValue({ + buffer: Buffer.from('receipt-bytes'), + contentType: 'application/pdf', + }) + mocks.createTarget.mockResolvedValue({ id: 'receipt-1', uri: 'https://upload.example/file' }) + mocks.uploadReceipt.mockResolvedValue(undefined) + }) + + it('authorizes provenance and carries cancellation through matching and file upload', async () => { + const controller = new AbortController() + const response = await executeBrexMatchReceipt( + { apiKey: 'token', file: rawFile, receiptName: 'dinner.pdf' }, + { userId: 'user-1', requestId: 'request-1', signal: controller.signal } + ) + + expect(mocks.assertAccess).toHaveBeenCalledWith( + userFile.key, + 'user-1', + 'request-1', + expect.anything() + ) + expect(mocks.downloadStorage).toHaveBeenCalledWith(userFile, 'request-1', expect.anything(), { + maxBytes: MAX_BREX_RECEIPT_BYTES, + signal: controller.signal, + }) + expect(mocks.clientConstructed).toHaveBeenCalledWith('token', controller.signal) + expect(mocks.createTarget).toHaveBeenCalledWith('dinner.pdf', undefined) + expect(mocks.uploadReceipt).toHaveBeenCalledWith( + 'https://upload.example/file', + Buffer.from('receipt-bytes') + ) + expect(await response.json()).toEqual({ + success: true, + output: { receiptId: 'receipt-1', receiptName: 'dinner.pdf', expenseId: null }, + }) + }) + + it('preserves expense upload semantics and trims the validated expense ID', async () => { + const response = await executeBrexUploadReceipt( + { apiKey: 'token', expenseId: 'expense-1', file: rawFile }, + { userId: 'user-1', requestId: 'request-1' } + ) + + expect(mocks.createTarget).toHaveBeenCalledWith('receipt.pdf', 'expense-1') + expect(await response.json()).toEqual({ + success: true, + output: { receiptId: 'receipt-1', receiptName: 'receipt.pdf', expenseId: 'expense-1' }, + }) + }) + + it('does not load receipt bytes when file authorization fails', async () => { + mocks.assertAccess.mockResolvedValue( + Response.json({ success: false, error: 'File not found' }, { status: 404 }) + ) + const response = await executeBrexMatchReceipt( + { apiKey: 'token', file: rawFile }, + { userId: 'user-1', requestId: 'request-1' } + ) + + expect(response.status).toBe(404) + expect(mocks.downloadStorage).not.toHaveBeenCalled() + expect(mocks.createTarget).not.toHaveBeenCalled() + }) + + it('preserves the 50 MB receipt error surface', async () => { + mocks.downloadStorage.mockRejectedValue( + new PayloadSizeLimitError({ + label: 'receipt', + maxBytes: MAX_BREX_RECEIPT_BYTES, + observedBytes: MAX_BREX_RECEIPT_BYTES + 1, + }) + ) + const response = await executeBrexMatchReceipt( + { apiKey: 'token', file: rawFile }, + { userId: 'user-1', requestId: 'request-1' } + ) + + expect(response.status).toBe(400) + expect(await response.json()).toEqual({ + success: false, + error: 'Receipt file exceeds the 50 MB limit', + }) + expect(mocks.createTarget).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/brex/operations.ts b/apps/sim/lib/internal/brex/operations.ts new file mode 100644 index 00000000000..637f3a2700d --- /dev/null +++ b/apps/sim/lib/internal/brex/operations.ts @@ -0,0 +1,99 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { BrexReceiptClient, BrexReceiptError } from '@/lib/internal/brex/client' +import type { BrexMatchReceiptInput, BrexUploadReceiptInput } from '@/lib/internal/brex/schema' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +const logger = createLogger('BrexReceiptOperations') +export const MAX_BREX_RECEIPT_BYTES = 50 * 1024 * 1024 + +export interface BrexReceiptOperationContext { + userId: string + requestId: string + signal?: AbortSignal +} + +function failureResponse(error: string, status: number): Response { + return Response.json({ success: false, error }, { status }) +} + +async function executeReceipt( + input: BrexMatchReceiptInput | BrexUploadReceiptInput, + context: BrexReceiptOperationContext +): Promise { + context.signal?.throwIfAborted() + const userFiles = processFilesToUserFiles([input.file], context.requestId, logger) + const userFile = userFiles[0] + if (!userFile) return failureResponse('Invalid file input', 400) + + const denied = await assertToolFileAccess(userFile.key, context.userId, context.requestId, logger) + context.signal?.throwIfAborted() + if (denied) return denied + + let fileBuffer: Buffer + try { + const resolved = await downloadServableFileFromStorage(userFile, context.requestId, logger, { + maxBytes: MAX_BREX_RECEIPT_BYTES, + signal: context.signal, + }) + fileBuffer = resolved.buffer + } catch (error) { + context.signal?.throwIfAborted() + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + return failureResponse('Receipt file exceeds the 50 MB limit', 400) + } + logger.error('Failed to download Brex receipt file', { + error: getErrorMessage(error), + requestId: context.requestId, + }) + return failureResponse(getErrorMessage(error, 'Unknown error'), 500) + } + + const expenseId = 'expenseId' in input ? input.expenseId : undefined + const receiptName = input.receiptName || userFile.name + try { + const client = new BrexReceiptClient(input.apiKey, context.signal) + const target = await client.createUploadTarget(receiptName, expenseId) + if (!target.uri || !target.id) { + return failureResponse('Brex did not return an upload URL', 502) + } + await client.uploadReceipt(target.uri, fileBuffer) + context.signal?.throwIfAborted() + return Response.json({ + success: true, + output: { + receiptId: target.id, + receiptName, + expenseId: expenseId ?? null, + }, + }) + } catch (error) { + context.signal?.throwIfAborted() + if (error instanceof BrexReceiptError) return failureResponse(error.message, error.status) + logger.error('Unexpected Brex receipt error', { + error: getErrorMessage(error), + requestId: context.requestId, + }) + return failureResponse(getErrorMessage(error, 'Unknown error'), 500) + } +} + +export function executeBrexMatchReceipt( + input: BrexMatchReceiptInput, + context: BrexReceiptOperationContext +): Promise { + return executeReceipt(input, context) +} + +export function executeBrexUploadReceipt( + input: BrexUploadReceiptInput, + context: BrexReceiptOperationContext +): Promise { + return executeReceipt(input, context) +} diff --git a/apps/sim/lib/internal/brex/schema.ts b/apps/sim/lib/internal/brex/schema.ts new file mode 100644 index 00000000000..57fc5148108 --- /dev/null +++ b/apps/sim/lib/internal/brex/schema.ts @@ -0,0 +1,35 @@ +import { z } from 'zod' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' + +const apiKeySchema = z + .string() + .min(1, 'API key is required') + .max(512, 'API key is too long') + .regex(/^[\x21-\x7e]+$/, 'API key contains invalid characters') + +const receiptNameSchema = z + .string() + .trim() + .min(1, 'Receipt name cannot be empty') + .max(255, 'Receipt name must be at most 255 characters') + .optional() + +const receiptInputShape = { + apiKey: apiKeySchema, + file: RawFileInputSchema, + receiptName: receiptNameSchema, +} + +export const brexMatchReceiptInputSchema = z.object(receiptInputShape) + +export const brexUploadReceiptInputSchema = z.object({ + ...receiptInputShape, + expenseId: z + .string() + .trim() + .min(1, 'Expense ID cannot be empty') + .max(255, 'Expense ID must be at most 255 characters'), +}) + +export type BrexMatchReceiptInput = z.output +export type BrexUploadReceiptInput = z.output diff --git a/apps/sim/lib/internal/buffer/errors.ts b/apps/sim/lib/internal/buffer/errors.ts new file mode 100644 index 00000000000..6d1da94edff --- /dev/null +++ b/apps/sim/lib/internal/buffer/errors.ts @@ -0,0 +1,9 @@ +export class BufferOperationError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'BufferOperationError' + } +} diff --git a/apps/sim/lib/internal/buffer/execute-tool.test.ts b/apps/sim/lib/internal/buffer/execute-tool.test.ts new file mode 100644 index 00000000000..ad8d3876d66 --- /dev/null +++ b/apps/sim/lib/internal/buffer/execute-tool.test.ts @@ -0,0 +1,91 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + createBufferPost: vi.fn(), + editBufferPost: vi.fn(), +})) + +vi.mock('@/lib/internal/buffer/operations', () => ({ + createBufferPost: mocks.createBufferPost, + editBufferPost: mocks.editBufferPost, +})) + +import { BufferOperationError } from '@/lib/internal/buffer/errors' +import { executeBufferTool } from '@/lib/internal/buffer/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +function request(overrides: Partial = {}): InternalToolOperationCall { + return { + toolId: 'buffer_create_post', + input: { + apiKey: 'buffer-key', + channelId: 'channel-1', + text: 'Hello', + mode: 'addToQueue', + }, + headers: new Headers(), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeBufferTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.createBufferPost.mockResolvedValue({ success: true, output: { post: { id: 'post-1' } } }) + mocks.editBufferPost.mockResolvedValue({ success: true, output: { post: { id: 'post-1' } } }) + }) + + it('dispatches create with trusted execution identity and defaults', async () => { + const controller = new AbortController() + const response = await executeBufferTool(request({ signal: controller.signal })) + + expect(response.status).toBe(200) + expect(mocks.createBufferPost).toHaveBeenCalledWith( + expect.objectContaining({ schedulingType: 'automatic', mediaType: 'auto' }), + { + userId: 'user-1', + requestId: 'request-1', + signal: controller.signal, + } + ) + }) + + it('dispatches edit', async () => { + const response = await executeBufferTool( + request({ + toolId: 'buffer_edit_post', + input: { apiKey: 'buffer-key', postId: 'post-1', mode: 'shareNow' }, + }) + ) + + expect(response.status).toBe(200) + expect(mocks.editBufferPost).toHaveBeenCalledOnce() + }) + + it('rejects invalid scheduling before provider work', async () => { + const response = await executeBufferTool( + request({ input: { apiKey: 'buffer-key', channelId: 'channel-1', mode: 'customScheduled' } }) + ) + + expect(response.status).toBe(400) + expect(mocks.createBufferPost).not.toHaveBeenCalled() + }) + + it('preserves operation error status', async () => { + mocks.createBufferPost.mockRejectedValue(new BufferOperationError('File not found', 404)) + + const response = await executeBufferTool(request()) + + expect(response.status).toBe(404) + await expect(response.json()).resolves.toEqual({ success: false, error: 'File not found' }) + }) +}) diff --git a/apps/sim/lib/internal/buffer/execute-tool.ts b/apps/sim/lib/internal/buffer/execute-tool.ts new file mode 100644 index 00000000000..7018e177aef --- /dev/null +++ b/apps/sim/lib/internal/buffer/execute-tool.ts @@ -0,0 +1,44 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { BufferOperationError } from '@/lib/internal/buffer/errors' +import { bufferCreatePostInputSchema, bufferEditPostInputSchema } from '@/lib/internal/buffer/input' +import { createBufferPost, editBufferPost } from '@/lib/internal/buffer/operations' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +export const executeBufferTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + const userId = request.context.userId + if (!userId) { + return Response.json({ success: false, error: 'Unauthorized' }, { status: 401 }) + } + + try { + const context = { userId, requestId: request.requestId, signal: request.signal } + if (request.toolId === 'buffer_create_post') { + const parsed = bufferCreatePostInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + return Response.json(await createBufferPost(parsed.data, context)) + } + if (request.toolId === 'buffer_edit_post') { + const parsed = bufferEditPostInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + return Response.json(await editBufferPost(parsed.data, context)) + } + return Response.json( + { success: false, error: `Unsupported Buffer tool: ${request.toolId}` }, + { status: 500 } + ) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof BufferOperationError) { + return Response.json({ success: false, error: error.message }, { status: error.status }) + } + return Response.json( + { success: false, error: getErrorMessage(error, 'Buffer operation failed') }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/buffer/input.ts b/apps/sim/lib/internal/buffer/input.ts new file mode 100644 index 00000000000..f9e440cd6a2 --- /dev/null +++ b/apps/sim/lib/internal/buffer/input.ts @@ -0,0 +1,54 @@ +import { z } from 'zod' +import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' + +const sharedFields = { + apiKey: z.string().min(1, 'API key is required'), + text: z.string().max(50000, 'text is too long').optional().nullable(), + mode: z.enum(['addToQueue', 'shareNext', 'shareNow', 'customScheduled']), + schedulingType: z.enum(['automatic', 'notification']).default('automatic'), + dueAt: z + .string() + .datetime({ offset: true, message: 'dueAt must be an ISO 8601 timestamp' }) + .optional() + .nullable(), + saveToDraft: z.boolean().optional().nullable(), + media: FileInputSchema.optional().nullable(), + mediaType: z.enum(['auto', 'image', 'video']).default('auto'), + mediaAltText: z.string().max(1000, 'mediaAltText is too long').optional().nullable(), +} + +function validateDueAt(body: { mode: string; dueAt?: string | null }, ctx: z.RefinementCtx): void { + if (body.mode === 'customScheduled' && !body.dueAt) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['dueAt'], + message: 'dueAt is required when mode is customScheduled', + }) + } +} + +export const bufferCreatePostInputSchema = z + .object({ + ...sharedFields, + channelId: z.string().min(1, 'channelId is required'), + }) + .superRefine((body, ctx) => { + validateDueAt(body, ctx) + if (!body.text?.trim() && !body.media) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['text'], + message: 'Either text or media is required', + }) + } + }) + +export const bufferEditPostInputSchema = z + .object({ + ...sharedFields, + postId: z.string().min(1, 'postId is required'), + }) + .superRefine(validateDueAt) + +export type BufferCreatePostInput = z.output +export type BufferEditPostInput = z.output diff --git a/apps/sim/lib/internal/buffer/operations.test.ts b/apps/sim/lib/internal/buffer/operations.test.ts new file mode 100644 index 00000000000..d03661da628 --- /dev/null +++ b/apps/sim/lib/internal/buffer/operations.test.ts @@ -0,0 +1,90 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + resolveFileInputToUrl: vi.fn(), +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + resolveFileInputToUrl: mocks.resolveFileInputToUrl, +})) + +import { createBufferPost } from '@/lib/internal/buffer/operations' + +describe('Buffer operations', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.unstubAllGlobals() + }) + + it('sends exactly one provider mutation with the operation signal', async () => { + const controller = new AbortController() + const fetchMock = vi.fn().mockResolvedValue( + Response.json({ + data: { + createPost: { + __typename: 'PostActionSuccess', + post: { id: 'post-1', text: 'Hello' }, + }, + }, + }) + ) + vi.stubGlobal('fetch', fetchMock) + + const result = await createBufferPost( + { + apiKey: 'buffer-key', + channelId: 'channel-1', + text: 'Hello', + mode: 'addToQueue', + schedulingType: 'automatic', + mediaType: 'auto', + }, + { userId: 'user-1', requestId: 'request-1', signal: controller.signal } + ) + + expect(result.output.post.id).toBe('post-1') + expect(fetchMock).toHaveBeenCalledOnce() + expect(fetchMock).toHaveBeenCalledWith( + 'https://api.buffer.com', + expect.objectContaining({ signal: controller.signal }) + ) + }) + + it('resolves stored media with trusted user context before the provider call', async () => { + mocks.resolveFileInputToUrl.mockResolvedValue({ fileUrl: 'https://files.example/image.png' }) + const fetchMock = vi.fn().mockResolvedValue( + Response.json({ + data: { + createPost: { + __typename: 'PostActionSuccess', + post: { id: 'post-1' }, + }, + }, + }) + ) + vi.stubGlobal('fetch', fetchMock) + + await createBufferPost( + { + apiKey: 'buffer-key', + channelId: 'channel-1', + mode: 'addToQueue', + schedulingType: 'automatic', + mediaType: 'auto', + media: { key: 'workspace/ws/file-1', name: 'image.png', type: 'image/png' }, + }, + { userId: 'user-1', requestId: 'request-1' } + ) + + expect(mocks.resolveFileInputToUrl).toHaveBeenCalledWith( + expect.objectContaining({ userId: 'user-1', presignExpirySeconds: 604800 }) + ) + const request = fetchMock.mock.calls[0][1] + expect(JSON.parse(request.body).variables.input.assets).toEqual([ + { image: { url: 'https://files.example/image.png' } }, + ]) + }) +}) diff --git a/apps/sim/lib/internal/buffer/operations.ts b/apps/sim/lib/internal/buffer/operations.ts new file mode 100644 index 00000000000..71c7f6c0262 --- /dev/null +++ b/apps/sim/lib/internal/buffer/operations.ts @@ -0,0 +1,214 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { BufferOperationError } from '@/lib/internal/buffer/errors' +import type { BufferCreatePostInput, BufferEditPostInput } from '@/lib/internal/buffer/input' +import { resolveFileInputToUrl } from '@/lib/uploads/utils/file-utils.server' +import { + BUFFER_API_URL, + BUFFER_POST_SELECTION, + type BufferPostResponse, + bufferHeaders, + mapBufferPost, + parseBufferGraphQLResponse, +} from '@/tools/buffer/types' + +const logger = createLogger('BufferOperations') +const VIDEO_EXTENSIONS = ['.mp4', '.mov', '.m4v', '.webm', '.avi'] +const IMAGE_EXTENSIONS = ['.png', '.jpg', '.jpeg', '.gif', '.webp'] +const MEDIA_PROBE_TIMEOUT_MS = 5000 +const MEDIA_PRESIGN_EXPIRY_SECONDS = 7 * 24 * 60 * 60 + +const CREATE_POST_MUTATION = ` + mutation CreatePost($input: CreatePostInput!) { + createPost(input: $input) { + __typename + ... on PostActionSuccess { post { ${BUFFER_POST_SELECTION} } } + ... on MutationError { message } + } + } +` + +const EDIT_POST_MUTATION = ` + mutation EditPost($input: EditPostInput!) { + editPost(input: $input) { + __typename + ... on PostActionSuccess { post { ${BUFFER_POST_SELECTION} } } + ... on MutationError { message } + } + } +` + +export interface BufferOperationContext { + userId: string + requestId: string + signal?: AbortSignal +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function mediaKindFromExtension(pathOrName: string): 'image' | 'video' | null { + const lowered = pathOrName.toLowerCase().split(/[?#]/)[0] + if (VIDEO_EXTENSIONS.some((extension) => lowered.endsWith(extension))) return 'video' + if (IMAGE_EXTENSIONS.some((extension) => lowered.endsWith(extension))) return 'image' + return null +} + +async function resolveMediaKind(args: { + mimeType?: string + pathOrName: string + fileUrl: string + context: BufferOperationContext +}): Promise<'image' | 'video' | null> { + const { mimeType, pathOrName, fileUrl, context } = args + if (mimeType?.startsWith('video/')) return 'video' + if (mimeType?.startsWith('image/')) return 'image' + const extensionKind = mediaKindFromExtension(pathOrName) + if (extensionKind) return extensionKind + + try { + const validation = await validateUrlWithDNS(fileUrl, 'media') + context.signal?.throwIfAborted() + if (validation.isValid && validation.resolvedIP) { + const probe = await secureFetchWithPinnedIP(fileUrl, validation.resolvedIP, { + method: 'HEAD', + timeout: MEDIA_PROBE_TIMEOUT_MS, + signal: context.signal, + }) + const contentType = probe.headers.get('content-type') || '' + if (contentType.startsWith('video/')) return 'video' + if (contentType.startsWith('image/')) return 'image' + } + } catch (error) { + context.signal?.throwIfAborted() + logger.warn(`[${context.requestId}] Media content-type probe was inconclusive`, { + error: getErrorMessage(error, 'probe failed'), + }) + } + return null +} + +async function resolveMediaAsset( + input: BufferCreatePostInput | BufferEditPostInput, + context: BufferOperationContext +): Promise | undefined> { + if (!input.media) return undefined + context.signal?.throwIfAborted() + const media = input.media + const isFileInput = typeof media === 'object' + const resolution = await resolveFileInputToUrl({ + file: isFileInput ? media : undefined, + filePath: isFileInput ? undefined : media, + userId: context.userId, + requestId: context.requestId, + logger, + presignExpirySeconds: MEDIA_PRESIGN_EXPIRY_SECONDS, + }) + context.signal?.throwIfAborted() + if (resolution.error || !resolution.fileUrl) { + throw new BufferOperationError( + resolution.error?.message || 'Failed to resolve media file', + resolution.error?.status || 400 + ) + } + + const kind = + input.mediaType === 'image' || input.mediaType === 'video' + ? input.mediaType + : await resolveMediaKind({ + mimeType: isFileInput ? media.type : undefined, + pathOrName: isFileInput ? media.name || '' : media, + fileUrl: resolution.fileUrl, + context, + }) + if (!kind) { + throw new BufferOperationError( + 'Could not determine whether the media is an image or a video. Set mediaType to "image" or "video".', + 400 + ) + } + if (kind === 'video') return { video: { url: resolution.fileUrl } } + + const image: Record = { url: resolution.fileUrl } + if (input.mediaAltText?.trim()) image.metadata = { altText: input.mediaAltText.trim() } + return { image } +} + +async function executePostMutation(args: { + apiKey: string + mutation: string + input: Record + context: BufferOperationContext +}): Promise { + const { apiKey, mutation, input, context } = args + let result: Record + try { + const response = await fetch(BUFFER_API_URL, { + method: 'POST', + headers: bufferHeaders(apiKey), + body: JSON.stringify({ query: mutation, variables: { input } }), + signal: context.signal, + }) + const data = await parseBufferGraphQLResponse(response) + const candidate = data.createPost ?? data.editPost + result = isRecord(candidate) ? candidate : {} + } catch (error) { + context.signal?.throwIfAborted() + const message = getErrorMessage(error, 'Buffer API request failed') + logger.error(`[${context.requestId}] Buffer post mutation failed`, { error: message }) + throw new BufferOperationError(message, 502) + } + + if (result.__typename !== 'PostActionSuccess' || !isRecord(result.post)) { + const message = typeof result.message === 'string' ? result.message : 'Buffer rejected the post' + throw new BufferOperationError(message, 400) + } + return { success: true, output: { post: mapBufferPost(result.post) } } +} + +async function mutatePost( + input: BufferCreatePostInput | BufferEditPostInput, + context: BufferOperationContext +): Promise { + context.signal?.throwIfAborted() + const isEdit = 'postId' in input + const mutationInput: Record = { + mode: input.mode, + schedulingType: input.schedulingType, + } + if (isEdit) mutationInput.id = input.postId + else { + mutationInput.channelId = input.channelId + mutationInput.assets = [] + } + if (input.text != null && input.text !== '') mutationInput.text = input.text + if (input.dueAt) mutationInput.dueAt = input.dueAt + if (input.saveToDraft != null) mutationInput.saveToDraft = input.saveToDraft + const asset = await resolveMediaAsset(input, context) + if (asset) mutationInput.assets = [asset] + return executePostMutation({ + apiKey: input.apiKey, + mutation: isEdit ? EDIT_POST_MUTATION : CREATE_POST_MUTATION, + input: mutationInput, + context, + }) +} + +export function createBufferPost( + input: BufferCreatePostInput, + context: BufferOperationContext +): Promise { + return mutatePost(input, context) +} + +export function editBufferPost( + input: BufferEditPostInput, + context: BufferOperationContext +): Promise { + return mutatePost(input, context) +} diff --git a/apps/sim/lib/internal/clickhouse/client.test.ts b/apps/sim/lib/internal/clickhouse/client.test.ts new file mode 100644 index 00000000000..2c77749162e --- /dev/null +++ b/apps/sim/lib/internal/clickhouse/client.test.ts @@ -0,0 +1,166 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { ClickHouseConnectionConfig } from '@/lib/internal/clickhouse/client' + +const { mockValidateDatabaseHost, mockSecureFetchWithPinnedIP, mockValidateSqlWhereClause } = + vi.hoisted(() => ({ + mockValidateDatabaseHost: vi.fn(), + mockSecureFetchWithPinnedIP: vi.fn(), + mockValidateSqlWhereClause: vi.fn(), + })) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + MAX_JSON_API_RESPONSE_BYTES: 10 * 1024 * 1024, + validateDatabaseHost: mockValidateDatabaseHost, + secureFetchWithPinnedIP: mockSecureFetchWithPinnedIP, + validateSqlWhereClause: mockValidateSqlWhereClause, +})) + +import { executeClickHouseInsert, executeClickHouseQuery } from '@/lib/internal/clickhouse/sql' + +function makeConfig( + overrides: Partial = {} +): ClickHouseConnectionConfig { + return { + host: 'clickhouse.example.com', + port: 8123, + database: 'default', + username: 'default', + password: 'secret', + secure: false, + ...overrides, + } +} + +function okResponse(body: string, summary?: string) { + return { + ok: true, + status: 200, + statusText: 'OK', + text: async () => body, + headers: { + get: (name: string) => + name.toLowerCase() === 'x-clickhouse-summary' ? (summary ?? null) : null, + }, + } +} + +describe('clickhouseRequest DNS pinning', () => { + beforeEach(() => { + vi.clearAllMocks() + mockValidateDatabaseHost.mockResolvedValue({ + isValid: true, + resolvedIP: '93.184.216.34', + originalHostname: 'clickhouse.example.com', + }) + mockValidateSqlWhereClause.mockReturnValue({ isValid: true }) + mockSecureFetchWithPinnedIP.mockResolvedValue(okResponse('{"data":[{"x":1}],"rows":1}')) + }) + + it('pins the connection to the validated IP, not the attacker-controlled hostname', async () => { + await executeClickHouseQuery(makeConfig({ host: 'rebind.attacker.example' }), 'SELECT 1') + + expect(mockValidateDatabaseHost).toHaveBeenCalledWith('rebind.attacker.example', 'host') + expect(mockSecureFetchWithPinnedIP).toHaveBeenCalledTimes(1) + + const [url, pinnedIP, options] = mockSecureFetchWithPinnedIP.mock.calls[0] + expect(pinnedIP).toBe('93.184.216.34') + expect(url).toContain('rebind.attacker.example') + expect(options.method).toBe('POST') + expect(options.maxResponseBytes).toBe(10 * 1024 * 1024) + expect(options.timeout).toBe(30_000) + expect(options.redirectPolicy).toEqual({ + mode: 'standard', + sendCredentialsOnCrossOriginRedirect: false, + sensitiveHeaders: ['X-ClickHouse-User', 'X-ClickHouse-Key'], + }) + }) + + it('never issues the request when host validation fails (no SSRF window)', async () => { + mockValidateDatabaseHost.mockResolvedValue({ + isValid: false, + error: 'host resolves to a blocked IP address', + }) + + await expect(executeClickHouseQuery(makeConfig(), 'SELECT 1')).rejects.toThrow( + 'host resolves to a blocked IP address' + ) + expect(mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() + }) + + it('uses https and disallows http redirects when secure is true', async () => { + await executeClickHouseQuery(makeConfig({ secure: true, port: 8443 }), 'SELECT 1') + + const [url, , options] = mockSecureFetchWithPinnedIP.mock.calls[0] + expect(url).toMatch(/^https:\/\//) + expect(options.allowHttp).toBe(false) + }) + + it('allows http for the initial request when secure is false', async () => { + await executeClickHouseQuery(makeConfig({ secure: false }), 'SELECT 1') + + const [url, , options] = mockSecureFetchWithPinnedIP.mock.calls[0] + expect(url).toMatch(/^http:\/\//) + expect(options.allowHttp).toBe(true) + }) + + it('brackets an unbracketed IPv6 literal when constructing the request URL', async () => { + mockValidateDatabaseHost.mockResolvedValue({ + isValid: true, + resolvedIP: '2001:4860:4860::8888', + originalHostname: '2001:4860:4860::8888', + }) + + await executeClickHouseQuery(makeConfig({ host: '2001:4860:4860::8888' }), 'SELECT 1') + + const [url, pinnedIP] = mockSecureFetchWithPinnedIP.mock.calls[0] + expect(url).toMatch(/^http:\/\/\[2001:4860:4860::8888\]:8123\//) + expect(pinnedIP).toBe('2001:4860:4860::8888') + }) + + it('passes cancellation through to the pinned request', async () => { + const controller = new AbortController() + + await executeClickHouseQuery(makeConfig(), 'SELECT 1', {}, controller.signal) + + const [, , options] = mockSecureFetchWithPinnedIP.mock.calls[0] + expect(options.signal).toBe(controller.signal) + }) + + it('stops before host validation when execution is already cancelled', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeClickHouseQuery(makeConfig(), 'SELECT 1', {}, controller.signal) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mockValidateDatabaseHost).not.toHaveBeenCalled() + expect(mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() + }) + + it('sends the statement as the body with a matching Content-Length and auth headers', async () => { + await executeClickHouseInsert(makeConfig(), 'events', { id: 1 }) + + const [, , options] = mockSecureFetchWithPinnedIP.mock.calls[0] + expect(options.body).toContain('INSERT INTO `events` FORMAT JSONEachRow') + expect(options.headers['Content-Length']).toBe(String(Buffer.byteLength(options.body, 'utf-8'))) + expect(options.headers['X-ClickHouse-User']).toBe('default') + expect(options.headers['X-ClickHouse-Key']).toBe('secret') + }) + + it('propagates non-ok responses as errors with the body text', async () => { + mockSecureFetchWithPinnedIP.mockResolvedValue({ + ok: false, + status: 400, + statusText: 'Bad Request', + text: async () => 'Code: 62. DB::Exception: Syntax error', + headers: { get: () => null }, + }) + + await expect(executeClickHouseQuery(makeConfig(), 'SELECT 1')).rejects.toThrow( + 'Code: 62. DB::Exception: Syntax error' + ) + }) +}) diff --git a/apps/sim/lib/internal/clickhouse/client.ts b/apps/sim/lib/internal/clickhouse/client.ts new file mode 100644 index 00000000000..cde12f9ff3f --- /dev/null +++ b/apps/sim/lib/internal/clickhouse/client.ts @@ -0,0 +1,94 @@ +import { isIP } from 'node:net' +import { + MAX_JSON_API_RESPONSE_BYTES, + secureFetchWithPinnedIP, + validateDatabaseHost, +} from '@/lib/core/security/input-validation.server' + +const REQUEST_TIMEOUT_MS = 30_000 + +export interface ClickHouseConnectionConfig { + host: string + port: number + database: string + username: string + password: string + secure: boolean +} + +interface ClickHouseSummary { + read_rows?: string + written_rows?: string + result_rows?: string +} + +export interface ClickHouseHttpResult { + text: string + summary: ClickHouseSummary | null +} + +export interface ClickHouseRequestOptions { + readOnly?: boolean + signal?: AbortSignal +} + +function parseSummary(header: string | null): ClickHouseSummary | null { + if (!header) return null + try { + return JSON.parse(header) as ClickHouseSummary + } catch { + return null + } +} + +function formatUrlHost(host: string): string { + const unbracketed = host.startsWith('[') && host.endsWith(']') ? host.slice(1, -1) : host + return isIP(unbracketed) === 6 ? `[${unbracketed}]` : host +} + +/** Sends one bounded, DNS-pinned statement through ClickHouse's HTTP interface. */ +export async function requestClickHouse( + config: ClickHouseConnectionConfig, + statement: string, + options: ClickHouseRequestOptions = {} +): Promise { + options.signal?.throwIfAborted() + const hostValidation = await validateDatabaseHost(config.host, 'host') + options.signal?.throwIfAborted() + if (!hostValidation.isValid) { + throw new Error(hostValidation.error) + } + + const protocol = config.secure ? 'https' : 'http' + const url = new URL(`${protocol}://${formatUrlHost(config.host)}:${config.port}/`) + url.searchParams.set('database', config.database) + if (options.readOnly) url.searchParams.set('readonly', '1') + + const response = await secureFetchWithPinnedIP(url.toString(), hostValidation.resolvedIP!, { + method: 'POST', + headers: { + 'X-ClickHouse-User': config.username, + 'X-ClickHouse-Key': config.password, + 'Content-Type': 'text/plain; charset=utf-8', + 'Content-Length': String(Buffer.byteLength(statement, 'utf-8')), + }, + body: statement, + timeout: REQUEST_TIMEOUT_MS, + allowHttp: !config.secure, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + redirectPolicy: { + mode: 'standard', + sendCredentialsOnCrossOriginRedirect: false, + sensitiveHeaders: ['X-ClickHouse-User', 'X-ClickHouse-Key'], + }, + signal: options.signal, + }) + + const text = await response.text() + options.signal?.throwIfAborted() + if (!response.ok) { + throw new Error(text.trim() || `ClickHouse request failed with status ${response.status}`) + } + + return { text, summary: parseSummary(response.headers.get('x-clickhouse-summary')) } +} diff --git a/apps/sim/lib/internal/clickhouse/execute-tool.test.ts b/apps/sim/lib/internal/clickhouse/execute-tool.test.ts new file mode 100644 index 00000000000..a218fb8d393 --- /dev/null +++ b/apps/sim/lib/internal/clickhouse/execute-tool.test.ts @@ -0,0 +1,202 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operationMocks = vi.hoisted(() => ({ + executeClickHouseCountRows: vi.fn(), + executeClickHouseCreateDatabase: vi.fn(), + executeClickHouseCreateTable: vi.fn(), + executeClickHouseDelete: vi.fn(), + executeClickHouseDescribeTable: vi.fn(), + executeClickHouseDropDatabase: vi.fn(), + executeClickHouseDropPartition: vi.fn(), + executeClickHouseDropTable: vi.fn(), + executeClickHouseInsert: vi.fn(), + executeClickHouseInsertRows: vi.fn(), + executeClickHouseIntrospection: vi.fn(), + executeClickHouseKillQuery: vi.fn(), + executeClickHouseListClusters: vi.fn(), + executeClickHouseListDatabases: vi.fn(), + executeClickHouseListMutations: vi.fn(), + executeClickHouseListPartitions: vi.fn(), + executeClickHouseListRunningQueries: vi.fn(), + executeClickHouseListTables: vi.fn(), + executeClickHouseOptimizeTable: vi.fn(), + executeClickHouseQuery: vi.fn(), + executeClickHouseRenameTable: vi.fn(), + executeClickHouseShowCreateTable: vi.fn(), + executeClickHouseStatement: vi.fn(), + executeClickHouseTableStats: vi.fn(), + executeClickHouseTruncateTable: vi.fn(), + executeClickHouseUpdate: vi.fn(), +})) + +vi.mock('@/lib/internal/clickhouse/operations', () => operationMocks) + +import { executeClickHouseTool } from '@/lib/internal/clickhouse/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const CLICKHOUSE_TOOL_IDS = [ + 'clickhouse_count_rows', + 'clickhouse_create_database', + 'clickhouse_create_table', + 'clickhouse_delete', + 'clickhouse_describe_table', + 'clickhouse_drop_database', + 'clickhouse_drop_partition', + 'clickhouse_drop_table', + 'clickhouse_execute', + 'clickhouse_insert_rows', + 'clickhouse_insert', + 'clickhouse_introspect', + 'clickhouse_kill_query', + 'clickhouse_list_clusters', + 'clickhouse_list_databases', + 'clickhouse_list_mutations', + 'clickhouse_list_partitions', + 'clickhouse_list_running_queries', + 'clickhouse_list_tables', + 'clickhouse_optimize_table', + 'clickhouse_query', + 'clickhouse_rename_table', + 'clickhouse_show_create_table', + 'clickhouse_table_stats', + 'clickhouse_truncate_table', + 'clickhouse_update', +] as const + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'clickhouse_query', + input: { + host: 'clickhouse.example.com', + port: 8443, + database: 'analytics', + username: 'default', + password: 'secret', + secure: true, + query: 'SELECT 1', + }, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeClickHouseTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('validates the canonical contract and executes the matching operation', async () => { + const controller = new AbortController() + operationMocks.executeClickHouseQuery.mockResolvedValue({ + message: 'Query executed successfully. 1 row(s) returned.', + rows: [{ value: 1 }], + rowCount: 1, + }) + + const response = await executeClickHouseTool(createRequest({ signal: controller.signal })) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + message: 'Query executed successfully. 1 row(s) returned.', + rows: [{ value: 1 }], + rowCount: 1, + }) + expect(operationMocks.executeClickHouseQuery).toHaveBeenCalledWith( + { + host: 'clickhouse.example.com', + port: 8443, + database: 'analytics', + username: 'default', + password: 'secret', + secure: true, + query: 'SELECT 1', + }, + controller.signal + ) + }) + + it('returns the canonical validation envelope before provider work', async () => { + const response = await executeClickHouseTool(createRequest({ input: { host: '' } })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(operationMocks.executeClickHouseQuery).not.toHaveBeenCalled() + }) + + it('rejects non-object operation input', async () => { + const response = await executeClickHouseTool(createRequest({ input: '{' })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(operationMocks.executeClickHouseQuery).not.toHaveBeenCalled() + }) + + it.each(CLICKHOUSE_TOOL_IDS)('recognizes the canonical tool ID %s', async (toolId) => { + const response = await executeClickHouseTool(createRequest({ toolId, input: {} })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + }) + }) + + it('preserves the route-compatible provider error prefix', async () => { + operationMocks.executeClickHouseIntrospection.mockRejectedValue(new Error('connection refused')) + + const response = await executeClickHouseTool( + createRequest({ + toolId: 'clickhouse_introspect', + input: { + host: 'clickhouse.example.com', + port: 8443, + database: 'analytics', + username: 'default', + password: 'secret', + secure: true, + }, + }) + ) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'ClickHouse introspection failed: connection refused', + }) + }) + + it('rejects unsupported ClickHouse IDs without provider work', async () => { + const response = await executeClickHouseTool(createRequest({ toolId: 'clickhouse_unknown' })) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'Unsupported ClickHouse tool: clickhouse_unknown', + }) + }) + + it('propagates cancellation without converting it into a provider failure', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeClickHouseTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(operationMocks.executeClickHouseQuery).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/clickhouse/execute-tool.ts b/apps/sim/lib/internal/clickhouse/execute-tool.ts new file mode 100644 index 00000000000..a946b1607d8 --- /dev/null +++ b/apps/sim/lib/internal/clickhouse/execute-tool.ts @@ -0,0 +1,308 @@ +import { getErrorMessage } from '@sim/utils/errors' +import type { z } from 'zod' +import { + executeClickHouseCountRows, + executeClickHouseCreateDatabase, + executeClickHouseCreateTable, + executeClickHouseDelete, + executeClickHouseDescribeTable, + executeClickHouseDropDatabase, + executeClickHouseDropPartition, + executeClickHouseDropTable, + executeClickHouseInsert, + executeClickHouseInsertRows, + executeClickHouseIntrospection, + executeClickHouseKillQuery, + executeClickHouseListClusters, + executeClickHouseListDatabases, + executeClickHouseListMutations, + executeClickHouseListPartitions, + executeClickHouseListRunningQueries, + executeClickHouseListTables, + executeClickHouseOptimizeTable, + executeClickHouseQuery, + executeClickHouseRenameTable, + executeClickHouseShowCreateTable, + executeClickHouseStatement, + executeClickHouseTableStats, + executeClickHouseTruncateTable, + executeClickHouseUpdate, +} from '@/lib/internal/clickhouse/operations' +import { + clickhouseCountRowsInputSchema, + clickhouseCreateDatabaseInputSchema, + clickhouseCreateTableInputSchema, + clickhouseDeleteInputSchema, + clickhouseDescribeTableInputSchema, + clickhouseDropDatabaseInputSchema, + clickhouseDropPartitionInputSchema, + clickhouseDropTableInputSchema, + clickhouseExecuteInputSchema, + clickhouseInsertInputSchema, + clickhouseInsertRowsInputSchema, + clickhouseIntrospectInputSchema, + clickhouseKillQueryInputSchema, + clickhouseListClustersInputSchema, + clickhouseListDatabasesInputSchema, + clickhouseListMutationsInputSchema, + clickhouseListPartitionsInputSchema, + clickhouseListRunningQueriesInputSchema, + clickhouseListTablesInputSchema, + clickhouseOptimizeTableInputSchema, + clickhouseQueryInputSchema, + clickhouseRenameTableInputSchema, + clickhouseShowCreateTableInputSchema, + clickhouseTableStatsInputSchema, + clickhouseTruncateTableInputSchema, + clickhouseUpdateInputSchema, +} from '@/lib/internal/clickhouse/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +async function executeOperation( + schema: z.ZodType, + input: unknown, + execute: (input: TInput, signal?: AbortSignal) => Promise, + errorPrefix: string, + signal?: AbortSignal +): Promise { + const parsed = schema.safeParse(input) + if (!parsed.success) { + return Response.json( + { error: 'Invalid request data', details: parsed.error.issues }, + { status: 400 } + ) + } + + try { + const result = await execute(parsed.data, signal) + signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + signal?.throwIfAborted() + return Response.json( + { error: `${errorPrefix}: ${getErrorMessage(error, 'Unknown error occurred')}` }, + { status: 500 } + ) + } +} + +export const executeClickHouseTool: InternalToolOperationHandler = async ({ + toolId, + input, + signal, +}) => { + signal?.throwIfAborted() + + switch (toolId) { + case 'clickhouse_query': + return executeOperation( + clickhouseQueryInputSchema, + input, + executeClickHouseQuery, + 'ClickHouse query failed', + signal + ) + case 'clickhouse_execute': + return executeOperation( + clickhouseExecuteInputSchema, + input, + executeClickHouseStatement, + 'ClickHouse execute failed', + signal + ) + case 'clickhouse_insert': + return executeOperation( + clickhouseInsertInputSchema, + input, + executeClickHouseInsert, + 'ClickHouse insert failed', + signal + ) + case 'clickhouse_update': + return executeOperation( + clickhouseUpdateInputSchema, + input, + executeClickHouseUpdate, + 'ClickHouse update failed', + signal + ) + case 'clickhouse_delete': + return executeOperation( + clickhouseDeleteInputSchema, + input, + executeClickHouseDelete, + 'ClickHouse delete failed', + signal + ) + case 'clickhouse_introspect': + return executeOperation( + clickhouseIntrospectInputSchema, + input, + executeClickHouseIntrospection, + 'ClickHouse introspection failed', + signal + ) + case 'clickhouse_list_databases': + return executeOperation( + clickhouseListDatabasesInputSchema, + input, + executeClickHouseListDatabases, + 'ClickHouse list databases failed', + signal + ) + case 'clickhouse_list_tables': + return executeOperation( + clickhouseListTablesInputSchema, + input, + executeClickHouseListTables, + 'ClickHouse list tables failed', + signal + ) + case 'clickhouse_describe_table': + return executeOperation( + clickhouseDescribeTableInputSchema, + input, + executeClickHouseDescribeTable, + 'ClickHouse describe table failed', + signal + ) + case 'clickhouse_show_create_table': + return executeOperation( + clickhouseShowCreateTableInputSchema, + input, + executeClickHouseShowCreateTable, + 'ClickHouse show create table failed', + signal + ) + case 'clickhouse_count_rows': + return executeOperation( + clickhouseCountRowsInputSchema, + input, + executeClickHouseCountRows, + 'ClickHouse count rows failed', + signal + ) + case 'clickhouse_list_partitions': + return executeOperation( + clickhouseListPartitionsInputSchema, + input, + executeClickHouseListPartitions, + 'ClickHouse list partitions failed', + signal + ) + case 'clickhouse_list_mutations': + return executeOperation( + clickhouseListMutationsInputSchema, + input, + executeClickHouseListMutations, + 'ClickHouse list mutations failed', + signal + ) + case 'clickhouse_list_running_queries': + return executeOperation( + clickhouseListRunningQueriesInputSchema, + input, + executeClickHouseListRunningQueries, + 'ClickHouse list running queries failed', + signal + ) + case 'clickhouse_table_stats': + return executeOperation( + clickhouseTableStatsInputSchema, + input, + executeClickHouseTableStats, + 'ClickHouse table stats failed', + signal + ) + case 'clickhouse_list_clusters': + return executeOperation( + clickhouseListClustersInputSchema, + input, + executeClickHouseListClusters, + 'ClickHouse list clusters failed', + signal + ) + case 'clickhouse_create_database': + return executeOperation( + clickhouseCreateDatabaseInputSchema, + input, + executeClickHouseCreateDatabase, + 'ClickHouse create database failed', + signal + ) + case 'clickhouse_drop_database': + return executeOperation( + clickhouseDropDatabaseInputSchema, + input, + executeClickHouseDropDatabase, + 'ClickHouse drop database failed', + signal + ) + case 'clickhouse_create_table': + return executeOperation( + clickhouseCreateTableInputSchema, + input, + executeClickHouseCreateTable, + 'ClickHouse create table failed', + signal + ) + case 'clickhouse_drop_table': + return executeOperation( + clickhouseDropTableInputSchema, + input, + executeClickHouseDropTable, + 'ClickHouse drop table failed', + signal + ) + case 'clickhouse_truncate_table': + return executeOperation( + clickhouseTruncateTableInputSchema, + input, + executeClickHouseTruncateTable, + 'ClickHouse truncate table failed', + signal + ) + case 'clickhouse_rename_table': + return executeOperation( + clickhouseRenameTableInputSchema, + input, + executeClickHouseRenameTable, + 'ClickHouse rename table failed', + signal + ) + case 'clickhouse_optimize_table': + return executeOperation( + clickhouseOptimizeTableInputSchema, + input, + executeClickHouseOptimizeTable, + 'ClickHouse optimize table failed', + signal + ) + case 'clickhouse_drop_partition': + return executeOperation( + clickhouseDropPartitionInputSchema, + input, + executeClickHouseDropPartition, + 'ClickHouse drop partition failed', + signal + ) + case 'clickhouse_kill_query': + return executeOperation( + clickhouseKillQueryInputSchema, + input, + executeClickHouseKillQuery, + 'ClickHouse kill query failed', + signal + ) + case 'clickhouse_insert_rows': + return executeOperation( + clickhouseInsertRowsInputSchema, + input, + executeClickHouseInsertRows, + 'ClickHouse insert rows failed', + signal + ) + default: + return Response.json({ error: `Unsupported ClickHouse tool: ${toolId}` }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/clickhouse/operations.test.ts b/apps/sim/lib/internal/clickhouse/operations.test.ts new file mode 100644 index 00000000000..dce1dbce419 --- /dev/null +++ b/apps/sim/lib/internal/clickhouse/operations.test.ts @@ -0,0 +1,180 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const sqlMocks = vi.hoisted(() => ({ + executeClickHouseCountRows: vi.fn(), + executeClickHouseCreateDatabase: vi.fn(), + executeClickHouseCreateTable: vi.fn(), + executeClickHouseDelete: vi.fn(), + executeClickHouseDescribeTable: vi.fn(), + executeClickHouseDropDatabase: vi.fn(), + executeClickHouseDropPartition: vi.fn(), + executeClickHouseDropTable: vi.fn(), + executeClickHouseInsert: vi.fn(), + executeClickHouseInsertRows: vi.fn(), + executeClickHouseIntrospect: vi.fn(), + executeClickHouseKillQuery: vi.fn(), + executeClickHouseListClusters: vi.fn(), + executeClickHouseListDatabases: vi.fn(), + executeClickHouseListMutations: vi.fn(), + executeClickHouseListPartitions: vi.fn(), + executeClickHouseListRunningQueries: vi.fn(), + executeClickHouseListTables: vi.fn(), + executeClickHouseOptimizeTable: vi.fn(), + executeClickHouseQuery: vi.fn(), + executeClickHouseRenameTable: vi.fn(), + executeClickHouseShowCreateTable: vi.fn(), + executeClickHouseTableStats: vi.fn(), + executeClickHouseTruncateTable: vi.fn(), + executeClickHouseUpdate: vi.fn(), +})) + +vi.mock('@/lib/internal/clickhouse/sql', () => sqlMocks) + +import { + executeClickHouseCountRows, + executeClickHouseCreateTable, + executeClickHouseIntrospection, + executeClickHouseQuery, + executeClickHouseStatement, + executeClickHouseUpdate, +} from '@/lib/internal/clickhouse/operations' + +const CONNECTION = { + host: 'clickhouse.example.com', + port: 8443, + database: 'analytics', + username: 'default', + password: 'secret', + secure: true, +} as const + +describe('ClickHouse operations', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('enforces read-only query execution and preserves its route response', async () => { + const controller = new AbortController() + sqlMocks.executeClickHouseQuery.mockResolvedValue({ + rows: [{ value: 1 }], + rowCount: 1, + }) + + await expect( + executeClickHouseQuery({ ...CONNECTION, query: 'SELECT 1' }, controller.signal) + ).resolves.toEqual({ + message: 'Query executed successfully. 1 row(s) returned.', + rows: [{ value: 1 }], + rowCount: 1, + }) + expect(sqlMocks.executeClickHouseQuery).toHaveBeenCalledWith( + { ...CONNECTION, query: 'SELECT 1' }, + 'SELECT 1', + { enforceReadOnly: true }, + controller.signal + ) + }) + + it('keeps raw execution distinct from the read-only query operation', async () => { + sqlMocks.executeClickHouseQuery.mockResolvedValue({ rows: [], rowCount: 4 }) + + await expect( + executeClickHouseStatement({ ...CONNECTION, query: 'ALTER TABLE events DELETE WHERE id=1' }) + ).resolves.toEqual({ + message: 'Statement executed successfully. 4 row(s) returned or affected.', + rows: [], + rowCount: 4, + }) + expect(sqlMocks.executeClickHouseQuery).toHaveBeenCalledWith( + { ...CONNECTION, query: 'ALTER TABLE events DELETE WHERE id=1' }, + 'ALTER TABLE events DELETE WHERE id=1', + {}, + undefined + ) + }) + + it('preserves introspection tables and the database-specific message', async () => { + sqlMocks.executeClickHouseIntrospect.mockResolvedValue({ + tables: [ + { + name: 'events', + database: 'analytics', + engine: 'MergeTree', + columns: [], + }, + ], + }) + + await expect(executeClickHouseIntrospection(CONNECTION)).resolves.toEqual({ + message: "Schema introspection completed. Found 1 table(s) in database 'analytics'.", + tables: [ + { + name: 'events', + database: 'analytics', + engine: 'MergeTree', + columns: [], + }, + ], + }) + }) + + it('preserves asynchronous mutation response semantics', async () => { + sqlMocks.executeClickHouseUpdate.mockResolvedValue({ rows: [], rowCount: 3 }) + + await expect( + executeClickHouseUpdate({ + ...CONNECTION, + table: 'events', + data: { status: 'done' }, + where: 'id = 1', + }) + ).resolves.toEqual({ + message: + 'Update mutation submitted. ClickHouse mutations run asynchronously. 3 row(s) written.', + rows: [], + rowCount: 3, + }) + }) + + it('forwards all create-table fields and cancellation', async () => { + const controller = new AbortController() + sqlMocks.executeClickHouseCreateTable.mockResolvedValue(undefined) + const input = { + ...CONNECTION, + table: 'events', + columns: [ + { name: 'id', type: 'UInt64' }, + { name: 'created_at', type: 'DateTime' }, + ], + engine: 'MergeTree', + orderBy: 'id', + partitionBy: 'toYYYYMM(created_at)', + } + + await expect(executeClickHouseCreateTable(input, controller.signal)).resolves.toEqual({ + message: "Table 'events' created.", + rows: [], + rowCount: 0, + }) + expect(sqlMocks.executeClickHouseCreateTable).toHaveBeenCalledWith( + input, + 'events', + input.columns, + 'MergeTree', + 'id', + 'toYYYYMM(created_at)', + controller.signal + ) + }) + + it('preserves count response semantics', async () => { + sqlMocks.executeClickHouseCountRows.mockResolvedValue(12) + + await expect( + executeClickHouseCountRows({ ...CONNECTION, table: 'events', where: 'active = 1' }) + ).resolves.toEqual({ message: 'Table contains 12 row(s).', count: 12 }) + }) +}) diff --git a/apps/sim/lib/internal/clickhouse/operations.ts b/apps/sim/lib/internal/clickhouse/operations.ts new file mode 100644 index 00000000000..f7558a0d9a0 --- /dev/null +++ b/apps/sim/lib/internal/clickhouse/operations.ts @@ -0,0 +1,322 @@ +import type { + ClickhouseCountRowsInput, + ClickhouseCreateDatabaseInput, + ClickhouseCreateTableInput, + ClickhouseDeleteInput, + ClickhouseDescribeTableInput, + ClickhouseDropDatabaseInput, + ClickhouseDropPartitionInput, + ClickhouseDropTableInput, + ClickhouseExecuteInput, + ClickhouseInsertInput, + ClickhouseInsertRowsInput, + ClickhouseIntrospectInput, + ClickhouseKillQueryInput, + ClickhouseListClustersInput, + ClickhouseListDatabasesInput, + ClickhouseListMutationsInput, + ClickhouseListPartitionsInput, + ClickhouseListRunningQueriesInput, + ClickhouseListTablesInput, + ClickhouseOptimizeTableInput, + ClickhouseQueryInput, + ClickhouseRenameTableInput, + ClickhouseShowCreateTableInput, + ClickhouseTableStatsInput, + ClickhouseTruncateTableInput, + ClickhouseUpdateInput, +} from '@/lib/internal/clickhouse/schema' +import { + executeClickHouseCountRows as countRows, + executeClickHouseCreateDatabase as createDatabase, + executeClickHouseCreateTable as createTable, + executeClickHouseDelete as deleteRows, + executeClickHouseDescribeTable as describeTable, + executeClickHouseDropDatabase as dropDatabase, + executeClickHouseDropPartition as dropPartition, + executeClickHouseDropTable as dropTable, + executeClickHouseQuery as executeQuery, + executeClickHouseInsert as insertRow, + executeClickHouseInsertRows as insertRows, + executeClickHouseIntrospect as introspect, + executeClickHouseKillQuery as killQuery, + executeClickHouseListClusters as listClusters, + executeClickHouseListDatabases as listDatabases, + executeClickHouseListMutations as listMutations, + executeClickHouseListPartitions as listPartitions, + executeClickHouseListRunningQueries as listRunningQueries, + executeClickHouseListTables as listTables, + executeClickHouseOptimizeTable as optimizeTable, + executeClickHouseRenameTable as renameTable, + executeClickHouseShowCreateTable as showCreateTable, + executeClickHouseTableStats as tableStats, + executeClickHouseTruncateTable as truncateTable, + executeClickHouseUpdate as updateRows, +} from '@/lib/internal/clickhouse/sql' + +interface RowsResponse { + message: string + rows: unknown[] + rowCount: number +} + +function rowsResponse(message: string, rows: unknown[], rowCount: number): RowsResponse { + return { message, rows, rowCount } +} + +export async function executeClickHouseQuery( + input: ClickhouseQueryInput, + signal?: AbortSignal +): Promise { + const result = await executeQuery(input, input.query, { enforceReadOnly: true }, signal) + return rowsResponse( + `Query executed successfully. ${result.rowCount} row(s) returned.`, + result.rows, + result.rowCount + ) +} + +export async function executeClickHouseStatement( + input: ClickhouseExecuteInput, + signal?: AbortSignal +): Promise { + const result = await executeQuery(input, input.query, {}, signal) + return rowsResponse( + `Statement executed successfully. ${result.rowCount} row(s) returned or affected.`, + result.rows, + result.rowCount + ) +} + +export async function executeClickHouseInsert( + input: ClickhouseInsertInput, + signal?: AbortSignal +): Promise { + const result = await insertRow(input, input.table, input.data, signal) + return rowsResponse( + `Data inserted successfully. ${result.rowCount} row(s) affected.`, + result.rows, + result.rowCount + ) +} + +export async function executeClickHouseUpdate( + input: ClickhouseUpdateInput, + signal?: AbortSignal +): Promise { + const result = await updateRows(input, input.table, input.data, input.where, signal) + return rowsResponse( + `Update mutation submitted. ClickHouse mutations run asynchronously. ${result.rowCount} row(s) written.`, + result.rows, + result.rowCount + ) +} + +export async function executeClickHouseDelete( + input: ClickhouseDeleteInput, + signal?: AbortSignal +): Promise { + const result = await deleteRows(input, input.table, input.where, signal) + return rowsResponse( + `Delete mutation submitted. ClickHouse mutations run asynchronously. ${result.rowCount} row(s) affected.`, + result.rows, + result.rowCount + ) +} + +export async function executeClickHouseIntrospection( + input: ClickhouseIntrospectInput, + signal?: AbortSignal +) { + const result = await introspect(input, signal) + return { + message: `Schema introspection completed. Found ${result.tables.length} table(s) in database '${input.database}'.`, + tables: result.tables, + } +} + +export async function executeClickHouseListDatabases( + input: ClickhouseListDatabasesInput, + signal?: AbortSignal +): Promise { + const result = await listDatabases(input, signal) + return rowsResponse(`Found ${result.rowCount} database(s).`, result.rows, result.rowCount) +} + +export async function executeClickHouseListTables( + input: ClickhouseListTablesInput, + signal?: AbortSignal +): Promise { + const result = await listTables(input, signal) + return rowsResponse(`Found ${result.rowCount} table(s).`, result.rows, result.rowCount) +} + +export async function executeClickHouseDescribeTable( + input: ClickhouseDescribeTableInput, + signal?: AbortSignal +): Promise { + const result = await describeTable(input, input.table, signal) + return rowsResponse( + `Described table with ${result.rowCount} column(s).`, + result.rows, + result.rowCount + ) +} + +export async function executeClickHouseShowCreateTable( + input: ClickhouseShowCreateTableInput, + signal?: AbortSignal +) { + return { + message: 'Retrieved CREATE statement.', + ddl: await showCreateTable(input, input.table, signal), + } +} + +export async function executeClickHouseCountRows( + input: ClickhouseCountRowsInput, + signal?: AbortSignal +) { + const count = await countRows(input, input.table, input.where, signal) + return { message: `Table contains ${count} row(s).`, count } +} + +export async function executeClickHouseListPartitions( + input: ClickhouseListPartitionsInput, + signal?: AbortSignal +): Promise { + const result = await listPartitions(input, input.table, signal) + return rowsResponse(`Found ${result.rowCount} partition(s).`, result.rows, result.rowCount) +} + +export async function executeClickHouseListMutations( + input: ClickhouseListMutationsInput, + signal?: AbortSignal +): Promise { + const result = await listMutations(input, input.table, input.onlyRunning, signal) + return rowsResponse(`Found ${result.rowCount} mutation(s).`, result.rows, result.rowCount) +} + +export async function executeClickHouseListRunningQueries( + input: ClickhouseListRunningQueriesInput, + signal?: AbortSignal +): Promise { + const result = await listRunningQueries(input, signal) + return rowsResponse(`Found ${result.rowCount} running query(ies).`, result.rows, result.rowCount) +} + +export async function executeClickHouseTableStats( + input: ClickhouseTableStatsInput, + signal?: AbortSignal +): Promise { + const result = await tableStats(input, input.table, signal) + return rowsResponse( + `Retrieved stats for ${result.rowCount} table(s).`, + result.rows, + result.rowCount + ) +} + +export async function executeClickHouseListClusters( + input: ClickhouseListClustersInput, + signal?: AbortSignal +): Promise { + const result = await listClusters(input, signal) + return rowsResponse(`Found ${result.rowCount} cluster node(s).`, result.rows, result.rowCount) +} + +export async function executeClickHouseCreateDatabase( + input: ClickhouseCreateDatabaseInput, + signal?: AbortSignal +): Promise { + await createDatabase(input, input.name, signal) + return rowsResponse(`Database '${input.name}' created.`, [], 0) +} + +export async function executeClickHouseDropDatabase( + input: ClickhouseDropDatabaseInput, + signal?: AbortSignal +): Promise { + await dropDatabase(input, input.name, signal) + return rowsResponse(`Database '${input.name}' dropped.`, [], 0) +} + +export async function executeClickHouseCreateTable( + input: ClickhouseCreateTableInput, + signal?: AbortSignal +): Promise { + await createTable( + input, + input.table, + input.columns, + input.engine, + input.orderBy, + input.partitionBy, + signal + ) + return rowsResponse(`Table '${input.table}' created.`, [], 0) +} + +export async function executeClickHouseDropTable( + input: ClickhouseDropTableInput, + signal?: AbortSignal +): Promise { + await dropTable(input, input.table, signal) + return rowsResponse(`Table '${input.table}' dropped.`, [], 0) +} + +export async function executeClickHouseTruncateTable( + input: ClickhouseTruncateTableInput, + signal?: AbortSignal +): Promise { + await truncateTable(input, input.table, signal) + return rowsResponse(`Table '${input.table}' truncated.`, [], 0) +} + +export async function executeClickHouseRenameTable( + input: ClickhouseRenameTableInput, + signal?: AbortSignal +): Promise { + await renameTable(input, input.table, input.newTable, signal) + return rowsResponse(`Renamed table '${input.table}' to '${input.newTable}'.`, [], 0) +} + +export async function executeClickHouseOptimizeTable( + input: ClickhouseOptimizeTableInput, + signal?: AbortSignal +): Promise { + await optimizeTable(input, input.table, input.final, signal) + return rowsResponse(`Optimize submitted for table '${input.table}'.`, [], 0) +} + +export async function executeClickHouseDropPartition( + input: ClickhouseDropPartitionInput, + signal?: AbortSignal +): Promise { + await dropPartition(input, input.table, input.partition, signal) + return rowsResponse(`Dropped partition from table '${input.table}'.`, [], 0) +} + +export async function executeClickHouseKillQuery( + input: ClickhouseKillQueryInput, + signal?: AbortSignal +): Promise { + const result = await killQuery(input, input.queryId, signal) + return rowsResponse( + `Kill command executed for query '${input.queryId}'.`, + result.rows, + result.rowCount + ) +} + +export async function executeClickHouseInsertRows( + input: ClickhouseInsertRowsInput, + signal?: AbortSignal +): Promise { + const result = await insertRows(input, input.table, input.rows, signal) + return rowsResponse( + `Inserted ${result.rowCount} row(s) into '${input.table}'.`, + result.rows, + result.rowCount + ) +} diff --git a/apps/sim/lib/internal/clickhouse/schema.ts b/apps/sim/lib/internal/clickhouse/schema.ts new file mode 100644 index 00000000000..b7f6da37697 --- /dev/null +++ b/apps/sim/lib/internal/clickhouse/schema.ts @@ -0,0 +1,125 @@ +import { z } from 'zod' + +const booleanFlagSchema = (defaultValue: boolean) => + z + .union([z.boolean(), z.string()]) + .transform((value) => (typeof value === 'string' ? value.toLowerCase() === 'true' : value)) + .default(defaultValue) + +const nonEmptyRecordSchema = (message: string) => + z.record(z.string(), z.unknown()).refine((value) => Object.keys(value).length > 0, { message }) + +export const clickhouseConnectionInputSchema = z.object({ + host: z.string().min(1, 'Host is required'), + port: z.coerce.number().int().positive('Port must be a positive integer'), + database: z.string().min(1, 'Database name is required'), + username: z.string().min(1, 'Username is required'), + password: z.string().default(''), + secure: booleanFlagSchema(true), +}) + +const tableInputSchema = clickhouseConnectionInputSchema.extend({ + table: z.string().min(1, 'Table name is required'), +}) + +export const clickhouseQueryInputSchema = clickhouseConnectionInputSchema.extend({ + query: z.string().min(1, 'Query is required'), +}) +export const clickhouseExecuteInputSchema = clickhouseQueryInputSchema +export const clickhouseInsertInputSchema = clickhouseConnectionInputSchema.extend({ + table: z.string().min(1, 'Table name is required'), + data: nonEmptyRecordSchema('Data object cannot be empty'), +}) +export const clickhouseUpdateInputSchema = clickhouseConnectionInputSchema.extend({ + table: z.string().min(1, 'Table name is required'), + data: nonEmptyRecordSchema('Data object cannot be empty'), + where: z.string().min(1, 'WHERE clause is required'), +}) +export const clickhouseDeleteInputSchema = clickhouseConnectionInputSchema.extend({ + table: z.string().min(1, 'Table name is required'), + where: z.string().min(1, 'WHERE clause is required'), +}) +export const clickhouseIntrospectInputSchema = clickhouseConnectionInputSchema +export const clickhouseListDatabasesInputSchema = clickhouseConnectionInputSchema +export const clickhouseListTablesInputSchema = clickhouseConnectionInputSchema +export const clickhouseDescribeTableInputSchema = tableInputSchema +export const clickhouseShowCreateTableInputSchema = tableInputSchema +export const clickhouseCountRowsInputSchema = tableInputSchema.extend({ + where: z.string().optional(), +}) +export const clickhouseListPartitionsInputSchema = tableInputSchema +export const clickhouseListMutationsInputSchema = clickhouseConnectionInputSchema.extend({ + table: z.string().optional(), + onlyRunning: booleanFlagSchema(false), +}) +export const clickhouseListRunningQueriesInputSchema = clickhouseConnectionInputSchema +export const clickhouseTableStatsInputSchema = clickhouseConnectionInputSchema.extend({ + table: z.string().optional(), +}) +export const clickhouseListClustersInputSchema = clickhouseConnectionInputSchema +export const clickhouseCreateDatabaseInputSchema = clickhouseConnectionInputSchema.extend({ + name: z.string().min(1, 'Database name is required'), +}) +export const clickhouseDropDatabaseInputSchema = clickhouseConnectionInputSchema.extend({ + name: z.string().min(1, 'Database name is required'), +}) +export const clickhouseCreateTableInputSchema = clickhouseConnectionInputSchema.extend({ + table: z.string().min(1, 'Table name is required'), + columns: z + .array( + z.object({ + name: z.string().min(1, 'Column name is required'), + type: z.string().min(1, 'Column type is required'), + }) + ) + .min(1, 'At least one column is required'), + engine: z.string().min(1).default('MergeTree'), + orderBy: z.string().min(1, 'ORDER BY expression is required'), + partitionBy: z.string().optional(), +}) +export const clickhouseDropTableInputSchema = tableInputSchema +export const clickhouseTruncateTableInputSchema = tableInputSchema +export const clickhouseRenameTableInputSchema = tableInputSchema.extend({ + newTable: z.string().min(1, 'New table name is required'), +}) +export const clickhouseOptimizeTableInputSchema = tableInputSchema.extend({ + final: booleanFlagSchema(false), +}) +export const clickhouseDropPartitionInputSchema = tableInputSchema.extend({ + partition: z.string().min(1, 'Partition expression is required'), +}) +export const clickhouseKillQueryInputSchema = clickhouseConnectionInputSchema.extend({ + queryId: z.string().min(1, 'Query ID is required'), +}) +export const clickhouseInsertRowsInputSchema = tableInputSchema.extend({ + rows: z.array(z.record(z.string(), z.unknown())).min(1, 'At least one row is required'), +}) + +export type ClickhouseQueryInput = z.output +export type ClickhouseExecuteInput = z.output +export type ClickhouseInsertInput = z.output +export type ClickhouseUpdateInput = z.output +export type ClickhouseDeleteInput = z.output +export type ClickhouseIntrospectInput = z.output +export type ClickhouseListDatabasesInput = z.output +export type ClickhouseListTablesInput = z.output +export type ClickhouseDescribeTableInput = z.output +export type ClickhouseShowCreateTableInput = z.output +export type ClickhouseCountRowsInput = z.output +export type ClickhouseListPartitionsInput = z.output +export type ClickhouseListMutationsInput = z.output +export type ClickhouseListRunningQueriesInput = z.output< + typeof clickhouseListRunningQueriesInputSchema +> +export type ClickhouseTableStatsInput = z.output +export type ClickhouseListClustersInput = z.output +export type ClickhouseCreateDatabaseInput = z.output +export type ClickhouseDropDatabaseInput = z.output +export type ClickhouseCreateTableInput = z.output +export type ClickhouseDropTableInput = z.output +export type ClickhouseTruncateTableInput = z.output +export type ClickhouseRenameTableInput = z.output +export type ClickhouseOptimizeTableInput = z.output +export type ClickhouseDropPartitionInput = z.output +export type ClickhouseKillQueryInput = z.output +export type ClickhouseInsertRowsInput = z.output diff --git a/apps/sim/lib/internal/clickhouse/sql.ts b/apps/sim/lib/internal/clickhouse/sql.ts new file mode 100644 index 00000000000..fed3f169070 --- /dev/null +++ b/apps/sim/lib/internal/clickhouse/sql.ts @@ -0,0 +1,825 @@ +import { validateSqlWhereClause } from '@/lib/core/security/input-validation.server' +import { + type ClickHouseConnectionConfig, + type ClickHouseHttpResult, + requestClickHouse, +} from '@/lib/internal/clickhouse/client' + +export interface ClickHouseRowsResult { + rows: unknown[] + rowCount: number +} + +interface ClickHouseColumnRow { + table: string + name: string + type: string + default_kind?: string + default_expression?: string + is_in_primary_key?: number | string + is_in_sorting_key?: number | string + position?: number | string +} + +interface ClickHouseTableRow { + name: string + engine?: string + total_rows?: number | string | null +} + +export interface ClickHouseIntrospectionResult { + tables: Array<{ + name: string + database: string + engine: string + totalRows?: number + columns: Array<{ + name: string + type: string + defaultKind?: string + defaultExpression?: string + isInPrimaryKey: boolean + isInSortingKey: boolean + }> + }> +} + +/** + * Parses a ClickHouse `FORMAT JSON` response body into rows, falling back to the + * summary header's row counts for statements that do not return a result set. + */ +function parseRowsResult(result: ClickHouseHttpResult): ClickHouseRowsResult { + const trimmed = result.text.trim() + if (trimmed) { + try { + const parsed = JSON.parse(trimmed) as { data?: unknown[]; rows?: number } + if (parsed && Array.isArray(parsed.data)) { + const rowCount = typeof parsed.rows === 'number' ? parsed.rows : parsed.data.length + return { rows: parsed.data, rowCount } + } + } catch {} + } + + const written = Number(result.summary?.written_rows ?? 0) + const read = Number(result.summary?.read_rows ?? 0) + return { rows: [], rowCount: written || read || 0 } +} + +/** Read-only statement leaders that return a result set and never mutate data. */ +const READ_ONLY_STATEMENT = /^(select|with|show|describe|desc|explain|exists)\b/i + +/** + * Normalizes the output format of a read statement to JSON so the HTTP response + * can always be parsed into rows. Strips every `FORMAT ` clause — wherever + * it sits relative to a trailing `SETTINGS` clause — and appends a single canonical + * `FORMAT JSON`. The `format()` function and `FORMAT`/format names appearing inside + * strings or comments are ignored (the scan runs on comment/string-masked SQL). + * Non-read statements are returned untouched (their own FORMAT, e.g. JSONEachRow + * for inserts, is preserved). + */ +function ensureJsonFormat(query: string): string { + const trimmed = query.trim().replace(/;+\s*$/, '') + if (!READ_ONLY_STATEMENT.test(trimmed)) { + return trimmed + } + const masked = maskSqlNoise(trimmed) + const formatClause = /\bformat\s+[a-z0-9_]+\b/gi + const spans: Array<[number, number]> = [] + for (let match = formatClause.exec(masked); match !== null; match = formatClause.exec(masked)) { + spans.push([match.index, match.index + match[0].length]) + } + let result = trimmed + for (let i = spans.length - 1; i >= 0; i--) { + result = result.slice(0, spans[i][0]) + result.slice(spans[i][1]) + } + return `${result.replace(/\s+$/, '')}\nFORMAT JSON` +} + +/** + * Replaces string literals ('...'), quoted identifiers ("..." / `...`), and SQL + * comments (`-- …` and `/* … *​/`) with spaces so that structural scans (e.g. for + * statement-chaining semicolons) only see actual SQL code, not data or comments. + */ +function maskSqlNoise(sql: string): string { + let out = '' + let i = 0 + while (i < sql.length) { + const ch = sql[i] + if (ch === "'" || ch === '"' || ch === '`') { + out += ' ' + i++ + while (i < sql.length && sql[i] !== ch) { + if (ch !== '`' && sql[i] === '\\') { + out += ' ' + i += 2 + continue + } + out += ' ' + i++ + } + if (i < sql.length) { + out += ' ' + i++ + } + continue + } + if (ch === '-' && sql[i + 1] === '-') { + const newline = sql.indexOf('\n', i + 2) + const end = newline === -1 ? sql.length : newline + out += ' '.repeat(end - i) + i = end + continue + } + if (ch === '/' && sql[i + 1] === '*') { + const close = sql.indexOf('*/', i + 2) + const end = close === -1 ? sql.length : close + 2 + out += ' '.repeat(end - i) + i = end + continue + } + out += ch + i++ + } + return out +} + +/** + * Detects whether a statement chains a second statement after a `;`, ignoring + * semicolons inside string literals, quoted identifiers, and comments. A trailing + * semicolon (with only whitespace/comments after it) is allowed. + */ +function hasChainedStatement(sql: string): boolean { + return /;\s*\S/.test(maskSqlNoise(sql)) +} + +/** + * Write/DDL statement shapes that must never run under the read-only query + * operation, even when wrapped by a leading `WITH` CTE (e.g. `WITH … INSERT INTO …`). + * Patterns require the keyword's statement context (e.g. `insert into`, `alter table`) + * so SQL functions/columns like `truncate(x)` or `created_at` are not false-positives. + */ +const MUTATING_STATEMENT = [ + /\binsert\s+into\b/i, + /\bdelete\s+from\b/i, + /\bupdate\s+[\w.`"]+\s+set\b/i, + /\balter\s+table\b/i, + /\b(?:create|attach)\s+(?:or\s+replace\s+)?(?:temporary\s+)?(?:table|database|dictionary|view|materialized\s+view|live\s+view|function|user|role)\b/i, + /\bdrop\s+(?:table|database|dictionary|view|column|partition|index|function|user|role)\b/i, + /\btruncate\s+table\b/i, + /\brename\s+(?:table|database|dictionary)\b/i, + /\bdetach\s+(?:table|database|dictionary|view|permanently)\b/i, + /\b(?:grant|revoke)\b/i, + /\boptimize\s+table\b/i, +] + +/** Whether a statement performs a write/DDL anywhere (comments and strings masked out). */ +function isMutatingStatement(sql: string): boolean { + const masked = maskSqlNoise(sql) + return MUTATING_STATEMENT.some((pattern) => pattern.test(masked)) +} + +/** + * Strips leading whitespace, `--`/`/* … *​/` comments, and opening parens from a + * statement so the read-only leader keyword can be detected even when a query + * starts with a comment (e.g. `-- note\nSELECT …`) or wrapping parens. + */ +function stripLeadingNoise(sql: string): string { + let s = sql.trim() + for (;;) { + if (s.startsWith('--')) { + const newline = s.indexOf('\n') + s = (newline === -1 ? '' : s.slice(newline + 1)).trim() + } else if (s.startsWith('/*')) { + const close = s.indexOf('*/') + s = (close === -1 ? '' : s.slice(close + 2)).trim() + } else if (s.startsWith('(')) { + s = s.slice(1).trim() + } else { + return s + } + } +} + +export async function executeClickHouseQuery( + config: ClickHouseConnectionConfig, + query: string, + options: { enforceReadOnly?: boolean } = {}, + signal?: AbortSignal +): Promise { + if (options.enforceReadOnly) { + const leader = stripLeadingNoise(query) + if (!READ_ONLY_STATEMENT.test(leader)) { + throw new Error( + 'The query operation only allows read-only statements (SELECT, WITH, SHOW, DESCRIBE, EXPLAIN, EXISTS). Use the Execute Raw SQL operation to run writes or DDL.' + ) + } + if (hasChainedStatement(query)) { + throw new Error( + 'The query operation only allows a single statement; chained statements separated by ";" are not allowed. Use the Execute Raw SQL operation to run multiple statements.' + ) + } + if (isMutatingStatement(query)) { + throw new Error( + 'The query operation only allows read-only statements; a write or DDL statement (e.g. INSERT/ALTER/DROP, including after a WITH clause) was detected. Use the Execute Raw SQL operation instead.' + ) + } + } + const result = await requestClickHouse(config, ensureJsonFormat(query), { + readOnly: options.enforceReadOnly, + signal, + }) + return parseRowsResult(result) +} + +export async function executeClickHouseInsert( + config: ClickHouseConnectionConfig, + table: string, + data: Record, + signal?: AbortSignal +): Promise { + const sanitizedTable = sanitizeIdentifier(table) + const statement = `INSERT INTO ${sanitizedTable} FORMAT JSONEachRow\n${JSON.stringify(data)}` + const result = await requestClickHouse(config, statement, { signal }) + const written = Number(result.summary?.written_rows ?? 0) + return { rows: [], rowCount: written || 1 } +} + +export async function executeClickHouseUpdate( + config: ClickHouseConnectionConfig, + table: string, + data: Record, + where: string, + signal?: AbortSignal +): Promise { + validateWhereClause(where) + const sanitizedTable = sanitizeIdentifier(table) + const assignments = Object.entries(data) + .map(([column, value]) => `${sanitizeIdentifier(column)} = ${formatValue(value)}`) + .join(', ') + + if (!assignments) { + throw new Error('Update data object cannot be empty') + } + + const statement = `ALTER TABLE ${sanitizedTable} UPDATE ${assignments} WHERE ${where}` + const result = await requestClickHouse(config, statement, { signal }) + return { rows: [], rowCount: Number(result.summary?.written_rows ?? 0) } +} + +export async function executeClickHouseDelete( + config: ClickHouseConnectionConfig, + table: string, + where: string, + signal?: AbortSignal +): Promise { + validateWhereClause(where) + const sanitizedTable = sanitizeIdentifier(table) + const statement = `ALTER TABLE ${sanitizedTable} DELETE WHERE ${where}` + const result = await requestClickHouse(config, statement, { signal }) + return { rows: [], rowCount: Number(result.summary?.written_rows ?? 0) } +} + +export async function executeClickHouseIntrospect( + config: ClickHouseConnectionConfig, + signal?: AbortSignal +): Promise { + const database = quoteString(config.database) + + const tablesResult = await requestClickHouse( + config, + `SELECT name, engine, total_rows FROM system.tables WHERE database = ${database} ORDER BY name FORMAT JSON`, + { signal } + ) + const tableRows = parseDataArray(tablesResult.text) + + const columnsResult = await requestClickHouse( + config, + `SELECT table, name, type, default_kind, default_expression, is_in_primary_key, is_in_sorting_key, position FROM system.columns WHERE database = ${database} ORDER BY table, position FORMAT JSON`, + { signal } + ) + const columnRows = parseDataArray(columnsResult.text) + + const columnsByTable = new Map< + string, + ClickHouseIntrospectionResult['tables'][number]['columns'] + >() + for (const column of columnRows) { + const columns = columnsByTable.get(column.table) ?? [] + columns.push({ + name: column.name, + type: column.type, + defaultKind: column.default_kind || undefined, + defaultExpression: column.default_expression || undefined, + isInPrimaryKey: toBoolean(column.is_in_primary_key), + isInSortingKey: toBoolean(column.is_in_sorting_key), + }) + columnsByTable.set(column.table, columns) + } + + const tables = tableRows.map((table) => ({ + name: table.name, + database: config.database, + engine: table.engine ?? '', + totalRows: table.total_rows != null ? Number(table.total_rows) : undefined, + columns: columnsByTable.get(table.name) ?? [], + })) + + return { tables } +} + +function parseDataArray(text: string): T[] { + const trimmed = text.trim() + if (!trimmed) return [] + try { + const parsed = JSON.parse(trimmed) as { data?: T[] } + return Array.isArray(parsed.data) ? parsed.data : [] + } catch { + return [] + } +} + +function toBoolean(value: number | string | undefined): boolean { + return value === 1 || value === '1' +} + +/** + * Quotes and escapes a value for inline use in a ClickHouse statement. + * Strings use ClickHouse's backslash escaping for single quotes and backslashes. + */ +function formatValue(value: unknown): string { + if (value === null || value === undefined) { + return 'NULL' + } + if (typeof value === 'number') { + return Number.isFinite(value) ? String(value) : 'NULL' + } + if (typeof value === 'boolean') { + return value ? '1' : '0' + } + if (typeof value === 'object') { + return quoteString(JSON.stringify(value)) + } + return quoteString(String(value)) +} + +function quoteString(value: string): string { + return `'${value.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'` +} + +/** + * Validates and backtick-quotes a ClickHouse identifier, supporting + * `database.table` qualified names. + */ +export function sanitizeIdentifier(identifier: string): string { + if (identifier.includes('.')) { + return identifier + .split('.') + .map((part) => sanitizeSingleIdentifier(part)) + .join('.') + } + return sanitizeSingleIdentifier(identifier) +} + +function sanitizeSingleIdentifier(identifier: string): string { + const cleaned = identifier.replace(/`/g, '') + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(cleaned)) { + throw new Error( + `Invalid identifier: ${identifier}. Identifiers must start with a letter or underscore and contain only letters, numbers, and underscores.` + ) + } + return `\`${cleaned}\`` +} + +/** + * Rejects WHERE clauses containing SQL-injection or always-true tautology + * patterns so user-supplied conditions cannot broaden a mutation to every row. + * Delegates to the shared {@link validateSqlWhereClause} guard (defense-in-depth). + */ +function validateWhereClause(where: string): void { + const result = validateSqlWhereClause(where, 'WHERE clause') + if (!result.isValid) { + throw new Error(result.error) + } +} + +/** + * Runs a SELECT statement (which must already include `FORMAT JSON`) and returns + * the parsed rows and row count. + */ +async function runSelect( + config: ClickHouseConnectionConfig, + statement: string, + signal?: AbortSignal +): Promise { + const result = await requestClickHouse(config, statement, { signal }) + return parseRowsResult(result) +} + +/** + * Runs a statement that does not return a result set (DDL or mutation) and + * returns the number of written rows reported by the summary header. + */ +async function runStatement( + config: ClickHouseConnectionConfig, + statement: string, + signal?: AbortSignal +): Promise { + const result = await requestClickHouse(config, statement, { signal }) + return Number(result.summary?.written_rows ?? 0) +} + +/** + * Validates a free-form SQL expression (ORDER BY, PARTITION BY, engine args) + * rejecting statement terminators and comment sequences. + */ +function validateExpression(expression: string, label: string): void { + if (/;|--|\/\*|\*\//.test(expression)) { + throw new Error(`${label} contains a disallowed character`) + } +} + +/** + * Validates an ORDER BY / PARTITION BY expression that is spliced inside wrapping + * parentheses in the generated DDL. In addition to rejecting terminators/comments, + * it requires balanced parentheses (quote-aware) so the expression cannot close + * the wrapping `(...)` early and append extra clauses (e.g. `id) SETTINGS …`). + */ +function validateClauseExpression(expression: string, label: string): void { + const trimmed = expression.trim() + if (!trimmed) { + throw new Error(`${label} is required`) + } + if (/;|--|\/\*|\*\//.test(trimmed)) { + throw new Error(`${label} contains a disallowed sequence`) + } + let depth = 0 + let inString = false + for (let i = 0; i < trimmed.length; i++) { + const ch = trimmed[i] + if (inString) { + if (ch === '\\') i++ + else if (ch === "'") inString = false + continue + } + if (ch === "'") inString = true + else if (ch === '(') depth++ + else if (ch === ')') { + depth-- + if (depth < 0) { + throw new Error(`${label} has unbalanced parentheses`) + } + } + } + if (inString || depth !== 0) { + throw new Error(`${label} has unbalanced parentheses or quotes`) + } +} + +/** + * Validates a partition value for `DROP PARTITION`. ClickHouse partition values + * are literals (signed numbers or single-quoted strings) or a parenthesised tuple + * of such literals, so anything else is rejected — barewords like `ALL`, function + * calls, operators, and extra tokens that could broaden the statement beyond + * dropping a single partition. + */ +function validatePartitionExpression(partition: string): void { + const partitionPattern = + /^\(?\s*(?:'(?:[^'\\]|\\.)*'|-?\d+(?:\.\d+)?)(?:\s*,\s*(?:'(?:[^'\\]|\\.)*'|-?\d+(?:\.\d+)?))*\s*\)?$/ + if (!partitionPattern.test(partition.trim())) { + throw new Error( + "Partition must be a literal value or a tuple of literals (number or single-quoted string), e.g. 202401, '2024-01', or (2024, 'EU')" + ) + } +} + +export function executeClickHouseListDatabases( + config: ClickHouseConnectionConfig, + signal?: AbortSignal +): Promise { + return runSelect( + config, + 'SELECT name, engine, comment FROM system.databases ORDER BY name FORMAT JSON', + signal + ) +} + +export function executeClickHouseListTables( + config: ClickHouseConnectionConfig, + signal?: AbortSignal +): Promise { + return runSelect( + config, + `SELECT name, engine, total_rows AS totalRows, total_bytes AS totalBytes, comment FROM system.tables WHERE database = ${quoteString(config.database)} ORDER BY name FORMAT JSON`, + signal + ) +} + +export function executeClickHouseDescribeTable( + config: ClickHouseConnectionConfig, + table: string, + signal?: AbortSignal +): Promise { + const tableName = stripDatabasePrefix(table) + return runSelect( + config, + `SELECT name, type, default_kind AS defaultKind, default_expression AS defaultExpression, comment, is_in_primary_key AS isInPrimaryKey, is_in_sorting_key AS isInSortingKey FROM system.columns WHERE database = ${quoteString(config.database)} AND table = ${quoteString(tableName)} ORDER BY position FORMAT JSON`, + signal + ) +} + +export async function executeClickHouseShowCreateTable( + config: ClickHouseConnectionConfig, + table: string, + signal?: AbortSignal +): Promise { + const result = await runSelect( + config, + `SHOW CREATE TABLE ${sanitizeIdentifier(table)} FORMAT JSON`, + signal + ) + const firstRow = result.rows[0] as Record | undefined + if (!firstRow) { + return '' + } + const value = firstRow.statement ?? Object.values(firstRow)[0] + return typeof value === 'string' ? value : '' +} + +export async function executeClickHouseCountRows( + config: ClickHouseConnectionConfig, + table: string, + where?: string, + signal?: AbortSignal +): Promise { + let statement = `SELECT count() AS count FROM ${sanitizeIdentifier(table)}` + if (where?.trim()) { + validateWhereClause(where) + statement += ` WHERE ${where}` + } + const result = await runSelect(config, `${statement} FORMAT JSON`, signal) + const firstRow = result.rows[0] as { count?: number | string } | undefined + return firstRow?.count != null ? Number(firstRow.count) : 0 +} + +export function executeClickHouseListPartitions( + config: ClickHouseConnectionConfig, + table: string, + signal?: AbortSignal +): Promise { + const tableName = stripDatabasePrefix(table) + return runSelect( + config, + `SELECT partition, count() AS parts, sum(rows) AS rows, sum(bytes_on_disk) AS bytesOnDisk FROM system.parts WHERE database = ${quoteString(config.database)} AND table = ${quoteString(tableName)} AND active GROUP BY partition ORDER BY partition FORMAT JSON`, + signal + ) +} + +export function executeClickHouseListMutations( + config: ClickHouseConnectionConfig, + table?: string, + onlyRunning = false, + signal?: AbortSignal +): Promise { + const filters = [`database = ${quoteString(config.database)}`] + if (table?.trim()) { + filters.push(`table = ${quoteString(stripDatabasePrefix(table))}`) + } + if (onlyRunning) { + filters.push('is_done = 0') + } + return runSelect( + config, + `SELECT table, mutation_id AS mutationId, command, create_time AS createTime, is_done AS isDone, parts_to_do AS partsToDo, latest_fail_reason AS latestFailReason FROM system.mutations WHERE ${filters.join(' AND ')} ORDER BY create_time DESC FORMAT JSON`, + signal + ) +} + +export function executeClickHouseListRunningQueries( + config: ClickHouseConnectionConfig, + signal?: AbortSignal +): Promise { + return runSelect( + config, + 'SELECT query_id AS queryId, user, toFloat64(elapsed) AS elapsedSeconds, formatReadableSize(memory_usage) AS memoryUsage, query FROM system.processes ORDER BY elapsed DESC FORMAT JSON', + signal + ) +} + +export function executeClickHouseTableStats( + config: ClickHouseConnectionConfig, + table?: string, + signal?: AbortSignal +): Promise { + const filters = ['active', `database = ${quoteString(config.database)}`] + if (table?.trim()) { + filters.push(`table = ${quoteString(stripDatabasePrefix(table))}`) + } + return runSelect( + config, + `SELECT database, table, sum(rows) AS rows, sum(bytes_on_disk) AS bytesOnDisk, formatReadableSize(sum(bytes_on_disk)) AS sizeOnDisk, count() AS parts FROM system.parts WHERE ${filters.join(' AND ')} GROUP BY database, table ORDER BY sum(bytes_on_disk) DESC FORMAT JSON`, + signal + ) +} + +export function executeClickHouseListClusters( + config: ClickHouseConnectionConfig, + signal?: AbortSignal +): Promise { + return runSelect( + config, + 'SELECT cluster, shard_num AS shardNum, replica_num AS replicaNum, host_name AS hostName, port, is_local AS isLocal FROM system.clusters ORDER BY cluster, shard_num, replica_num FORMAT JSON', + signal + ) +} + +export async function executeClickHouseCreateDatabase( + config: ClickHouseConnectionConfig, + name: string, + signal?: AbortSignal +): Promise { + await requestClickHouse(config, `CREATE DATABASE IF NOT EXISTS ${sanitizeIdentifier(name)}`, { + signal, + }) +} + +export async function executeClickHouseDropDatabase( + config: ClickHouseConnectionConfig, + name: string, + signal?: AbortSignal +): Promise { + await requestClickHouse(config, `DROP DATABASE IF EXISTS ${sanitizeIdentifier(name)}`, { + signal, + }) +} + +/** + * Validates a single ClickHouse column type. Types may legitimately contain + * commas, single-quoted strings, `=`, and `-` inside their parameter parentheses + * (e.g. `Decimal(10, 2)`, `Enum8('a' = 1, 'b' = -2)`, `Map(String, UInt64)`, + * `Array(Tuple(a UInt8, b String))`). We allow those but reject anything that + * could break out of the single type literal and inject another column or SQL: + * comment/terminator sequences, a top-level (unparenthesised) comma, or an + * unbalanced closing paren. + */ +function validateColumnType(type: string): void { + const trimmed = type.trim() + if (!trimmed || !/^[A-Za-z_]/.test(trimmed)) { + throw new Error(`Invalid column type: ${type}`) + } + if (!/^[A-Za-z0-9_(),.\s'"=-]+$/.test(trimmed) || /--|;/.test(trimmed)) { + throw new Error(`Invalid column type: ${type}`) + } + let depth = 0 + let inString = false + for (let i = 0; i < trimmed.length; i++) { + const ch = trimmed[i] + if (inString) { + if (ch === '\\') i++ + else if (ch === "'") inString = false + continue + } + if (ch === "'") inString = true + else if (ch === '(') depth++ + else if (ch === ')') { + depth-- + if (depth < 0) throw new Error(`Invalid column type: ${type}`) + } else if (ch === ',' && depth === 0) { + throw new Error(`Invalid column type: ${type}`) + } + } + if (inString || depth !== 0) { + throw new Error(`Invalid column type: ${type}`) + } +} + +export async function executeClickHouseCreateTable( + config: ClickHouseConnectionConfig, + table: string, + columns: Array<{ name: string; type: string }>, + engine: string, + orderBy: string, + partitionBy?: string, + signal?: AbortSignal +): Promise { + if (!Array.isArray(columns) || columns.length === 0) { + throw new Error('At least one column definition is required') + } + + const columnDefs = columns.map((column) => { + if (!column?.name || !column?.type) { + throw new Error('Each column requires a name and type') + } + validateColumnType(column.type) + return `${sanitizeIdentifier(column.name)} ${column.type.trim()}` + }) + + if (!/^[A-Za-z][A-Za-z0-9]*(\(.*\))?$/.test(engine.trim())) { + throw new Error(`Invalid table engine: ${engine}`) + } + validateExpression(engine, 'Engine') + + if (!orderBy?.trim()) { + throw new Error('ORDER BY expression is required') + } + validateClauseExpression(orderBy, 'ORDER BY') + + let statement = `CREATE TABLE IF NOT EXISTS ${sanitizeIdentifier(table)} (${columnDefs.join(', ')}) ENGINE = ${engine.trim()}` + if (partitionBy?.trim()) { + validateClauseExpression(partitionBy, 'PARTITION BY') + statement += ` PARTITION BY (${partitionBy.trim()})` + } + statement += ` ORDER BY (${orderBy.trim()})` + + await requestClickHouse(config, statement, { signal }) +} + +export async function executeClickHouseDropTable( + config: ClickHouseConnectionConfig, + table: string, + signal?: AbortSignal +): Promise { + await requestClickHouse(config, `DROP TABLE IF EXISTS ${sanitizeIdentifier(table)}`, { signal }) +} + +export async function executeClickHouseTruncateTable( + config: ClickHouseConnectionConfig, + table: string, + signal?: AbortSignal +): Promise { + await requestClickHouse(config, `TRUNCATE TABLE IF EXISTS ${sanitizeIdentifier(table)}`, { + signal, + }) +} + +export async function executeClickHouseRenameTable( + config: ClickHouseConnectionConfig, + fromTable: string, + toTable: string, + signal?: AbortSignal +): Promise { + await requestClickHouse( + config, + `RENAME TABLE ${sanitizeIdentifier(fromTable)} TO ${sanitizeIdentifier(toTable)}`, + { signal } + ) +} + +export async function executeClickHouseOptimizeTable( + config: ClickHouseConnectionConfig, + table: string, + final: boolean, + signal?: AbortSignal +): Promise { + await requestClickHouse( + config, + `OPTIMIZE TABLE ${sanitizeIdentifier(table)}${final ? ' FINAL' : ''}`, + { signal } + ) +} + +export async function executeClickHouseDropPartition( + config: ClickHouseConnectionConfig, + table: string, + partition: string, + signal?: AbortSignal +): Promise { + validatePartitionExpression(partition) + await requestClickHouse( + config, + `ALTER TABLE ${sanitizeIdentifier(table)} DROP PARTITION ${partition.trim()}`, + { signal } + ) +} + +export function executeClickHouseKillQuery( + config: ClickHouseConnectionConfig, + queryId: string, + signal?: AbortSignal +): Promise { + return runSelect( + config, + `KILL QUERY WHERE query_id = ${quoteString(queryId)} SYNC FORMAT JSON`, + signal + ) +} + +export async function executeClickHouseInsertRows( + config: ClickHouseConnectionConfig, + table: string, + rows: Array>, + signal?: AbortSignal +): Promise { + if (!Array.isArray(rows) || rows.length === 0) { + throw new Error('At least one row is required') + } + const sanitizedTable = sanitizeIdentifier(table) + const payload = rows.map((row) => JSON.stringify(row)).join('\n') + const statement = `INSERT INTO ${sanitizedTable} FORMAT JSONEachRow\n${payload}` + const written = await runStatement(config, statement, signal) + return { rows: [], rowCount: written || rows.length } +} + +function stripDatabasePrefix(table: string): string { + const parts = table.split('.') + return parts[parts.length - 1].replace(/`/g, '') +} diff --git a/apps/sim/lib/internal/clickup/client.test.ts b/apps/sim/lib/internal/clickup/client.test.ts new file mode 100644 index 00000000000..e77484d265f --- /dev/null +++ b/apps/sim/lib/internal/clickup/client.test.ts @@ -0,0 +1,71 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + isPayloadSizeLimitError: vi.fn(), + readResponseJsonWithLimit: vi.fn(), +})) + +vi.mock('@/lib/core/utils/stream-limits', () => ({ + isPayloadSizeLimitError: mocks.isPayloadSizeLimitError, + readResponseJsonWithLimit: mocks.readResponseJsonWithLimit, +})) + +import { uploadClickUpAttachment } from '@/lib/internal/clickup/client' +import { ClickUpOperationError } from '@/lib/internal/clickup/errors' + +describe('uploadClickUpAttachment', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(null, { status: 200 }))) + mocks.readResponseJsonWithLimit.mockResolvedValue({ id: 'attachment-1' }) + mocks.isPayloadSizeLimitError.mockReturnValue(false) + }) + + it('maps oversized provider responses to a bounded operation error', async () => { + const sizeError = Object.assign(new Error('response too large'), { + maxBytes: 64, + observedBytes: 65, + }) + mocks.readResponseJsonWithLimit.mockRejectedValueOnce(sizeError) + mocks.isPayloadSizeLimitError.mockReturnValueOnce(true) + + const error = await uploadClickUpAttachment('token', 'task-1', new FormData()).catch( + (caught: unknown) => caught + ) + + expect(error).toBeInstanceOf(ClickUpOperationError) + expect(error).toMatchObject({ status: 413 }) + }) + + it('preserves provider status when a bodyless response has no declared size', async () => { + const unavailableBodyError = Object.assign(new Error('response body unavailable'), { + maxBytes: 64, + observedBytes: undefined, + }) + vi.mocked(fetch).mockResolvedValueOnce(new Response(null, { status: 503 })) + mocks.readResponseJsonWithLimit.mockRejectedValueOnce(unavailableBodyError) + mocks.isPayloadSizeLimitError.mockReturnValueOnce(true) + + const error = await uploadClickUpAttachment('token', 'task-1', new FormData()).catch( + (caught: unknown) => caught + ) + + expect(error).toBeInstanceOf(ClickUpOperationError) + expect(error).toMatchObject({ status: 503 }) + }) + + it('does not swallow cancellation while reading the provider response', async () => { + const controller = new AbortController() + mocks.readResponseJsonWithLimit.mockImplementationOnce(async () => { + controller.abort(new DOMException('cancelled', 'AbortError')) + throw controller.signal.reason + }) + + await expect( + uploadClickUpAttachment('token', 'task-1', new FormData(), controller.signal) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) +}) diff --git a/apps/sim/lib/internal/clickup/client.ts b/apps/sim/lib/internal/clickup/client.ts new file mode 100644 index 00000000000..78da954d554 --- /dev/null +++ b/apps/sim/lib/internal/clickup/client.ts @@ -0,0 +1,51 @@ +import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' +import { ClickUpOperationError } from '@/lib/internal/clickup/errors' +import { + CLICKUP_API_BASE_URL, + clickupAuthorizationHeader, + extractClickUpErrorMessage, +} from '@/tools/clickup/shared' + +export async function uploadClickUpAttachment( + accessToken: string, + taskId: string, + formData: FormData, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const response = await fetch( + `${CLICKUP_API_BASE_URL}/task/${encodeURIComponent(taskId)}/attachment`, + { + method: 'POST', + headers: { Authorization: clickupAuthorizationHeader(accessToken) }, + body: formData, + signal, + } + ) + let data: unknown + try { + data = await readResponseJsonWithLimit(response, { + maxBytes: MAX_JSON_API_RESPONSE_BYTES, + label: 'ClickUp attachment response', + signal, + }) + } catch (error) { + signal?.throwIfAborted() + if ( + isPayloadSizeLimitError(error) && + error.observedBytes !== undefined && + error.observedBytes > error.maxBytes + ) { + throw new ClickUpOperationError('ClickUp attachment response exceeded the size limit', 413) + } + data = null + } + if (!response.ok) { + throw new ClickUpOperationError( + extractClickUpErrorMessage(response, data, 'Failed to upload ClickUp attachment'), + response.status + ) + } + return data +} diff --git a/apps/sim/lib/internal/clickup/errors.ts b/apps/sim/lib/internal/clickup/errors.ts new file mode 100644 index 00000000000..60a7f61281c --- /dev/null +++ b/apps/sim/lib/internal/clickup/errors.ts @@ -0,0 +1,10 @@ +export class ClickUpOperationError extends Error { + constructor( + message: string, + readonly status: number, + readonly body: Record = { success: false, error: message } + ) { + super(message) + this.name = 'ClickUpOperationError' + } +} diff --git a/apps/sim/lib/internal/clickup/execute-tool.ts b/apps/sim/lib/internal/clickup/execute-tool.ts new file mode 100644 index 00000000000..b7bba66717b --- /dev/null +++ b/apps/sim/lib/internal/clickup/execute-tool.ts @@ -0,0 +1,68 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { ClickUpOperationError } from '@/lib/internal/clickup/errors' +import { executeClickUpUploadAttachment } from '@/lib/internal/clickup/operations' +import { clickupUploadAttachmentInputSchema } from '@/lib/internal/clickup/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const logger = createLogger('ClickUpToolExecution') + +function inputSizeError(input: unknown): Response | null { + let serialized: string + try { + serialized = JSON.stringify(input) ?? '' + } catch { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + return Buffer.byteLength(serialized) > DEFAULT_MAX_JSON_BODY_BYTES + ? Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + : null +} + +export const executeClickUpTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'clickup_upload_attachment') { + return Response.json( + { success: false, error: `Unsupported ClickUp tool: ${request.toolId}` }, + { status: 500 } + ) + } + const userId = request.context.userId + if (!userId) return Response.json({ success: false, error: 'Unauthorized' }, { status: 401 }) + const sizeError = inputSizeError(request.input) + if (sizeError) return sizeError + const parsed = clickupUploadAttachmentInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { success: false, error: getValidationErrorMessage(parsed.error, 'Invalid request data') }, + { status: 400 } + ) + } + try { + const result = await executeClickUpUploadAttachment(parsed.data, { + requestId: request.requestId, + signal: request.signal, + userId, + }) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof ClickUpOperationError) { + return Response.json(error.body, { status: error.status }) + } + const message = getErrorMessage(error, 'Internal server error') + logger.error('ClickUp attachment upload failed', { + error: message, + requestId: request.requestId, + }) + return Response.json({ success: false, error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/clickup/operations.test.ts b/apps/sim/lib/internal/clickup/operations.test.ts new file mode 100644 index 00000000000..b8c2a308ef0 --- /dev/null +++ b/apps/sim/lib/internal/clickup/operations.test.ts @@ -0,0 +1,61 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + assertToolFileAccess: vi.fn(), + downloadServableFileFromStorage: vi.fn(), + processFilesToUserFiles: vi.fn(), + uploadClickUpAttachment: vi.fn(), +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertToolFileAccess, +})) +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + processFilesToUserFiles: mocks.processFilesToUserFiles, +})) +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mocks.downloadServableFileFromStorage, +})) +vi.mock('@/lib/internal/clickup/client', () => ({ + uploadClickUpAttachment: mocks.uploadClickUpAttachment, +})) + +import { executeClickUpUploadAttachment } from '@/lib/internal/clickup/operations' + +const rawFile = { key: 'uploads/file.txt', name: 'file.txt', size: 4 } +const userFile = { ...rawFile, type: 'text/plain' } + +describe('executeClickUpUploadAttachment', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.processFilesToUserFiles.mockReturnValue([userFile]) + mocks.assertToolFileAccess.mockResolvedValue(null) + mocks.downloadServableFileFromStorage.mockResolvedValue({ + buffer: Buffer.from('file'), + contentType: 'text/plain', + }) + mocks.uploadClickUpAttachment.mockResolvedValue({ id: 'attachment-1' }) + }) + + it('accepts serialized advanced-mode file inputs without weakening authorization', async () => { + await executeClickUpUploadAttachment( + { accessToken: 'token', taskId: 'task-1', file: JSON.stringify(rawFile) }, + { requestId: 'request-1', userId: 'user-1' } + ) + + expect(mocks.processFilesToUserFiles).toHaveBeenCalledWith( + [rawFile], + 'request-1', + expect.anything() + ) + expect(mocks.assertToolFileAccess).toHaveBeenCalledWith( + userFile.key, + 'user-1', + 'request-1', + expect.anything() + ) + }) +}) diff --git a/apps/sim/lib/internal/clickup/operations.ts b/apps/sim/lib/internal/clickup/operations.ts new file mode 100644 index 00000000000..2d96af66413 --- /dev/null +++ b/apps/sim/lib/internal/clickup/operations.ts @@ -0,0 +1,94 @@ +import { createLogger } from '@sim/logger' +import { isRecordLike } from '@sim/utils/object' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { uploadClickUpAttachment } from '@/lib/internal/clickup/client' +import { ClickUpOperationError } from '@/lib/internal/clickup/errors' +import type { ClickUpUploadAttachmentInput } from '@/lib/internal/clickup/schema' +import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/doc-not-ready' +import { parseRawFileInput } from '@/lib/uploads/utils/file-schemas' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' +import { mapClickUpAttachment } from '@/tools/clickup/shared' + +const logger = createLogger('ClickUpOperations') +const MAX_UPLOAD_SIZE_BYTES = 100 * 1024 * 1024 + +export interface ClickUpOperationContext { + requestId: string + signal?: AbortSignal + userId: string +} + +async function deniedBody(response: Response): Promise> { + const body: unknown = await response.json().catch(() => null) + return isRecordLike(body) ? body : { success: false, error: 'File not found' } +} + +function uploadSizeError(bytes: number): ClickUpOperationError { + const sizeMB = (bytes / (1024 * 1024)).toFixed(2) + return new ClickUpOperationError(`File size (${sizeMB}MB) exceeds upload limit of 100MB`, 400) +} + +export async function executeClickUpUploadAttachment( + input: ClickUpUploadAttachmentInput, + context: ClickUpOperationContext +) { + context.signal?.throwIfAborted() + const parsedFile = parseRawFileInput(input.file) + if (!parsedFile) { + throw new ClickUpOperationError('No valid file provided for upload', 400) + } + const userFiles = processFilesToUserFiles([parsedFile], context.requestId, logger) + const userFile = userFiles[0] + if (!userFile) { + throw new ClickUpOperationError('No valid file provided for upload', 400) + } + + const denied = await assertToolFileAccess(userFile.key, context.userId, context.requestId, logger) + context.signal?.throwIfAborted() + if (denied) { + throw new ClickUpOperationError('File not found', denied.status, await deniedBody(denied)) + } + if (userFile.size > MAX_UPLOAD_SIZE_BYTES) throw uploadSizeError(userFile.size) + + let buffer: Buffer + let downloadedContentType: string + try { + const downloaded = await downloadServableFileFromStorage(userFile, context.requestId, logger, { + maxBytes: MAX_UPLOAD_SIZE_BYTES, + signal: context.signal, + }) + buffer = downloaded.buffer + downloadedContentType = downloaded.contentType + } catch (error) { + context.signal?.throwIfAborted() + if (isDocNotReadyError(error)) { + throw new ClickUpOperationError(docNotReadyMessage(), 409) + } + if (isPayloadSizeLimitError(error)) { + throw uploadSizeError(error.observedBytes ?? userFile.size) + } + throw error + } + if (buffer.length > MAX_UPLOAD_SIZE_BYTES) throw uploadSizeError(buffer.length) + + const formData = new FormData() + const mimeType = downloadedContentType || userFile.type || 'application/octet-stream' + formData.append( + 'attachment', + new Blob([new Uint8Array(buffer)], { type: mimeType }), + userFile.name + ) + const data = await uploadClickUpAttachment( + input.accessToken, + input.taskId, + formData, + context.signal + ) + context.signal?.throwIfAborted() + return { + success: true, + output: { attachment: mapClickUpAttachment(data), files: userFiles }, + } +} diff --git a/apps/sim/lib/internal/clickup/schema.ts b/apps/sim/lib/internal/clickup/schema.ts new file mode 100644 index 00000000000..c703ccb97d8 --- /dev/null +++ b/apps/sim/lib/internal/clickup/schema.ts @@ -0,0 +1,10 @@ +import { z } from 'zod' +import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' + +export const clickupUploadAttachmentInputSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + taskId: z.string().min(1, 'Task ID is required'), + file: FileInputSchema, +}) + +export type ClickUpUploadAttachmentInput = z.output diff --git a/apps/sim/lib/internal/cloudformation/client.ts b/apps/sim/lib/internal/cloudformation/client.ts new file mode 100644 index 00000000000..fb2eb8bca66 --- /dev/null +++ b/apps/sim/lib/internal/cloudformation/client.ts @@ -0,0 +1,59 @@ +import { + type Capability, + CloudFormationClient, + type Parameter, + type Tag, +} from '@aws-sdk/client-cloudformation' + +export interface CloudFormationConnectionConfig { + region: string + accessKeyId: string + secretAccessKey: string +} + +export function createCloudFormationClient( + config: CloudFormationConnectionConfig +): CloudFormationClient { + return new CloudFormationClient({ + region: config.region, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + }) +} + +/** + * Parses a comma-separated capabilities string (e.g. "CAPABILITY_IAM,CAPABILITY_NAMED_IAM") + * into the array shape the CloudFormation SDK expects. + */ +export function parseCapabilities(value?: string): Capability[] | undefined { + if (!value) return undefined + const capabilities = value + .split(',') + .map((c) => c.trim()) + .filter(Boolean) + return capabilities.length > 0 ? (capabilities as Capability[]) : undefined +} + +/** + * Maps camelCase stack parameter inputs to the PascalCase `Parameter` shape CloudFormation expects. + */ +export function toStackParameters( + parameters?: { parameterKey: string; parameterValue?: string; usePreviousValue?: boolean }[] +): Parameter[] | undefined { + if (!parameters || parameters.length === 0) return undefined + return parameters.map((p) => ({ + ParameterKey: p.parameterKey, + ParameterValue: p.parameterValue, + UsePreviousValue: p.usePreviousValue, + })) +} + +/** + * Maps camelCase tag inputs to the PascalCase `Tag` shape CloudFormation expects. + */ +export function toStackTags(tags?: { key: string; value: string }[]): Tag[] | undefined { + if (!tags || tags.length === 0) return undefined + return tags.map((t) => ({ Key: t.key, Value: t.value })) +} diff --git a/apps/sim/lib/internal/cloudformation/execute-tool.test.ts b/apps/sim/lib/internal/cloudformation/execute-tool.test.ts new file mode 100644 index 00000000000..c2a8aa3b526 --- /dev/null +++ b/apps/sim/lib/internal/cloudformation/execute-tool.test.ts @@ -0,0 +1,173 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mockOperations = vi.hoisted(() => ({ + executeCloudformationCancelUpdateStack: vi.fn(), + executeCloudformationCreateChangeSet: vi.fn(), + executeCloudformationCreateStack: vi.fn(), + executeCloudformationDeleteStack: vi.fn(), + executeCloudformationDescribeChangeSet: vi.fn(), + executeCloudformationDescribeStackDriftDetectionStatus: vi.fn(), + executeCloudformationDescribeStackEvents: vi.fn(), + executeCloudformationDescribeStacks: vi.fn(), + executeCloudformationDetectStackDrift: vi.fn(), + executeCloudformationExecuteChangeSet: vi.fn(), + executeCloudformationGetTemplate: vi.fn(), + executeCloudformationGetTemplateSummary: vi.fn(), + executeCloudformationListStackResources: vi.fn(), + executeCloudformationUpdateStack: vi.fn(), + executeCloudformationValidateTemplate: vi.fn(), +})) + +vi.mock('@/lib/internal/cloudformation/operations', () => mockOperations) + +import { executeCloudformationTool } from '@/lib/internal/cloudformation/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const CONNECTION = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', +} + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'cloudformation_describe_stacks', + input: CONNECTION, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + userId: 'user-1', + metadata: {}, + }, + requestId: 'request-1', + ...overrides, + } +} + +const STACK = { ...CONNECTION, stackName: 'stack' } +const CHANGE_SET = { ...CONNECTION, changeSetName: 'change-set' } + +const TOOL_CASES = [ + [ + 'cloudformation_cancel_update_stack', + STACK, + mockOperations.executeCloudformationCancelUpdateStack, + ], + [ + 'cloudformation_create_change_set', + { ...STACK, changeSetName: 'change-set', templateBody: '{}' }, + mockOperations.executeCloudformationCreateChangeSet, + ], + [ + 'cloudformation_create_stack', + { ...STACK, templateBody: '{}' }, + mockOperations.executeCloudformationCreateStack, + ], + ['cloudformation_delete_stack', STACK, mockOperations.executeCloudformationDeleteStack], + [ + 'cloudformation_describe_change_set', + CHANGE_SET, + mockOperations.executeCloudformationDescribeChangeSet, + ], + [ + 'cloudformation_describe_stack_drift_detection_status', + { ...CONNECTION, stackDriftDetectionId: 'drift-id' }, + mockOperations.executeCloudformationDescribeStackDriftDetectionStatus, + ], + [ + 'cloudformation_describe_stack_events', + STACK, + mockOperations.executeCloudformationDescribeStackEvents, + ], + [ + 'cloudformation_describe_stacks', + CONNECTION, + mockOperations.executeCloudformationDescribeStacks, + ], + [ + 'cloudformation_detect_stack_drift', + STACK, + mockOperations.executeCloudformationDetectStackDrift, + ], + [ + 'cloudformation_execute_change_set', + CHANGE_SET, + mockOperations.executeCloudformationExecuteChangeSet, + ], + ['cloudformation_get_template', STACK, mockOperations.executeCloudformationGetTemplate], + [ + 'cloudformation_get_template_summary', + STACK, + mockOperations.executeCloudformationGetTemplateSummary, + ], + [ + 'cloudformation_list_stack_resources', + STACK, + mockOperations.executeCloudformationListStackResources, + ], + [ + 'cloudformation_update_stack', + { ...STACK, usePreviousTemplate: true }, + mockOperations.executeCloudformationUpdateStack, + ], + [ + 'cloudformation_validate_template', + { ...CONNECTION, templateBody: '{}' }, + mockOperations.executeCloudformationValidateTemplate, + ], +] as const + +describe('executeCloudformationTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it.each(TOOL_CASES)('validates and dispatches %s', async (toolId, input, operation) => { + const controller = new AbortController() + operation.mockResolvedValue({ toolId }) + + const response = await executeCloudformationTool( + createRequest({ toolId, input, signal: controller.signal }) + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ toolId }) + expect(operation).toHaveBeenCalledWith(input, controller.signal) + }) + + it('returns the canonical validation envelope before provider work', async () => { + const response = await executeCloudformationTool(createRequest({ input: { region: '' } })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(mockOperations.executeCloudformationDescribeStacks).not.toHaveBeenCalled() + }) + + it('preserves the provider error envelope', async () => { + mockOperations.executeCloudformationDescribeStacks.mockRejectedValue(new Error('AWS rejected')) + + const response = await executeCloudformationTool(createRequest()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ error: 'AWS rejected' }) + }) + + it('propagates cancellation without starting provider work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeCloudformationTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mockOperations.executeCloudformationDescribeStacks).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/cloudformation/execute-tool.ts b/apps/sim/lib/internal/cloudformation/execute-tool.ts new file mode 100644 index 00000000000..a36d84ea3cc --- /dev/null +++ b/apps/sim/lib/internal/cloudformation/execute-tool.ts @@ -0,0 +1,189 @@ +import { getErrorMessage } from '@sim/utils/errors' +import type { AnyApiRouteContract, ContractBody } from '@/lib/api/contracts' +import { awsCloudformationCancelUpdateStackContract } from '@/lib/api/contracts/tools/aws/cloudformation-cancel-update-stack' +import { awsCloudformationCreateChangeSetContract } from '@/lib/api/contracts/tools/aws/cloudformation-create-change-set' +import { awsCloudformationCreateStackContract } from '@/lib/api/contracts/tools/aws/cloudformation-create-stack' +import { awsCloudformationDeleteStackContract } from '@/lib/api/contracts/tools/aws/cloudformation-delete-stack' +import { awsCloudformationDescribeChangeSetContract } from '@/lib/api/contracts/tools/aws/cloudformation-describe-change-set' +import { awsCloudformationDescribeStackDriftDetectionStatusContract } from '@/lib/api/contracts/tools/aws/cloudformation-describe-stack-drift-detection-status' +import { awsCloudformationDescribeStackEventsContract } from '@/lib/api/contracts/tools/aws/cloudformation-describe-stack-events' +import { awsCloudformationDescribeStacksContract } from '@/lib/api/contracts/tools/aws/cloudformation-describe-stacks' +import { awsCloudformationDetectStackDriftContract } from '@/lib/api/contracts/tools/aws/cloudformation-detect-stack-drift' +import { awsCloudformationExecuteChangeSetContract } from '@/lib/api/contracts/tools/aws/cloudformation-execute-change-set' +import { awsCloudformationGetTemplateContract } from '@/lib/api/contracts/tools/aws/cloudformation-get-template' +import { awsCloudformationGetTemplateSummaryContract } from '@/lib/api/contracts/tools/aws/cloudformation-get-template-summary' +import { awsCloudformationListStackResourcesContract } from '@/lib/api/contracts/tools/aws/cloudformation-list-stack-resources' +import { awsCloudformationUpdateStackContract } from '@/lib/api/contracts/tools/aws/cloudformation-update-stack' +import { awsCloudformationValidateTemplateContract } from '@/lib/api/contracts/tools/aws/cloudformation-validate-template' +import { + executeCloudformationCancelUpdateStack, + executeCloudformationCreateChangeSet, + executeCloudformationCreateStack, + executeCloudformationDeleteStack, + executeCloudformationDescribeChangeSet, + executeCloudformationDescribeStackDriftDetectionStatus, + executeCloudformationDescribeStackEvents, + executeCloudformationDescribeStacks, + executeCloudformationDetectStackDrift, + executeCloudformationExecuteChangeSet, + executeCloudformationGetTemplate, + executeCloudformationGetTemplateSummary, + executeCloudformationListStackResources, + executeCloudformationUpdateStack, + executeCloudformationValidateTemplate, +} from '@/lib/internal/cloudformation/operations' +import { parseInternalToolInput } from '@/lib/internal/tool-operations/parse-input' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +async function executeOperation( + contract: C, + input: unknown, + execute: (input: ContractBody, signal?: AbortSignal) => Promise, + fallbackError: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const parsed = parseInternalToolInput(contract, input) + if (!parsed.success) return parsed.response + + try { + const result = await execute(parsed.data, signal) + signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + signal?.throwIfAborted() + return Response.json({ error: getErrorMessage(error, fallbackError) }, { status: 500 }) + } +} + +export const executeCloudformationTool: InternalToolOperationHandler = async ({ + toolId, + input, + signal, +}) => { + signal?.throwIfAborted() + switch (toolId) { + case 'cloudformation_cancel_update_stack': + return executeOperation( + awsCloudformationCancelUpdateStackContract, + input, + executeCloudformationCancelUpdateStack, + 'Failed to cancel CloudFormation stack update', + signal + ) + case 'cloudformation_create_change_set': + return executeOperation( + awsCloudformationCreateChangeSetContract, + input, + executeCloudformationCreateChangeSet, + 'Failed to create CloudFormation change set', + signal + ) + case 'cloudformation_create_stack': + return executeOperation( + awsCloudformationCreateStackContract, + input, + executeCloudformationCreateStack, + 'Failed to create CloudFormation stack', + signal + ) + case 'cloudformation_delete_stack': + return executeOperation( + awsCloudformationDeleteStackContract, + input, + executeCloudformationDeleteStack, + 'Failed to delete CloudFormation stack', + signal + ) + case 'cloudformation_describe_change_set': + return executeOperation( + awsCloudformationDescribeChangeSetContract, + input, + executeCloudformationDescribeChangeSet, + 'Failed to describe CloudFormation change set', + signal + ) + case 'cloudformation_describe_stack_drift_detection_status': + return executeOperation( + awsCloudformationDescribeStackDriftDetectionStatusContract, + input, + executeCloudformationDescribeStackDriftDetectionStatus, + 'Failed to describe stack drift detection status', + signal + ) + case 'cloudformation_describe_stack_events': + return executeOperation( + awsCloudformationDescribeStackEventsContract, + input, + executeCloudformationDescribeStackEvents, + 'Failed to describe CloudFormation stack events', + signal + ) + case 'cloudformation_describe_stacks': + return executeOperation( + awsCloudformationDescribeStacksContract, + input, + executeCloudformationDescribeStacks, + 'Failed to describe CloudFormation stacks', + signal + ) + case 'cloudformation_detect_stack_drift': + return executeOperation( + awsCloudformationDetectStackDriftContract, + input, + executeCloudformationDetectStackDrift, + 'Failed to detect CloudFormation stack drift', + signal + ) + case 'cloudformation_execute_change_set': + return executeOperation( + awsCloudformationExecuteChangeSetContract, + input, + executeCloudformationExecuteChangeSet, + 'Failed to execute CloudFormation change set', + signal + ) + case 'cloudformation_get_template_summary': + return executeOperation( + awsCloudformationGetTemplateSummaryContract, + input, + executeCloudformationGetTemplateSummary, + 'Failed to get CloudFormation template summary', + signal + ) + case 'cloudformation_get_template': + return executeOperation( + awsCloudformationGetTemplateContract, + input, + executeCloudformationGetTemplate, + 'Failed to get CloudFormation template', + signal + ) + case 'cloudformation_list_stack_resources': + return executeOperation( + awsCloudformationListStackResourcesContract, + input, + executeCloudformationListStackResources, + 'Failed to list CloudFormation stack resources', + signal + ) + case 'cloudformation_update_stack': + return executeOperation( + awsCloudformationUpdateStackContract, + input, + executeCloudformationUpdateStack, + 'Failed to update CloudFormation stack', + signal + ) + case 'cloudformation_validate_template': + return executeOperation( + awsCloudformationValidateTemplateContract, + input, + executeCloudformationValidateTemplate, + 'Failed to validate CloudFormation template', + signal + ) + default: + return Response.json({ error: `Unsupported CloudFormation tool: ${toolId}` }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/cloudformation/operations.test.ts b/apps/sim/lib/internal/cloudformation/operations.test.ts new file mode 100644 index 00000000000..e8b07689dd1 --- /dev/null +++ b/apps/sim/lib/internal/cloudformation/operations.test.ts @@ -0,0 +1,143 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + createCloudFormationClient: vi.fn(), + destroy: vi.fn(), + parseCapabilities: vi.fn(), + send: vi.fn(), + toStackParameters: vi.fn(), + toStackTags: vi.fn(), +})) + +vi.mock('@/lib/internal/cloudformation/client', () => ({ + createCloudFormationClient: mocks.createCloudFormationClient, + parseCapabilities: mocks.parseCapabilities, + toStackParameters: mocks.toStackParameters, + toStackTags: mocks.toStackTags, +})) + +import { + executeCloudformationCreateStack, + executeCloudformationDescribeStacks, +} from '@/lib/internal/cloudformation/operations' + +const CONNECTION = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', +} + +describe('CloudFormation operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.createCloudFormationClient.mockReturnValue({ + send: mocks.send, + destroy: mocks.destroy, + }) + }) + + it('forwards cancellation across paginated requests and destroys the client', async () => { + const controller = new AbortController() + mocks.send + .mockResolvedValueOnce({ + Stacks: [{ StackName: 'first', StackId: 'first-id', StackStatus: 'CREATE_COMPLETE' }], + NextToken: 'next-page', + }) + .mockResolvedValueOnce({ + Stacks: [{ StackName: 'second', StackId: 'second-id', StackStatus: 'UPDATE_COMPLETE' }], + }) + + await expect( + executeCloudformationDescribeStacks(CONNECTION, controller.signal) + ).resolves.toEqual({ + success: true, + output: { + stacks: [ + { + stackName: 'first', + stackId: 'first-id', + stackStatus: 'CREATE_COMPLETE', + stackStatusReason: undefined, + creationTime: undefined, + lastUpdatedTime: undefined, + description: undefined, + enableTerminationProtection: undefined, + driftInformation: null, + outputs: [], + tags: [], + }, + { + stackName: 'second', + stackId: 'second-id', + stackStatus: 'UPDATE_COMPLETE', + stackStatusReason: undefined, + creationTime: undefined, + lastUpdatedTime: undefined, + description: undefined, + enableTerminationProtection: undefined, + driftInformation: null, + outputs: [], + tags: [], + }, + ], + }, + }) + expect(mocks.send).toHaveBeenCalledTimes(2) + expect(mocks.send.mock.calls[0]?.[1]).toEqual({ abortSignal: controller.signal }) + expect(mocks.send.mock.calls[1]?.[1]).toEqual({ abortSignal: controller.signal }) + expect(mocks.destroy).toHaveBeenCalledOnce() + }) + + it('passes transformed stack inputs and cancellation to the SDK', async () => { + const controller = new AbortController() + const parameters = [{ ParameterKey: 'Environment', ParameterValue: 'test' }] + const capabilities = ['CAPABILITY_IAM'] + const tags = [{ Key: 'service', Value: 'sim' }] + mocks.toStackParameters.mockReturnValue(parameters) + mocks.parseCapabilities.mockReturnValue(capabilities) + mocks.toStackTags.mockReturnValue(tags) + mocks.send.mockResolvedValue({ StackId: 'stack-id' }) + + await expect( + executeCloudformationCreateStack( + { + ...CONNECTION, + stackName: 'stack', + templateBody: '{}', + parameters: [ + { parameterKey: 'Environment', parameterValue: 'test', usePreviousValue: false }, + ], + capabilities: 'CAPABILITY_IAM', + tags: [{ key: 'service', value: 'sim' }], + onFailure: 'ROLLBACK', + timeoutInMinutes: 10, + }, + controller.signal + ) + ).resolves.toEqual({ success: true, output: { stackId: 'stack-id' } }) + + expect(mocks.send.mock.calls[0]?.[0].input).toEqual({ + StackName: 'stack', + TemplateBody: '{}', + Parameters: parameters, + Capabilities: capabilities, + Tags: tags, + OnFailure: 'ROLLBACK', + TimeoutInMinutes: 10, + }) + expect(mocks.send.mock.calls[0]?.[1]).toEqual({ abortSignal: controller.signal }) + expect(mocks.destroy).toHaveBeenCalledOnce() + }) + + it('destroys the client when provider execution fails', async () => { + mocks.send.mockRejectedValue(new Error('provider failure')) + + await expect(executeCloudformationDescribeStacks(CONNECTION)).rejects.toThrow( + 'provider failure' + ) + expect(mocks.destroy).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/internal/cloudformation/operations.ts b/apps/sim/lib/internal/cloudformation/operations.ts new file mode 100644 index 00000000000..f208c873621 --- /dev/null +++ b/apps/sim/lib/internal/cloudformation/operations.ts @@ -0,0 +1,498 @@ +import { + CancelUpdateStackCommand, + CreateChangeSetCommand, + CreateStackCommand, + DeleteStackCommand, + DescribeChangeSetCommand, + DescribeStackDriftDetectionStatusCommand, + DescribeStackEventsCommand, + DescribeStacksCommand, + DetectStackDriftCommand, + ExecuteChangeSetCommand, + GetTemplateCommand, + GetTemplateSummaryCommand, + ListStackResourcesCommand, + type Stack, + type StackEvent, + type StackResourceSummary, + UpdateStackCommand, + ValidateTemplateCommand, +} from '@aws-sdk/client-cloudformation' +import type { AwsCloudformationCancelUpdateStackBody } from '@/lib/api/contracts/tools/aws/cloudformation-cancel-update-stack' +import type { AwsCloudformationCreateChangeSetBody } from '@/lib/api/contracts/tools/aws/cloudformation-create-change-set' +import type { AwsCloudformationCreateStackBody } from '@/lib/api/contracts/tools/aws/cloudformation-create-stack' +import type { AwsCloudformationDeleteStackBody } from '@/lib/api/contracts/tools/aws/cloudformation-delete-stack' +import type { AwsCloudformationDescribeChangeSetBody } from '@/lib/api/contracts/tools/aws/cloudformation-describe-change-set' +import type { AwsCloudformationDescribeStackDriftDetectionStatusBody } from '@/lib/api/contracts/tools/aws/cloudformation-describe-stack-drift-detection-status' +import type { AwsCloudformationDescribeStackEventsBody } from '@/lib/api/contracts/tools/aws/cloudformation-describe-stack-events' +import type { AwsCloudformationDescribeStacksBody } from '@/lib/api/contracts/tools/aws/cloudformation-describe-stacks' +import type { AwsCloudformationDetectStackDriftBody } from '@/lib/api/contracts/tools/aws/cloudformation-detect-stack-drift' +import type { AwsCloudformationExecuteChangeSetBody } from '@/lib/api/contracts/tools/aws/cloudformation-execute-change-set' +import type { AwsCloudformationGetTemplateBody } from '@/lib/api/contracts/tools/aws/cloudformation-get-template' +import type { AwsCloudformationGetTemplateSummaryBody } from '@/lib/api/contracts/tools/aws/cloudformation-get-template-summary' +import type { AwsCloudformationListStackResourcesBody } from '@/lib/api/contracts/tools/aws/cloudformation-list-stack-resources' +import type { AwsCloudformationUpdateStackBody } from '@/lib/api/contracts/tools/aws/cloudformation-update-stack' +import type { AwsCloudformationValidateTemplateBody } from '@/lib/api/contracts/tools/aws/cloudformation-validate-template' +import { + createCloudFormationClient, + parseCapabilities, + toStackParameters, + toStackTags, +} from '@/lib/internal/cloudformation/client' + +export async function executeCloudformationCancelUpdateStack( + input: AwsCloudformationCancelUpdateStackBody, + signal?: AbortSignal +) { + const client = createCloudFormationClient(input) + try { + await client.send(new CancelUpdateStackCommand({ StackName: input.stackName }), { + abortSignal: signal, + }) + return { + success: true, + output: { + message: `Update for stack "${input.stackName}" is being cancelled and rolled back`, + }, + } + } finally { + client.destroy() + } +} + +export async function executeCloudformationCreateChangeSet( + input: AwsCloudformationCreateChangeSetBody, + signal?: AbortSignal +) { + const client = createCloudFormationClient(input) + try { + const response = await client.send( + new CreateChangeSetCommand({ + StackName: input.stackName, + ChangeSetName: input.changeSetName, + TemplateBody: input.templateBody, + UsePreviousTemplate: input.usePreviousTemplate, + Parameters: toStackParameters(input.parameters), + Capabilities: parseCapabilities(input.capabilities), + ChangeSetType: input.changeSetType, + Description: input.description, + }), + { abortSignal: signal } + ) + return { + success: true, + output: { + changeSetId: response.Id ?? '', + stackId: response.StackId ?? '', + }, + } + } finally { + client.destroy() + } +} + +export async function executeCloudformationCreateStack( + input: AwsCloudformationCreateStackBody, + signal?: AbortSignal +) { + const client = createCloudFormationClient(input) + try { + const response = await client.send( + new CreateStackCommand({ + StackName: input.stackName, + TemplateBody: input.templateBody, + Parameters: toStackParameters(input.parameters), + Capabilities: parseCapabilities(input.capabilities), + Tags: toStackTags(input.tags), + OnFailure: input.onFailure, + TimeoutInMinutes: input.timeoutInMinutes, + }), + { abortSignal: signal } + ) + return { success: true, output: { stackId: response.StackId ?? '' } } + } finally { + client.destroy() + } +} + +export async function executeCloudformationDeleteStack( + input: AwsCloudformationDeleteStackBody, + signal?: AbortSignal +) { + const client = createCloudFormationClient(input) + try { + const retainResources = input.retainResources + ?.split(',') + .map((resource) => resource.trim()) + .filter(Boolean) + await client.send( + new DeleteStackCommand({ + StackName: input.stackName, + ...(retainResources && retainResources.length > 0 + ? { RetainResources: retainResources } + : {}), + }), + { abortSignal: signal } + ) + return { + success: true, + output: { message: `Deletion of stack "${input.stackName}" has been initiated` }, + } + } finally { + client.destroy() + } +} + +export async function executeCloudformationDescribeChangeSet( + input: AwsCloudformationDescribeChangeSetBody, + signal?: AbortSignal +) { + const client = createCloudFormationClient(input) + try { + const response = await client.send( + new DescribeChangeSetCommand({ + ChangeSetName: input.changeSetName, + ...(input.stackName ? { StackName: input.stackName } : {}), + }), + { abortSignal: signal } + ) + const changes = (response.Changes ?? []).map((change) => ({ + action: change.ResourceChange?.Action, + logicalResourceId: change.ResourceChange?.LogicalResourceId, + physicalResourceId: change.ResourceChange?.PhysicalResourceId, + resourceType: change.ResourceChange?.ResourceType, + replacement: change.ResourceChange?.Replacement, + })) + return { + success: true, + output: { + changeSetName: response.ChangeSetName, + changeSetId: response.ChangeSetId, + stackId: response.StackId, + stackName: response.StackName, + description: response.Description, + executionStatus: response.ExecutionStatus, + status: response.Status, + statusReason: response.StatusReason, + creationTime: response.CreationTime?.getTime(), + capabilities: response.Capabilities ?? [], + changes, + }, + } + } finally { + client.destroy() + } +} + +export async function executeCloudformationDescribeStackDriftDetectionStatus( + input: AwsCloudformationDescribeStackDriftDetectionStatusBody, + signal?: AbortSignal +) { + const client = createCloudFormationClient(input) + try { + const response = await client.send( + new DescribeStackDriftDetectionStatusCommand({ + StackDriftDetectionId: input.stackDriftDetectionId, + }), + { abortSignal: signal } + ) + return { + success: true, + output: { + stackId: response.StackId ?? '', + stackDriftDetectionId: response.StackDriftDetectionId ?? '', + stackDriftStatus: response.StackDriftStatus, + detectionStatus: response.DetectionStatus ?? 'UNKNOWN', + detectionStatusReason: response.DetectionStatusReason, + driftedStackResourceCount: response.DriftedStackResourceCount, + timestamp: response.Timestamp?.getTime(), + }, + } + } finally { + client.destroy() + } +} + +export async function executeCloudformationDescribeStackEvents( + input: AwsCloudformationDescribeStackEventsBody, + signal?: AbortSignal +) { + const client = createCloudFormationClient(input) + try { + const limit = input.limit ?? 50 + const allEvents: StackEvent[] = [] + let nextToken: string | undefined + do { + const response = await client.send( + new DescribeStackEventsCommand({ + StackName: input.stackName, + ...(nextToken ? { NextToken: nextToken } : {}), + }), + { abortSignal: signal } + ) + allEvents.push(...(response.StackEvents ?? [])) + nextToken = allEvents.length >= limit ? undefined : response.NextToken + } while (nextToken) + + const events = allEvents.slice(0, limit).map((event) => ({ + stackId: event.StackId ?? '', + eventId: event.EventId ?? '', + stackName: event.StackName ?? '', + logicalResourceId: event.LogicalResourceId, + physicalResourceId: event.PhysicalResourceId, + resourceType: event.ResourceType, + resourceStatus: event.ResourceStatus, + resourceStatusReason: event.ResourceStatusReason, + timestamp: event.Timestamp?.getTime(), + })) + return { success: true, output: { events } } + } finally { + client.destroy() + } +} + +export async function executeCloudformationDescribeStacks( + input: AwsCloudformationDescribeStacksBody, + signal?: AbortSignal +) { + const client = createCloudFormationClient(input) + try { + const allStacks: Stack[] = [] + let nextToken: string | undefined + do { + const response = await client.send( + new DescribeStacksCommand({ + ...(input.stackName ? { StackName: input.stackName } : {}), + ...(nextToken ? { NextToken: nextToken } : {}), + }), + { abortSignal: signal } + ) + allStacks.push(...(response.Stacks ?? [])) + nextToken = response.NextToken + } while (nextToken) + + const stacks = allStacks.map((stack) => ({ + stackName: stack.StackName ?? '', + stackId: stack.StackId ?? '', + stackStatus: stack.StackStatus ?? 'UNKNOWN', + stackStatusReason: stack.StackStatusReason, + creationTime: stack.CreationTime?.getTime(), + lastUpdatedTime: stack.LastUpdatedTime?.getTime(), + description: stack.Description, + enableTerminationProtection: stack.EnableTerminationProtection, + driftInformation: stack.DriftInformation + ? { + stackDriftStatus: stack.DriftInformation.StackDriftStatus, + lastCheckTimestamp: stack.DriftInformation.LastCheckTimestamp?.getTime(), + } + : null, + outputs: (stack.Outputs ?? []).map((output) => ({ + outputKey: output.OutputKey ?? '', + outputValue: output.OutputValue ?? '', + description: output.Description, + })), + tags: (stack.Tags ?? []).map((tag) => ({ key: tag.Key ?? '', value: tag.Value ?? '' })), + })) + return { success: true, output: { stacks } } + } finally { + client.destroy() + } +} + +export async function executeCloudformationDetectStackDrift( + input: AwsCloudformationDetectStackDriftBody, + signal?: AbortSignal +) { + const client = createCloudFormationClient(input) + try { + const response = await client.send( + new DetectStackDriftCommand({ StackName: input.stackName }), + { + abortSignal: signal, + } + ) + if (!response.StackDriftDetectionId) throw new Error('No drift detection ID returned') + return { + success: true, + output: { stackDriftDetectionId: response.StackDriftDetectionId }, + } + } finally { + client.destroy() + } +} + +export async function executeCloudformationExecuteChangeSet( + input: AwsCloudformationExecuteChangeSetBody, + signal?: AbortSignal +) { + const client = createCloudFormationClient(input) + try { + await client.send( + new ExecuteChangeSetCommand({ + ChangeSetName: input.changeSetName, + ...(input.stackName ? { StackName: input.stackName } : {}), + }), + { abortSignal: signal } + ) + return { + success: true, + output: { message: `Change set "${input.changeSetName}" execution has been initiated` }, + } + } finally { + client.destroy() + } +} + +export async function executeCloudformationGetTemplateSummary( + input: AwsCloudformationGetTemplateSummaryBody, + signal?: AbortSignal +) { + const client = createCloudFormationClient(input) + try { + const response = await client.send( + new GetTemplateSummaryCommand({ + ...(input.templateBody ? { TemplateBody: input.templateBody } : {}), + ...(input.stackName ? { StackName: input.stackName } : {}), + }), + { abortSignal: signal } + ) + return { + success: true, + output: { + description: response.Description, + parameters: (response.Parameters ?? []).map((parameter) => ({ + parameterKey: parameter.ParameterKey, + defaultValue: parameter.DefaultValue, + parameterType: parameter.ParameterType, + noEcho: parameter.NoEcho, + description: parameter.Description, + })), + capabilities: response.Capabilities ?? [], + capabilitiesReason: response.CapabilitiesReason, + resourceTypes: response.ResourceTypes ?? [], + version: response.Version, + declaredTransforms: response.DeclaredTransforms ?? [], + }, + } + } finally { + client.destroy() + } +} + +export async function executeCloudformationGetTemplate( + input: AwsCloudformationGetTemplateBody, + signal?: AbortSignal +) { + const client = createCloudFormationClient(input) + try { + const response = await client.send( + new GetTemplateCommand({ + StackName: input.stackName, + ...(input.templateStage ? { TemplateStage: input.templateStage } : {}), + }), + { abortSignal: signal } + ) + return { + success: true, + output: { + templateBody: response.TemplateBody ?? '', + stagesAvailable: response.StagesAvailable ?? [], + }, + } + } finally { + client.destroy() + } +} + +export async function executeCloudformationListStackResources( + input: AwsCloudformationListStackResourcesBody, + signal?: AbortSignal +) { + const client = createCloudFormationClient(input) + try { + const allSummaries: StackResourceSummary[] = [] + let nextToken: string | undefined + do { + const response = await client.send( + new ListStackResourcesCommand({ + StackName: input.stackName, + ...(nextToken ? { NextToken: nextToken } : {}), + }), + { abortSignal: signal } + ) + allSummaries.push(...(response.StackResourceSummaries ?? [])) + nextToken = response.NextToken + } while (nextToken) + + const resources = allSummaries.map((resource) => ({ + logicalResourceId: resource.LogicalResourceId ?? '', + physicalResourceId: resource.PhysicalResourceId, + resourceType: resource.ResourceType ?? '', + resourceStatus: resource.ResourceStatus ?? 'UNKNOWN', + resourceStatusReason: resource.ResourceStatusReason, + lastUpdatedTimestamp: resource.LastUpdatedTimestamp?.getTime(), + driftInformation: resource.DriftInformation + ? { + stackResourceDriftStatus: resource.DriftInformation.StackResourceDriftStatus, + lastCheckTimestamp: resource.DriftInformation.LastCheckTimestamp?.getTime(), + } + : null, + })) + return { success: true, output: { resources } } + } finally { + client.destroy() + } +} + +export async function executeCloudformationUpdateStack( + input: AwsCloudformationUpdateStackBody, + signal?: AbortSignal +) { + const client = createCloudFormationClient(input) + try { + const response = await client.send( + new UpdateStackCommand({ + StackName: input.stackName, + TemplateBody: input.templateBody, + UsePreviousTemplate: input.usePreviousTemplate, + Parameters: toStackParameters(input.parameters), + Capabilities: parseCapabilities(input.capabilities), + Tags: toStackTags(input.tags), + }), + { abortSignal: signal } + ) + return { success: true, output: { stackId: response.StackId ?? '' } } + } finally { + client.destroy() + } +} + +export async function executeCloudformationValidateTemplate( + input: AwsCloudformationValidateTemplateBody, + signal?: AbortSignal +) { + const client = createCloudFormationClient(input) + try { + const response = await client.send( + new ValidateTemplateCommand({ TemplateBody: input.templateBody }), + { abortSignal: signal } + ) + return { + success: true, + output: { + description: response.Description, + parameters: (response.Parameters ?? []).map((parameter) => ({ + parameterKey: parameter.ParameterKey, + defaultValue: parameter.DefaultValue, + noEcho: parameter.NoEcho, + description: parameter.Description, + })), + capabilities: response.Capabilities ?? [], + capabilitiesReason: response.CapabilitiesReason, + declaredTransforms: response.DeclaredTransforms ?? [], + }, + } + } finally { + client.destroy() + } +} diff --git a/apps/sim/lib/internal/cloudwatch/client.ts b/apps/sim/lib/internal/cloudwatch/client.ts new file mode 100644 index 00000000000..7d6182e709a --- /dev/null +++ b/apps/sim/lib/internal/cloudwatch/client.ts @@ -0,0 +1,313 @@ +import { CloudWatchClient } from '@aws-sdk/client-cloudwatch' +import { + CloudWatchLogsClient, + DescribeLogStreamsCommand, + FilterLogEventsCommand, + GetLogEventsCommand, + GetQueryResultsCommand, + type ResultField, +} from '@aws-sdk/client-cloudwatch-logs' +import { createLogger } from '@sim/logger' +import { sleep } from '@sim/utils/helpers' +import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' + +interface AwsCredentials { + region: string + accessKeyId: string + secretAccessKey: string +} + +export function createCloudWatchClient( + config: AwsCredentials, + options?: { maxAttempts?: number } +): CloudWatchClient { + return new CloudWatchClient({ + region: config.region, + ...(options?.maxAttempts !== undefined && { maxAttempts: options.maxAttempts }), + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + }) +} + +export function createCloudWatchLogsClient(config: AwsCredentials): CloudWatchLogsClient { + return new CloudWatchLogsClient({ + region: config.region, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + }) +} + +interface PollOptions { + maxWaitMs?: number + pollIntervalMs?: number +} + +interface PollResult { + results: Record[] + statistics: { + bytesScanned: number + recordsMatched: number + recordsScanned: number + } + status: string +} + +function parseResultFields(fields: ResultField[] | undefined): Record { + const record: Record = {} + if (!fields) return record + for (const field of fields) { + if (field.field && field.value !== undefined) { + record[field.field] = field.value ?? '' + } + } + return record +} + +export async function pollQueryResults( + client: CloudWatchLogsClient, + queryId: string, + options: PollOptions = {}, + signal?: AbortSignal +): Promise { + const { maxWaitMs = DEFAULT_EXECUTION_TIMEOUT_MS, pollIntervalMs = 1_000 } = options + const startTime = Date.now() + + while (Date.now() - startTime < maxWaitMs) { + signal?.throwIfAborted() + const command = new GetQueryResultsCommand({ queryId }) + const response = await client.send(command, { abortSignal: signal }) + + const status = response.status ?? 'Unknown' + + if (status === 'Complete') { + return { + results: (response.results ?? []).map(parseResultFields), + statistics: { + bytesScanned: response.statistics?.bytesScanned ?? 0, + recordsMatched: response.statistics?.recordsMatched ?? 0, + recordsScanned: response.statistics?.recordsScanned ?? 0, + }, + status, + } + } + + if (status === 'Failed' || status === 'Cancelled') { + throw new Error(`CloudWatch Log Insights query ${status.toLowerCase()}`) + } + + await sleep(pollIntervalMs) + } + + signal?.throwIfAborted() + const finalResponse = await client.send(new GetQueryResultsCommand({ queryId }), { + abortSignal: signal, + }) + return { + results: (finalResponse.results ?? []).map(parseResultFields), + statistics: { + bytesScanned: finalResponse.statistics?.bytesScanned ?? 0, + recordsMatched: finalResponse.statistics?.recordsMatched ?? 0, + recordsScanned: finalResponse.statistics?.recordsScanned ?? 0, + }, + status: `Timeout (last status: ${finalResponse.status ?? 'Unknown'})`, + } +} + +/** AWS DescribeLogStreams caps `limit` at 50 items per page. */ +const LOG_STREAMS_PAGE_SIZE = 50 + +/** Upper bound on pages drained to avoid unbounded loops on log groups with many streams. */ +const MAX_LOG_STREAMS_PAGES = 20 + +const logger = createLogger('CloudWatchUtils') + +interface DescribedLogStream { + logStreamName: string + lastEventTimestamp: number | undefined + firstEventTimestamp: number | undefined + creationTime: number | undefined + storedBytes: number +} + +/** + * Lists log streams for a log group, following `nextToken` so the complete set + * is returned rather than just the first page. Bounded by + * `MAX_LOG_STREAMS_PAGES`; logs a warning rather than silently dropping streams + * when the cap is hit. Ordering/prefix inputs are preserved across all pages. + * + * When `limit` is provided it is treated as a total result cap: draining stops + * once enough streams have been collected. When omitted, every page is drained. + */ +export async function describeLogStreams( + client: CloudWatchLogsClient, + logGroupName: string, + options?: { prefix?: string; limit?: number }, + signal?: AbortSignal +): Promise<{ logStreams: DescribedLogStream[] }> { + const hasPrefix = Boolean(options?.prefix) + const totalLimit = options?.limit + const logStreams: DescribedLogStream[] = [] + let nextToken: string | undefined + + for (let page = 0; page < MAX_LOG_STREAMS_PAGES; page++) { + const pageLimit = + totalLimit !== undefined + ? Math.min(LOG_STREAMS_PAGE_SIZE, totalLimit - logStreams.length) + : LOG_STREAMS_PAGE_SIZE + + const command = new DescribeLogStreamsCommand({ + logGroupName, + ...(hasPrefix + ? { orderBy: 'LogStreamName', logStreamNamePrefix: options!.prefix } + : { orderBy: 'LastEventTime', descending: true }), + limit: pageLimit, + ...(nextToken && { nextToken }), + }) + + const response = await client.send(command, { abortSignal: signal }) + + for (const ls of response.logStreams ?? []) { + logStreams.push({ + logStreamName: ls.logStreamName ?? '', + lastEventTimestamp: ls.lastEventTimestamp, + firstEventTimestamp: ls.firstEventTimestamp, + creationTime: ls.creationTime, + storedBytes: ls.storedBytes ?? 0, + }) + } + + nextToken = response.nextToken + if (!nextToken) break + if (totalLimit !== undefined && logStreams.length >= totalLimit) break + + if (page === MAX_LOG_STREAMS_PAGES - 1) { + logger.warn( + `DescribeLogStreams hit pagination cap of ${MAX_LOG_STREAMS_PAGES} pages; log stream list may be incomplete`, + { logGroupName } + ) + } + } + + return { + logStreams: totalLimit !== undefined ? logStreams.slice(0, totalLimit) : logStreams, + } +} + +/** AWS FilterLogEvents caps `limit` at 10,000 events per page. */ +const FILTER_LOG_EVENTS_PAGE_SIZE = 10_000 + +/** Upper bound on pages drained to avoid unbounded loops on very active log groups. */ +const MAX_FILTER_LOG_EVENTS_PAGES = 20 + +interface FilteredLogEventResult { + logStreamName: string | undefined + timestamp: number | undefined + message: string | undefined + ingestionTime: number | undefined +} + +/** + * Searches log events across all streams (or a prefix-matched subset) in a log + * group, following `nextToken` so the complete matching set is returned rather + * than just the first page. Bounded by `MAX_FILTER_LOG_EVENTS_PAGES`. + * + * When `limit` is provided it is treated as a total result cap: draining stops + * once enough events have been collected. When omitted, every page is drained. + */ +export async function filterLogEvents( + client: CloudWatchLogsClient, + logGroupName: string, + options?: { + filterPattern?: string + logStreamNamePrefix?: string + startTime?: number + endTime?: number + startFromHead?: boolean + limit?: number + }, + signal?: AbortSignal +): Promise<{ events: FilteredLogEventResult[] }> { + const totalLimit = options?.limit + const events: FilteredLogEventResult[] = [] + let nextToken: string | undefined + + for (let page = 0; page < MAX_FILTER_LOG_EVENTS_PAGES; page++) { + const pageLimit = + totalLimit !== undefined + ? Math.min(FILTER_LOG_EVENTS_PAGE_SIZE, totalLimit - events.length) + : FILTER_LOG_EVENTS_PAGE_SIZE + + const command = new FilterLogEventsCommand({ + logGroupName, + ...(options?.filterPattern && { filterPattern: options.filterPattern }), + ...(options?.logStreamNamePrefix && { logStreamNamePrefix: options.logStreamNamePrefix }), + ...(options?.startTime !== undefined && { startTime: options.startTime }), + ...(options?.endTime !== undefined && { endTime: options.endTime }), + ...(options?.startFromHead !== undefined && { startFromHead: options.startFromHead }), + limit: pageLimit, + ...(nextToken && { nextToken }), + }) + + const response = await client.send(command, { abortSignal: signal }) + + for (const e of response.events ?? []) { + events.push({ + logStreamName: e.logStreamName, + timestamp: e.timestamp, + message: e.message, + ingestionTime: e.ingestionTime, + }) + } + + nextToken = response.nextToken + if (!nextToken) break + if (totalLimit !== undefined && events.length >= totalLimit) break + + if (page === MAX_FILTER_LOG_EVENTS_PAGES - 1) { + logger.warn( + `FilterLogEvents hit pagination cap of ${MAX_FILTER_LOG_EVENTS_PAGES} pages; event list may be incomplete`, + { logGroupName } + ) + } + } + + return { + events: totalLimit !== undefined ? events.slice(0, totalLimit) : events, + } +} + +export async function getLogEvents( + client: CloudWatchLogsClient, + logGroupName: string, + logStreamName: string, + options?: { startTime?: number; endTime?: number; limit?: number }, + signal?: AbortSignal +): Promise<{ + events: { + timestamp: number | undefined + message: string | undefined + ingestionTime: number | undefined + }[] +}> { + const command = new GetLogEventsCommand({ + logGroupName, + logStreamName, + ...(options?.startTime !== undefined && { startTime: options.startTime * 1000 }), + ...(options?.endTime !== undefined && { endTime: options.endTime * 1000 }), + ...(options?.limit !== undefined && { limit: options.limit }), + startFromHead: true, + }) + + const response = await client.send(command, { abortSignal: signal }) + return { + events: (response.events ?? []).map((e) => ({ + timestamp: e.timestamp, + message: e.message, + ingestionTime: e.ingestionTime, + })), + } +} diff --git a/apps/sim/lib/internal/cloudwatch/execute-tool.test.ts b/apps/sim/lib/internal/cloudwatch/execute-tool.test.ts new file mode 100644 index 00000000000..7f41a9acbd7 --- /dev/null +++ b/apps/sim/lib/internal/cloudwatch/execute-tool.test.ts @@ -0,0 +1,193 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mockOperations = vi.hoisted(() => ({ + executeCloudwatchDescribeAlarmHistory: vi.fn(), + executeCloudwatchDescribeAlarms: vi.fn(), + executeCloudwatchDescribeLogGroups: vi.fn(), + executeCloudwatchDescribeLogStreams: vi.fn(), + executeCloudwatchFilterLogEvents: vi.fn(), + executeCloudwatchGetLogEvents: vi.fn(), + executeCloudwatchGetMetricStatistics: vi.fn(), + executeCloudwatchListMetrics: vi.fn(), + executeCloudwatchMuteAlarm: vi.fn(), + executeCloudwatchPutLogGroupRetention: vi.fn(), + executeCloudwatchPutMetricData: vi.fn(), + executeCloudwatchQueryLogs: vi.fn(), + executeCloudwatchUnmuteAlarm: vi.fn(), +})) + +vi.mock('@/lib/internal/cloudwatch/operations', () => ({ + CloudWatchInputError: class CloudWatchInputError extends Error { + readonly status = 400 + }, + ...mockOperations, +})) + +import { executeCloudwatchTool } from '@/lib/internal/cloudwatch/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const CONNECTION = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', +} + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'cloudwatch_list_metrics', + input: CONNECTION, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + userId: 'user-1', + metadata: {}, + }, + requestId: 'request-1', + ...overrides, + } +} + +const TOOL_CASES = [ + [ + 'cloudwatch_describe_alarm_history', + CONNECTION, + mockOperations.executeCloudwatchDescribeAlarmHistory, + ], + ['cloudwatch_describe_alarms', CONNECTION, mockOperations.executeCloudwatchDescribeAlarms], + ['cloudwatch_describe_log_groups', CONNECTION, mockOperations.executeCloudwatchDescribeLogGroups], + [ + 'cloudwatch_describe_log_streams', + { ...CONNECTION, logGroupName: 'group' }, + mockOperations.executeCloudwatchDescribeLogStreams, + ], + [ + 'cloudwatch_filter_log_events', + { ...CONNECTION, logGroupName: 'group' }, + mockOperations.executeCloudwatchFilterLogEvents, + ], + [ + 'cloudwatch_get_log_events', + { ...CONNECTION, logGroupName: 'group', logStreamName: 'stream' }, + mockOperations.executeCloudwatchGetLogEvents, + ], + [ + 'cloudwatch_get_metric_statistics', + { + ...CONNECTION, + namespace: 'Sim/Test', + metricName: 'Requests', + startTime: 1, + endTime: 2, + period: 60, + statistics: ['Average'], + }, + mockOperations.executeCloudwatchGetMetricStatistics, + ], + ['cloudwatch_list_metrics', CONNECTION, mockOperations.executeCloudwatchListMetrics], + [ + 'cloudwatch_mute_alarm', + { + ...CONNECTION, + muteRuleName: 'maintenance', + alarmNames: ['alarm-1'], + durationValue: 1, + durationUnit: 'hours', + }, + mockOperations.executeCloudwatchMuteAlarm, + ], + [ + 'cloudwatch_put_log_group_retention', + { ...CONNECTION, logGroupName: 'group', retentionInDays: 30 }, + mockOperations.executeCloudwatchPutLogGroupRetention, + ], + [ + 'cloudwatch_put_metric_data', + { ...CONNECTION, namespace: 'Sim/Test', metricName: 'Requests', value: 1 }, + mockOperations.executeCloudwatchPutMetricData, + ], + [ + 'cloudwatch_query_logs', + { + ...CONNECTION, + logGroupNames: ['group'], + queryString: 'fields @message', + startTime: 1, + endTime: 2, + }, + mockOperations.executeCloudwatchQueryLogs, + ], + [ + 'cloudwatch_unmute_alarm', + { ...CONNECTION, muteRuleName: 'maintenance' }, + mockOperations.executeCloudwatchUnmuteAlarm, + ], +] as const + +describe('executeCloudwatchTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it.each(TOOL_CASES)('validates and dispatches %s', async (toolId, input, operation) => { + const controller = new AbortController() + operation.mockResolvedValue({ toolId }) + + const response = await executeCloudwatchTool( + createRequest({ toolId, input, signal: controller.signal }) + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ toolId }) + expect(operation).toHaveBeenCalledWith(input, controller.signal) + }) + + it('returns the canonical validation envelope before provider work', async () => { + const response = await executeCloudwatchTool(createRequest({ input: { region: 'invalid' } })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(mockOperations.executeCloudwatchListMetrics).not.toHaveBeenCalled() + }) + + it('preserves provider error envelopes', async () => { + mockOperations.executeCloudwatchListMetrics.mockRejectedValue(new Error('AWS rejected')) + + const response = await executeCloudwatchTool(createRequest()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'Failed to list CloudWatch metrics: AWS rejected', + }) + }) + + it('propagates cancellation without starting provider work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeCloudwatchTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mockOperations.executeCloudwatchListMetrics).not.toHaveBeenCalled() + }) + + it('rethrows cancellation that arrives while provider work is in flight', async () => { + const controller = new AbortController() + mockOperations.executeCloudwatchListMetrics.mockImplementationOnce(async () => { + controller.abort(new DOMException('cancelled', 'AbortError')) + throw new Error('AWS request failed') + }) + + await expect( + executeCloudwatchTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) +}) diff --git a/apps/sim/lib/internal/cloudwatch/execute-tool.ts b/apps/sim/lib/internal/cloudwatch/execute-tool.ts new file mode 100644 index 00000000000..8a26ef1cef7 --- /dev/null +++ b/apps/sim/lib/internal/cloudwatch/execute-tool.ts @@ -0,0 +1,173 @@ +import { toError } from '@sim/utils/errors' +import type { AnyApiRouteContract, ContractBody } from '@/lib/api/contracts' +import { + cloudwatchLogGroupsSelectorContract, + cloudwatchLogStreamsSelectorContract, +} from '@/lib/api/contracts/selectors/cloudwatch' +import { awsCloudwatchDescribeAlarmHistoryContract } from '@/lib/api/contracts/tools/aws/cloudwatch-describe-alarm-history' +import { awsCloudwatchDescribeAlarmsContract } from '@/lib/api/contracts/tools/aws/cloudwatch-describe-alarms' +import { awsCloudwatchFilterLogEventsContract } from '@/lib/api/contracts/tools/aws/cloudwatch-filter-log-events' +import { awsCloudwatchGetLogEventsContract } from '@/lib/api/contracts/tools/aws/cloudwatch-get-log-events' +import { awsCloudwatchGetMetricStatisticsContract } from '@/lib/api/contracts/tools/aws/cloudwatch-get-metric-statistics' +import { awsCloudwatchListMetricsContract } from '@/lib/api/contracts/tools/aws/cloudwatch-list-metrics' +import { awsCloudwatchMuteAlarmContract } from '@/lib/api/contracts/tools/aws/cloudwatch-mute-alarm' +import { awsCloudwatchPutLogGroupRetentionContract } from '@/lib/api/contracts/tools/aws/cloudwatch-put-log-group-retention' +import { awsCloudwatchPutMetricDataContract } from '@/lib/api/contracts/tools/aws/cloudwatch-put-metric-data' +import { awsCloudwatchQueryLogsContract } from '@/lib/api/contracts/tools/aws/cloudwatch-query-logs' +import { awsCloudwatchUnmuteAlarmContract } from '@/lib/api/contracts/tools/aws/cloudwatch-unmute-alarm' +import { + CloudWatchInputError, + executeCloudwatchDescribeAlarmHistory, + executeCloudwatchDescribeAlarms, + executeCloudwatchDescribeLogGroups, + executeCloudwatchDescribeLogStreams, + executeCloudwatchFilterLogEvents, + executeCloudwatchGetLogEvents, + executeCloudwatchGetMetricStatistics, + executeCloudwatchListMetrics, + executeCloudwatchMuteAlarm, + executeCloudwatchPutLogGroupRetention, + executeCloudwatchPutMetricData, + executeCloudwatchQueryLogs, + executeCloudwatchUnmuteAlarm, +} from '@/lib/internal/cloudwatch/operations' +import { parseInternalToolInput } from '@/lib/internal/tool-operations/parse-input' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +async function executeOperation( + contract: C, + input: unknown, + execute: (input: ContractBody, signal?: AbortSignal) => Promise, + errorMessage: string, + signal?: AbortSignal +): Promise { + const parsed = parseInternalToolInput(contract, input) + if (!parsed.success) return parsed.response + try { + const result = await execute(parsed.data, signal) + signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + signal?.throwIfAborted() + if (error instanceof CloudWatchInputError) { + return Response.json({ error: error.message }, { status: error.status }) + } + return Response.json({ error: `${errorMessage}: ${toError(error).message}` }, { status: 500 }) + } +} + +export const executeCloudwatchTool: InternalToolOperationHandler = async ({ + toolId, + input, + signal, +}) => { + signal?.throwIfAborted() + switch (toolId) { + case 'cloudwatch_describe_alarm_history': + return executeOperation( + awsCloudwatchDescribeAlarmHistoryContract, + input, + executeCloudwatchDescribeAlarmHistory, + 'Failed to describe CloudWatch alarm history', + signal + ) + case 'cloudwatch_describe_alarms': + return executeOperation( + awsCloudwatchDescribeAlarmsContract, + input, + executeCloudwatchDescribeAlarms, + 'Failed to describe CloudWatch alarms', + signal + ) + case 'cloudwatch_describe_log_groups': + return executeOperation( + cloudwatchLogGroupsSelectorContract, + input, + executeCloudwatchDescribeLogGroups, + 'Failed to describe CloudWatch log groups', + signal + ) + case 'cloudwatch_describe_log_streams': + return executeOperation( + cloudwatchLogStreamsSelectorContract, + input, + executeCloudwatchDescribeLogStreams, + 'Failed to describe CloudWatch log streams', + signal + ) + case 'cloudwatch_filter_log_events': + return executeOperation( + awsCloudwatchFilterLogEventsContract, + input, + executeCloudwatchFilterLogEvents, + 'Failed to filter CloudWatch log events', + signal + ) + case 'cloudwatch_get_log_events': + return executeOperation( + awsCloudwatchGetLogEventsContract, + input, + executeCloudwatchGetLogEvents, + 'Failed to get CloudWatch log events', + signal + ) + case 'cloudwatch_get_metric_statistics': + return executeOperation( + awsCloudwatchGetMetricStatisticsContract, + input, + executeCloudwatchGetMetricStatistics, + 'Failed to get CloudWatch metric statistics', + signal + ) + case 'cloudwatch_list_metrics': + return executeOperation( + awsCloudwatchListMetricsContract, + input, + executeCloudwatchListMetrics, + 'Failed to list CloudWatch metrics', + signal + ) + case 'cloudwatch_mute_alarm': + return executeOperation( + awsCloudwatchMuteAlarmContract, + input, + executeCloudwatchMuteAlarm, + 'Failed to create CloudWatch alarm mute rule', + signal + ) + case 'cloudwatch_put_log_group_retention': + return executeOperation( + awsCloudwatchPutLogGroupRetentionContract, + input, + executeCloudwatchPutLogGroupRetention, + 'Failed to set CloudWatch log group retention', + signal + ) + case 'cloudwatch_put_metric_data': + return executeOperation( + awsCloudwatchPutMetricDataContract, + input, + executeCloudwatchPutMetricData, + 'Failed to publish CloudWatch metric', + signal + ) + case 'cloudwatch_query_logs': + return executeOperation( + awsCloudwatchQueryLogsContract, + input, + executeCloudwatchQueryLogs, + 'CloudWatch Log Insights query failed', + signal + ) + case 'cloudwatch_unmute_alarm': + return executeOperation( + awsCloudwatchUnmuteAlarmContract, + input, + executeCloudwatchUnmuteAlarm, + 'Failed to delete CloudWatch alarm mute rule', + signal + ) + default: + return Response.json({ error: `Unsupported CloudWatch tool: ${toolId}` }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/cloudwatch/http-route.ts b/apps/sim/lib/internal/cloudwatch/http-route.ts new file mode 100644 index 00000000000..fe9b4de4cc2 --- /dev/null +++ b/apps/sim/lib/internal/cloudwatch/http-route.ts @@ -0,0 +1,54 @@ +import type { Logger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { type NextRequest, NextResponse } from 'next/server' +import { checkInternalAuth, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { CloudWatchInputError } from '@/lib/internal/cloudwatch/operations' + +type CloudWatchRequestParseResult = + | { success: true; data: { body: T } } + | { success: false; response: Response } + +interface CloudWatchHttpRouteConfig { + logger: Logger + parse: (request: NextRequest) => Promise> + execute: (input: T, signal?: AbortSignal) => Promise + errorMessage: string + auth?: 'internal' | 'session-or-internal' + logError?: (error: unknown) => void +} + +export function createCloudWatchHttpRoute({ + logger, + parse, + execute, + errorMessage, + auth = 'internal', + logError, +}: CloudWatchHttpRouteConfig) { + return withRouteHandler(async (request: NextRequest) => { + try { + const result = + auth === 'session-or-internal' + ? await checkSessionOrInternalAuth(request) + : await checkInternalAuth(request) + if (!result.success || !result.userId) { + return NextResponse.json({ error: result.error || 'Unauthorized' }, { status: 401 }) + } + + const parsed = await parse(request) + if (!parsed.success) return parsed.response + return NextResponse.json(await execute(parsed.data.body, request.signal)) + } catch (error) { + if (error instanceof CloudWatchInputError) { + return NextResponse.json({ error: error.message }, { status: error.status }) + } + if (logError) logError(error) + else logger.error(errorMessage, { error: toError(error).message }) + return NextResponse.json( + { error: `${errorMessage}: ${toError(error).message}` }, + { status: 500 } + ) + } + }) +} diff --git a/apps/sim/lib/internal/cloudwatch/operations.test.ts b/apps/sim/lib/internal/cloudwatch/operations.test.ts new file mode 100644 index 00000000000..a4ddbe0229c --- /dev/null +++ b/apps/sim/lib/internal/cloudwatch/operations.test.ts @@ -0,0 +1,88 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + createCloudWatchClient: vi.fn(), + createCloudWatchLogsClient: vi.fn(), + destroy: vi.fn(), + send: vi.fn(), +})) + +vi.mock('@/lib/internal/cloudwatch/client', () => ({ + createCloudWatchClient: mocks.createCloudWatchClient, + createCloudWatchLogsClient: mocks.createCloudWatchLogsClient, + describeLogStreams: vi.fn(), + filterLogEvents: vi.fn(), + getLogEvents: vi.fn(), + pollQueryResults: vi.fn(), +})) + +import { + CloudWatchInputError, + executeCloudwatchGetMetricStatistics, + executeCloudwatchListMetrics, +} from '@/lib/internal/cloudwatch/operations' + +const CONNECTION = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', +} + +describe('CloudWatch operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.createCloudWatchClient.mockReturnValue({ send: mocks.send, destroy: mocks.destroy }) + }) + + it('forwards cancellation across paginated metric requests and destroys the client', async () => { + const controller = new AbortController() + mocks.send + .mockResolvedValueOnce({ + Metrics: [{ Namespace: 'Sim/Test', MetricName: 'Requests', Dimensions: [] }], + NextToken: 'next-page', + }) + .mockResolvedValueOnce({ + Metrics: [{ Namespace: 'Sim/Test', MetricName: 'Errors', Dimensions: [] }], + }) + + await expect(executeCloudwatchListMetrics(CONNECTION, controller.signal)).resolves.toEqual({ + success: true, + output: { + metrics: [ + { namespace: 'Sim/Test', metricName: 'Requests', dimensions: [] }, + { namespace: 'Sim/Test', metricName: 'Errors', dimensions: [] }, + ], + }, + }) + expect(mocks.send).toHaveBeenCalledTimes(2) + expect(mocks.send.mock.calls[0]?.[1]).toEqual({ abortSignal: controller.signal }) + expect(mocks.send.mock.calls[1]?.[1]).toEqual({ abortSignal: controller.signal }) + expect(mocks.destroy).toHaveBeenCalledOnce() + }) + + it('destroys the client when provider execution fails', async () => { + mocks.send.mockRejectedValue(new Error('provider failure')) + + await expect(executeCloudwatchListMetrics(CONNECTION)).rejects.toThrow('provider failure') + expect(mocks.destroy).toHaveBeenCalledOnce() + }) + + it('rejects invalid metric dimensions before creating a client', async () => { + await expect( + executeCloudwatchGetMetricStatistics({ + ...CONNECTION, + namespace: 'Sim/Test', + metricName: 'Requests', + startTime: 1, + endTime: 2, + period: 60, + statistics: ['Average'], + dimensions: '{invalid', + }) + ).rejects.toBeInstanceOf(CloudWatchInputError) + expect(mocks.createCloudWatchClient).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/cloudwatch/operations.ts b/apps/sim/lib/internal/cloudwatch/operations.ts new file mode 100644 index 00000000000..51082581223 --- /dev/null +++ b/apps/sim/lib/internal/cloudwatch/operations.ts @@ -0,0 +1,563 @@ +import { + type AlarmType, + DeleteAlarmMuteRuleCommand, + DescribeAlarmHistoryCommand, + DescribeAlarmsCommand, + GetMetricStatisticsCommand, + ListMetricsCommand, + PutAlarmMuteRuleCommand, + PutMetricDataCommand, + type StandardUnit, + type StateValue, +} from '@aws-sdk/client-cloudwatch' +import { + DeleteRetentionPolicyCommand, + DescribeLogGroupsCommand, + PutRetentionPolicyCommand, + StartQueryCommand, +} from '@aws-sdk/client-cloudwatch-logs' +import { createLogger } from '@sim/logger' +import type { + CloudwatchLogGroupsSelectorBody, + CloudwatchLogStreamsSelectorBody, +} from '@/lib/api/contracts/selectors/cloudwatch' +import type { AwsCloudwatchDescribeAlarmHistoryBody } from '@/lib/api/contracts/tools/aws/cloudwatch-describe-alarm-history' +import type { AwsCloudwatchDescribeAlarmsBody } from '@/lib/api/contracts/tools/aws/cloudwatch-describe-alarms' +import type { AwsCloudwatchFilterLogEventsBody } from '@/lib/api/contracts/tools/aws/cloudwatch-filter-log-events' +import type { AwsCloudwatchGetLogEventsBody } from '@/lib/api/contracts/tools/aws/cloudwatch-get-log-events' +import type { AwsCloudwatchGetMetricStatisticsBody } from '@/lib/api/contracts/tools/aws/cloudwatch-get-metric-statistics' +import type { AwsCloudwatchListMetricsBody } from '@/lib/api/contracts/tools/aws/cloudwatch-list-metrics' +import type { AwsCloudwatchMuteAlarmBody } from '@/lib/api/contracts/tools/aws/cloudwatch-mute-alarm' +import type { AwsCloudwatchPutLogGroupRetentionBody } from '@/lib/api/contracts/tools/aws/cloudwatch-put-log-group-retention' +import type { AwsCloudwatchPutMetricDataBody } from '@/lib/api/contracts/tools/aws/cloudwatch-put-metric-data' +import type { AwsCloudwatchQueryLogsBody } from '@/lib/api/contracts/tools/aws/cloudwatch-query-logs' +import type { AwsCloudwatchUnmuteAlarmBody } from '@/lib/api/contracts/tools/aws/cloudwatch-unmute-alarm' +import { + createCloudWatchClient, + createCloudWatchLogsClient, + describeLogStreams, + filterLogEvents, + getLogEvents, + pollQueryResults, +} from '@/lib/internal/cloudwatch/client' + +const logger = createLogger('CloudWatchOperations') +const ALARM_HISTORY_PAGE_SIZE = 100 +const MAX_ALARM_HISTORY_PAGES = 20 +const LOG_GROUPS_PAGE_SIZE = 50 +const MAX_LOG_GROUPS_PAGES = 20 +const METRICS_PAGE_SIZE = 500 +const MAX_METRICS_PAGES = 20 +const NON_IDEMPOTENT_MAX_ATTEMPTS = 1 + +export class CloudWatchInputError extends Error { + readonly status = 400 +} + +export async function executeCloudwatchDescribeAlarmHistory( + input: AwsCloudwatchDescribeAlarmHistoryBody, + signal?: AbortSignal +) { + const client = createCloudWatchClient(input) + try { + const items: { + alarmName: string | undefined + alarmType: string | undefined + timestamp: number | undefined + historyItemType: string | undefined + historySummary: string | undefined + }[] = [] + let nextToken: string | undefined + + for (let page = 0; page < MAX_ALARM_HISTORY_PAGES; page++) { + const pageLimit = + input.limit !== undefined + ? Math.min(ALARM_HISTORY_PAGE_SIZE, input.limit - items.length) + : ALARM_HISTORY_PAGE_SIZE + const response = await client.send( + new DescribeAlarmHistoryCommand({ + ...(input.alarmName && { AlarmName: input.alarmName }), + AlarmTypes: ['MetricAlarm', 'CompositeAlarm'] as AlarmType[], + ...(input.historyItemType && { HistoryItemType: input.historyItemType }), + ...(input.startDate !== undefined && { StartDate: new Date(input.startDate * 1000) }), + ...(input.endDate !== undefined && { EndDate: new Date(input.endDate * 1000) }), + ScanBy: input.scanBy ?? 'TimestampDescending', + MaxRecords: pageLimit, + ...(nextToken && { NextToken: nextToken }), + }), + { abortSignal: signal } + ) + + for (const item of response.AlarmHistoryItems ?? []) { + items.push({ + alarmName: item.AlarmName, + alarmType: item.AlarmType, + timestamp: item.Timestamp?.getTime(), + historyItemType: item.HistoryItemType, + historySummary: item.HistorySummary, + }) + } + nextToken = response.NextToken + if (!nextToken || (input.limit !== undefined && items.length >= input.limit)) break + if (page === MAX_ALARM_HISTORY_PAGES - 1) { + logger.warn( + `DescribeAlarmHistory hit pagination cap of ${MAX_ALARM_HISTORY_PAGES} pages; history may be incomplete` + ) + } + } + + return { + success: true, + output: { + alarmHistoryItems: input.limit !== undefined ? items.slice(0, input.limit) : items, + }, + } + } finally { + client.destroy() + } +} + +export async function executeCloudwatchDescribeAlarms( + input: AwsCloudwatchDescribeAlarmsBody, + signal?: AbortSignal +) { + const client = createCloudWatchClient(input) + try { + const response = await client.send( + new DescribeAlarmsCommand({ + ...(input.alarmNamePrefix && { AlarmNamePrefix: input.alarmNamePrefix }), + ...(input.stateValue && { StateValue: input.stateValue as StateValue }), + AlarmTypes: input.alarmType + ? [input.alarmType as AlarmType] + : (['MetricAlarm', 'CompositeAlarm'] as AlarmType[]), + ...(input.limit !== undefined && { MaxRecords: input.limit }), + }), + { abortSignal: signal } + ) + const metricAlarms = (response.MetricAlarms ?? []).map((alarm) => ({ + alarmName: alarm.AlarmName ?? '', + alarmArn: alarm.AlarmArn ?? '', + stateValue: alarm.StateValue ?? 'UNKNOWN', + stateReason: alarm.StateReason ?? '', + metricName: alarm.MetricName, + namespace: alarm.Namespace, + comparisonOperator: alarm.ComparisonOperator, + threshold: alarm.Threshold, + evaluationPeriods: alarm.EvaluationPeriods, + stateUpdatedTimestamp: alarm.StateUpdatedTimestamp?.getTime(), + })) + const compositeAlarms = (response.CompositeAlarms ?? []).map((alarm) => ({ + alarmName: alarm.AlarmName ?? '', + alarmArn: alarm.AlarmArn ?? '', + stateValue: alarm.StateValue ?? 'UNKNOWN', + stateReason: alarm.StateReason ?? '', + metricName: undefined, + namespace: undefined, + comparisonOperator: undefined, + threshold: undefined, + evaluationPeriods: undefined, + stateUpdatedTimestamp: alarm.StateUpdatedTimestamp?.getTime(), + })) + return { success: true, output: { alarms: [...metricAlarms, ...compositeAlarms] } } + } finally { + client.destroy() + } +} + +export async function executeCloudwatchDescribeLogGroups( + input: CloudwatchLogGroupsSelectorBody, + signal?: AbortSignal +) { + const client = createCloudWatchLogsClient(input) + try { + const logGroups: { + logGroupName: string + arn: string + storedBytes: number + retentionInDays: number | undefined + creationTime: number | undefined + }[] = [] + let nextToken: string | undefined + for (let page = 0; page < MAX_LOG_GROUPS_PAGES; page++) { + const pageLimit = + input.limit !== undefined + ? Math.min(LOG_GROUPS_PAGE_SIZE, input.limit - logGroups.length) + : LOG_GROUPS_PAGE_SIZE + const response = await client.send( + new DescribeLogGroupsCommand({ + ...(input.prefix && { logGroupNamePrefix: input.prefix }), + limit: pageLimit, + ...(nextToken && { nextToken }), + }), + { abortSignal: signal } + ) + for (const group of response.logGroups ?? []) { + logGroups.push({ + logGroupName: group.logGroupName ?? '', + arn: group.arn ?? '', + storedBytes: group.storedBytes ?? 0, + retentionInDays: group.retentionInDays, + creationTime: group.creationTime, + }) + } + nextToken = response.nextToken + if (!nextToken || (input.limit !== undefined && logGroups.length >= input.limit)) break + if (page === MAX_LOG_GROUPS_PAGES - 1) { + logger.warn( + `DescribeLogGroups hit pagination cap of ${MAX_LOG_GROUPS_PAGES} pages; log group list may be incomplete` + ) + } + } + return { + success: true, + output: { + logGroups: input.limit !== undefined ? logGroups.slice(0, input.limit) : logGroups, + }, + } + } finally { + client.destroy() + } +} + +export async function executeCloudwatchDescribeLogStreams( + input: CloudwatchLogStreamsSelectorBody, + signal?: AbortSignal +) { + const client = createCloudWatchLogsClient(input) + try { + const result = await describeLogStreams( + client, + input.logGroupName, + { prefix: input.prefix, limit: input.limit }, + signal + ) + return { success: true, output: { logStreams: result.logStreams } } + } finally { + client.destroy() + } +} + +export async function executeCloudwatchFilterLogEvents( + input: AwsCloudwatchFilterLogEventsBody, + signal?: AbortSignal +) { + const client = createCloudWatchLogsClient(input) + try { + const result = await filterLogEvents( + client, + input.logGroupName, + { + filterPattern: input.filterPattern, + logStreamNamePrefix: input.logStreamNamePrefix, + startTime: input.startTime !== undefined ? input.startTime * 1000 : undefined, + endTime: input.endTime !== undefined ? input.endTime * 1000 : undefined, + startFromHead: input.startFromHead, + limit: input.limit, + }, + signal + ) + return { success: true, output: { events: result.events } } + } finally { + client.destroy() + } +} + +export async function executeCloudwatchGetLogEvents( + input: AwsCloudwatchGetLogEventsBody, + signal?: AbortSignal +) { + const client = createCloudWatchLogsClient(input) + try { + const result = await getLogEvents( + client, + input.logGroupName, + input.logStreamName, + { startTime: input.startTime, endTime: input.endTime, limit: input.limit }, + signal + ) + return { success: true, output: { events: result.events } } + } finally { + client.destroy() + } +} + +export async function executeCloudwatchGetMetricStatistics( + input: AwsCloudwatchGetMetricStatisticsBody, + signal?: AbortSignal +) { + let dimensions: { Name: string; Value: string }[] | undefined + if (input.dimensions) { + try { + const parsed: unknown = JSON.parse(input.dimensions) + if (Array.isArray(parsed)) { + dimensions = parsed.map((dimension: Record) => ({ + Name: dimension.name, + Value: dimension.value, + })) + } else if (typeof parsed === 'object' && parsed !== null) { + dimensions = Object.entries(parsed).map(([name, value]) => ({ + Name: name, + Value: String(value), + })) + } + } catch { + throw new CloudWatchInputError('Invalid dimensions JSON format') + } + } + + const client = createCloudWatchClient(input) + try { + const response = await client.send( + new GetMetricStatisticsCommand({ + Namespace: input.namespace, + MetricName: input.metricName, + StartTime: new Date(input.startTime * 1000), + EndTime: new Date(input.endTime * 1000), + Period: input.period, + Statistics: input.statistics, + ...(dimensions && { Dimensions: dimensions }), + }), + { abortSignal: signal } + ) + const datapoints = (response.Datapoints ?? []) + .sort((a, b) => (a.Timestamp?.getTime() ?? 0) - (b.Timestamp?.getTime() ?? 0)) + .map((datapoint) => ({ + timestamp: datapoint.Timestamp?.getTime() ?? 0, + average: datapoint.Average, + sum: datapoint.Sum, + minimum: datapoint.Minimum, + maximum: datapoint.Maximum, + sampleCount: datapoint.SampleCount, + unit: datapoint.Unit, + })) + return { + success: true, + output: { label: response.Label ?? input.metricName, datapoints }, + } + } finally { + client.destroy() + } +} + +export async function executeCloudwatchListMetrics( + input: AwsCloudwatchListMetricsBody, + signal?: AbortSignal +) { + const client = createCloudWatchClient(input) + try { + const totalLimit = input.limit ?? METRICS_PAGE_SIZE + const metrics: { + namespace: string + metricName: string + dimensions: { name: string; value: string }[] + }[] = [] + let nextToken: string | undefined + for (let page = 0; page < MAX_METRICS_PAGES; page++) { + const response = await client.send( + new ListMetricsCommand({ + ...(input.namespace && { Namespace: input.namespace }), + ...(input.metricName && { MetricName: input.metricName }), + ...(input.recentlyActive && { RecentlyActive: 'PT3H' }), + ...(nextToken && { NextToken: nextToken }), + }), + { abortSignal: signal } + ) + for (const metric of response.Metrics ?? []) { + metrics.push({ + namespace: metric.Namespace ?? '', + metricName: metric.MetricName ?? '', + dimensions: (metric.Dimensions ?? []).map((dimension) => ({ + name: dimension.Name ?? '', + value: dimension.Value ?? '', + })), + }) + } + nextToken = response.NextToken + if (!nextToken || metrics.length >= totalLimit) break + if (page === MAX_METRICS_PAGES - 1) { + logger.warn( + `ListMetrics hit pagination cap of ${MAX_METRICS_PAGES} pages; metric list may be incomplete` + ) + } + } + return { success: true, output: { metrics: metrics.slice(0, totalLimit) } } + } finally { + client.destroy() + } +} + +function toAtExpression(date: Date): string { + const yyyy = date.getUTCFullYear() + const mm = String(date.getUTCMonth() + 1).padStart(2, '0') + const dd = String(date.getUTCDate()).padStart(2, '0') + const hh = String(date.getUTCHours()).padStart(2, '0') + const min = String(date.getUTCMinutes()).padStart(2, '0') + return `at(${yyyy}-${mm}-${dd}T${hh}:${min})` +} + +function toIsoDuration(value: number, unit: 'minutes' | 'hours' | 'days'): string { + if (unit === 'minutes') return `PT${value}M` + if (unit === 'hours') return `PT${value}H` + return `P${value}D` +} + +export async function executeCloudwatchMuteAlarm( + input: AwsCloudwatchMuteAlarmBody, + signal?: AbortSignal +) { + const startDate = input.startDate !== undefined ? new Date(input.startDate * 1000) : new Date() + const expression = toAtExpression(startDate) + const duration = toIsoDuration(input.durationValue, input.durationUnit) + const client = createCloudWatchClient(input) + try { + await client.send( + new PutAlarmMuteRuleCommand({ + Name: input.muteRuleName, + ...(input.description && { Description: input.description }), + Rule: { Schedule: { Expression: expression, Duration: duration } }, + MuteTargets: { AlarmNames: input.alarmNames }, + }), + { abortSignal: signal } + ) + return { + success: true, + output: { + success: true, + muteRuleName: input.muteRuleName, + alarmNames: input.alarmNames, + expression, + duration, + }, + } + } finally { + client.destroy() + } +} + +export async function executeCloudwatchPutLogGroupRetention( + input: AwsCloudwatchPutLogGroupRetentionBody, + signal?: AbortSignal +) { + const client = createCloudWatchLogsClient(input) + try { + if (input.retentionInDays !== undefined) { + await client.send( + new PutRetentionPolicyCommand({ + logGroupName: input.logGroupName, + retentionInDays: input.retentionInDays, + }), + { abortSignal: signal } + ) + } else { + await client.send(new DeleteRetentionPolicyCommand({ logGroupName: input.logGroupName }), { + abortSignal: signal, + }) + } + return { + success: true, + output: { + success: true, + logGroupName: input.logGroupName, + retentionInDays: input.retentionInDays ?? null, + }, + } + } finally { + client.destroy() + } +} + +export async function executeCloudwatchPutMetricData( + input: AwsCloudwatchPutMetricDataBody, + signal?: AbortSignal +) { + const client = createCloudWatchClient(input, { maxAttempts: NON_IDEMPOTENT_MAX_ATTEMPTS }) + try { + const timestamp = new Date() + const dimensions: { Name: string; Value: string }[] = [] + if (input.dimensions) { + const parsed = JSON.parse(input.dimensions) as Record + for (const [name, value] of Object.entries(parsed)) { + dimensions.push({ Name: name, Value: String(value) }) + } + } + await client.send( + new PutMetricDataCommand({ + Namespace: input.namespace, + MetricData: [ + { + MetricName: input.metricName, + Value: input.value, + Timestamp: timestamp, + ...(input.unit && { Unit: input.unit as StandardUnit }), + ...(dimensions.length > 0 && { Dimensions: dimensions }), + }, + ], + }), + { abortSignal: signal } + ) + return { + success: true, + output: { + success: true, + namespace: input.namespace, + metricName: input.metricName, + value: input.value, + unit: input.unit ?? 'None', + timestamp: timestamp.toISOString(), + }, + } + } finally { + client.destroy() + } +} + +export async function executeCloudwatchQueryLogs( + input: AwsCloudwatchQueryLogsBody, + signal?: AbortSignal +) { + const client = createCloudWatchLogsClient(input) + try { + const response = await client.send( + new StartQueryCommand({ + logGroupNames: input.logGroupNames, + queryString: input.queryString, + startTime: input.startTime, + endTime: input.endTime, + ...(input.limit !== undefined && { limit: input.limit }), + }), + { abortSignal: signal } + ) + if (!response.queryId) { + throw new Error('Failed to start CloudWatch Log Insights query: no queryId returned') + } + const result = await pollQueryResults(client, response.queryId, {}, signal) + return { + success: true, + output: { + results: result.results, + statistics: result.statistics, + status: result.status, + }, + } + } finally { + client.destroy() + } +} + +export async function executeCloudwatchUnmuteAlarm( + input: AwsCloudwatchUnmuteAlarmBody, + signal?: AbortSignal +) { + const client = createCloudWatchClient(input) + try { + await client.send(new DeleteAlarmMuteRuleCommand({ AlarmMuteRuleName: input.muteRuleName }), { + abortSignal: signal, + }) + return { + success: true, + output: { success: true, muteRuleName: input.muteRuleName }, + } + } finally { + client.destroy() + } +} diff --git a/apps/sim/lib/internal/codepipeline/client.ts b/apps/sim/lib/internal/codepipeline/client.ts new file mode 100644 index 00000000000..0061c45688d --- /dev/null +++ b/apps/sim/lib/internal/codepipeline/client.ts @@ -0,0 +1,17 @@ +import { CodePipelineClient } from '@aws-sdk/client-codepipeline' + +export interface CodePipelineConnectionConfig { + region: string + accessKeyId: string + secretAccessKey: string +} + +export function createCodePipelineClient(config: CodePipelineConnectionConfig): CodePipelineClient { + return new CodePipelineClient({ + region: config.region, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + }) +} diff --git a/apps/sim/app/api/tools/codepipeline/utils.ts b/apps/sim/lib/internal/codepipeline/errors.ts similarity index 100% rename from apps/sim/app/api/tools/codepipeline/utils.ts rename to apps/sim/lib/internal/codepipeline/errors.ts diff --git a/apps/sim/lib/internal/codepipeline/execute-tool.test.ts b/apps/sim/lib/internal/codepipeline/execute-tool.test.ts new file mode 100644 index 00000000000..01095845907 --- /dev/null +++ b/apps/sim/lib/internal/codepipeline/execute-tool.test.ts @@ -0,0 +1,160 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mockOperations = vi.hoisted(() => ({ + executeCodepipelineDisableStageTransition: vi.fn(), + executeCodepipelineEnableStageTransition: vi.fn(), + executeCodepipelineGetPipeline: vi.fn(), + executeCodepipelineGetPipelineExecution: vi.fn(), + executeCodepipelineGetPipelineState: vi.fn(), + executeCodepipelineListActionExecutions: vi.fn(), + executeCodepipelineListPipelineExecutions: vi.fn(), + executeCodepipelineListPipelines: vi.fn(), + executeCodepipelinePutApprovalResult: vi.fn(), + executeCodepipelineRetryStageExecution: vi.fn(), + executeCodepipelineStartExecution: vi.fn(), + executeCodepipelineStopExecution: vi.fn(), +})) + +vi.mock('@/lib/internal/codepipeline/operations', () => mockOperations) + +import { executeCodepipelineTool } from '@/lib/internal/codepipeline/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const CONNECTION = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', +} + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'codepipeline_list_pipelines', + input: CONNECTION, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + userId: 'user-1', + metadata: {}, + }, + requestId: 'request-1', + ...overrides, + } +} + +const PIPELINE = { ...CONNECTION, pipelineName: 'pipeline' } +const EXECUTION = { ...PIPELINE, pipelineExecutionId: 'execution-id' } + +const TOOL_CASES = [ + [ + 'codepipeline_disable_stage_transition', + { + ...PIPELINE, + stageName: 'Deploy', + transitionType: 'Inbound', + reason: 'maintenance', + }, + mockOperations.executeCodepipelineDisableStageTransition, + ], + [ + 'codepipeline_enable_stage_transition', + { ...PIPELINE, stageName: 'Deploy', transitionType: 'Outbound' }, + mockOperations.executeCodepipelineEnableStageTransition, + ], + [ + 'codepipeline_get_pipeline_execution', + EXECUTION, + mockOperations.executeCodepipelineGetPipelineExecution, + ], + ['codepipeline_get_pipeline_state', PIPELINE, mockOperations.executeCodepipelineGetPipelineState], + ['codepipeline_get_pipeline', PIPELINE, mockOperations.executeCodepipelineGetPipeline], + [ + 'codepipeline_list_action_executions', + PIPELINE, + mockOperations.executeCodepipelineListActionExecutions, + ], + [ + 'codepipeline_list_pipeline_executions', + PIPELINE, + mockOperations.executeCodepipelineListPipelineExecutions, + ], + ['codepipeline_list_pipelines', CONNECTION, mockOperations.executeCodepipelineListPipelines], + [ + 'codepipeline_put_approval_result', + { + ...PIPELINE, + stageName: 'Approval', + actionName: 'Approve', + token: 'approval-token', + status: 'Approved', + summary: 'approved', + }, + mockOperations.executeCodepipelinePutApprovalResult, + ], + [ + 'codepipeline_retry_stage_execution', + { ...EXECUTION, stageName: 'Deploy', retryMode: 'FAILED_ACTIONS' }, + mockOperations.executeCodepipelineRetryStageExecution, + ], + ['codepipeline_start_execution', PIPELINE, mockOperations.executeCodepipelineStartExecution], + ['codepipeline_stop_execution', EXECUTION, mockOperations.executeCodepipelineStopExecution], +] as const + +describe('executeCodepipelineTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it.each(TOOL_CASES)('validates and dispatches %s', async (toolId, input, operation) => { + const controller = new AbortController() + operation.mockResolvedValue({ toolId }) + + const response = await executeCodepipelineTool( + createRequest({ toolId, input, signal: controller.signal }) + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ toolId }) + expect(operation).toHaveBeenCalledWith(input, controller.signal) + }) + + it('returns the canonical validation envelope before provider work', async () => { + const response = await executeCodepipelineTool(createRequest({ input: { region: '' } })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(mockOperations.executeCodepipelineListPipelines).not.toHaveBeenCalled() + }) + + it('preserves AWS client status and the provider error envelope', async () => { + const error = Object.assign(new Error('Pipeline missing'), { + $metadata: { httpStatusCode: 404 }, + }) + mockOperations.executeCodepipelineListPipelines.mockRejectedValue(error) + + const response = await executeCodepipelineTool(createRequest()) + + expect(response.status).toBe(404) + await expect(response.json()).resolves.toEqual({ + error: 'Failed to list CodePipeline pipelines: Pipeline missing', + }) + }) + + it('propagates cancellation without starting provider work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeCodepipelineTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mockOperations.executeCodepipelineListPipelines).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/codepipeline/execute-tool.ts b/apps/sim/lib/internal/codepipeline/execute-tool.ts new file mode 100644 index 00000000000..6f3a173ed4e --- /dev/null +++ b/apps/sim/lib/internal/codepipeline/execute-tool.ts @@ -0,0 +1,163 @@ +import { toError } from '@sim/utils/errors' +import type { AnyApiRouteContract, ContractBody } from '@/lib/api/contracts' +import { awsCodepipelineDisableStageTransitionContract } from '@/lib/api/contracts/tools/aws/codepipeline-disable-stage-transition' +import { awsCodepipelineEnableStageTransitionContract } from '@/lib/api/contracts/tools/aws/codepipeline-enable-stage-transition' +import { awsCodepipelineGetPipelineContract } from '@/lib/api/contracts/tools/aws/codepipeline-get-pipeline' +import { awsCodepipelineGetPipelineExecutionContract } from '@/lib/api/contracts/tools/aws/codepipeline-get-pipeline-execution' +import { awsCodepipelineGetPipelineStateContract } from '@/lib/api/contracts/tools/aws/codepipeline-get-pipeline-state' +import { awsCodepipelineListActionExecutionsContract } from '@/lib/api/contracts/tools/aws/codepipeline-list-action-executions' +import { awsCodepipelineListPipelineExecutionsContract } from '@/lib/api/contracts/tools/aws/codepipeline-list-pipeline-executions' +import { awsCodepipelineListPipelinesContract } from '@/lib/api/contracts/tools/aws/codepipeline-list-pipelines' +import { awsCodepipelinePutApprovalResultContract } from '@/lib/api/contracts/tools/aws/codepipeline-put-approval-result' +import { awsCodepipelineRetryStageExecutionContract } from '@/lib/api/contracts/tools/aws/codepipeline-retry-stage-execution' +import { awsCodepipelineStartExecutionContract } from '@/lib/api/contracts/tools/aws/codepipeline-start-execution' +import { awsCodepipelineStopExecutionContract } from '@/lib/api/contracts/tools/aws/codepipeline-stop-execution' +import { awsErrorStatus } from '@/lib/internal/codepipeline/errors' +import { + executeCodepipelineDisableStageTransition, + executeCodepipelineEnableStageTransition, + executeCodepipelineGetPipeline, + executeCodepipelineGetPipelineExecution, + executeCodepipelineGetPipelineState, + executeCodepipelineListActionExecutions, + executeCodepipelineListPipelineExecutions, + executeCodepipelineListPipelines, + executeCodepipelinePutApprovalResult, + executeCodepipelineRetryStageExecution, + executeCodepipelineStartExecution, + executeCodepipelineStopExecution, +} from '@/lib/internal/codepipeline/operations' +import { parseInternalToolInput } from '@/lib/internal/tool-operations/parse-input' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +async function executeOperation( + contract: C, + input: unknown, + execute: (input: ContractBody, signal?: AbortSignal) => Promise, + errorMessage: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const parsed = parseInternalToolInput(contract, input) + if (!parsed.success) return parsed.response + + try { + const result = await execute(parsed.data, signal) + signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + signal?.throwIfAborted() + return Response.json( + { error: `${errorMessage}: ${toError(error).message}` }, + { status: awsErrorStatus(error) } + ) + } +} + +export const executeCodepipelineTool: InternalToolOperationHandler = async ({ + toolId, + input, + signal, +}) => { + signal?.throwIfAborted() + switch (toolId) { + case 'codepipeline_disable_stage_transition': + return executeOperation( + awsCodepipelineDisableStageTransitionContract, + input, + executeCodepipelineDisableStageTransition, + 'Failed to disable CodePipeline stage transition', + signal + ) + case 'codepipeline_enable_stage_transition': + return executeOperation( + awsCodepipelineEnableStageTransitionContract, + input, + executeCodepipelineEnableStageTransition, + 'Failed to enable CodePipeline stage transition', + signal + ) + case 'codepipeline_get_pipeline_execution': + return executeOperation( + awsCodepipelineGetPipelineExecutionContract, + input, + executeCodepipelineGetPipelineExecution, + 'Failed to get CodePipeline pipeline execution', + signal + ) + case 'codepipeline_get_pipeline_state': + return executeOperation( + awsCodepipelineGetPipelineStateContract, + input, + executeCodepipelineGetPipelineState, + 'Failed to get CodePipeline pipeline state', + signal + ) + case 'codepipeline_get_pipeline': + return executeOperation( + awsCodepipelineGetPipelineContract, + input, + executeCodepipelineGetPipeline, + 'Failed to get CodePipeline pipeline', + signal + ) + case 'codepipeline_list_action_executions': + return executeOperation( + awsCodepipelineListActionExecutionsContract, + input, + executeCodepipelineListActionExecutions, + 'Failed to list CodePipeline action executions', + signal + ) + case 'codepipeline_list_pipeline_executions': + return executeOperation( + awsCodepipelineListPipelineExecutionsContract, + input, + executeCodepipelineListPipelineExecutions, + 'Failed to list CodePipeline pipeline executions', + signal + ) + case 'codepipeline_list_pipelines': + return executeOperation( + awsCodepipelineListPipelinesContract, + input, + executeCodepipelineListPipelines, + 'Failed to list CodePipeline pipelines', + signal + ) + case 'codepipeline_put_approval_result': + return executeOperation( + awsCodepipelinePutApprovalResultContract, + input, + executeCodepipelinePutApprovalResult, + 'Failed to submit CodePipeline approval result', + signal + ) + case 'codepipeline_retry_stage_execution': + return executeOperation( + awsCodepipelineRetryStageExecutionContract, + input, + executeCodepipelineRetryStageExecution, + 'Failed to retry CodePipeline stage execution', + signal + ) + case 'codepipeline_start_execution': + return executeOperation( + awsCodepipelineStartExecutionContract, + input, + executeCodepipelineStartExecution, + 'Failed to start CodePipeline pipeline execution', + signal + ) + case 'codepipeline_stop_execution': + return executeOperation( + awsCodepipelineStopExecutionContract, + input, + executeCodepipelineStopExecution, + 'Failed to stop CodePipeline pipeline execution', + signal + ) + default: + return Response.json({ error: `Unsupported CodePipeline tool: ${toolId}` }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/codepipeline/operations.test.ts b/apps/sim/lib/internal/codepipeline/operations.test.ts new file mode 100644 index 00000000000..ce10e8d5499 --- /dev/null +++ b/apps/sim/lib/internal/codepipeline/operations.test.ts @@ -0,0 +1,96 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + createCodePipelineClient: vi.fn(), + destroy: vi.fn(), + send: vi.fn(), +})) + +vi.mock('@/lib/internal/codepipeline/client', () => ({ + createCodePipelineClient: mocks.createCodePipelineClient, +})) + +import { + executeCodepipelineListPipelines, + executeCodepipelineStartExecution, +} from '@/lib/internal/codepipeline/operations' + +const CONNECTION = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', +} + +describe('CodePipeline operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.createCodePipelineClient.mockReturnValue({ + send: mocks.send, + destroy: mocks.destroy, + }) + }) + + it('forwards pagination inputs and cancellation to the SDK and destroys the client', async () => { + const controller = new AbortController() + mocks.send.mockResolvedValue({ + pipelines: [ + { + name: 'pipeline', + version: 3, + pipelineType: 'V2', + executionMode: 'QUEUED', + created: new Date(100), + updated: new Date(200), + }, + ], + nextToken: 'next-page', + }) + + await expect( + executeCodepipelineListPipelines( + { ...CONNECTION, maxResults: 20, nextToken: 'current-page' }, + controller.signal + ) + ).resolves.toEqual({ + success: true, + output: { + pipelines: [ + { + name: 'pipeline', + version: 3, + pipelineType: 'V2', + executionMode: 'QUEUED', + created: 100, + updated: 200, + }, + ], + nextToken: 'next-page', + }, + }) + expect(mocks.send.mock.calls[0]?.[0].input).toEqual({ + maxResults: 20, + nextToken: 'current-page', + }) + expect(mocks.send.mock.calls[0]?.[1]).toEqual({ abortSignal: controller.signal }) + expect(mocks.destroy).toHaveBeenCalledOnce() + }) + + it('preserves missing execution ID behavior and destroys the client', async () => { + mocks.send.mockResolvedValue({}) + + await expect( + executeCodepipelineStartExecution({ ...CONNECTION, pipelineName: 'pipeline' }) + ).rejects.toThrow('No pipeline execution ID returned') + expect(mocks.destroy).toHaveBeenCalledOnce() + }) + + it('destroys the client when provider execution fails', async () => { + mocks.send.mockRejectedValue(new Error('provider failure')) + + await expect(executeCodepipelineListPipelines(CONNECTION)).rejects.toThrow('provider failure') + expect(mocks.destroy).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/internal/codepipeline/operations.ts b/apps/sim/lib/internal/codepipeline/operations.ts new file mode 100644 index 00000000000..9cd8249e775 --- /dev/null +++ b/apps/sim/lib/internal/codepipeline/operations.ts @@ -0,0 +1,432 @@ +import { + type ApprovalStatus, + type CodePipelineClient, + DisableStageTransitionCommand, + EnableStageTransitionCommand, + GetPipelineCommand, + GetPipelineExecutionCommand, + GetPipelineStateCommand, + ListActionExecutionsCommand, + ListPipelineExecutionsCommand, + ListPipelinesCommand, + PutApprovalResultCommand, + RetryStageExecutionCommand, + type StageRetryMode, + type StageTransitionType, + StartPipelineExecutionCommand, + StopPipelineExecutionCommand, +} from '@aws-sdk/client-codepipeline' +import type { AwsCodepipelineDisableStageTransitionBody } from '@/lib/api/contracts/tools/aws/codepipeline-disable-stage-transition' +import type { AwsCodepipelineEnableStageTransitionBody } from '@/lib/api/contracts/tools/aws/codepipeline-enable-stage-transition' +import type { AwsCodepipelineGetPipelineBody } from '@/lib/api/contracts/tools/aws/codepipeline-get-pipeline' +import type { AwsCodepipelineGetPipelineExecutionBody } from '@/lib/api/contracts/tools/aws/codepipeline-get-pipeline-execution' +import type { AwsCodepipelineGetPipelineStateBody } from '@/lib/api/contracts/tools/aws/codepipeline-get-pipeline-state' +import type { AwsCodepipelineListActionExecutionsBody } from '@/lib/api/contracts/tools/aws/codepipeline-list-action-executions' +import type { AwsCodepipelineListPipelineExecutionsBody } from '@/lib/api/contracts/tools/aws/codepipeline-list-pipeline-executions' +import type { AwsCodepipelineListPipelinesBody } from '@/lib/api/contracts/tools/aws/codepipeline-list-pipelines' +import type { AwsCodepipelinePutApprovalResultBody } from '@/lib/api/contracts/tools/aws/codepipeline-put-approval-result' +import type { AwsCodepipelineRetryStageExecutionBody } from '@/lib/api/contracts/tools/aws/codepipeline-retry-stage-execution' +import type { AwsCodepipelineStartExecutionBody } from '@/lib/api/contracts/tools/aws/codepipeline-start-execution' +import type { AwsCodepipelineStopExecutionBody } from '@/lib/api/contracts/tools/aws/codepipeline-stop-execution' +import { + type CodePipelineConnectionConfig, + createCodePipelineClient, +} from '@/lib/internal/codepipeline/client' + +async function withCodePipelineClient( + input: CodePipelineConnectionConfig, + execute: (client: CodePipelineClient) => Promise +): Promise { + const client = createCodePipelineClient(input) + try { + return await execute(client) + } finally { + client.destroy() + } +} + +export async function executeCodepipelineDisableStageTransition( + input: AwsCodepipelineDisableStageTransitionBody, + signal?: AbortSignal +) { + return withCodePipelineClient(input, async (client) => { + await client.send( + new DisableStageTransitionCommand({ + pipelineName: input.pipelineName, + stageName: input.stageName, + transitionType: input.transitionType as StageTransitionType, + reason: input.reason, + }), + { abortSignal: signal } + ) + return { + success: true, + output: { + pipelineName: input.pipelineName, + stageName: input.stageName, + transitionType: input.transitionType, + }, + } + }) +} + +export async function executeCodepipelineEnableStageTransition( + input: AwsCodepipelineEnableStageTransitionBody, + signal?: AbortSignal +) { + return withCodePipelineClient(input, async (client) => { + await client.send( + new EnableStageTransitionCommand({ + pipelineName: input.pipelineName, + stageName: input.stageName, + transitionType: input.transitionType as StageTransitionType, + }), + { abortSignal: signal } + ) + return { + success: true, + output: { + pipelineName: input.pipelineName, + stageName: input.stageName, + transitionType: input.transitionType, + }, + } + }) +} + +export async function executeCodepipelineGetPipelineExecution( + input: AwsCodepipelineGetPipelineExecutionBody, + signal?: AbortSignal +) { + return withCodePipelineClient(input, async (client) => { + const response = await client.send( + new GetPipelineExecutionCommand({ + pipelineName: input.pipelineName, + pipelineExecutionId: input.pipelineExecutionId, + }), + { abortSignal: signal } + ) + const execution = response.pipelineExecution + if (!execution) throw new Error('Pipeline execution not found in response') + + return { + success: true, + output: { + pipelineExecutionId: execution.pipelineExecutionId ?? input.pipelineExecutionId, + pipelineName: execution.pipelineName ?? input.pipelineName, + pipelineVersion: execution.pipelineVersion, + status: execution.status ?? 'Unknown', + statusSummary: execution.statusSummary, + executionMode: execution.executionMode, + executionType: execution.executionType, + triggerType: execution.trigger?.triggerType, + triggerDetail: execution.trigger?.triggerDetail, + artifactRevisions: (execution.artifactRevisions ?? []).map((revision) => ({ + name: revision.name ?? '', + revisionId: revision.revisionId, + revisionSummary: revision.revisionSummary, + revisionUrl: revision.revisionUrl, + created: revision.created?.getTime(), + })), + variables: (execution.variables ?? []).map((variable) => ({ + name: variable.name ?? '', + resolvedValue: variable.resolvedValue ?? '', + })), + }, + } + }) +} + +export async function executeCodepipelineGetPipelineState( + input: AwsCodepipelineGetPipelineStateBody, + signal?: AbortSignal +) { + return withCodePipelineClient(input, async (client) => { + const response = await client.send(new GetPipelineStateCommand({ name: input.pipelineName }), { + abortSignal: signal, + }) + const stageStates = (response.stageStates ?? []).map((stage) => ({ + stageName: stage.stageName ?? '', + status: stage.latestExecution?.status, + pipelineExecutionId: stage.latestExecution?.pipelineExecutionId, + inboundTransitionEnabled: stage.inboundTransitionState?.enabled, + actionStates: (stage.actionStates ?? []).map((action) => ({ + actionName: action.actionName ?? '', + status: action.latestExecution?.status, + summary: action.latestExecution?.summary, + lastStatusChange: action.latestExecution?.lastStatusChange?.getTime(), + externalExecutionId: action.latestExecution?.externalExecutionId, + externalExecutionUrl: action.latestExecution?.externalExecutionUrl, + errorCode: action.latestExecution?.errorDetails?.code, + errorMessage: action.latestExecution?.errorDetails?.message, + percentComplete: action.latestExecution?.percentComplete, + token: action.latestExecution?.token, + revisionId: action.currentRevision?.revisionId, + entityUrl: action.entityUrl, + })), + })) + return { + success: true, + output: { + pipelineName: response.pipelineName ?? input.pipelineName, + pipelineVersion: response.pipelineVersion, + created: response.created?.getTime(), + updated: response.updated?.getTime(), + stageStates, + }, + } + }) +} + +export async function executeCodepipelineGetPipeline( + input: AwsCodepipelineGetPipelineBody, + signal?: AbortSignal +) { + return withCodePipelineClient(input, async (client) => { + const response = await client.send( + new GetPipelineCommand({ + name: input.pipelineName, + ...(input.version !== undefined ? { version: input.version } : {}), + }), + { abortSignal: signal } + ) + const pipeline = response.pipeline + if (!pipeline) throw new Error('Pipeline structure not found in response') + + const stages = (pipeline.stages ?? []).map((stage) => ({ + stageName: stage.name ?? '', + actions: (stage.actions ?? []).map((action) => ({ + name: action.name ?? '', + category: action.actionTypeId?.category ?? '', + owner: action.actionTypeId?.owner ?? '', + provider: action.actionTypeId?.provider ?? '', + version: action.actionTypeId?.version ?? '', + runOrder: action.runOrder, + configuration: action.configuration ?? {}, + inputArtifacts: (action.inputArtifacts ?? []).map((artifact) => artifact.name ?? ''), + outputArtifacts: (action.outputArtifacts ?? []).map((artifact) => artifact.name ?? ''), + })), + })) + return { + success: true, + output: { + pipelineName: pipeline.name ?? input.pipelineName, + pipelineArn: response.metadata?.pipelineArn, + roleArn: pipeline.roleArn ?? '', + version: pipeline.version, + pipelineType: pipeline.pipelineType, + executionMode: pipeline.executionMode, + artifactStoreType: pipeline.artifactStore?.type, + artifactStoreLocation: pipeline.artifactStore?.location, + stages, + variables: (pipeline.variables ?? []).map((variable) => ({ + name: variable.name ?? '', + defaultValue: variable.defaultValue, + description: variable.description, + })), + created: response.metadata?.created?.getTime(), + updated: response.metadata?.updated?.getTime(), + }, + } + }) +} + +export async function executeCodepipelineListActionExecutions( + input: AwsCodepipelineListActionExecutionsBody, + signal?: AbortSignal +) { + return withCodePipelineClient(input, async (client) => { + const response = await client.send( + new ListActionExecutionsCommand({ + pipelineName: input.pipelineName, + ...(input.pipelineExecutionId + ? { filter: { pipelineExecutionId: input.pipelineExecutionId } } + : {}), + ...(input.maxResults !== undefined ? { maxResults: input.maxResults } : {}), + ...(input.nextToken ? { nextToken: input.nextToken } : {}), + }), + { abortSignal: signal } + ) + const actionExecutionDetails = (response.actionExecutionDetails ?? []).map((detail) => ({ + pipelineExecutionId: detail.pipelineExecutionId, + actionExecutionId: detail.actionExecutionId, + pipelineVersion: detail.pipelineVersion, + stageName: detail.stageName, + actionName: detail.actionName, + startTime: detail.startTime?.getTime(), + lastUpdateTime: detail.lastUpdateTime?.getTime(), + updatedBy: detail.updatedBy, + status: detail.status, + externalExecutionId: detail.output?.executionResult?.externalExecutionId, + externalExecutionSummary: detail.output?.executionResult?.externalExecutionSummary, + externalExecutionUrl: detail.output?.executionResult?.externalExecutionUrl, + errorCode: detail.output?.executionResult?.errorDetails?.code, + errorMessage: detail.output?.executionResult?.errorDetails?.message, + })) + return { + success: true, + output: { + actionExecutionDetails, + ...(response.nextToken ? { nextToken: response.nextToken } : {}), + }, + } + }) +} + +export async function executeCodepipelineListPipelineExecutions( + input: AwsCodepipelineListPipelineExecutionsBody, + signal?: AbortSignal +) { + return withCodePipelineClient(input, async (client) => { + const response = await client.send( + new ListPipelineExecutionsCommand({ + pipelineName: input.pipelineName, + ...(input.maxResults !== undefined ? { maxResults: input.maxResults } : {}), + ...(input.nextToken ? { nextToken: input.nextToken } : {}), + ...(input.succeededInStage + ? { filter: { succeededInStage: { stageName: input.succeededInStage } } } + : {}), + }), + { abortSignal: signal } + ) + const executions = (response.pipelineExecutionSummaries ?? []).map((execution) => ({ + pipelineExecutionId: execution.pipelineExecutionId ?? '', + status: execution.status ?? 'Unknown', + statusSummary: execution.statusSummary, + startTime: execution.startTime?.getTime(), + lastUpdateTime: execution.lastUpdateTime?.getTime(), + executionMode: execution.executionMode, + executionType: execution.executionType, + stopTriggerReason: execution.stopTrigger?.reason, + triggerType: execution.trigger?.triggerType, + triggerDetail: execution.trigger?.triggerDetail, + rollbackTargetPipelineExecutionId: + execution.rollbackMetadata?.rollbackTargetPipelineExecutionId, + sourceRevisions: (execution.sourceRevisions ?? []).map((revision) => ({ + actionName: revision.actionName ?? '', + revisionId: revision.revisionId, + revisionSummary: revision.revisionSummary, + revisionUrl: revision.revisionUrl, + })), + })) + return { + success: true, + output: { + executions, + ...(response.nextToken ? { nextToken: response.nextToken } : {}), + }, + } + }) +} + +export async function executeCodepipelineListPipelines( + input: AwsCodepipelineListPipelinesBody, + signal?: AbortSignal +) { + return withCodePipelineClient(input, async (client) => { + const response = await client.send( + new ListPipelinesCommand({ + ...(input.maxResults !== undefined ? { maxResults: input.maxResults } : {}), + ...(input.nextToken ? { nextToken: input.nextToken } : {}), + }), + { abortSignal: signal } + ) + const pipelines = (response.pipelines ?? []).map((pipeline) => ({ + name: pipeline.name ?? '', + version: pipeline.version, + pipelineType: pipeline.pipelineType, + executionMode: pipeline.executionMode, + created: pipeline.created?.getTime(), + updated: pipeline.updated?.getTime(), + })) + return { + success: true, + output: { + pipelines, + ...(response.nextToken ? { nextToken: response.nextToken } : {}), + }, + } + }) +} + +export async function executeCodepipelinePutApprovalResult( + input: AwsCodepipelinePutApprovalResultBody, + signal?: AbortSignal +) { + return withCodePipelineClient(input, async (client) => { + const response = await client.send( + new PutApprovalResultCommand({ + pipelineName: input.pipelineName, + stageName: input.stageName, + actionName: input.actionName, + token: input.token, + result: { status: input.status as ApprovalStatus, summary: input.summary }, + }), + { abortSignal: signal } + ) + return { + success: true, + output: { approvedAt: response.approvedAt?.getTime(), status: input.status }, + } + }) +} + +export async function executeCodepipelineRetryStageExecution( + input: AwsCodepipelineRetryStageExecutionBody, + signal?: AbortSignal +) { + return withCodePipelineClient(input, async (client) => { + const response = await client.send( + new RetryStageExecutionCommand({ + pipelineName: input.pipelineName, + stageName: input.stageName, + pipelineExecutionId: input.pipelineExecutionId, + retryMode: input.retryMode as StageRetryMode, + }), + { abortSignal: signal } + ) + return { + success: true, + output: { pipelineExecutionId: response.pipelineExecutionId ?? input.pipelineExecutionId }, + } + }) +} + +export async function executeCodepipelineStartExecution( + input: AwsCodepipelineStartExecutionBody, + signal?: AbortSignal +) { + return withCodePipelineClient(input, async (client) => { + const response = await client.send( + new StartPipelineExecutionCommand({ + name: input.pipelineName, + ...(input.clientRequestToken ? { clientRequestToken: input.clientRequestToken } : {}), + ...(input.variables && input.variables.length > 0 ? { variables: input.variables } : {}), + }), + { abortSignal: signal } + ) + if (!response.pipelineExecutionId) throw new Error('No pipeline execution ID returned') + return { success: true, output: { pipelineExecutionId: response.pipelineExecutionId } } + }) +} + +export async function executeCodepipelineStopExecution( + input: AwsCodepipelineStopExecutionBody, + signal?: AbortSignal +) { + return withCodePipelineClient(input, async (client) => { + const response = await client.send( + new StopPipelineExecutionCommand({ + pipelineName: input.pipelineName, + pipelineExecutionId: input.pipelineExecutionId, + ...(input.abandon !== undefined ? { abandon: input.abandon } : {}), + ...(input.reason ? { reason: input.reason } : {}), + }), + { abortSignal: signal } + ) + return { + success: true, + output: { pipelineExecutionId: response.pipelineExecutionId ?? input.pipelineExecutionId }, + } + }) +} diff --git a/apps/sim/lib/internal/confluence/client.test.ts b/apps/sim/lib/internal/confluence/client.test.ts new file mode 100644 index 00000000000..324c227a55d --- /dev/null +++ b/apps/sim/lib/internal/confluence/client.test.ts @@ -0,0 +1,118 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockGetConfluenceCloudId } = vi.hoisted(() => ({ + mockGetConfluenceCloudId: vi.fn(), +})) + +vi.mock('@/tools/confluence/utils', () => ({ + getConfluenceCloudId: mockGetConfluenceCloudId, +})) + +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { ConfluenceClient, createConfluenceClient } from '@/lib/internal/confluence/client' +import { ConfluenceOperationError } from '@/lib/internal/confluence/errors' + +const CLOUD_ID = '12345678-1234-1234-1234-123456789012' + +describe('ConfluenceClient', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.unstubAllGlobals() + }) + + it('applies Atlassian authentication and forwards cancellation', async () => { + const response = Response.json({ id: 'page-1' }) + const fetchMock = vi.fn().mockResolvedValue(response) + vi.stubGlobal('fetch', fetchMock) + const controller = new AbortController() + const client = new ConfluenceClient(CLOUD_ID, 'access-token') + + await expect( + client.json(client.apiV2('/pages/page-1'), {}, controller.signal) + ).resolves.toEqual({ id: 'page-1' }) + + expect(fetchMock).toHaveBeenCalledWith( + `https://api.atlassian.com/ex/confluence/${CLOUD_ID}/wiki/api/v2/pages/page-1`, + expect.objectContaining({ + headers: { + Accept: 'application/json', + Authorization: 'Bearer access-token', + }, + signal: controller.signal, + }) + ) + expect(response.bodyUsed).toBe(true) + }) + + it('consumes provider error bodies and preserves provider status', async () => { + const response = new Response(JSON.stringify({ message: 'Page not found' }), { + status: 404, + statusText: 'Not Found', + }) + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(response)) + const client = new ConfluenceClient(CLOUD_ID, 'access-token') + + let caught: unknown + try { + await client.json(client.apiV2('/pages/missing')) + } catch (error) { + caught = error + } + + expect(caught).toBeInstanceOf(ConfluenceOperationError) + expect(caught).toMatchObject({ status: 404 }) + expect(response.bodyUsed).toBe(true) + }) + + it.each([ + { status: 200, maxBytes: 10 * 1024 * 1024, label: 'Confluence response' }, + { status: 502, maxBytes: 64 * 1024, label: 'Confluence error response' }, + ])('bounds and cancels $status provider responses', async ({ status, maxBytes, label }) => { + let cancelled = false + const response = new Response( + new ReadableStream({ + cancel: () => { + cancelled = true + }, + }), + { status, headers: { 'content-length': String(maxBytes + 1) } } + ) + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(response)) + const client = new ConfluenceClient(CLOUD_ID, 'access-token') + + await expect(client.json(client.apiV2('/pages/page-1'))).rejects.toEqual( + new PayloadSizeLimitError({ label, maxBytes, observedBytes: maxBytes + 1 }) + ) + expect(cancelled).toBe(true) + }) + + it('does not issue a request after cancellation', async () => { + const fetchMock = vi.fn() + vi.stubGlobal('fetch', fetchMock) + const controller = new AbortController() + controller.abort() + const client = new ConfluenceClient(CLOUD_ID, 'access-token') + + await expect( + client.fetch(client.apiV2('/pages/page-1'), {}, controller.signal) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('stops waiting for cloud discovery when execution is cancelled', async () => { + mockGetConfluenceCloudId.mockReturnValue(new Promise(() => {})) + const controller = new AbortController() + const pending = createConfluenceClient( + { domain: 'example.atlassian.net', accessToken: 'access-token' }, + controller.signal + ) + + controller.abort() + + await expect(pending).rejects.toMatchObject({ name: 'AbortError' }) + expect(mockGetConfluenceCloudId).toHaveBeenCalledWith('example.atlassian.net', 'access-token') + }) +}) diff --git a/apps/sim/lib/internal/confluence/client.ts b/apps/sim/lib/internal/confluence/client.ts new file mode 100644 index 00000000000..603110ba36c --- /dev/null +++ b/apps/sim/lib/internal/confluence/client.ts @@ -0,0 +1,158 @@ +import { validateJiraCloudId } from '@/lib/core/security/input-validation' +import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + readResponseTextWithLimit, +} from '@/lib/core/utils/stream-limits' +import { ConfluenceOperationError } from '@/lib/internal/confluence/errors' +import { getConfluenceCloudId } from '@/tools/confluence/utils' +import { parseAtlassianErrorMessage } from '@/tools/jira/utils' + +export interface ConfluenceConnectionConfig { + domain: string + accessToken: string + cloudId?: string +} + +export type JsonObject = Record + +export function asObject(value: unknown): JsonObject { + return value && typeof value === 'object' && !Array.isArray(value) ? (value as JsonObject) : {} +} + +export function asArray(value: unknown): unknown[] { + return Array.isArray(value) ? value : [] +} + +export function nested(object: JsonObject, ...keys: string[]): unknown { + let value: unknown = object + for (const key of keys) value = asObject(value)[key] + return value +} + +export function nextCursor(data: JsonObject): string | null { + const next = nested(data, '_links', 'next') + return typeof next === 'string' + ? new URL(next, 'https://placeholder').searchParams.get('cursor') + : null +} + +export class ConfluenceClient { + constructor( + readonly cloudId: string, + private readonly accessToken: string + ) {} + + apiV2(path: string): string { + return `https://api.atlassian.com/ex/confluence/${this.cloudId}/wiki/api/v2${path}` + } + + rest(path: string): string { + return `https://api.atlassian.com/ex/confluence/${this.cloudId}/wiki/rest/api${path}` + } + + async fetch(path: string, init: RequestInit = {}, signal?: AbortSignal): Promise { + signal?.throwIfAborted() + return fetch(path, { + ...init, + headers: { + Accept: 'application/json', + Authorization: `Bearer ${this.accessToken}`, + ...init.headers, + }, + signal, + }) + } + + async json(path: string, init: RequestInit = {}, signal?: AbortSignal): Promise { + const response = await this.fetch(path, init, signal) + if (!response.ok) await throwConfluenceResponseError(response, signal) + return readConfluenceResponseObject(response, signal) + } + + async delete(path: string, signal?: AbortSignal): Promise { + const response = await this.fetch(path, { method: 'DELETE' }, signal) + if (!response.ok) await throwConfluenceResponseError(response, signal) + await readConfluenceResponseText(response, signal, 'Confluence delete response', 'DELETE') + } +} + +function waitForConfluenceCloudId(promise: Promise, signal?: AbortSignal): Promise { + if (!signal) return promise + signal.throwIfAborted() + return new Promise((resolve, reject) => { + const cleanup = () => signal.removeEventListener('abort', onAbort) + const onAbort = () => { + cleanup() + reject(signal.reason) + } + signal.addEventListener('abort', onAbort, { once: true }) + promise.then( + (cloudId) => { + cleanup() + resolve(cloudId) + }, + (error: unknown) => { + cleanup() + reject(error) + } + ) + }) +} + +export async function readConfluenceResponseText( + response: Response, + signal?: AbortSignal, + label = 'Confluence response', + requestMethod?: string +): Promise { + return readResponseTextWithLimit(response, { + maxBytes: MAX_JSON_API_RESPONSE_BYTES, + label, + requestMethod, + signal, + }) +} + +export async function readConfluenceResponseObject( + response: Response, + signal?: AbortSignal, + label = 'Confluence response' +): Promise { + const text = await readConfluenceResponseText(response, signal, label) + return text ? asObject(JSON.parse(text)) : {} +} + +export async function throwConfluenceResponseError( + response: Response, + signal?: AbortSignal +): Promise { + const errorText = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Confluence error response', + signal, + }) + throw new ConfluenceOperationError( + parseAtlassianErrorMessage(response.status, response.statusText, errorText), + response.status + ) +} + +export async function createConfluenceClient( + config: ConfluenceConnectionConfig, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const cloudId = + config.cloudId || + (await waitForConfluenceCloudId( + getConfluenceCloudId(config.domain, config.accessToken), + signal + )) + signal?.throwIfAborted() + const validation = validateJiraCloudId(cloudId, 'cloudId') + if (!validation.isValid) { + throw new ConfluenceOperationError(validation.error || 'Invalid cloudId', 400) + } + return new ConfluenceClient(validation.sanitized ?? cloudId, config.accessToken) +} diff --git a/apps/sim/lib/internal/confluence/errors.ts b/apps/sim/lib/internal/confluence/errors.ts new file mode 100644 index 00000000000..d7e98000179 --- /dev/null +++ b/apps/sim/lib/internal/confluence/errors.ts @@ -0,0 +1,10 @@ +export class ConfluenceOperationError extends Error { + constructor( + message: string, + readonly status: number, + readonly body?: Record + ) { + super(message) + this.name = 'ConfluenceOperationError' + } +} diff --git a/apps/sim/lib/internal/confluence/execute-tool.test.ts b/apps/sim/lib/internal/confluence/execute-tool.test.ts new file mode 100644 index 00000000000..2f0ef547379 --- /dev/null +++ b/apps/sim/lib/internal/confluence/execute-tool.test.ts @@ -0,0 +1,438 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { ConfluenceOperationError } from '@/lib/internal/confluence/errors' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' +import type { ExecutionContext } from '@/executor/types' + +const mockOperations = vi.hoisted(() => ({ + executeConfluenceAddLabel: vi.fn(), + executeConfluenceCreateBlogPost: vi.fn(), + executeConfluenceCreateComment: vi.fn(), + executeConfluenceCreatePage: vi.fn(), + executeConfluenceCreatePageProperty: vi.fn(), + executeConfluenceCreateSpace: vi.fn(), + executeConfluenceDeleteAttachment: vi.fn(), + executeConfluenceDeleteBlogPost: vi.fn(), + executeConfluenceDeleteComment: vi.fn(), + executeConfluenceDeleteLabel: vi.fn(), + executeConfluenceDeletePage: vi.fn(), + executeConfluenceDeletePageProperty: vi.fn(), + executeConfluenceDeleteSpace: vi.fn(), + executeConfluenceGetBlogPost: vi.fn(), + executeConfluenceGetPageAncestors: vi.fn(), + executeConfluenceGetPageChildren: vi.fn(), + executeConfluenceGetPageDescendants: vi.fn(), + executeConfluenceGetPagesByLabel: vi.fn(), + executeConfluenceGetSpace: vi.fn(), + executeConfluenceGetUser: vi.fn(), + executeConfluenceListAttachments: vi.fn(), + executeConfluenceListBlogPosts: vi.fn(), + executeConfluenceListBlogPostsInSpace: vi.fn(), + executeConfluenceListComments: vi.fn(), + executeConfluenceListLabels: vi.fn(), + executeConfluenceListPageProperties: vi.fn(), + executeConfluenceListPagesInSpace: vi.fn(), + executeConfluenceListSpaceLabels: vi.fn(), + executeConfluenceListSpacePermissions: vi.fn(), + executeConfluenceListSpaces: vi.fn(), + executeConfluencePageVersions: vi.fn(), + executeConfluenceRetrievePage: vi.fn(), + executeConfluenceSearch: vi.fn(), + executeConfluenceSearchInSpace: vi.fn(), + executeConfluenceSpaceProperties: vi.fn(), + executeConfluenceTasks: vi.fn(), + executeConfluenceUpdateBlogPost: vi.fn(), + executeConfluenceUpdateComment: vi.fn(), + executeConfluenceUpdatePage: vi.fn(), + executeConfluenceUpdateSpace: vi.fn(), + executeConfluenceUploadAttachment: vi.fn(), +})) + +vi.mock('@/lib/internal/confluence/operations', () => mockOperations) + +import { executeConfluenceTool } from '@/lib/internal/confluence/execute-tool' + +const BASE = { + domain: 'example.atlassian.net', + accessToken: 'access-token', + cloudId: '12345678-1234-1234-1234-123456789012', +} +const PAGE = { ...BASE, pageId: '123' } +const SPACE = { ...BASE, spaceId: '456' } +const BLOG_POST = { ...BASE, blogPostId: '789' } +const COMMENT = { ...BASE, commentId: '321' } + +type OperationName = keyof typeof mockOperations + +interface DispatchCase { + toolId: string + operation: OperationName + input: Record + query?: boolean +} + +const DISPATCH_CASES: DispatchCase[] = [ + { + toolId: 'confluence_add_label', + operation: 'executeConfluenceAddLabel', + input: { ...PAGE, labelName: 'release' }, + }, + { + toolId: 'confluence_create_blogpost', + operation: 'executeConfluenceCreateBlogPost', + input: { ...SPACE, title: 'Title', content: '

Body

' }, + }, + { + toolId: 'confluence_create_comment', + operation: 'executeConfluenceCreateComment', + input: { ...PAGE, comment: '

Comment

' }, + }, + { + toolId: 'confluence_create_page', + operation: 'executeConfluenceCreatePage', + input: { ...SPACE, title: 'Title', content: '

Body

' }, + }, + { + toolId: 'confluence_create_page_property', + operation: 'executeConfluenceCreatePageProperty', + input: { ...PAGE, key: 'owner', value: { id: 'user-1' } }, + }, + { + toolId: 'confluence_create_space', + operation: 'executeConfluenceCreateSpace', + input: { ...BASE, name: 'Engineering', key: 'ENG' }, + }, + { + toolId: 'confluence_create_space_property', + operation: 'executeConfluenceSpaceProperties', + input: { ...SPACE, action: 'create', key: 'owner', value: 'user-1' }, + }, + { + toolId: 'confluence_delete_attachment', + operation: 'executeConfluenceDeleteAttachment', + input: { ...BASE, attachmentId: '111' }, + }, + { + toolId: 'confluence_delete_blogpost', + operation: 'executeConfluenceDeleteBlogPost', + input: BLOG_POST, + }, + { + toolId: 'confluence_delete_comment', + operation: 'executeConfluenceDeleteComment', + input: COMMENT, + }, + { + toolId: 'confluence_delete_label', + operation: 'executeConfluenceDeleteLabel', + input: { ...PAGE, labelName: 'release' }, + }, + { + toolId: 'confluence_delete_page', + operation: 'executeConfluenceDeletePage', + input: PAGE, + }, + { + toolId: 'confluence_delete_page_property', + operation: 'executeConfluenceDeletePageProperty', + input: { ...PAGE, propertyId: '222' }, + }, + { + toolId: 'confluence_delete_space', + operation: 'executeConfluenceDeleteSpace', + input: SPACE, + }, + { + toolId: 'confluence_delete_space_property', + operation: 'executeConfluenceSpaceProperties', + input: { ...SPACE, action: 'delete', propertyId: '333' }, + }, + { + toolId: 'confluence_get_blogpost', + operation: 'executeConfluenceGetBlogPost', + input: BLOG_POST, + }, + { + toolId: 'confluence_get_page_ancestors', + operation: 'executeConfluenceGetPageAncestors', + input: PAGE, + }, + { + toolId: 'confluence_get_page_children', + operation: 'executeConfluenceGetPageChildren', + input: PAGE, + }, + { + toolId: 'confluence_get_page_descendants', + operation: 'executeConfluenceGetPageDescendants', + input: PAGE, + }, + { + toolId: 'confluence_get_page_version', + operation: 'executeConfluencePageVersions', + input: { ...PAGE, versionNumber: 2 }, + }, + { + toolId: 'confluence_get_pages_by_label', + operation: 'executeConfluenceGetPagesByLabel', + input: { ...BASE, labelId: '444' }, + query: true, + }, + { + toolId: 'confluence_get_space', + operation: 'executeConfluenceGetSpace', + input: SPACE, + query: true, + }, + { + toolId: 'confluence_get_task', + operation: 'executeConfluenceTasks', + input: { ...BASE, taskId: '555' }, + }, + { + toolId: 'confluence_get_user', + operation: 'executeConfluenceGetUser', + input: { ...BASE, accountId: 'account-1' }, + }, + { + toolId: 'confluence_list_attachments', + operation: 'executeConfluenceListAttachments', + input: PAGE, + query: true, + }, + { + toolId: 'confluence_list_blogposts', + operation: 'executeConfluenceListBlogPosts', + input: BASE, + query: true, + }, + { + toolId: 'confluence_list_blogposts_in_space', + operation: 'executeConfluenceListBlogPostsInSpace', + input: SPACE, + }, + { + toolId: 'confluence_list_comments', + operation: 'executeConfluenceListComments', + input: PAGE, + query: true, + }, + { + toolId: 'confluence_list_labels', + operation: 'executeConfluenceListLabels', + input: PAGE, + query: true, + }, + { + toolId: 'confluence_list_page_properties', + operation: 'executeConfluenceListPageProperties', + input: PAGE, + query: true, + }, + { + toolId: 'confluence_list_page_versions', + operation: 'executeConfluencePageVersions', + input: PAGE, + }, + { + toolId: 'confluence_list_pages_in_space', + operation: 'executeConfluenceListPagesInSpace', + input: SPACE, + }, + { + toolId: 'confluence_list_space_labels', + operation: 'executeConfluenceListSpaceLabels', + input: SPACE, + query: true, + }, + { + toolId: 'confluence_list_space_permissions', + operation: 'executeConfluenceListSpacePermissions', + input: SPACE, + }, + { + toolId: 'confluence_list_space_properties', + operation: 'executeConfluenceSpaceProperties', + input: SPACE, + }, + { + toolId: 'confluence_list_spaces', + operation: 'executeConfluenceListSpaces', + input: BASE, + query: true, + }, + { + toolId: 'confluence_list_tasks', + operation: 'executeConfluenceTasks', + input: BASE, + }, + { + toolId: 'confluence_retrieve', + operation: 'executeConfluenceRetrievePage', + input: PAGE, + }, + { + toolId: 'confluence_search', + operation: 'executeConfluenceSearch', + input: { ...BASE, query: 'release notes' }, + }, + { + toolId: 'confluence_search_in_space', + operation: 'executeConfluenceSearchInSpace', + input: { ...BASE, spaceKey: 'ENG' }, + }, + { + toolId: 'confluence_update', + operation: 'executeConfluenceUpdatePage', + input: PAGE, + }, + { + toolId: 'confluence_update_blogpost', + operation: 'executeConfluenceUpdateBlogPost', + input: BLOG_POST, + }, + { + toolId: 'confluence_update_comment', + operation: 'executeConfluenceUpdateComment', + input: { ...COMMENT, comment: '

Updated

' }, + }, + { + toolId: 'confluence_update_space', + operation: 'executeConfluenceUpdateSpace', + input: { ...SPACE, name: 'Engineering' }, + }, + { + toolId: 'confluence_update_task', + operation: 'executeConfluenceTasks', + input: { ...BASE, action: 'update', taskId: '555', status: 'complete' }, + }, + { + toolId: 'confluence_upload_attachment', + operation: 'executeConfluenceUploadAttachment', + input: { ...PAGE, file: { key: 'uploads/file.txt', name: 'file.txt', size: 4 } }, + }, +] + +function makeRequest( + toolId: string, + input: Record, + _query = false, + signal?: AbortSignal +): InternalToolOperationCall { + return { + toolId, + input, + headers: new Headers({ 'x-execution-id': 'execution-1' }), + context: { + workflowId: 'workflow-1', + userId: 'user-1', + } as ExecutionContext, + requestId: 'request-1', + signal, + } +} + +describe('executeConfluenceTool', () => { + beforeEach(() => { + vi.clearAllMocks() + for (const [operation, mock] of Object.entries(mockOperations)) { + mock.mockResolvedValue({ operation }) + } + }) + + it.each(DISPATCH_CASES)('dispatches $toolId through $operation', async (testCase) => { + const response = await executeConfluenceTool( + makeRequest(testCase.toolId, testCase.input, testCase.query) + ) + + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ operation: testCase.operation }) + expect(mockOperations[testCase.operation]).toHaveBeenCalledOnce() + expect(mockOperations[testCase.operation]).toHaveBeenCalledWith( + expect.objectContaining(testCase.input), + { + headers: expect.any(Headers), + requestId: 'request-1', + signal: undefined, + userId: 'user-1', + } + ) + }) + + it('returns the canonical invalid operation input envelope', async () => { + const request = makeRequest('confluence_retrieve', {}) + request.input = '{' + const response = await executeConfluenceTool(request) + + expect(response.status).toBe(400) + expect(await response.json()).toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + }) + + it('returns the canonical validation envelope', async () => { + const response = await executeConfluenceTool( + makeRequest('confluence_retrieve', { domain: 'example.atlassian.net' }) + ) + + expect(response.status).toBe(400) + expect(await response.json()).toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + }) + + it('preserves typed operation status and error envelopes', async () => { + mockOperations.executeConfluenceCreateSpace.mockRejectedValueOnce( + new ConfluenceOperationError('Confluence rejected the request', 409) + ) + + const response = await executeConfluenceTool( + makeRequest('confluence_create_space', { ...BASE, name: 'Engineering', key: 'ENG' }) + ) + + expect(response.status).toBe(409) + expect(await response.json()).toEqual({ error: 'Confluence rejected the request' }) + }) + + it('preserves specialized upload error envelopes', async () => { + mockOperations.executeConfluenceUploadAttachment.mockRejectedValueOnce( + new ConfluenceOperationError('File not found', 404, { + success: false, + error: 'File not found', + }) + ) + + const response = await executeConfluenceTool( + makeRequest('confluence_upload_attachment', { + ...PAGE, + file: { key: 'uploads/file.txt' }, + }) + ) + + expect(response.status).toBe(404) + expect(await response.json()).toEqual({ success: false, error: 'File not found' }) + }) + + it('stops before dispatch when execution is cancelled', async () => { + const controller = new AbortController() + controller.abort() + + await expect( + executeConfluenceTool(makeRequest('confluence_retrieve', PAGE, false, controller.signal)) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mockOperations.executeConfluenceRetrievePage).not.toHaveBeenCalled() + }) + + it('returns an explicit error for unknown Confluence tools', async () => { + const response = await executeConfluenceTool(makeRequest('confluence_unknown', BASE)) + + expect(response.status).toBe(500) + expect(await response.json()).toEqual({ + error: 'Unsupported Confluence tool: confluence_unknown', + }) + }) +}) diff --git a/apps/sim/lib/internal/confluence/execute-tool.ts b/apps/sim/lib/internal/confluence/execute-tool.ts new file mode 100644 index 00000000000..671902f4a8a --- /dev/null +++ b/apps/sim/lib/internal/confluence/execute-tool.ts @@ -0,0 +1,357 @@ +import { getErrorMessage } from '@sim/utils/errors' +import type { AnyApiRouteContract, ContractBody, ContractQuery } from '@/lib/api/contracts' +import { + confluenceBlogPostOperationContract, + confluenceCreateCommentContract, + confluenceCreatePageContract, + confluenceCreatePagePropertyContract, + confluenceCreateSpaceContract, + confluenceDeleteAttachmentContract, + confluenceDeleteBlogPostContract, + confluenceDeleteCommentContract, + confluenceDeleteLabelContract, + confluenceDeletePageContract, + confluenceDeletePagePropertyContract, + confluenceDeleteSpaceContract, + confluenceGetSpaceContract, + confluenceLabelMutationContract, + confluenceListAttachmentsContract, + confluenceListBlogPostsContract, + confluenceListCommentsContract, + confluenceListLabelsContract, + confluenceListPagePropertiesContract, + confluenceListSpacesContract, + confluencePageAncestorsContract, + confluencePageChildrenContract, + confluencePageDescendantsContract, + confluencePageSelectorContract, + confluencePagesByLabelContract, + confluencePageVersionsContract, + confluenceSearchContract, + confluenceSearchInSpaceContract, + confluenceSpaceBlogPostsContract, + confluenceSpaceLabelsContract, + confluenceSpacePagesContract, + confluenceSpacePermissionsContract, + confluenceSpacePropertiesContract, + confluenceTasksContract, + confluenceUpdateBlogPostContract, + confluenceUpdateCommentContract, + confluenceUpdatePageContract, + confluenceUpdateSpaceContract, + confluenceUploadAttachmentContract, + confluenceUserContract, +} from '@/lib/api/contracts/selectors/confluence' +import { ConfluenceOperationError } from '@/lib/internal/confluence/errors' +import { + type ConfluenceOperationContext, + executeConfluenceAddLabel, + executeConfluenceCreateBlogPost, + executeConfluenceCreateComment, + executeConfluenceCreatePage, + executeConfluenceCreatePageProperty, + executeConfluenceCreateSpace, + executeConfluenceDeleteAttachment, + executeConfluenceDeleteBlogPost, + executeConfluenceDeleteComment, + executeConfluenceDeleteLabel, + executeConfluenceDeletePage, + executeConfluenceDeletePageProperty, + executeConfluenceDeleteSpace, + executeConfluenceGetBlogPost, + executeConfluenceGetPageAncestors, + executeConfluenceGetPageChildren, + executeConfluenceGetPageDescendants, + executeConfluenceGetPagesByLabel, + executeConfluenceGetSpace, + executeConfluenceGetUser, + executeConfluenceListAttachments, + executeConfluenceListBlogPosts, + executeConfluenceListBlogPostsInSpace, + executeConfluenceListComments, + executeConfluenceListLabels, + executeConfluenceListPageProperties, + executeConfluenceListPagesInSpace, + executeConfluenceListSpaceLabels, + executeConfluenceListSpacePermissions, + executeConfluenceListSpaces, + executeConfluencePageVersions, + executeConfluenceRetrievePage, + executeConfluenceSearch, + executeConfluenceSearchInSpace, + executeConfluenceSpaceProperties, + executeConfluenceTasks, + executeConfluenceUpdateBlogPost, + executeConfluenceUpdateComment, + executeConfluenceUpdatePage, + executeConfluenceUpdateSpace, + executeConfluenceUploadAttachment, +} from '@/lib/internal/confluence/operations' +import type { + InternalToolOperationCall, + InternalToolOperationHandler, +} from '@/lib/internal/tool-operations/types' + +type ContractInput = NonNullable | ContractQuery> + +function parsePreparedRequest( + contract: C, + request: InternalToolOperationCall +): { success: true; data: ContractInput } | { success: false; response: Response } { + const schema = contract.query ?? contract.body + if (!schema) throw new Error(`Confluence contract ${contract.path} has no request input`) + const parsed = schema.safeParse(request.input) + if (!parsed.success) { + return { + success: false, + response: Response.json( + { error: 'Invalid request data', details: parsed.error.issues }, + { status: 400 } + ), + } + } + return { success: true, data: parsed.data as ContractInput } +} + +async function executeOperation( + contract: C, + request: InternalToolOperationCall, + execute: (input: ContractInput, context: ConfluenceOperationContext) => Promise +): Promise { + request.signal?.throwIfAborted() + const parsed = parsePreparedRequest(contract, request) + if (!parsed.success) return parsed.response + try { + const result = await execute(parsed.data, { + headers: request.headers, + requestId: request.requestId, + signal: request.signal, + userId: request.context.userId, + }) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + return Response.json( + error instanceof ConfluenceOperationError && error.body + ? error.body + : { error: getErrorMessage(error, 'Internal server error') }, + { status: error instanceof ConfluenceOperationError ? error.status : 500 } + ) + } +} + +export const executeConfluenceTool: InternalToolOperationHandler = async (request) => { + switch (request.toolId) { + case 'confluence_add_label': + return executeOperation(confluenceLabelMutationContract, request, executeConfluenceAddLabel) + case 'confluence_create_blogpost': + return executeOperation( + confluenceBlogPostOperationContract, + request, + executeConfluenceCreateBlogPost + ) + case 'confluence_create_comment': + return executeOperation( + confluenceCreateCommentContract, + request, + executeConfluenceCreateComment + ) + case 'confluence_create_page': + return executeOperation(confluenceCreatePageContract, request, executeConfluenceCreatePage) + case 'confluence_create_page_property': + return executeOperation( + confluenceCreatePagePropertyContract, + request, + executeConfluenceCreatePageProperty + ) + case 'confluence_create_space': + return executeOperation(confluenceCreateSpaceContract, request, executeConfluenceCreateSpace) + case 'confluence_create_space_property': + return executeOperation( + confluenceSpacePropertiesContract, + request, + executeConfluenceSpaceProperties + ) + case 'confluence_delete_attachment': + return executeOperation( + confluenceDeleteAttachmentContract, + request, + executeConfluenceDeleteAttachment + ) + case 'confluence_delete_blogpost': + return executeOperation( + confluenceDeleteBlogPostContract, + request, + executeConfluenceDeleteBlogPost + ) + case 'confluence_delete_comment': + return executeOperation( + confluenceDeleteCommentContract, + request, + executeConfluenceDeleteComment + ) + case 'confluence_delete_label': + return executeOperation(confluenceDeleteLabelContract, request, executeConfluenceDeleteLabel) + case 'confluence_delete_page': + return executeOperation(confluenceDeletePageContract, request, executeConfluenceDeletePage) + case 'confluence_delete_page_property': + return executeOperation( + confluenceDeletePagePropertyContract, + request, + executeConfluenceDeletePageProperty + ) + case 'confluence_delete_space': + return executeOperation(confluenceDeleteSpaceContract, request, executeConfluenceDeleteSpace) + case 'confluence_delete_space_property': + return executeOperation( + confluenceSpacePropertiesContract, + request, + executeConfluenceSpaceProperties + ) + case 'confluence_get_blogpost': + return executeOperation( + confluenceBlogPostOperationContract, + request, + executeConfluenceGetBlogPost + ) + case 'confluence_get_page_ancestors': + return executeOperation( + confluencePageAncestorsContract, + request, + executeConfluenceGetPageAncestors + ) + case 'confluence_get_page_children': + return executeOperation( + confluencePageChildrenContract, + request, + executeConfluenceGetPageChildren + ) + case 'confluence_get_page_descendants': + return executeOperation( + confluencePageDescendantsContract, + request, + executeConfluenceGetPageDescendants + ) + case 'confluence_get_page_version': + case 'confluence_list_page_versions': + return executeOperation( + confluencePageVersionsContract, + request, + executeConfluencePageVersions + ) + case 'confluence_get_pages_by_label': + return executeOperation( + confluencePagesByLabelContract, + request, + executeConfluenceGetPagesByLabel + ) + case 'confluence_get_space': + return executeOperation(confluenceGetSpaceContract, request, executeConfluenceGetSpace) + case 'confluence_get_task': + case 'confluence_list_tasks': + case 'confluence_update_task': + return executeOperation(confluenceTasksContract, request, executeConfluenceTasks) + case 'confluence_get_user': + return executeOperation(confluenceUserContract, request, executeConfluenceGetUser) + case 'confluence_list_attachments': + return executeOperation( + confluenceListAttachmentsContract, + request, + executeConfluenceListAttachments + ) + case 'confluence_list_blogposts': + return executeOperation( + confluenceListBlogPostsContract, + request, + executeConfluenceListBlogPosts + ) + case 'confluence_list_blogposts_in_space': + return executeOperation( + confluenceSpaceBlogPostsContract, + request, + executeConfluenceListBlogPostsInSpace + ) + case 'confluence_list_comments': + return executeOperation( + confluenceListCommentsContract, + request, + executeConfluenceListComments + ) + case 'confluence_list_labels': + return executeOperation(confluenceListLabelsContract, request, executeConfluenceListLabels) + case 'confluence_list_page_properties': + return executeOperation( + confluenceListPagePropertiesContract, + request, + executeConfluenceListPageProperties + ) + case 'confluence_list_pages_in_space': + return executeOperation( + confluenceSpacePagesContract, + request, + executeConfluenceListPagesInSpace + ) + case 'confluence_list_space_labels': + return executeOperation( + confluenceSpaceLabelsContract, + request, + executeConfluenceListSpaceLabels + ) + case 'confluence_list_space_permissions': + return executeOperation( + confluenceSpacePermissionsContract, + request, + executeConfluenceListSpacePermissions + ) + case 'confluence_list_space_properties': + return executeOperation( + confluenceSpacePropertiesContract, + request, + executeConfluenceSpaceProperties + ) + case 'confluence_list_spaces': + return executeOperation(confluenceListSpacesContract, request, executeConfluenceListSpaces) + case 'confluence_retrieve': + return executeOperation( + confluencePageSelectorContract, + request, + executeConfluenceRetrievePage + ) + case 'confluence_search': + return executeOperation(confluenceSearchContract, request, executeConfluenceSearch) + case 'confluence_search_in_space': + return executeOperation( + confluenceSearchInSpaceContract, + request, + executeConfluenceSearchInSpace + ) + case 'confluence_update': + return executeOperation(confluenceUpdatePageContract, request, executeConfluenceUpdatePage) + case 'confluence_update_blogpost': + return executeOperation( + confluenceUpdateBlogPostContract, + request, + executeConfluenceUpdateBlogPost + ) + case 'confluence_update_comment': + return executeOperation( + confluenceUpdateCommentContract, + request, + executeConfluenceUpdateComment + ) + case 'confluence_update_space': + return executeOperation(confluenceUpdateSpaceContract, request, executeConfluenceUpdateSpace) + case 'confluence_upload_attachment': + return executeOperation( + confluenceUploadAttachmentContract, + request, + executeConfluenceUploadAttachment + ) + default: + return Response.json( + { error: `Unsupported Confluence tool: ${request.toolId}` }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/confluence/http-route.ts b/apps/sim/lib/internal/confluence/http-route.ts new file mode 100644 index 00000000000..a9e168fea47 --- /dev/null +++ b/apps/sim/lib/internal/confluence/http-route.ts @@ -0,0 +1,53 @@ +import type { Logger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { type NextRequest, NextResponse } from 'next/server' +import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { ConfluenceOperationError } from '@/lib/internal/confluence/errors' +import type { ConfluenceOperationContext } from '@/lib/internal/confluence/operations' + +type ConfluenceRequestParseResult = + | { success: true; data: { body?: T; query?: T } } + | { success: false; response: Response } + +interface ConfluenceHttpRouteConfig { + logger: Logger + parse: (request: NextRequest) => Promise> + execute: (input: T, context: ConfluenceOperationContext) => Promise +} + +export function createConfluenceHttpRoute({ + logger, + parse, + execute, +}: ConfluenceHttpRouteConfig) { + return withRouteHandler(async (request: NextRequest) => { + try { + const auth = await checkSessionOrInternalAuth(request) + if (!auth.success || !auth.userId) { + return NextResponse.json({ error: auth.error || 'Unauthorized' }, { status: 401 }) + } + const parsed = await parse(request) + if (!parsed.success) return parsed.response + const input = parsed.data.body ?? parsed.data.query + if (!input) throw new Error('Parsed Confluence request is missing input') + return NextResponse.json( + await execute(input, { + headers: request.headers, + requestId: request.headers.get('x-request-id') || 'confluence-http', + signal: request.signal, + userId: auth.userId, + }) + ) + } catch (error) { + request.signal.throwIfAborted() + const status = error instanceof ConfluenceOperationError ? error.status : 500 + const message = getErrorMessage(error, 'Internal server error') + logger.error('Confluence operation failed', { error: message }) + return NextResponse.json( + error instanceof ConfluenceOperationError && error.body ? error.body : { error: message }, + { status } + ) + } + }) +} diff --git a/apps/sim/lib/internal/confluence/operations.test.ts b/apps/sim/lib/internal/confluence/operations.test.ts new file mode 100644 index 00000000000..460cf35682c --- /dev/null +++ b/apps/sim/lib/internal/confluence/operations.test.ts @@ -0,0 +1,193 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const uploadMocks = vi.hoisted(() => ({ + assertToolFileAccess: vi.fn(), + downloadServableFileFromStorage: vi.fn(), + processSingleFileToUserFile: vi.fn(), +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: uploadMocks.assertToolFileAccess, +})) + +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + processSingleFileToUserFile: uploadMocks.processSingleFileToUserFile, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: uploadMocks.downloadServableFileFromStorage, +})) + +import { ConfluenceOperationError } from '@/lib/internal/confluence/errors' +import { + executeConfluenceListLabels, + executeConfluenceUploadAttachment, +} from '@/lib/internal/confluence/operations' + +const CONNECTION = { + domain: 'example.atlassian.net', + accessToken: 'access-token', + cloudId: '12345678-1234-1234-1234-123456789012', +} + +describe('Confluence operations', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.unstubAllGlobals() + uploadMocks.processSingleFileToUserFile.mockReturnValue({ + id: 'file-1', + key: 'uploads/file.txt', + name: 'file.txt', + size: 4, + type: 'text/plain', + url: '', + }) + uploadMocks.assertToolFileAccess.mockResolvedValue(null) + uploadMocks.downloadServableFileFromStorage.mockResolvedValue({ + buffer: Buffer.from('test'), + contentType: 'text/plain', + }) + }) + + it('preserves list pagination and forwards cancellation to Atlassian', async () => { + const response = Response.json({ + results: [{ id: 'label-1', name: 'release', prefix: 'global' }], + _links: { next: '/wiki/api/v2/pages/123/labels?cursor=next-cursor' }, + }) + const fetchMock = vi.fn().mockResolvedValue(response) + vi.stubGlobal('fetch', fetchMock) + const controller = new AbortController() + + await expect( + executeConfluenceListLabels( + { ...CONNECTION, pageId: '123', limit: '20', cursor: 'current-cursor' }, + { + headers: new Headers(), + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + } + ) + ).resolves.toEqual({ + labels: [{ id: 'label-1', name: 'release', prefix: 'global' }], + nextCursor: 'next-cursor', + }) + + expect(fetchMock).toHaveBeenCalledWith( + expect.stringContaining('/pages/123/labels?limit=20&cursor=current-cursor'), + expect.objectContaining({ signal: controller.signal }) + ) + expect(response.bodyUsed).toBe(true) + }) + + it('fails closed before downloading a stored file without an acting user', async () => { + let caught: unknown + try { + await executeConfluenceUploadAttachment( + { ...CONNECTION, pageId: '123', file: { key: 'uploads/file.txt' } }, + { headers: new Headers(), requestId: 'request-1' } + ) + } catch (error) { + caught = error + } + + expect(caught).toEqual(new ConfluenceOperationError('Unauthorized', 401)) + expect(uploadMocks.processSingleFileToUserFile).not.toHaveBeenCalled() + expect(uploadMocks.downloadServableFileFromStorage).not.toHaveBeenCalled() + }) + + it('fails closed when stored-file authorization denies access', async () => { + uploadMocks.assertToolFileAccess.mockResolvedValueOnce(Response.json({ error: 'Forbidden' })) + + let caught: unknown + try { + await executeConfluenceUploadAttachment( + { ...CONNECTION, pageId: '123', file: { key: 'uploads/file.txt' } }, + { + headers: new Headers(), + requestId: 'request-1', + userId: 'user-1', + } + ) + } catch (error) { + caught = error + } + + expect(uploadMocks.assertToolFileAccess).toHaveBeenCalledWith( + 'uploads/file.txt', + 'user-1', + 'confluence-upload', + expect.anything() + ) + expect(caught).toEqual( + new ConfluenceOperationError('File not found', 404, { + success: false, + error: 'File not found', + }) + ) + expect(uploadMocks.downloadServableFileFromStorage).not.toHaveBeenCalled() + }) + + it('uploads only the authorized first file and consumes the provider response', async () => { + const response = Response.json({ + results: [ + { + id: 'attachment-1', + title: 'file.txt', + extensions: { fileSize: 4, mediaType: 'text/plain' }, + _links: { download: '/download/file.txt' }, + }, + ], + }) + const fetchMock = vi.fn().mockResolvedValue(response) + vi.stubGlobal('fetch', fetchMock) + const controller = new AbortController() + + await expect( + executeConfluenceUploadAttachment( + { + ...CONNECTION, + pageId: '123', + file: [{ key: 'uploads/file.txt' }, { key: 'uploads/ignored.txt' }], + comment: 'Release notes', + }, + { + headers: new Headers(), + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + } + ) + ).resolves.toEqual({ + attachmentId: 'attachment-1', + title: 'file.txt', + fileSize: 4, + mediaType: 'text/plain', + downloadUrl: '/download/file.txt', + pageId: '123', + }) + + expect(uploadMocks.processSingleFileToUserFile).toHaveBeenCalledWith( + { key: 'uploads/file.txt' }, + 'confluence-upload', + expect.anything() + ) + expect(fetchMock).toHaveBeenCalledWith( + expect.stringContaining('/wiki/rest/api/content/123/child/attachment'), + expect.objectContaining({ + method: 'POST', + headers: expect.objectContaining({ + Accept: 'application/json', + Authorization: 'Bearer access-token', + 'X-Atlassian-Token': 'nocheck', + }), + body: expect.any(FormData), + signal: controller.signal, + }) + ) + expect(response.bodyUsed).toBe(true) + }) +}) diff --git a/apps/sim/lib/internal/confluence/operations.ts b/apps/sim/lib/internal/confluence/operations.ts new file mode 100644 index 00000000000..e78940adccd --- /dev/null +++ b/apps/sim/lib/internal/confluence/operations.ts @@ -0,0 +1,1353 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { + ConfluenceBlogPostOperationBody, + ConfluenceCreateCommentBody, + ConfluenceCreatePageBody, + ConfluenceCreatePagePropertyBody, + ConfluenceCreateSpaceBody, + ConfluenceDeleteAttachmentBody, + ConfluenceDeleteBlogPostBody, + ConfluenceDeleteCommentBody, + ConfluenceDeleteLabelBody, + ConfluenceDeletePageBody, + ConfluenceDeletePagePropertyBody, + ConfluenceDeleteSpaceBody, + ConfluenceGetSpaceQuery, + ConfluenceLabelMutationBody, + ConfluenceListAttachmentsQuery, + ConfluenceListBlogPostsQuery, + ConfluenceListCommentsQuery, + ConfluenceListLabelsQuery, + ConfluenceListPagePropertiesQuery, + ConfluenceListSpacesQuery, + ConfluencePageAncestorsBody, + ConfluencePageBody, + ConfluencePageChildrenBody, + ConfluencePageDescendantsBody, + ConfluencePagesByLabelQuery, + ConfluencePageVersionsBody, + ConfluenceSearchBody, + ConfluenceSearchInSpaceBody, + ConfluenceSpaceBlogPostsBody, + ConfluenceSpaceLabelsQuery, + ConfluenceSpacePagesBody, + ConfluenceSpacePermissionsBody, + ConfluenceSpacePropertiesBody, + ConfluenceTasksBody, + ConfluenceUpdateBlogPostBody, + ConfluenceUpdateCommentBody, + ConfluenceUpdatePageBody, + ConfluenceUpdateSpaceBody, + ConfluenceUploadAttachmentBody, + ConfluenceUserBody, +} from '@/lib/api/contracts/selectors/confluence' +import { + validateAlphanumericId, + validateNumericId, + validatePaginationCursor, + validatePathSegment, +} from '@/lib/core/security/input-validation' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { + asArray, + asObject, + createConfluenceClient, + type JsonObject, + nested, + nextCursor, + readConfluenceResponseObject, + readConfluenceResponseText, + throwConfluenceResponseError, +} from '@/lib/internal/confluence/client' +import { ConfluenceOperationError } from '@/lib/internal/confluence/errors' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/doc-not-ready' +import { processSingleFileToUserFile, type RawFileInput } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' +import { cleanHtmlContent } from '@/tools/confluence/utils' +import { parseAtlassianErrorMessage } from '@/tools/jira/utils' + +const logger = createLogger('ConfluenceOperations') + +export interface ConfluenceOperationContext { + headers: Headers + requestId: string + signal?: AbortSignal + userId?: string +} + +type Connection = { domain: string; accessToken: string; cloudId?: string } + +function jsonInit(method: 'POST' | 'PUT', body: unknown): RequestInit { + return { + method, + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + } +} + +function assertId(value: string, field: string): void { + const validation = validateAlphanumericId(value, field, 255) + if (!validation.isValid) { + throw new ConfluenceOperationError(validation.error || `Invalid ${field}`, 400) + } +} + +function assertCursor(value: string | undefined): void { + if (!value) return + const validation = validatePaginationCursor(value, 'cursor') + if (!validation.isValid) { + throw new ConfluenceOperationError(validation.error || 'Invalid cursor', 400) + } +} + +function cappedLimit(value: string | number): string { + return String(Math.min(Number(value), 250)) +} + +function mappedPage(value: unknown): JsonObject { + const page = asObject(value) + return { + id: page.id, + title: page.title, + status: page.status ?? null, + spaceId: page.spaceId ?? null, + parentId: page.parentId ?? null, + authorId: page.authorId ?? null, + createdAt: page.createdAt ?? null, + version: page.version ?? null, + webUrl: nested(page, '_links', 'webui') ?? null, + } +} + +export async function executeConfluenceRetrievePage( + input: ConfluencePageBody, + context: ConfluenceOperationContext +) { + const client = await createConfluenceClient(input, context.signal) + const data = await client.json( + client.apiV2(`/pages/${input.pageId}?body-format=storage`), + {}, + context.signal + ) + return { + id: data.id, + title: data.title, + body: { + storage: { + value: nested(data, 'body', 'storage', 'value') ?? null, + representation: 'storage', + }, + }, + status: data.status ?? null, + spaceId: data.spaceId ?? null, + parentId: data.parentId ?? null, + authorId: data.authorId ?? null, + createdAt: data.createdAt ?? null, + version: data.version ?? null, + _links: data._links ?? null, + } +} + +export async function executeConfluenceUpdatePage( + input: ConfluenceUpdatePageBody, + context: ConfluenceOperationContext +) { + const client = await createConfluenceClient(input, context.signal) + const url = client.apiV2(`/pages/${input.pageId}?body-format=storage`) + const currentResponse = await client.fetch(url, {}, context.signal) + if (!currentResponse.ok) { + const text = await readConfluenceResponseText( + currentResponse, + context.signal, + 'Confluence page response' + ) + throw new Error( + parseAtlassianErrorMessage(currentResponse.status, currentResponse.statusText, text) + ) + } + const current = await readConfluenceResponseObject( + currentResponse, + context.signal, + 'Confluence page response' + ) + const currentVersion = Number(nested(current, 'version', 'number')) + const title = input.title || current.title + const value = input.body?.value || nested(current, 'body', 'storage', 'value') || '' + return client.json( + url, + jsonInit('PUT', { + id: input.pageId, + version: { + number: currentVersion + 1, + message: input.version?.message || 'Updated via API', + }, + status: 'current', + title, + body: { representation: 'storage', value }, + }), + context.signal + ) +} + +export async function executeConfluenceDeletePage( + input: ConfluenceDeletePageBody, + context: ConfluenceOperationContext +) { + const client = await createConfluenceClient(input, context.signal) + await client.delete( + client.apiV2(`/pages/${input.pageId}${input.purge ? '?purge=true' : ''}`), + context.signal + ) + return { pageId: input.pageId, deleted: true } +} + +export async function executeConfluenceCreatePage( + input: ConfluenceCreatePageBody, + context: ConfluenceOperationContext +) { + if (!/^\d+$/.test(String(input.spaceId))) { + throw new ConfluenceOperationError( + 'Invalid Space ID. The Space ID must be a numeric value, not the space key from the URL. Use the "list" operation to get all spaces with their numeric IDs.', + 400 + ) + } + assertId(input.spaceId, 'spaceId') + if (input.parentId) assertId(input.parentId, 'parentId') + const client = await createConfluenceClient(input, context.signal) + const body: JsonObject = { + spaceId: input.spaceId, + status: 'current', + title: input.title, + body: { representation: 'storage', value: input.content }, + } + if (input.parentId) body.parentId = input.parentId + try { + return await client.json(client.apiV2('/pages'), jsonInit('POST', body), context.signal) + } catch (error) { + if ( + error instanceof ConfluenceOperationError && + error.message.includes("'spaceId'") && + error.message.includes('Long') + ) { + throw new ConfluenceOperationError( + 'Invalid Space ID. Use the list spaces operation to find valid space IDs.', + error.status + ) + } + throw error + } +} + +export async function executeConfluenceListAttachments( + input: ConfluenceListAttachmentsQuery, + context: ConfluenceOperationContext +) { + assertId(input.pageId, 'pageId') + const client = await createConfluenceClient(input, context.signal) + const query = new URLSearchParams({ limit: cappedLimit(input.limit) }) + if (input.cursor) query.set('cursor', input.cursor) + const data = await client.json( + client.apiV2(`/pages/${input.pageId}/attachments?${query}`), + {}, + context.signal + ) + return { + attachments: asArray(data.results).map((value) => { + const attachment = asObject(value) + return { + id: attachment.id, + title: attachment.title, + fileSize: attachment.fileSize || 0, + mediaType: attachment.mediaType || '', + downloadUrl: attachment.downloadLink || nested(attachment, '_links', 'download') || '', + status: attachment.status ?? null, + webuiUrl: nested(attachment, '_links', 'webui') ?? null, + pageId: attachment.pageId ?? null, + blogPostId: attachment.blogPostId ?? null, + comment: attachment.comment ?? null, + version: attachment.version ?? null, + } + }), + nextCursor: nextCursor(data), + } +} + +export async function executeConfluenceDeleteAttachment( + input: ConfluenceDeleteAttachmentBody, + context: ConfluenceOperationContext +) { + assertId(input.attachmentId, 'attachmentId') + const client = await createConfluenceClient(input, context.signal) + await client.delete(client.apiV2(`/attachments/${input.attachmentId}`), context.signal) + return { attachmentId: input.attachmentId, deleted: true } +} + +export async function executeConfluenceUploadAttachment( + input: ConfluenceUploadAttachmentBody, + context: ConfluenceOperationContext +) { + const { signal, userId } = context + if (!userId) throw new ConfluenceOperationError('Unauthorized', 401) + assertId(input.pageId, 'pageId') + let file = input.file as RawFileInput | RawFileInput[] + if (Array.isArray(file)) { + if (file.length === 0) throw new ConfluenceOperationError('No file provided', 400) + file = file[0] + } + let userFile: ReturnType + try { + userFile = processSingleFileToUserFile(file, 'confluence-upload', logger) + } catch (error) { + throw new ConfluenceOperationError(getErrorMessage(error, 'Failed to process file'), 400) + } + const denied = await assertToolFileAccess(userFile.key, userId, 'confluence-upload', logger) + if (denied) { + throw new ConfluenceOperationError('File not found', 404, { + success: false, + error: 'File not found', + }) + } + signal?.throwIfAborted() + let fileBuffer: Buffer + let contentType: string + try { + const servable = await downloadServableFileFromStorage(userFile, 'confluence-upload', logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) + fileBuffer = servable.buffer + contentType = servable.contentType + } catch (error) { + signal?.throwIfAborted() + if (isDocNotReadyError(error)) { + const message = docNotReadyMessage() + throw new ConfluenceOperationError(message, 409, { success: false, error: message }) + } + throw new ConfluenceOperationError( + `Failed to download file: ${getErrorMessage(error, 'Unknown error')}`, + isPayloadSizeLimitError(error) ? 413 : 500 + ) + } + const client = await createConfluenceClient(input, signal) + const mimeType = contentType || userFile.type || 'application/octet-stream' + const form = new FormData() + form.append( + 'file', + new Blob([new Uint8Array(fileBuffer)], { type: mimeType }), + input.fileName || userFile.name || 'attachment' + ) + if (input.comment) form.append('comment', input.comment) + form.append('minorEdit', 'false') + const response = await client.fetch( + client.rest(`/content/${input.pageId}/child/attachment`), + { + method: 'POST', + headers: { 'X-Atlassian-Token': 'nocheck' }, + body: form, + }, + signal + ) + if (!response.ok) await throwConfluenceResponseError(response, signal) + const data = await readConfluenceResponseObject( + response, + signal, + 'Confluence attachment response' + ) + const attachment = asObject(asArray(data.results)[0] || data) + return { + attachmentId: attachment.id, + title: attachment.title, + fileSize: nested(attachment, 'extensions', 'fileSize') || 0, + mediaType: nested(attachment, 'extensions', 'mediaType') || mimeType, + downloadUrl: nested(attachment, '_links', 'download') || '', + pageId: input.pageId, + } +} + +export async function executeConfluenceAddLabel( + input: ConfluenceLabelMutationBody, + context: ConfluenceOperationContext +) { + assertId(input.pageId, 'pageId') + const client = await createConfluenceClient(input, context.signal) + const data = await client.json( + client.rest(`/content/${input.pageId}/label`), + jsonInit('POST', [{ prefix: input.prefix || 'global', name: input.labelName }]), + context.signal + ) + const label = asObject(asArray(data.results)[0] || asArray(data)[0] || data) + return { + id: label.id ?? '', + name: label.name ?? input.labelName, + prefix: label.prefix ?? input.prefix ?? 'global', + pageId: input.pageId, + labelName: input.labelName, + } +} + +export async function executeConfluenceListLabels( + input: ConfluenceListLabelsQuery, + context: ConfluenceOperationContext +) { + assertId(input.pageId, 'pageId') + const client = await createConfluenceClient(input, context.signal) + const query = new URLSearchParams({ limit: cappedLimit(input.limit) }) + if (input.cursor) query.set('cursor', input.cursor) + const data = await client.json( + client.apiV2(`/pages/${input.pageId}/labels?${query}`), + {}, + context.signal + ) + return { + labels: asArray(data.results).map((value) => { + const label = asObject(value) + return { id: label.id, name: label.name, prefix: label.prefix || 'global' } + }), + nextCursor: nextCursor(data), + } +} + +export async function executeConfluenceDeleteLabel( + input: ConfluenceDeleteLabelBody, + context: ConfluenceOperationContext +) { + assertId(input.pageId, 'pageId') + const client = await createConfluenceClient(input, context.signal) + await client.delete( + client.rest( + `/content/${input.pageId}/label?name=${encodeURIComponent(input.labelName.trim())}` + ), + context.signal + ) + return { pageId: input.pageId, labelName: input.labelName, deleted: true } +} + +export async function executeConfluenceCreateComment( + input: ConfluenceCreateCommentBody, + context: ConfluenceOperationContext +) { + assertId(input.pageId, 'pageId') + const client = await createConfluenceClient(input, context.signal) + const data = await client.json( + client.apiV2('/footer-comments'), + jsonInit('POST', { + pageId: input.pageId, + body: { representation: 'storage', value: input.comment }, + }), + context.signal + ) + return { ...data, pageId: input.pageId } +} + +export async function executeConfluenceListComments( + input: ConfluenceListCommentsQuery, + context: ConfluenceOperationContext +) { + assertId(input.pageId, 'pageId') + const client = await createConfluenceClient(input, context.signal) + const query = new URLSearchParams({ + limit: cappedLimit(input.limit), + 'body-format': input.bodyFormat, + }) + if (input.cursor) query.set('cursor', input.cursor) + const data = await client.json( + client.apiV2(`/pages/${input.pageId}/footer-comments?${query}`), + {}, + context.signal + ) + return { + comments: asArray(data.results).map((value) => { + const comment = asObject(value) + return { + id: comment.id, + body: { + value: + nested(comment, 'body', 'storage', 'value') || + nested(comment, 'body', 'view', 'value') || + '', + representation: input.bodyFormat, + }, + createdAt: comment.createdAt || '', + authorId: comment.authorId || '', + status: comment.status ?? null, + title: comment.title ?? null, + pageId: comment.pageId ?? null, + blogPostId: comment.blogPostId ?? null, + parentCommentId: comment.parentCommentId ?? null, + version: comment.version ?? null, + } + }), + nextCursor: nextCursor(data), + } +} + +async function detectCommentEndpoint( + input: Connection & { commentId: string }, + context: ConfluenceOperationContext +) { + const client = await createConfluenceClient(input, context.signal) + let endpoint = 'footer-comments' + let response = await client.fetch( + client.apiV2(`/footer-comments/${input.commentId}`), + {}, + context.signal + ) + if (response.status === 404) { + await readConfluenceResponseText(response, context.signal, 'Confluence comment lookup response') + endpoint = 'inline-comments' + response = await client.fetch( + client.apiV2(`/inline-comments/${input.commentId}`), + {}, + context.signal + ) + } + return { client, endpoint, response } +} + +export async function executeConfluenceUpdateComment( + input: ConfluenceUpdateCommentBody, + context: ConfluenceOperationContext +) { + const { client, endpoint, response } = await detectCommentEndpoint(input, context) + if (!response.ok) { + const text = await readConfluenceResponseText( + response, + context.signal, + 'Confluence comment response' + ) + throw new Error(parseAtlassianErrorMessage(response.status, response.statusText, text)) + } + const current = await readConfluenceResponseObject( + response, + context.signal, + 'Confluence comment response' + ) + return client.json( + client.apiV2(`/${endpoint}/${input.commentId}`), + jsonInit('PUT', { + body: { representation: 'storage', value: input.comment }, + version: { + number: Number(nested(current, 'version', 'number') || 1) + 1, + message: 'Updated via Sim', + }, + }), + context.signal + ) +} + +export async function executeConfluenceDeleteComment( + input: ConfluenceDeleteCommentBody, + context: ConfluenceOperationContext +) { + const { client, endpoint, response } = await detectCommentEndpoint(input, context) + if (!response.ok) await throwConfluenceResponseError(response, context.signal) + await client.delete(client.apiV2(`/${endpoint}/${input.commentId}`), context.signal) + return { commentId: input.commentId, deleted: true } +} + +export async function executeConfluenceListPageProperties( + input: ConfluenceListPagePropertiesQuery, + context: ConfluenceOperationContext +) { + assertId(input.pageId, 'pageId') + const client = await createConfluenceClient(input, context.signal) + const query = new URLSearchParams({ limit: cappedLimit(input.limit) }) + if (input.cursor) query.set('cursor', input.cursor) + const data = await client.json( + client.apiV2(`/pages/${input.pageId}/properties?${query}`), + {}, + context.signal + ) + return { + properties: asArray(data.results).map((value) => { + const property = asObject(value) + return { + id: property.id, + key: property.key, + value: property.value ?? null, + version: property.version ?? null, + } + }), + pageId: input.pageId, + nextCursor: nextCursor(data), + } +} + +export async function executeConfluenceCreatePageProperty( + input: ConfluenceCreatePagePropertyBody, + context: ConfluenceOperationContext +) { + assertId(input.pageId, 'pageId') + const client = await createConfluenceClient(input, context.signal) + const data = await client.json( + client.apiV2(`/pages/${input.pageId}/properties`), + jsonInit('POST', { key: input.key, value: input.value }), + context.signal + ) + return { + id: data.id, + key: data.key, + value: data.value, + version: data.version, + pageId: input.pageId, + } +} + +export async function executeConfluenceDeletePageProperty( + input: ConfluenceDeletePagePropertyBody, + context: ConfluenceOperationContext +) { + assertId(input.pageId, 'pageId') + assertId(input.propertyId, 'propertyId') + const client = await createConfluenceClient(input, context.signal) + await client.delete( + client.apiV2(`/pages/${input.pageId}/properties/${input.propertyId}`), + context.signal + ) + return { propertyId: input.propertyId, pageId: input.pageId, deleted: true } +} + +export async function executeConfluenceGetPageAncestors( + input: ConfluencePageAncestorsBody, + context: ConfluenceOperationContext +) { + assertId(input.pageId, 'pageId') + const client = await createConfluenceClient(input, context.signal) + const data = await client.json( + client.apiV2(`/pages/${input.pageId}/ancestors?limit=${cappedLimit(input.limit)}`), + {}, + context.signal + ) + return { + ancestors: asArray(data.results).map((value) => { + const page = asObject(value) + return { + id: page.id, + title: page.title, + status: page.status ?? null, + spaceId: page.spaceId ?? null, + webUrl: nested(page, '_links', 'webui') ?? null, + } + }), + pageId: input.pageId, + } +} + +export async function executeConfluenceGetPageChildren( + input: ConfluencePageChildrenBody, + context: ConfluenceOperationContext +) { + assertId(input.pageId, 'pageId') + const client = await createConfluenceClient(input, context.signal) + const query = new URLSearchParams({ limit: cappedLimit(input.limit) }) + if (input.cursor) query.set('cursor', input.cursor) + const data = await client.json( + client.apiV2(`/pages/${input.pageId}/children?${query}`), + {}, + context.signal + ) + return { + children: asArray(data.results).map((value) => { + const page = asObject(value) + return { + id: page.id, + title: page.title, + status: page.status ?? null, + spaceId: page.spaceId ?? null, + childPosition: page.childPosition ?? null, + webUrl: nested(page, '_links', 'webui') ?? null, + } + }), + parentId: input.pageId, + nextCursor: nextCursor(data), + } +} + +export async function executeConfluenceGetPageDescendants( + input: ConfluencePageDescendantsBody, + context: ConfluenceOperationContext +) { + assertId(input.pageId, 'pageId') + assertCursor(input.cursor) + const client = await createConfluenceClient(input, context.signal) + const query = new URLSearchParams({ limit: cappedLimit(input.limit) }) + if (input.cursor) query.set('cursor', input.cursor) + const data = await client.json( + client.apiV2(`/pages/${input.pageId}/descendants?${query}`), + {}, + context.signal + ) + return { + descendants: asArray(data.results).map((value) => { + const page = asObject(value) + return { + id: page.id, + title: page.title, + type: page.type ?? null, + status: page.status ?? null, + spaceId: page.spaceId ?? null, + parentId: page.parentId ?? null, + childPosition: page.childPosition ?? null, + depth: page.depth ?? null, + } + }), + pageId: input.pageId, + nextCursor: nextCursor(data), + } +} + +export async function executeConfluencePageVersions( + input: ConfluencePageVersionsBody, + context: ConfluenceOperationContext +) { + assertId(input.pageId, 'pageId') + const client = await createConfluenceClient(input, context.signal) + if (input.versionNumber !== undefined && input.versionNumber !== null) { + const validation = validateNumericId(input.versionNumber, 'versionNumber', { min: 1 }) + if (!validation.isValid) { + throw new ConfluenceOperationError(validation.error || 'Invalid versionNumber', 400) + } + const version = validation.sanitized + const [versionResponse, pageResponse] = await Promise.all([ + client.fetch(client.apiV2(`/pages/${input.pageId}/versions/${version}`), {}, context.signal), + client.fetch( + client.apiV2(`/pages/${input.pageId}?version=${version}&body-format=storage`), + {}, + context.signal + ), + ]) + if (!versionResponse.ok) { + await pageResponse.body?.cancel() + await throwConfluenceResponseError(versionResponse, context.signal) + } + const versionData = await readConfluenceResponseObject( + versionResponse, + context.signal, + 'Confluence page version response' + ) + let title: unknown = null + let body: unknown = null + let content: string | null = null + if (pageResponse.ok) { + const page = await readConfluenceResponseObject( + pageResponse, + context.signal, + 'Confluence page response' + ) + title = page.title ?? null + body = page.body ?? null + const raw = + nested(page, 'body', 'storage', 'value') || + nested(page, 'body', 'view', 'value') || + nested(page, 'body', 'atlas_doc_format', 'value') || + '' + if (typeof raw === 'string' && raw) content = cleanHtmlContent(raw) + } else { + await pageResponse.body?.cancel() + } + return { + version: { + number: versionData.number, + message: versionData.message ?? null, + minorEdit: versionData.minorEdit ?? false, + authorId: versionData.authorId ?? null, + createdAt: versionData.createdAt ?? null, + }, + pageId: input.pageId, + title, + content, + body, + } + } + assertCursor(input.cursor) + const query = new URLSearchParams({ limit: cappedLimit(input.limit) }) + if (input.cursor) query.set('cursor', input.cursor) + const data = await client.json( + client.apiV2(`/pages/${input.pageId}/versions?${query}`), + {}, + context.signal + ) + return { + versions: asArray(data.results).map((value) => { + const version = asObject(value) + return { + number: version.number, + message: version.message ?? null, + minorEdit: version.minorEdit ?? false, + authorId: version.authorId ?? null, + createdAt: version.createdAt ?? null, + } + }), + pageId: input.pageId, + nextCursor: nextCursor(data), + } +} + +export async function executeConfluenceGetPagesByLabel( + input: ConfluencePagesByLabelQuery, + context: ConfluenceOperationContext +) { + assertId(input.labelId, 'labelId') + const client = await createConfluenceClient(input, context.signal) + const query = new URLSearchParams({ limit: cappedLimit(input.limit) }) + if (input.cursor) query.set('cursor', input.cursor) + const data = await client.json( + client.apiV2(`/labels/${input.labelId}/pages?${query}`), + {}, + context.signal + ) + return { + pages: asArray(data.results).map(mappedPage), + labelId: input.labelId, + nextCursor: nextCursor(data), + } +} + +function escapeCql(value: string): string { + return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"') +} + +export async function executeConfluenceSearch( + input: ConfluenceSearchBody, + context: ConfluenceOperationContext +) { + const client = await createConfluenceClient(input, context.signal) + const query = new URLSearchParams({ + cql: `text ~ "${escapeCql(input.query)}"`, + limit: input.limit.toString(), + }) + const data = await client.json(client.rest(`/search?${query}`), {}, context.signal) + return { + results: asArray(data.results).map((value) => { + const result = asObject(value) + const content = asObject(result.content) + const globalContainer = asObject(result.resultGlobalContainer) + const contentSpace = asObject(content.space) + const space = Object.keys(globalContainer).length ? globalContainer : contentSpace + return { + id: content.id || result.id, + title: content.title || result.title, + type: content.type || result.type, + url: result.url || nested(result, '_links', 'webui') || '', + excerpt: result.excerpt || '', + status: content.status ?? null, + spaceKey: globalContainer.key ?? contentSpace.key ?? null, + space: Object.keys(space).length + ? { + id: space.id ?? null, + key: space.key ?? null, + name: space.name ?? space.title ?? null, + } + : null, + lastModified: + result.lastModified ?? nested(content, 'history', 'lastUpdated', 'when') ?? null, + entityType: result.entityType ?? null, + } + }), + } +} + +export async function executeConfluenceSearchInSpace( + input: ConfluenceSearchInSpaceBody, + context: ConfluenceOperationContext +) { + assertId(input.spaceKey, 'spaceKey') + const client = await createConfluenceClient(input, context.signal) + let cql = `space = "${escapeCql(input.spaceKey)}"` + if (input.query) cql += ` AND text ~ "${escapeCql(input.query)}"` + if (input.contentType) cql += ` AND type = "${escapeCql(input.contentType)}"` + const query = new URLSearchParams({ cql, limit: cappedLimit(input.limit) }) + const data = await client.json(client.rest(`/search?${query}`), {}, context.signal) + const results = asArray(data.results).map((value) => { + const result = asObject(value) + const content = asObject(result.content) + return { + id: content.id ?? result.id, + title: content.title ?? result.title, + type: content.type ?? result.type, + status: content.status ?? null, + url: result.url ?? nested(result, '_links', 'webui') ?? '', + excerpt: result.excerpt ?? '', + lastModified: result.lastModified ?? null, + } + }) + return { results, spaceKey: input.spaceKey, totalSize: data.totalSize ?? results.length } +} + +export async function executeConfluenceListBlogPostsInSpace( + input: ConfluenceSpaceBlogPostsBody, + context: ConfluenceOperationContext +) { + assertId(input.spaceId, 'spaceId') + const client = await createConfluenceClient(input, context.signal) + const query = new URLSearchParams({ limit: cappedLimit(input.limit) }) + if (input.status) query.set('status', input.status) + if (input.bodyFormat) query.set('body-format', input.bodyFormat) + if (input.cursor) query.set('cursor', input.cursor) + const data = await client.json( + client.apiV2(`/spaces/${input.spaceId}/blogposts?${query}`), + {}, + context.signal + ) + return { + blogPosts: asArray(data.results).map((value) => { + const post = asObject(value) + return { + id: post.id, + title: post.title, + status: post.status ?? null, + spaceId: post.spaceId ?? null, + authorId: post.authorId ?? null, + createdAt: post.createdAt ?? null, + version: post.version ?? null, + body: post.body ?? null, + webUrl: nested(post, '_links', 'webui') ?? null, + } + }), + nextCursor: nextCursor(data), + } +} + +export async function executeConfluenceListPagesInSpace( + input: ConfluenceSpacePagesBody, + context: ConfluenceOperationContext +) { + assertId(input.spaceId, 'spaceId') + const client = await createConfluenceClient(input, context.signal) + const query = new URLSearchParams({ limit: cappedLimit(input.limit) }) + if (input.status) query.set('status', input.status) + if (input.bodyFormat) query.set('body-format', input.bodyFormat) + if (input.cursor) query.set('cursor', input.cursor) + const data = await client.json( + client.apiV2(`/spaces/${input.spaceId}/pages?${query}`), + {}, + context.signal + ) + return { + pages: asArray(data.results).map((value) => { + const page = asObject(value) + return { ...mappedPage(page), body: page.body ?? null } + }), + nextCursor: nextCursor(data), + } +} + +export async function executeConfluenceListSpaceLabels( + input: ConfluenceSpaceLabelsQuery, + context: ConfluenceOperationContext +) { + assertId(input.spaceId, 'spaceId') + const client = await createConfluenceClient(input, context.signal) + const query = new URLSearchParams({ limit: cappedLimit(input.limit) }) + if (input.cursor) query.set('cursor', input.cursor) + const data = await client.json( + client.apiV2(`/spaces/${input.spaceId}/labels?${query}`), + {}, + context.signal + ) + return { + labels: asArray(data.results).map((value) => { + const label = asObject(value) + return { id: label.id, name: label.name, prefix: label.prefix || 'global' } + }), + spaceId: input.spaceId, + nextCursor: nextCursor(data), + } +} + +export async function executeConfluenceListSpacePermissions( + input: ConfluenceSpacePermissionsBody, + context: ConfluenceOperationContext +) { + assertId(input.spaceId, 'spaceId') + assertCursor(input.cursor) + const client = await createConfluenceClient(input, context.signal) + const query = new URLSearchParams({ limit: cappedLimit(input.limit) }) + if (input.cursor) query.set('cursor', input.cursor) + const data = await client.json( + client.apiV2(`/spaces/${input.spaceId}/permissions?${query}`), + {}, + context.signal + ) + return { + permissions: asArray(data.results).map((value) => { + const permission = asObject(value) + return { + id: permission.id, + principalType: nested(permission, 'principal', 'type') ?? null, + principalId: nested(permission, 'principal', 'id') ?? null, + operationKey: nested(permission, 'operation', 'key') ?? null, + operationTargetType: nested(permission, 'operation', 'targetType') ?? null, + anonymousAccess: permission.anonymousAccess ?? false, + unlicensedAccess: permission.unlicensedAccess ?? false, + } + }), + spaceId: input.spaceId, + nextCursor: nextCursor(data), + } +} + +export async function executeConfluenceListBlogPosts( + input: ConfluenceListBlogPostsQuery, + context: ConfluenceOperationContext +) { + const client = await createConfluenceClient(input, context.signal) + const query = new URLSearchParams({ limit: cappedLimit(input.limit) }) + if (input.status) query.set('status', input.status) + if (input.sort) query.set('sort', input.sort) + if (input.cursor) query.set('cursor', input.cursor) + const data = await client.json(client.apiV2(`/blogposts?${query}`), {}, context.signal) + return { + blogPosts: asArray(data.results).map((value) => { + const post = asObject(value) + return { + id: post.id, + title: post.title, + status: post.status ?? null, + spaceId: post.spaceId ?? null, + authorId: post.authorId ?? null, + createdAt: post.createdAt ?? null, + version: post.version ?? null, + webUrl: nested(post, '_links', 'webui') ?? null, + } + }), + nextCursor: nextCursor(data), + } +} + +export async function executeConfluenceCreateBlogPost( + input: ConfluenceBlogPostOperationBody, + context: ConfluenceOperationContext +) { + if (!('spaceId' in input) || !('title' in input) || !('content' in input)) { + throw new ConfluenceOperationError('Invalid create blog post request', 400) + } + const client = await createConfluenceClient(input, context.signal) + const data = await client.json( + client.apiV2('/blogposts'), + jsonInit('POST', { + spaceId: input.spaceId, + status: input.status || 'current', + title: input.title, + body: { representation: 'storage', value: input.content }, + }), + context.signal + ) + return { + id: data.id, + title: data.title, + spaceId: data.spaceId, + webUrl: nested(data, '_links', 'webui') ?? null, + } +} + +export async function executeConfluenceGetBlogPost( + input: ConfluenceBlogPostOperationBody, + context: ConfluenceOperationContext +) { + if (!('blogPostId' in input)) throw new ConfluenceOperationError('Blog post ID is required', 400) + const client = await createConfluenceClient(input, context.signal) + const query = input.bodyFormat ? `?body-format=${encodeURIComponent(input.bodyFormat)}` : '' + const data = await client.json( + client.apiV2(`/blogposts/${input.blogPostId}${query}`), + {}, + context.signal + ) + return { + id: data.id, + title: data.title, + status: data.status ?? null, + spaceId: data.spaceId ?? null, + authorId: data.authorId ?? null, + createdAt: data.createdAt ?? null, + version: data.version ?? null, + body: data.body ?? null, + webUrl: nested(data, '_links', 'webui') ?? null, + } +} + +export async function executeConfluenceUpdateBlogPost( + input: ConfluenceUpdateBlogPostBody, + context: ConfluenceOperationContext +) { + const client = await createConfluenceClient(input, context.signal) + const url = client.apiV2(`/blogposts/${input.blogPostId}?body-format=storage`) + const currentResponse = await client.fetch(url, {}, context.signal) + if (!currentResponse.ok) { + const text = await readConfluenceResponseText( + currentResponse, + context.signal, + 'Confluence blog post response' + ) + throw new Error( + parseAtlassianErrorMessage(currentResponse.status, currentResponse.statusText, text) + ) + } + const current = await readConfluenceResponseObject( + currentResponse, + context.signal, + 'Confluence blog post response' + ) + const version = nested(current, 'version', 'number') + if (typeof version !== 'number') { + throw new ConfluenceOperationError('Unable to determine current blog post version', 422) + } + return client.json( + url, + jsonInit('PUT', { + id: input.blogPostId, + version: { number: version + 1 }, + status: 'current', + title: input.title || current.title, + body: { + representation: 'storage', + value: input.content || nested(current, 'body', 'storage', 'value') || '', + }, + }), + context.signal + ) +} + +export async function executeConfluenceDeleteBlogPost( + input: ConfluenceDeleteBlogPostBody, + context: ConfluenceOperationContext +) { + const client = await createConfluenceClient(input, context.signal) + await client.delete(client.apiV2(`/blogposts/${input.blogPostId}`), context.signal) + return { blogPostId: input.blogPostId, deleted: true } +} + +export async function executeConfluenceGetSpace( + input: ConfluenceGetSpaceQuery, + context: ConfluenceOperationContext +) { + assertId(input.spaceId, 'spaceId') + const client = await createConfluenceClient(input, context.signal) + return client.json(client.apiV2(`/spaces/${input.spaceId}`), {}, context.signal) +} + +export async function executeConfluenceCreateSpace( + input: ConfluenceCreateSpaceBody, + context: ConfluenceOperationContext +) { + const client = await createConfluenceClient(input, context.signal) + const body: JsonObject = { name: input.name, key: input.key } + if (input.description) { + body.description = { plain: { value: input.description, representation: 'plain' } } + } + return client.json(client.apiV2('/spaces'), jsonInit('POST', body), context.signal) +} + +export async function executeConfluenceUpdateSpace( + input: ConfluenceUpdateSpaceBody, + context: ConfluenceOperationContext +) { + assertId(input.spaceId, 'spaceId') + if (!input.name && input.description === undefined) { + throw new ConfluenceOperationError( + 'At least one of name or description is required for update', + 400 + ) + } + const client = await createConfluenceClient(input, context.signal) + const current = await client.json(client.apiV2(`/spaces/${input.spaceId}`), {}, context.signal) + const body: JsonObject = { name: input.name || current.name } + if (input.description !== undefined) { + body.description = { plain: { value: input.description, representation: 'plain' } } + } + return client.json( + client.rest(`/space/${encodeURIComponent(String(current.key))}`), + jsonInit('PUT', body), + context.signal + ) +} + +export async function executeConfluenceDeleteSpace( + input: ConfluenceDeleteSpaceBody, + context: ConfluenceOperationContext +) { + assertId(input.spaceId, 'spaceId') + const client = await createConfluenceClient(input, context.signal) + const current = await client.json(client.apiV2(`/spaces/${input.spaceId}`), {}, context.signal) + const response = await client.fetch( + client.rest(`/space/${encodeURIComponent(String(current.key))}`), + { method: 'DELETE' }, + context.signal + ) + if (!response.ok) await throwConfluenceResponseError(response, context.signal) + let longTask: JsonObject = {} + try { + const text = await readConfluenceResponseText( + response, + context.signal, + 'Confluence delete space response', + 'DELETE' + ) + if (text) longTask = asObject(JSON.parse(text)) + } catch { + context.signal?.throwIfAborted() + } + return { + spaceId: input.spaceId, + deleted: true, + longTaskId: longTask.id, + longTaskStatusLink: nested(longTask, 'links', 'status'), + } +} + +export async function executeConfluenceListSpaces( + input: ConfluenceListSpacesQuery, + context: ConfluenceOperationContext +) { + const client = await createConfluenceClient(input, context.signal) + const query = new URLSearchParams({ limit: cappedLimit(input.limit) }) + if (input.cursor) query.set('cursor', input.cursor) + const data = await client.json(client.apiV2(`/spaces?${query}`), {}, context.signal) + return { + spaces: asArray(data.results).map((value) => { + const space = asObject(value) + return { + id: space.id, + name: space.name, + key: space.key, + type: space.type, + status: space.status, + authorId: space.authorId ?? null, + createdAt: space.createdAt ?? null, + homepageId: space.homepageId ?? null, + description: space.description ?? null, + } + }), + nextCursor: nextCursor(data), + } +} + +export async function executeConfluenceSpaceProperties( + input: ConfluenceSpacePropertiesBody, + context: ConfluenceOperationContext +) { + assertId(input.spaceId, 'spaceId') + const client = await createConfluenceClient(input, context.signal) + const base = client.apiV2(`/spaces/${input.spaceId}/properties`) + if (input.action === 'delete') { + if (!input.propertyId) { + throw new ConfluenceOperationError('Property ID is required for delete action', 400) + } + assertId(input.propertyId, 'propertyId') + await client.delete(`${base}/${encodeURIComponent(input.propertyId)}`, context.signal) + return { spaceId: input.spaceId, propertyId: input.propertyId, deleted: true } + } + if (input.action === 'create') { + if (!input.key) { + throw new ConfluenceOperationError('Property key is required for create action', 400) + } + const data = await client.json( + base, + jsonInit('POST', { key: input.key, value: input.value ?? {} }), + context.signal + ) + return { propertyId: data.id, key: data.key, value: data.value ?? null, spaceId: input.spaceId } + } + assertCursor(input.cursor) + const query = new URLSearchParams({ limit: cappedLimit(input.limit) }) + if (input.cursor) query.set('cursor', input.cursor) + const data = await client.json(`${base}?${query}`, {}, context.signal) + return { + properties: asArray(data.results).map((value) => { + const property = asObject(value) + return { id: property.id, key: property.key, value: property.value ?? null } + }), + spaceId: input.spaceId, + nextCursor: nextCursor(data), + } +} + +function mapTask(value: unknown): JsonObject { + const task = asObject(value) + return { + id: task.id, + localId: task.localId ?? null, + spaceId: task.spaceId ?? null, + pageId: task.pageId ?? null, + blogPostId: task.blogPostId ?? null, + status: task.status, + body: nested(task, 'body', 'storage', 'value') ?? null, + createdBy: task.createdBy ?? null, + assignedTo: task.assignedTo ?? null, + completedBy: task.completedBy ?? null, + createdAt: task.createdAt ?? null, + updatedAt: task.updatedAt ?? null, + dueAt: task.dueAt ?? null, + completedAt: task.completedAt ?? null, + } +} + +export async function executeConfluenceTasks( + input: ConfluenceTasksBody, + context: ConfluenceOperationContext +) { + const client = await createConfluenceClient(input, context.signal) + if (input.action === 'update' && input.taskId) { + assertId(input.taskId, 'taskId') + const url = client.apiV2(`/tasks/${input.taskId}`) + const current = await client.json(url, {}, context.signal) + const data = await client.json( + url, + jsonInit('PUT', { id: input.taskId, status: input.status || current.status }), + context.signal + ) + return { task: mapTask(data) } + } + if (input.taskId) { + assertId(input.taskId, 'taskId') + const data = await client.json(client.apiV2(`/tasks/${input.taskId}`), {}, context.signal) + return { task: mapTask(data) } + } + assertCursor(input.cursor) + const query = new URLSearchParams({ limit: cappedLimit(input.limit) }) + if (input.cursor) query.set('cursor', input.cursor) + if (input.status) query.set('status', input.status) + if (input.pageId) { + assertId(input.pageId, 'pageId') + query.set('page-id', input.pageId) + } + if (input.spaceId) { + assertId(input.spaceId, 'spaceId') + query.set('space-id', input.spaceId) + } + if (input.assignedTo) { + const validation = validatePathSegment(input.assignedTo, { + paramName: 'assignedTo', + maxLength: 128, + customPattern: /^[a-zA-Z0-9_|:-]+$/, + }) + if (!validation.isValid) { + throw new ConfluenceOperationError(validation.error || 'Invalid assignedTo', 400) + } + query.set('assigned-to', input.assignedTo) + } + const data = await client.json(client.apiV2(`/tasks?${query}`), {}, context.signal) + return { tasks: asArray(data.results).map(mapTask), nextCursor: nextCursor(data) } +} + +export async function executeConfluenceGetUser( + input: ConfluenceUserBody, + context: ConfluenceOperationContext +) { + const validation = validatePathSegment(input.accountId, { + paramName: 'accountId', + maxLength: 128, + customPattern: /^[a-zA-Z0-9_|:-]+$/, + }) + if (!validation.isValid) { + throw new ConfluenceOperationError(validation.error || 'Invalid accountId', 400) + } + const client = await createConfluenceClient(input, context.signal) + return client.json( + client.rest(`/user?accountId=${encodeURIComponent(input.accountId)}`), + {}, + context.signal + ) +} diff --git a/apps/sim/lib/internal/crowdstrike/client.test.ts b/apps/sim/lib/internal/crowdstrike/client.test.ts new file mode 100644 index 00000000000..d48fc77b60c --- /dev/null +++ b/apps/sim/lib/internal/crowdstrike/client.test.ts @@ -0,0 +1,91 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + CrowdStrikeAuthError, + callCrowdStrike, + getAccessToken, +} from '@/lib/internal/crowdstrike/client' + +const fetchMock = vi.fn() + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }) +} + +describe('CrowdStrike client', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', fetchMock) + }) + + it('forwards cancellation through token and provider requests', async () => { + const controller = new AbortController() + fetchMock + .mockResolvedValueOnce(jsonResponse({ access_token: 'token-1' })) + .mockResolvedValueOnce(jsonResponse({ resources: ['alert-1'] })) + + await expect( + getAccessToken( + { clientId: 'client-id', clientSecret: 'client-secret', cloud: 'us-1' }, + controller.signal + ) + ).resolves.toBe('token-1') + await callCrowdStrike( + 'https://api.crowdstrike.com', + 'token-1', + { method: 'GET', path: '/alerts/queries/alerts/v2' }, + controller.signal + ) + + expect(fetchMock.mock.calls[0]?.[1]).toMatchObject({ signal: controller.signal }) + expect(fetchMock.mock.calls[1]?.[1]).toMatchObject({ signal: controller.signal }) + }) + + it('maps Falcon authentication errors with the provider status', async () => { + fetchMock.mockResolvedValueOnce( + jsonResponse({ errors: [{ code: 401, message: 'invalid credentials' }] }, 401) + ) + + await expect( + getAccessToken({ clientId: 'client-id', clientSecret: 'bad-secret', cloud: 'us-1' }) + ).rejects.toEqual(new CrowdStrikeAuthError('invalid credentials', 401)) + }) + + it('caps provider response bodies and cancels an oversized stream', async () => { + let cancelled = false + const stream = new ReadableStream({ + cancel: () => { + cancelled = true + }, + }) + fetchMock.mockResolvedValueOnce( + new Response(stream, { + status: 200, + headers: { 'Content-Length': String(10 * 1024 * 1024 + 1) }, + }) + ) + + await expect( + getAccessToken({ clientId: 'client-id', clientSecret: 'client-secret', cloud: 'us-1' }) + ).rejects.toMatchObject({ name: 'PayloadSizeLimitError' }) + expect(cancelled).toBe(true) + }) + + it('does not start network work after cancellation', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + getAccessToken( + { clientId: 'client-id', clientSecret: 'client-secret', cloud: 'us-1' }, + controller.signal + ) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(fetchMock).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/crowdstrike/client.ts b/apps/sim/lib/internal/crowdstrike/client.ts new file mode 100644 index 00000000000..df44323b648 --- /dev/null +++ b/apps/sim/lib/internal/crowdstrike/client.ts @@ -0,0 +1,286 @@ +import { isRecordLike } from '@sim/utils/object' +import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' +import type { CrowdStrikeBaseParams, CrowdStrikeCloud } from '@/tools/crowdstrike/types' + +export type JsonRecord = Record + +const CLOUD_BASE_URLS: Record = { + 'eu-1': 'https://api.eu-1.crowdstrike.com', + 'us-1': 'https://api.crowdstrike.com', + 'us-2': 'https://api.us-2.crowdstrike.com', + 'us-3': 'https://api.us-3.crowdstrike.com', + 'us-gov-1': 'https://api.laggar.gcw.crowdstrike.com', + 'us-gov-2': 'https://api.us-gov-2.crowdstrike.mil', +} + +export function getCloudBaseUrl(cloud: CrowdStrikeCloud): string { + return CLOUD_BASE_URLS[cloud] +} + +export function getString(value: unknown): string | null { + return typeof value === 'string' ? value : null +} + +export function getNumber(value: unknown): number | null { + return typeof value === 'number' ? value : null +} + +export function getBoolean(value: unknown): boolean | null { + return typeof value === 'boolean' ? value : null +} + +export function getStringArray(value: unknown): string[] { + if (!Array.isArray(value)) { + return [] + } + + return value.filter((entry): entry is string => typeof entry === 'string') +} + +export function getRecordArray(value: unknown): JsonRecord[] { + if (!Array.isArray(value)) { + return [] + } + + return value.filter(isRecordLike) +} + +export function getRecord(value: unknown): JsonRecord | null { + return isRecordLike(value) ? value : null +} + +/** + * Every Falcon endpoint this integration calls answers with a flat + * `{ meta, resources, errors }` envelope, so the envelope readers below and + * `getFalconErrorMessage` both read the payload root directly. + */ +export function getResourcesArray(data: unknown): unknown[] { + if (!isRecordLike(data) || !Array.isArray(data.resources)) { + return [] + } + + return data.resources +} + +export function getRecordResources(data: unknown): JsonRecord[] { + return getResourcesArray(data).filter(isRecordLike) +} + +export function getStringResources(data: unknown): string[] { + return getStringArray(getResourcesArray(data)) +} + +export function getFirstRecordResource(data: unknown): JsonRecord | null { + return getRecordResources(data)[0] ?? null +} + +export function getPagination(data: unknown) { + if (!isRecordLike(data) || !isRecordLike(data.meta) || !isRecordLike(data.meta.pagination)) { + return null + } + + const { pagination } = data.meta + + return { + limit: getNumber(pagination.limit), + offset: getNumber(pagination.offset), + total: getNumber(pagination.total), + } +} + +/** Offset pagination plus the `after` cursor the IOC Management API returns. */ +export function getCursorPagination(data: unknown) { + if (!isRecordLike(data) || !isRecordLike(data.meta) || !isRecordLike(data.meta.pagination)) { + return null + } + + const { pagination } = data.meta + + return { + after: getString(pagination.after), + limit: getNumber(pagination.limit), + offset: getNumber(pagination.offset), + total: getNumber(pagination.total), + } +} + +/** Spotlight paginates by cursor only — it returns no offset. */ +export function getSpotlightPagination(data: unknown) { + if (!isRecordLike(data) || !isRecordLike(data.meta) || !isRecordLike(data.meta.pagination)) { + return null + } + + const { pagination } = data.meta + + return { + after: getString(pagination.after), + limit: getNumber(pagination.limit), + total: getNumber(pagination.total), + } +} + +/** + * CrowdStrike returns `{ meta, resources, errors }` on every endpoint, and a 200 + * can still carry a populated `errors` array for the IDs that failed. + */ +export function getEnvelopeErrors(data: unknown) { + if (!isRecordLike(data)) { + return [] + } + + return getRecordArray(data.errors).map((entry) => ({ + code: getNumber(entry.code), + id: getString(entry.id), + message: getString(entry.message), + })) +} + +export function getFalconErrorMessage(data: unknown, fallback: string): string { + if (!isRecordLike(data)) { + return fallback + } + + const errors = Array.isArray(data.errors) ? data.errors : [] + const firstError = errors[0] + if (isRecordLike(firstError)) { + const firstMessage = getString(firstError.message) ?? getString(firstError.code) + if (firstMessage) { + return firstMessage + } + } + + return ( + getString(data.message) ?? + getString(data.error_description) ?? + getString(data.error) ?? + fallback + ) +} + +/** + * Raised when the Falcon OAuth2 token exchange fails. Carries the Falcon status + * so the route can answer with the real cause (401 for bad credentials) instead + * of letting a credential problem fall through to a generic 500. + */ +export class CrowdStrikeAuthError extends Error { + readonly status: number + + constructor(message: string, status: number) { + super(message) + this.name = 'CrowdStrikeAuthError' + this.status = status >= 400 && status <= 599 ? status : 502 + } +} + +async function readFalconJson(response: Response): Promise { + const text = await readResponseTextWithLimit(response, { + maxBytes: MAX_JSON_API_RESPONSE_BYTES, + label: 'CrowdStrike response body', + }) + + try { + return JSON.parse(text) + } catch { + return null + } +} + +export async function getAccessToken( + params: CrowdStrikeBaseParams, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const baseUrl = getCloudBaseUrl(params.cloud) + const response = await fetch(`${baseUrl}/oauth2/token`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: new URLSearchParams({ + client_id: params.clientId, + client_secret: params.clientSecret, + grant_type: 'client_credentials', + }).toString(), + cache: 'no-store', + signal, + }) + + const data = await readFalconJson(response) + signal?.throwIfAborted() + if (!response.ok) { + throw new CrowdStrikeAuthError( + getFalconErrorMessage(data, 'Failed to authenticate with CrowdStrike'), + response.status + ) + } + + if (!isRecordLike(data) || typeof data.access_token !== 'string') { + throw new CrowdStrikeAuthError('CrowdStrike authentication did not return an access token', 502) + } + + return data.access_token +} + +interface CrowdStrikeRequestOptions { + method: 'GET' | 'POST' | 'PATCH' | 'DELETE' + path: string + query?: Record + repeatedQuery?: Record + body?: unknown +} + +export interface CrowdStrikeCallResult { + ok: boolean + status: number + data: unknown +} + +export function buildUrl(baseUrl: string, options: CrowdStrikeRequestOptions): string { + const url = new URL(options.path, baseUrl) + + for (const [key, value] of Object.entries(options.query ?? {})) { + if (value !== undefined) { + url.searchParams.set(key, String(value)) + } + } + + for (const [key, values] of Object.entries(options.repeatedQuery ?? {})) { + for (const value of values ?? []) { + url.searchParams.append(key, value) + } + } + + return url.toString() +} + +export async function callCrowdStrike( + baseUrl: string, + accessToken: string, + options: CrowdStrikeRequestOptions, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const headers: Record = { + Accept: 'application/json', + Authorization: `Bearer ${accessToken}`, + } + + if (options.body !== undefined) { + headers['Content-Type'] = 'application/json' + } + + const response = await fetch(buildUrl(baseUrl, options), { + method: options.method, + headers, + body: options.body === undefined ? undefined : JSON.stringify(options.body), + cache: 'no-store', + signal, + }) + + const data = await readFalconJson(response) + signal?.throwIfAborted() + + return { ok: response.ok, status: response.status, data } +} diff --git a/apps/sim/lib/internal/crowdstrike/execute-tool.test.ts b/apps/sim/lib/internal/crowdstrike/execute-tool.test.ts new file mode 100644 index 00000000000..2ccde6a38d8 --- /dev/null +++ b/apps/sim/lib/internal/crowdstrike/execute-tool.test.ts @@ -0,0 +1,161 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operationMocks = vi.hoisted(() => ({ + executeCrowdStrikeRequest: vi.fn(), +})) + +vi.mock('@/lib/internal/crowdstrike/operations', () => operationMocks) + +import { CrowdStrikeAuthError } from '@/lib/internal/crowdstrike/client' +import { executeCrowdStrikeTool } from '@/lib/internal/crowdstrike/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const CROWDSTRIKE_TOOL_IDS = [ + 'crowdstrike_create_indicators', + 'crowdstrike_delete_indicators', + 'crowdstrike_delete_rtr_session', + 'crowdstrike_execute_rtr_command', + 'crowdstrike_get_alert_details', + 'crowdstrike_get_case_details', + 'crowdstrike_get_host_group_details', + 'crowdstrike_get_indicator_details', + 'crowdstrike_get_rtr_command_status', + 'crowdstrike_get_sensor_aggregates', + 'crowdstrike_get_sensor_details', + 'crowdstrike_get_vulnerability_details', + 'crowdstrike_init_rtr_session', + 'crowdstrike_perform_host_action', + 'crowdstrike_perform_host_group_action', + 'crowdstrike_query_alerts', + 'crowdstrike_query_cases', + 'crowdstrike_query_host_groups', + 'crowdstrike_query_indicators', + 'crowdstrike_query_sensors', + 'crowdstrike_query_vulnerabilities', + 'crowdstrike_update_alerts', + 'crowdstrike_update_indicators', +] as const + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'crowdstrike_query_sensors', + input: { + operation: 'crowdstrike_query_sensors', + clientId: 'client-id', + clientSecret: 'client-secret', + cloud: 'us-1', + limit: 25, + }, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeCrowdStrikeTool', () => { + beforeEach(() => { + vi.clearAllMocks() + operationMocks.executeCrowdStrikeRequest.mockResolvedValue({ + ok: true, + output: { sensors: [], count: 0, errors: [], pagination: null }, + }) + }) + + it('validates the canonical contract and dispatches with cancellation', async () => { + const controller = new AbortController() + const response = await executeCrowdStrikeTool(createRequest({ signal: controller.signal })) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: true, + output: { sensors: [], count: 0, errors: [], pagination: null }, + }) + expect(operationMocks.executeCrowdStrikeRequest).toHaveBeenCalledWith( + { + operation: 'crowdstrike_query_sensors', + clientId: 'client-id', + clientSecret: 'client-secret', + cloud: 'us-1', + limit: 25, + }, + controller.signal + ) + }) + + it.each(CROWDSTRIKE_TOOL_IDS)('recognizes canonical tool ID %s', async (toolId) => { + const response = await executeCrowdStrikeTool(createRequest({ toolId })) + + expect(response.status).toBe(200) + }) + + it('preserves canonical validation details', async () => { + const response = await executeCrowdStrikeTool( + createRequest({ input: { operation: 'crowdstrike_query_sensors' } }) + ) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + success: false, + error: 'Invalid input: expected string, received undefined', + details: expect.any(Array), + }) + expect(operationMocks.executeCrowdStrikeRequest).not.toHaveBeenCalled() + }) + + it('preserves Falcon operation and authentication statuses', async () => { + operationMocks.executeCrowdStrikeRequest.mockResolvedValueOnce({ + ok: false, + status: 429, + error: 'rate limited', + }) + const providerResponse = await executeCrowdStrikeTool(createRequest()) + expect(providerResponse.status).toBe(429) + await expect(providerResponse.json()).resolves.toEqual({ + success: false, + error: 'rate limited', + }) + + operationMocks.executeCrowdStrikeRequest.mockRejectedValueOnce( + new CrowdStrikeAuthError('invalid credentials', 401) + ) + const authResponse = await executeCrowdStrikeTool(createRequest()) + expect(authResponse.status).toBe(401) + await expect(authResponse.json()).resolves.toEqual({ + success: false, + error: 'invalid credentials', + }) + }) + + it('preserves the route-compatible generic provider failure envelope', async () => { + operationMocks.executeCrowdStrikeRequest.mockRejectedValueOnce(new Error('network unavailable')) + + const response = await executeCrowdStrikeTool(createRequest()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'network unavailable', + }) + }) + + it('propagates cancellation instead of converting it into a provider error', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeCrowdStrikeTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(operationMocks.executeCrowdStrikeRequest).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/crowdstrike/execute-tool.ts b/apps/sim/lib/internal/crowdstrike/execute-tool.ts new file mode 100644 index 00000000000..b5da73fa63d --- /dev/null +++ b/apps/sim/lib/internal/crowdstrike/execute-tool.ts @@ -0,0 +1,90 @@ +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { crowdstrikeQueryBodySchema } from '@/lib/api/contracts/tools/crowdstrike' +import { CrowdStrikeAuthError } from '@/lib/internal/crowdstrike/client' +import { executeCrowdStrikeRequest } from '@/lib/internal/crowdstrike/operations' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const logger = createLogger('CrowdStrikeToolExecution') + +const CROWDSTRIKE_TOOL_IDS = new Set([ + 'crowdstrike_create_indicators', + 'crowdstrike_delete_indicators', + 'crowdstrike_delete_rtr_session', + 'crowdstrike_execute_rtr_command', + 'crowdstrike_get_alert_details', + 'crowdstrike_get_case_details', + 'crowdstrike_get_host_group_details', + 'crowdstrike_get_indicator_details', + 'crowdstrike_get_rtr_command_status', + 'crowdstrike_get_sensor_aggregates', + 'crowdstrike_get_sensor_details', + 'crowdstrike_get_vulnerability_details', + 'crowdstrike_init_rtr_session', + 'crowdstrike_perform_host_action', + 'crowdstrike_perform_host_group_action', + 'crowdstrike_query_alerts', + 'crowdstrike_query_cases', + 'crowdstrike_query_host_groups', + 'crowdstrike_query_indicators', + 'crowdstrike_query_sensors', + 'crowdstrike_query_vulnerabilities', + 'crowdstrike_update_alerts', + 'crowdstrike_update_indicators', +]) + +function parseCrowdStrikeInput(input: unknown) { + const parsed = crowdstrikeQueryBodySchema.safeParse(input) + if (!parsed.success) { + return { + success: false as const, + response: Response.json( + { + success: false, + error: parsed.error.issues[0]?.message || 'Invalid request data', + details: parsed.error.issues, + }, + { status: 400 } + ), + } + } + + return { success: true as const, data: parsed.data } +} + +export const executeCrowdStrikeTool: InternalToolOperationHandler = async ({ + toolId, + input, + signal, +}) => { + signal?.throwIfAborted() + if (!CROWDSTRIKE_TOOL_IDS.has(toolId)) { + return Response.json({ error: `Unsupported CrowdStrike tool: ${toolId}` }, { status: 500 }) + } + + const parsed = parseCrowdStrikeInput(input) + if (!parsed.success) return parsed.response + + try { + const result = await executeCrowdStrikeRequest(parsed.data, signal) + signal?.throwIfAborted() + if (!result.ok) { + return Response.json( + { success: false, error: result.error }, + { status: result.status || 502 } + ) + } + + return Response.json({ success: true, output: result.output }) + } catch (error) { + signal?.throwIfAborted() + const message = toError(error).message + if (error instanceof CrowdStrikeAuthError) { + logger.warn('CrowdStrike authentication failed', { error: message, status: error.status }) + return Response.json({ success: false, error: message }, { status: error.status }) + } + + logger.error('CrowdStrike request failed', { error: message }) + return Response.json({ success: false, error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/crowdstrike/normalizers.ts b/apps/sim/lib/internal/crowdstrike/normalizers.ts new file mode 100644 index 00000000000..ac6c6d03dd4 --- /dev/null +++ b/apps/sim/lib/internal/crowdstrike/normalizers.ts @@ -0,0 +1,266 @@ +import { + getBoolean, + getNumber, + getRecord, + getRecordArray, + getString, + getStringArray, + type JsonRecord, +} from '@/lib/internal/crowdstrike/client' +import type { + CrowdStrikeAffectedEntity, + CrowdStrikeAlert, + CrowdStrikeCase, + CrowdStrikeFalconUser, + CrowdStrikeHostGroup, + CrowdStrikeIndicator, + CrowdStrikeVulnerability, +} from '@/tools/crowdstrike/types' + +export function normalizeAlert(resource: JsonRecord): CrowdStrikeAlert { + const device = getRecord(resource.device) + + return { + compositeId: getString(resource.composite_id), + id: getString(resource.id), + cid: getString(resource.cid), + aggregateId: getString(resource.aggregate_id), + agentId: getString(resource.agent_id), + deviceId: device ? getString(device.device_id) : null, + hostname: device ? getString(device.hostname) : null, + name: getString(resource.name), + displayName: getString(resource.display_name), + description: getString(resource.description), + type: getString(resource.type), + product: getString(resource.product), + platform: getString(resource.platform), + severity: getNumber(resource.severity), + severityName: getString(resource.severity_name), + confidence: getNumber(resource.confidence), + status: getString(resource.status), + assignedToName: getString(resource.assigned_to_name), + assignedToUid: getString(resource.assigned_to_uid), + assignedToUuid: getString(resource.assigned_to_uuid), + tactic: getString(resource.tactic), + tacticId: getString(resource.tactic_id), + technique: getString(resource.technique), + techniqueId: getString(resource.technique_id), + scenario: getString(resource.scenario), + objective: getString(resource.objective), + resolution: getString(resource.resolution), + showInUi: getBoolean(resource.show_in_ui), + tags: getStringArray(resource.tags), + filename: getString(resource.filename), + filepath: getString(resource.filepath), + cmdline: getString(resource.cmdline), + sha256: getString(resource.sha256), + sha1: getString(resource.sha1), + md5: getString(resource.md5), + userName: getString(resource.user_name), + userId: getString(resource.user_id), + patternId: getNumber(resource.pattern_id), + falconHostLink: getString(resource.falcon_host_link), + controlGraphId: getString(resource.control_graph_id), + external: getBoolean(resource.external), + emailSent: getBoolean(resource.email_sent), + isAggregated: getBoolean(resource.is_aggregated), + isFalconPlatformIoa: getBoolean(resource.is_falcon_platform_ioa), + dataDomains: getStringArray(resource.data_domains), + iocValues: getStringArray(resource.ioc_values), + linkedCaseIds: getStringArray(resource.linked_case_ids), + linkedBehavioralDetections: getStringArray(resource.linked_behavioral_detections), + timestamp: getString(resource.timestamp), + createdTimestamp: getString(resource.created_timestamp), + updatedTimestamp: getString(resource.updated_timestamp), + crawledTimestamp: getString(resource.crawled_timestamp), + contextTimestamp: getString(resource.context_timestamp), + } +} + +export function normalizeAffectedEntity(resource: JsonRecord): CrowdStrikeAffectedEntity { + return { + id: getString(resource.id), + path: getString(resource.path), + } +} + +export function normalizeHostGroup(resource: JsonRecord): CrowdStrikeHostGroup { + return { + id: getString(resource.id), + name: getString(resource.name), + description: getString(resource.description), + groupType: getString(resource.group_type), + assignmentRule: getString(resource.assignment_rule), + createdBy: getString(resource.created_by), + createdTimestamp: getString(resource.created_timestamp), + modifiedBy: getString(resource.modified_by), + modifiedTimestamp: getString(resource.modified_timestamp), + } +} + +export function normalizeIndicator(resource: JsonRecord): CrowdStrikeIndicator { + const metadata = getRecord(resource.metadata) + + return { + id: getString(resource.id), + type: getString(resource.type), + value: getString(resource.value), + action: getString(resource.action), + mobileAction: getString(resource.mobile_action), + severity: getString(resource.severity), + description: getString(resource.description), + source: getString(resource.source), + appliedGlobally: getBoolean(resource.applied_globally), + platforms: getStringArray(resource.platforms), + hostGroups: getStringArray(resource.host_groups), + tags: getStringArray(resource.tags), + expiration: getString(resource.expiration), + expired: getBoolean(resource.expired), + deleted: getBoolean(resource.deleted), + fromParent: getBoolean(resource.from_parent), + parentCidName: getString(resource.parent_cid_name), + createdBy: getString(resource.created_by), + createdOn: getString(resource.created_on), + modifiedBy: getString(resource.modified_by), + modifiedOn: getString(resource.modified_on), + metadata: metadata + ? { + avHits: getNumber(metadata.av_hits), + companyName: getString(metadata.company_name), + fileDescription: getString(metadata.file_description), + fileVersion: getString(metadata.file_version), + filename: getString(metadata.filename), + originalFilename: getString(metadata.original_filename), + productName: getString(metadata.product_name), + productVersion: getString(metadata.product_version), + signed: getBoolean(metadata.signed), + } + : null, + } +} + +export function normalizeVulnerability(resource: JsonRecord): CrowdStrikeVulnerability { + const cve = getRecord(resource.cve) + const cisaInfo = cve ? getRecord(cve.cisa_info) : null + const app = getRecord(resource.app) + const hostInfo = getRecord(resource.host_info) + const remediation = getRecord(resource.remediation) + const suppressionInfo = getRecord(resource.suppression_info) + + return { + id: getString(resource.id), + aid: getString(resource.aid), + cid: getString(resource.cid), + status: getString(resource.status), + confidence: getString(resource.confidence), + vulnerabilityId: getString(resource.vulnerability_id), + createdTimestamp: getString(resource.created_timestamp), + updatedTimestamp: getString(resource.updated_timestamp), + closedTimestamp: getString(resource.closed_timestamp), + cve: cve + ? { + id: getString(cve.id), + baseScore: getNumber(cve.base_score), + severity: getString(cve.severity), + exprtRating: getString(cve.exprt_rating), + exploitStatus: getNumber(cve.exploit_status), + exploitabilityScore: getNumber(cve.exploitability_score), + impactScore: getNumber(cve.impact_score), + remediationLevel: getString(cve.remediation_level), + description: getString(cve.description), + publishedDate: getString(cve.published_date), + vector: getString(cve.vector), + types: getStringArray(cve.types), + isCisaKev: cisaInfo ? getBoolean(cisaInfo.is_cisa_kev) : null, + cisaDueDate: cisaInfo ? getString(cisaInfo.due_date) : null, + } + : null, + app: app + ? { + productNameNormalized: getString(app.product_name_normalized), + productNameVersion: getString(app.product_name_version), + vendorNormalized: getString(app.vendor_normalized), + } + : null, + hostInfo: hostInfo + ? { + hostname: getString(hostInfo.hostname), + localIp: getString(hostInfo.local_ip), + machineDomain: getString(hostInfo.machine_domain), + osVersion: getString(hostInfo.os_version), + platform: getString(hostInfo.platform), + productTypeDesc: getString(hostInfo.product_type_desc), + assetCriticality: getString(hostInfo.asset_criticality), + internetExposure: getString(hostInfo.internet_exposure), + tags: getStringArray(hostInfo.tags), + groups: getRecordArray(hostInfo.groups) + .map((group) => getString(group.name)) + .filter((name): name is string => name !== null), + } + : null, + remediationIds: remediation ? getStringArray(remediation.ids) : [], + remediations: remediation + ? getRecordArray(remediation.entities).map((entity) => ({ + id: getString(entity.id), + title: getString(entity.title), + action: getString(entity.action), + type: getString(entity.type), + link: getString(entity.link), + reference: getString(entity.reference), + vendorUrl: getString(entity.vendor_url), + })) + : [], + suppressionInfo: suppressionInfo + ? { + isSuppressed: getBoolean(suppressionInfo.is_suppressed), + reason: getString(suppressionInfo.reason), + } + : null, + } +} + +function normalizeFalconUser(value: unknown): CrowdStrikeFalconUser | null { + const user = getRecord(value) + if (!user) { + return null + } + + return { + uuid: getString(user.uuid), + email: getString(user.email), + fullName: getString(user.full_name), + } +} + +export function normalizeCase(resource: JsonRecord): CrowdStrikeCase { + const severityInfo = getRecord(resource.severity_info) + const template = getRecord(resource.template) + const sla = getRecord(resource.sla) + const readOnly = getRecord(resource.read_only) + + return { + id: getString(resource.id), + cid: getString(resource.cid), + name: getString(resource.name), + description: getString(resource.description), + descriptionFormat: getString(resource.description_format), + status: getString(resource.status), + severity: getNumber(resource.severity), + severityLevel: severityInfo ? getString(severityInfo.level) : null, + referenceId: getString(resource.reference_id), + version: getNumber(resource.version), + tags: getStringArray(resource.tags), + assignedTo: normalizeFalconUser(resource.assigned_to), + createdBy: normalizeFalconUser(resource.created_by), + lastUpdatedBy: normalizeFalconUser(resource.last_updated_by), + createdTimestamp: getString(resource.created_timestamp), + updatedTimestamp: getString(resource.updated_timestamp), + startTimestamp: getString(resource.start_timestamp), + endTimestamp: getString(resource.end_timestamp), + templateId: template ? getString(template.id) : null, + templateName: template ? getString(template.name) : null, + slaId: sla ? getString(sla.id) : null, + slaName: sla ? getString(sla.name) : null, + isReadOnly: readOnly ? getBoolean(readOnly.is_read_only) : null, + } +} diff --git a/apps/sim/lib/internal/crowdstrike/operations.test.ts b/apps/sim/lib/internal/crowdstrike/operations.test.ts new file mode 100644 index 00000000000..5a6672563a7 --- /dev/null +++ b/apps/sim/lib/internal/crowdstrike/operations.test.ts @@ -0,0 +1,121 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + chunkIdsByUrlBudget, + executeCrowdStrikeOperation, + executeCrowdStrikeRequest, +} from '@/lib/internal/crowdstrike/operations' + +const fetchMock = vi.fn() + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }) +} + +describe('CrowdStrike operations', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', fetchMock) + }) + + it('hydrates sensor queries with the same pagination envelope and signal', async () => { + const controller = new AbortController() + fetchMock + .mockResolvedValueOnce(jsonResponse({ access_token: 'token-1' })) + .mockResolvedValueOnce( + jsonResponse({ + meta: { pagination: { limit: 1, offset: 0, total: 1 } }, + resources: ['sensor-1'], + }) + ) + .mockResolvedValueOnce( + jsonResponse({ + resources: [ + { + device_id: 'sensor-1', + hostname: 'host-1', + status: 'protected', + status_causes: ['healthy'], + }, + ], + }) + ) + + const result = await executeCrowdStrikeRequest( + { + operation: 'crowdstrike_query_sensors', + clientId: 'client-id', + clientSecret: 'client-secret', + cloud: 'us-1', + limit: 1, + }, + controller.signal + ) + + expect(result).toMatchObject({ + ok: true, + output: { + count: 1, + pagination: { limit: 1, offset: 0, total: 1 }, + sensors: [{ deviceId: 'sensor-1', hostname: 'host-1', status: 'protected' }], + }, + }) + expect(fetchMock).toHaveBeenCalledTimes(3) + for (const call of fetchMock.mock.calls) { + expect(call[1]).toMatchObject({ signal: controller.signal }) + } + }) + + it('maps a resource-less 200 error envelope to its Falcon error status', async () => { + fetchMock + .mockResolvedValueOnce(jsonResponse({ access_token: 'token-1' })) + .mockResolvedValueOnce( + jsonResponse({ resources: [], errors: [{ code: 503, message: 'Falcon unavailable' }] }) + ) + + await expect( + executeCrowdStrikeRequest({ + operation: 'crowdstrike_query_sensors', + clientId: 'client-id', + clientSecret: 'client-secret', + cloud: 'us-1', + }) + ).resolves.toEqual({ ok: false, status: 503, error: 'Falcon unavailable' }) + }) + + it('keeps by-ID requests within the URL budget and executes batches sequentially', async () => { + const controller = new AbortController() + const indicatorIds = [`sha256:${'a'.repeat(4050)}`, `sha256:${'b'.repeat(4050)}`] + fetchMock + .mockResolvedValueOnce(jsonResponse({ resources: [{ id: indicatorIds[0] }] })) + .mockResolvedValueOnce(jsonResponse({ resources: [{ id: indicatorIds[1] }] })) + + const result = await executeCrowdStrikeOperation( + { + operation: 'crowdstrike_get_indicator_details', + clientId: 'client-id', + clientSecret: 'client-secret', + cloud: 'us-1', + indicatorIds, + }, + 'https://api.crowdstrike.com', + 'token-1', + controller.signal + ) + + expect(result).toMatchObject({ ok: true, output: { count: 2 } }) + expect(fetchMock).toHaveBeenCalledTimes(2) + expect(fetchMock.mock.invocationCallOrder[0]).toBeLessThan( + fetchMock.mock.invocationCallOrder[1] + ) + expect(chunkIdsByUrlBudget(indicatorIds, 4096)).toHaveLength(2) + for (const call of fetchMock.mock.calls) { + expect(call[1]).toMatchObject({ signal: controller.signal }) + } + }) +}) diff --git a/apps/sim/lib/internal/crowdstrike/operations.ts b/apps/sim/lib/internal/crowdstrike/operations.ts new file mode 100644 index 00000000000..d29450b425f --- /dev/null +++ b/apps/sim/lib/internal/crowdstrike/operations.ts @@ -0,0 +1,1028 @@ +import { createLogger } from '@sim/logger' +import { isRecordLike } from '@sim/utils/object' +import { truncate } from '@sim/utils/string' +import type { CrowdstrikeQueryBody } from '@/lib/api/contracts/tools/crowdstrike' +import { + buildUrl, + type CrowdStrikeCallResult, + callCrowdStrike, + getAccessToken, + getBoolean, + getCloudBaseUrl, + getCursorPagination, + getEnvelopeErrors, + getFalconErrorMessage, + getFirstRecordResource, + getNumber, + getPagination, + getRecordArray, + getRecordResources, + getResourcesArray, + getSpotlightPagination, + getString, + getStringArray, + getStringResources, +} from '@/lib/internal/crowdstrike/client' +import { + normalizeAffectedEntity, + normalizeAlert, + normalizeCase, + normalizeHostGroup, + normalizeIndicator, + normalizeVulnerability, +} from '@/lib/internal/crowdstrike/normalizers' +import type { + CrowdStrikeActionParameter, + CrowdStrikeQuerySensorsParams, + CrowdStrikeSensorAggregateBucket, + CrowdStrikeSensorAggregateResult, +} from '@/tools/crowdstrike/types' + +const logger = createLogger('CrowdStrikeOperations') + +type ExtendedOperation = Exclude< + CrowdstrikeQueryBody['operation'], + | 'crowdstrike_query_sensors' + | 'crowdstrike_get_sensor_details' + | 'crowdstrike_get_sensor_aggregates' +> + +type ExtendedBody = Extract + +type SensorBody = Exclude + +export interface OperationFailure { + ok: false + status: number + error: string +} + +export interface OperationSuccess { + ok: true + output: Record +} + +export type OperationResult = OperationSuccess | OperationFailure + +/** + * CrowdStrike can answer 200 while the envelope carries only errors. Reporting + * that as HTTP 200 would read as a success, so fall back to the per-item error + * code the envelope supplies, and to 502 when it supplies none. + */ +export function failureStatus(result: CrowdStrikeCallResult): number { + if (!result.ok) { + return result.status + } + + const envelopeCode = getEnvelopeErrors(result.data)[0]?.code + if (envelopeCode != null && envelopeCode >= 400 && envelopeCode <= 599) { + return envelopeCode + } + + return 502 +} + +function fail(result: CrowdStrikeCallResult, fallback: string): OperationFailure { + return { + ok: false, + status: failureStatus(result), + error: getFalconErrorMessage(result.data, fallback), + } +} + +/** + * CrowdStrike answers 200 with a populated `errors` array when only some IDs + * fail. Treat that as an outright failure only when nothing came back at all. + */ +export function failedWithoutResources( + result: CrowdStrikeCallResult, + resourceCount: number +): boolean { + return resourceCount === 0 && getEnvelopeErrors(result.data).length > 0 +} + +function normalizeSensor(resource: Record) { + return { + agentVersion: getString(resource.agent_version), + cid: getString(resource.cid), + deviceId: getString(resource.device_id), + heartbeatTime: getNumber(resource.heartbeat_time), + hostname: getString(resource.hostname), + idpPolicyId: getString(resource.idp_policy_id), + idpPolicyName: getString(resource.idp_policy_name), + ipAddress: getString(resource.local_ip), + kerberosConfig: getString(resource.kerberos_config), + ldapConfig: getString(resource.ldap_config), + ldapsConfig: getString(resource.ldaps_config), + machineDomain: getString(resource.machine_domain), + ntlmConfig: getString(resource.ntlm_config), + osVersion: getString(resource.os_version), + rdpToDcConfig: getString(resource.rdp_to_dc_config), + smbToDcConfig: getString(resource.smb_to_dc_config), + status: getString(resource.status), + statusCauses: getStringArray(resource.status_causes), + tiEnabled: getString(resource.ti_enabled), + } +} + +function normalizeSensorsOutput(data: unknown, paginationData?: unknown) { + const sensors = getRecordResources(data).map(normalizeSensor) + + return { + count: sensors.length, + errors: getEnvelopeErrors(data), + pagination: paginationData == null ? null : getPagination(paginationData), + sensors, + } +} + +function normalizeAggregationResult( + resource: Record +): CrowdStrikeSensorAggregateResult { + return { + buckets: getRecordArray(resource.buckets).map(normalizeAggregationBucket), + docCountErrorUpperBound: getNumber(resource.doc_count_error_upper_bound), + name: getString(resource.name), + sumOtherDocCount: getNumber(resource.sum_other_doc_count), + } +} + +function normalizeAggregationBucket( + resource: Record +): CrowdStrikeSensorAggregateBucket { + return { + count: getNumber(resource.count), + from: getNumber(resource.from), + keyAsString: getString(resource.key_as_string), + label: resource.label ?? null, + stringFrom: getString(resource.string_from), + stringTo: getString(resource.string_to), + subAggregates: getRecordArray(resource.sub_aggregates).map(normalizeAggregationResult), + to: getNumber(resource.to), + value: getNumber(resource.value), + valueAsString: getString(resource.value_as_string), + } +} + +function normalizeAggregatesOutput(data: unknown) { + const aggregates = getRecordResources(data).map(normalizeAggregationResult) + + return { + aggregates, + count: aggregates.length, + errors: getEnvelopeErrors(data), + } +} + +function sensorQuery(params: CrowdStrikeQuerySensorsParams) { + return { + filter: params.filter, + limit: params.limit, + offset: params.offset, + sort: params.sort, + } +} + +async function executeSensorOperation( + body: SensorBody, + baseUrl: string, + accessToken: string, + signal?: AbortSignal +): Promise { + const request = (options: Parameters[2]) => + callCrowdStrike(baseUrl, accessToken, options, signal) + + if (body.operation === 'crowdstrike_query_sensors') { + const queryResult = await request({ + method: 'GET', + path: '/identity-protection/queries/devices/v1', + query: sensorQuery(body), + }) + if (!queryResult.ok) return fail(queryResult, 'CrowdStrike request failed') + + const ids = getStringResources(queryResult.data) + if (failedWithoutResources(queryResult, ids.length)) { + return fail(queryResult, 'Failed to query CrowdStrike sensors') + } + + if (ids.length === 0) { + return { + ok: true, + output: normalizeSensorsOutput({ resources: [] }, queryResult.data), + } + } + + const detailResult = await request({ + method: 'POST', + path: '/identity-protection/entities/devices/GET/v1', + body: { ids }, + }) + if (!detailResult.ok) { + return fail(detailResult, 'Failed to fetch CrowdStrike sensor details') + } + if (failedWithoutResources(detailResult, getRecordResources(detailResult.data).length)) { + return fail(detailResult, 'Failed to fetch CrowdStrike sensor details') + } + + return { + ok: true, + output: normalizeSensorsOutput(detailResult.data, queryResult.data), + } + } + + if (body.operation === 'crowdstrike_get_sensor_details') { + const detailResult = await request({ + method: 'POST', + path: '/identity-protection/entities/devices/GET/v1', + body: { ids: body.ids }, + }) + if (!detailResult.ok) { + return fail(detailResult, 'Failed to fetch CrowdStrike sensor details') + } + if (failedWithoutResources(detailResult, getRecordResources(detailResult.data).length)) { + return fail(detailResult, 'Failed to fetch CrowdStrike sensor details') + } + + return { ok: true, output: normalizeSensorsOutput(detailResult.data) } + } + + const aggregateResult = await request({ + method: 'POST', + path: '/identity-protection/aggregates/devices/GET/v1', + body: body.aggregateQuery, + }) + if (!aggregateResult.ok) { + return fail(aggregateResult, 'Failed to fetch CrowdStrike sensor aggregates') + } + if (failedWithoutResources(aggregateResult, getRecordResources(aggregateResult.data).length)) { + return fail(aggregateResult, 'Failed to fetch CrowdStrike sensor aggregates') + } + + return { ok: true, output: normalizeAggregatesOutput(aggregateResult.data) } +} + +/** + * Falcon's by-ids lookups carry every ID in the query string, so a request at the + * contract maxima (1000 indicator IDs at ~68 bytes each) would generate a ~68 KB + * URL. Proxies and load balancers commonly cap the request line plus headers at + * 8 KB, so batches are sized to keep each generated URL at or under half of that, + * leaving the rest of the budget for headers. + */ +export const MAX_ID_URL_BYTES = 4096 + +/** + * Batches by cumulative encoded length rather than by a fixed count: Falcon IDs + * range from 32-character AIDs to long composite alert IDs, so a count-based cap + * would either waste the budget or blow past it. A single ID longer than the + * budget still gets its own batch — truncating the list would silently drop it. + */ +export function chunkIdsByUrlBudget(ids: string[], budget: number): string[][] { + const chunks: string[][] = [] + let current: string[] = [] + let used = 0 + + for (const id of ids) { + const cost = `&ids=${encodeURIComponent(id)}`.length + if (current.length > 0 && used + cost > budget) { + chunks.push(current) + current = [] + used = 0 + } + current.push(id) + used += cost + } + + if (current.length > 0) { + chunks.push(current) + } + + return chunks +} + +/** + * Caps how much of the already-committed ID list is spelled out in a partial-failure + * message. A by-ids delete can carry 1000 IDs at ~68 bytes each, so the full list + * would bury the actual failure under ~68 KB of text. + */ +const MAX_COMMITTED_IDS_IN_MESSAGE = 400 + +/** + * Rewrites a failed batch's envelope so the reported error names the deletions the + * earlier batches already committed. + * + * Batches run sequentially and Falcon has no way to roll back a deletion it already + * performed. Short-circuiting on a later batch would therefore report a bare failure + * over work that already happened, and a blind retry would target IDs that no longer + * exist. Only the message survives to the caller ({@link fail} keeps `status` and the + * message, not `data`), so the committed list is written onto `errors[0].message`, + * which is the first thing {@link getFalconErrorMessage} reads. + * + * `committed` holds the IDs Falcon echoed in `resources`, never the IDs that were + * requested, so an ID that failed inside an otherwise-200 batch is not reported as + * deleted. + */ +function withCommittedIds( + result: CrowdStrikeCallResult, + committed: string[] +): CrowdStrikeCallResult { + if (committed.length === 0) return result + + const envelope = isRecordLike(result.data) ? result.data : {} + const existing = getRecordArray(envelope.errors) + const reason = getFalconErrorMessage(result.data, 'CrowdStrike rejected a later batch.') + const message = + `${reason} This request was split into batches and ${committed.length} ID(s) were already deleted ` + + `before the failing batch; they were not rolled back, so retry only the remainder. ` + + `Deleted: ${truncate(committed.join(', '), MAX_COMMITTED_IDS_IN_MESSAGE)}` + + return { + ...result, + data: { ...envelope, errors: [{ ...(existing[0] ?? {}), message }, ...existing.slice(1)] }, + } +} + +interface ByIdsRequestOptions { + method: 'GET' | 'DELETE' + path: string + ids: string[] | undefined + query?: Record +} + +/** + * Issues a by-ids lookup as however many requests it takes to stay under + * `MAX_ID_URL_BYTES`, then presents the batches as one `{ meta, resources, errors }` + * envelope so callers read the same shape a single request returns. + * + * Batches run sequentially: resource order matches the caller's ID order, the + * endpoint's rate limit only ever sees one request at a time, and a failing batch + * short-circuits with its own status instead of being merged away. A `DELETE` that + * fails partway also carries the IDs its earlier batches already removed — see + * {@link withCommittedIds}. `meta` comes + * from the first batch — pagination is meaningless for a lookup that names every + * ID it wants, and no by-ids operation here reads it. + */ +async function callCrowdStrikeByIds( + baseUrl: string, + accessToken: string, + options: ByIdsRequestOptions, + signal?: AbortSignal +): Promise { + const prefix = buildUrl(baseUrl, { + method: options.method, + path: options.path, + query: options.query, + }) + const chunks = chunkIdsByUrlBudget( + options.ids ?? [], + Math.max(MAX_ID_URL_BYTES - prefix.length, 1) + ) + + if (chunks.length <= 1) { + return callCrowdStrike( + baseUrl, + accessToken, + { + method: options.method, + path: options.path, + query: options.query, + repeatedQuery: { ids: options.ids }, + }, + signal + ) + } + + const resources: unknown[] = [] + const errors: unknown[] = [] + const committed: string[] = [] + let meta: unknown + let status = 200 + + for (const [index, chunk] of chunks.entries()) { + const result = await callCrowdStrike( + baseUrl, + accessToken, + { + method: options.method, + path: options.path, + query: options.query, + repeatedQuery: { ids: chunk }, + }, + signal + ) + + if (!result.ok) { + return options.method === 'DELETE' ? withCommittedIds(result, committed) : result + } + + /** + * Only the IDs Falcon echoed in `resources` were actually deleted. A batch can + * answer 200 while reporting per-ID failures in `errors`, so recording the + * requested chunk would name indicators that are still live and tell the + * caller to drop them from the retry. + */ + if (options.method === 'DELETE') { + committed.push(...getStringResources(result.data)) + } + + if (index === 0) { + status = result.status + meta = isRecordLike(result.data) ? result.data.meta : undefined + } + + resources.push(...getResourcesArray(result.data)) + errors.push(...getRecordArray(isRecordLike(result.data) ? result.data.errors : undefined)) + } + + return { ok: true, status, data: { meta, resources, errors } } +} + +function buildAlertActionParameters( + body: Extract +) { + const parameters: CrowdStrikeActionParameter[] = [] + + const push = (name: string, value: string | undefined) => { + if (value !== undefined) { + parameters.push({ name, value }) + } + } + + push('update_status', body.updateStatus) + push('assign_to_uuid', body.assignToUuid) + push('assign_to_user_id', body.assignToUserId) + push('assign_to_name', body.assignToName) + push('append_comment', body.appendComment) + push('add_tag', body.addTag) + push('remove_tag', body.removeTag) + push('remove_tags_by_prefix', body.removeTagsByPrefix) + + if (body.unassign === true) { + parameters.push({ name: 'unassign', value: '' }) + } + + if (body.showInUi !== undefined) { + parameters.push({ name: 'show_in_ui', value: String(body.showInUi) }) + } + + for (const parameter of body.actionParameters ?? []) { + parameters.push({ name: parameter.name, value: parameter.value }) + } + + return parameters +} + +/** + * CrowdStrike's host-group action endpoint selects the hosts to add or remove + * with an FQL `device_id` filter rather than an ID list. + */ +function buildDeviceIdFilter(deviceIds: string[]): string { + const values = deviceIds.map((id) => `'${id.replaceAll("'", "\\'")}'`).join(',') + return `(device_id:[${values}])` +} + +export async function executeCrowdStrikeOperation( + body: ExtendedBody, + baseUrl: string, + accessToken: string, + signal?: AbortSignal +): Promise { + const request = (options: Parameters[2]) => + callCrowdStrike(baseUrl, accessToken, options, signal) + + switch (body.operation) { + case 'crowdstrike_query_alerts': { + const result = await request({ + method: 'GET', + path: '/alerts/queries/alerts/v2', + query: { + filter: body.filter, + include_hidden: body.includeHidden, + limit: body.limit, + offset: body.offset, + q: body.q, + sort: body.sort, + }, + }) + if (!result.ok) return fail(result, 'Failed to query CrowdStrike alerts') + + const alertIds = getStringResources(result.data) + if (failedWithoutResources(result, alertIds.length)) { + return fail(result, 'Failed to query CrowdStrike alerts') + } + + return { + ok: true, + output: { alertIds, count: alertIds.length, pagination: getPagination(result.data) }, + } + } + + case 'crowdstrike_get_alert_details': { + const result = await request({ + method: 'POST', + path: '/alerts/entities/alerts/v2', + query: { include_hidden: body.includeHidden }, + body: { composite_ids: body.compositeIds }, + }) + if (!result.ok) return fail(result, 'Failed to fetch CrowdStrike alert details') + + const alerts = getRecordResources(result.data).map(normalizeAlert) + if (failedWithoutResources(result, alerts.length)) { + return fail(result, 'Failed to fetch CrowdStrike alert details') + } + + return { + ok: true, + output: { alerts, count: alerts.length, errors: getEnvelopeErrors(result.data) }, + } + } + + case 'crowdstrike_update_alerts': { + const result = await request({ + method: 'PATCH', + path: '/alerts/entities/alerts/v3', + query: { include_hidden: body.includeHidden }, + body: { + action_parameters: buildAlertActionParameters(body), + composite_ids: body.compositeIds, + }, + }) + if (!result.ok) return fail(result, 'Failed to update CrowdStrike alerts') + + const errors = getEnvelopeErrors(result.data) + if (errors.length > 0) { + return fail(result, 'Failed to update CrowdStrike alerts') + } + + return { + ok: true, + output: { updatedIds: body.compositeIds, count: body.compositeIds.length, errors }, + } + } + + case 'crowdstrike_perform_host_action': { + const result = await request({ + method: 'POST', + path: '/devices/entities/devices-actions/v2', + query: { action_name: body.actionName }, + body: { ids: body.deviceIds }, + }) + if (!result.ok) return fail(result, 'Failed to perform CrowdStrike host action') + + const affected = getRecordResources(result.data).map(normalizeAffectedEntity) + if (failedWithoutResources(result, affected.length)) { + return fail(result, 'Failed to perform CrowdStrike host action') + } + + return { + ok: true, + output: { affected, count: affected.length, errors: getEnvelopeErrors(result.data) }, + } + } + + case 'crowdstrike_query_host_groups': { + const result = await request({ + method: 'GET', + path: '/devices/queries/host-groups/v1', + query: { + filter: body.filter, + limit: body.limit, + offset: body.offset, + sort: body.sort, + }, + }) + if (!result.ok) return fail(result, 'Failed to query CrowdStrike host groups') + + const hostGroupIds = getStringResources(result.data) + if (failedWithoutResources(result, hostGroupIds.length)) { + return fail(result, 'Failed to query CrowdStrike host groups') + } + + return { + ok: true, + output: { + hostGroupIds, + count: hostGroupIds.length, + pagination: getPagination(result.data), + }, + } + } + + case 'crowdstrike_get_host_group_details': { + const result = await callCrowdStrikeByIds( + baseUrl, + accessToken, + { + method: 'GET', + path: '/devices/entities/host-groups/v1', + ids: body.hostGroupIds, + }, + signal + ) + if (!result.ok) return fail(result, 'Failed to fetch CrowdStrike host group details') + + const hostGroups = getRecordResources(result.data).map(normalizeHostGroup) + if (failedWithoutResources(result, hostGroups.length)) { + return fail(result, 'Failed to fetch CrowdStrike host group details') + } + + return { + ok: true, + output: { + hostGroups, + count: hostGroups.length, + errors: getEnvelopeErrors(result.data), + }, + } + } + + case 'crowdstrike_perform_host_group_action': { + const result = await request({ + method: 'POST', + path: '/devices/entities/host-group-actions/v1', + query: { action_name: body.actionName }, + body: { + action_parameters: [{ name: 'filter', value: buildDeviceIdFilter(body.deviceIds) }], + ids: [body.hostGroupId], + }, + }) + if (!result.ok) return fail(result, 'Failed to perform CrowdStrike host group action') + + const hostGroups = getRecordResources(result.data).map(normalizeHostGroup) + if (failedWithoutResources(result, hostGroups.length)) { + return fail(result, 'Failed to perform CrowdStrike host group action') + } + + return { + ok: true, + output: { + hostGroups, + count: hostGroups.length, + errors: getEnvelopeErrors(result.data), + }, + } + } + + case 'crowdstrike_query_indicators': { + const result = await request({ + method: 'GET', + path: '/iocs/queries/indicators/v1', + query: { + after: body.after, + filter: body.filter, + limit: body.limit, + offset: body.offset, + sort: body.sort, + }, + }) + if (!result.ok) return fail(result, 'Failed to query CrowdStrike indicators') + + const indicatorIds = getStringResources(result.data) + if (failedWithoutResources(result, indicatorIds.length)) { + return fail(result, 'Failed to query CrowdStrike indicators') + } + + return { + ok: true, + output: { + indicatorIds, + count: indicatorIds.length, + pagination: getCursorPagination(result.data), + }, + } + } + + case 'crowdstrike_get_indicator_details': { + const result = await callCrowdStrikeByIds( + baseUrl, + accessToken, + { + method: 'GET', + path: '/iocs/entities/indicators/v1', + ids: body.indicatorIds, + }, + signal + ) + if (!result.ok) return fail(result, 'Failed to fetch CrowdStrike indicator details') + + const indicators = getRecordResources(result.data).map(normalizeIndicator) + if (failedWithoutResources(result, indicators.length)) { + return fail(result, 'Failed to fetch CrowdStrike indicator details') + } + + return { + ok: true, + output: { + indicators, + count: indicators.length, + errors: getEnvelopeErrors(result.data), + }, + } + } + + case 'crowdstrike_create_indicators': + case 'crowdstrike_update_indicators': { + const isCreate = body.operation === 'crowdstrike_create_indicators' + const result = await request({ + method: isCreate ? 'POST' : 'PATCH', + path: '/iocs/entities/indicators/v1', + query: { + ignore_warnings: body.ignoreWarnings, + retrodetects: body.retrodetects, + }, + body: { + comment: body.comment, + indicators: body.indicators, + }, + }) + if (!result.ok) { + return fail( + result, + isCreate + ? 'Failed to create CrowdStrike indicators' + : 'Failed to update CrowdStrike indicators' + ) + } + + const indicators = getRecordResources(result.data).map(normalizeIndicator) + if (failedWithoutResources(result, indicators.length)) { + return fail( + result, + isCreate + ? 'Failed to create CrowdStrike indicators' + : 'Failed to update CrowdStrike indicators' + ) + } + + return { + ok: true, + output: { + indicators, + count: indicators.length, + errors: getEnvelopeErrors(result.data), + }, + } + } + + case 'crowdstrike_delete_indicators': { + const result = await callCrowdStrikeByIds( + baseUrl, + accessToken, + { + method: 'DELETE', + path: '/iocs/entities/indicators/v1', + query: { comment: body.comment, filter: body.filter }, + ids: body.filter ? undefined : body.indicatorIds, + }, + signal + ) + if (!result.ok) return fail(result, 'Failed to delete CrowdStrike indicators') + + const deletedIds = getStringResources(result.data) + if (failedWithoutResources(result, deletedIds.length)) { + return fail(result, 'Failed to delete CrowdStrike indicators') + } + + return { + ok: true, + output: { + deletedIds, + count: deletedIds.length, + errors: getEnvelopeErrors(result.data), + }, + } + } + + case 'crowdstrike_query_vulnerabilities': { + const result = await request({ + method: 'GET', + path: '/spotlight/queries/vulnerabilities/v1', + query: { + after: body.after, + filter: body.filter, + limit: body.limit, + sort: body.sort, + }, + }) + if (!result.ok) return fail(result, 'Failed to query CrowdStrike vulnerabilities') + + const vulnerabilityIds = getStringResources(result.data) + if (failedWithoutResources(result, vulnerabilityIds.length)) { + return fail(result, 'Failed to query CrowdStrike vulnerabilities') + } + + return { + ok: true, + output: { + vulnerabilityIds, + count: vulnerabilityIds.length, + pagination: getSpotlightPagination(result.data), + }, + } + } + + case 'crowdstrike_get_vulnerability_details': { + const result = await callCrowdStrikeByIds( + baseUrl, + accessToken, + { + method: 'GET', + path: '/spotlight/entities/vulnerabilities/v2', + ids: body.vulnerabilityIds, + }, + signal + ) + if (!result.ok) return fail(result, 'Failed to fetch CrowdStrike vulnerability details') + + const vulnerabilities = getRecordResources(result.data).map(normalizeVulnerability) + if (failedWithoutResources(result, vulnerabilities.length)) { + return fail(result, 'Failed to fetch CrowdStrike vulnerability details') + } + + return { + ok: true, + output: { + vulnerabilities, + count: vulnerabilities.length, + errors: getEnvelopeErrors(result.data), + }, + } + } + + case 'crowdstrike_init_rtr_session': { + const result = await request({ + method: 'POST', + path: '/real-time-response/entities/sessions/v1', + body: { + device_id: body.deviceId, + origin: body.origin, + queue_offline: body.queueOffline, + }, + }) + if (!result.ok) return fail(result, 'Failed to initialize CrowdStrike RTR session') + + const session = getFirstRecordResource(result.data) + if (!session) return fail(result, 'CrowdStrike did not return an RTR session') + + return { + ok: true, + output: { + sessionId: getString(session.session_id), + deviceId: getString(session.device_id), + platform: getString(session.platform), + pwd: getString(session.pwd), + offlineQueued: getBoolean(session.offline_queued), + existingAidSessions: getNumber(session.existing_aid_sessions), + createdAt: getString(session.created_at), + errors: getEnvelopeErrors(result.data), + }, + } + } + + case 'crowdstrike_execute_rtr_command': { + const result = await request({ + method: 'POST', + path: '/real-time-response/entities/command/v1', + body: { + base_command: body.baseCommand, + command_string: body.commandString, + session_id: body.sessionId, + }, + }) + if (!result.ok) return fail(result, 'Failed to execute CrowdStrike RTR command') + + const command = getFirstRecordResource(result.data) + if (!command) return fail(result, 'CrowdStrike did not return an RTR command result') + + return { + ok: true, + output: { + cloudRequestId: getString(command.cloud_request_id), + sessionId: getString(command.session_id), + queuedCommandOffline: getBoolean(command.queued_command_offline), + errors: getEnvelopeErrors(result.data), + }, + } + } + + case 'crowdstrike_get_rtr_command_status': { + const result = await request({ + method: 'GET', + path: '/real-time-response/entities/command/v1', + query: { + cloud_request_id: body.cloudRequestId, + sequence_id: body.sequenceId ?? 0, + }, + }) + if (!result.ok) return fail(result, 'Failed to fetch CrowdStrike RTR command status') + + const status = getFirstRecordResource(result.data) + if (!status) return fail(result, 'CrowdStrike did not return an RTR command status') + + return { + ok: true, + output: { + complete: getBoolean(status.complete), + stdout: getString(status.stdout), + stderr: getString(status.stderr), + baseCommand: getString(status.base_command), + sessionId: getString(status.session_id), + taskId: getString(status.task_id), + sequenceId: getNumber(status.sequence_id), + errors: getEnvelopeErrors(result.data), + }, + } + } + + case 'crowdstrike_delete_rtr_session': { + const result = await request({ + method: 'DELETE', + path: '/real-time-response/entities/sessions/v1', + query: { session_id: body.sessionId }, + }) + if (!result.ok) return fail(result, 'Failed to delete CrowdStrike RTR session') + + const deleteErrors = getEnvelopeErrors(result.data) + if (deleteErrors.length > 0) { + return fail(result, 'Failed to delete CrowdStrike RTR session') + } + + return { + ok: true, + output: { + sessionId: body.sessionId, + deleted: true, + errors: deleteErrors, + }, + } + } + + case 'crowdstrike_query_cases': { + const result = await request({ + method: 'GET', + path: '/cases/queries/cases/v1', + query: { + filter: body.filter, + limit: body.limit, + offset: body.offset, + q: body.q, + sort: body.sort, + }, + }) + if (!result.ok) return fail(result, 'Failed to query CrowdStrike cases') + + const caseIds = getStringResources(result.data) + if (failedWithoutResources(result, caseIds.length)) { + return fail(result, 'Failed to query CrowdStrike cases') + } + + return { + ok: true, + output: { caseIds, count: caseIds.length, pagination: getPagination(result.data) }, + } + } + + case 'crowdstrike_get_case_details': { + const result = await request({ + method: 'POST', + path: '/cases/entities/cases/v2', + body: { ids: body.caseIds }, + }) + if (!result.ok) return fail(result, 'Failed to fetch CrowdStrike case details') + + const cases = getRecordResources(result.data).map(normalizeCase) + if (failedWithoutResources(result, cases.length)) { + return fail(result, 'Failed to fetch CrowdStrike case details') + } + + return { + ok: true, + output: { cases, count: cases.length, errors: getEnvelopeErrors(result.data) }, + } + } + } +} + +export async function executeCrowdStrikeRequest( + body: CrowdstrikeQueryBody, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const baseUrl = getCloudBaseUrl(body.cloud) + const accessToken = await getAccessToken(body, signal) + signal?.throwIfAborted() + + logger.info('CrowdStrike request', { + cloud: body.cloud, + operation: body.operation, + }) + + if ( + body.operation === 'crowdstrike_query_sensors' || + body.operation === 'crowdstrike_get_sensor_details' || + body.operation === 'crowdstrike_get_sensor_aggregates' + ) { + return executeSensorOperation(body, baseUrl, accessToken, signal) + } + + return executeCrowdStrikeOperation(body, baseUrl, accessToken, signal) +} diff --git a/apps/sim/lib/internal/cursor/errors.ts b/apps/sim/lib/internal/cursor/errors.ts new file mode 100644 index 00000000000..45354ebd4ea --- /dev/null +++ b/apps/sim/lib/internal/cursor/errors.ts @@ -0,0 +1,9 @@ +export class CursorOperationError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'CursorOperationError' + } +} diff --git a/apps/sim/lib/internal/cursor/execute-tool.test.ts b/apps/sim/lib/internal/cursor/execute-tool.test.ts new file mode 100644 index 00000000000..a290d6a7f06 --- /dev/null +++ b/apps/sim/lib/internal/cursor/execute-tool.test.ts @@ -0,0 +1,70 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ downloadCursorArtifact: vi.fn() })) + +vi.mock('@/lib/internal/cursor/operations', () => ({ + downloadCursorArtifact: mocks.downloadCursorArtifact, + cursorOperationErrorMessage: (error: unknown) => + typeof error === 'object' && error !== null && 'message' in error + ? String(error.message) + : 'Unknown error occurred', +})) + +import { CursorOperationError } from '@/lib/internal/cursor/errors' +import { executeCursorTool } from '@/lib/internal/cursor/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +function request(overrides: Partial = {}): InternalToolOperationCall { + return { + toolId: 'cursor_download_artifact', + input: { apiKey: 'cursor-key', agentId: 'agent-1', path: '/src/index.ts' }, + headers: new Headers(), + context: createExecutionContext({ workflowId: 'workflow-1' }), + requestId: 'request-1', + ...overrides, + } +} + +describe('executeCursorTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.downloadCursorArtifact.mockResolvedValue({ + success: true, + output: { file: { name: 'index.ts', mimeType: 'text/plain', data: 'YQ==', size: 1 } }, + }) + }) + + it.each(['cursor_download_artifact', 'cursor_download_artifact_v2'])( + 'dispatches %s without an HTTP route', + async (toolId) => { + const controller = new AbortController() + const response = await executeCursorTool(request({ toolId, signal: controller.signal })) + + expect(response.status).toBe(200) + expect(mocks.downloadCursorArtifact).toHaveBeenCalledWith( + { apiKey: 'cursor-key', agentId: 'agent-1', path: '/src/index.ts' }, + { requestId: 'request-1', signal: controller.signal } + ) + } + ) + + it('rejects invalid input before provider work', async () => { + const response = await executeCursorTool(request({ input: { apiKey: '' } })) + + expect(response.status).toBe(400) + expect(mocks.downloadCursorArtifact).not.toHaveBeenCalled() + }) + + it('preserves provider status errors', async () => { + mocks.downloadCursorArtifact.mockRejectedValue(new CursorOperationError('not found', 404)) + + const response = await executeCursorTool(request()) + + expect(response.status).toBe(404) + await expect(response.json()).resolves.toEqual({ success: false, error: 'not found' }) + }) +}) diff --git a/apps/sim/lib/internal/cursor/execute-tool.ts b/apps/sim/lib/internal/cursor/execute-tool.ts new file mode 100644 index 00000000000..9436106fdec --- /dev/null +++ b/apps/sim/lib/internal/cursor/execute-tool.ts @@ -0,0 +1,43 @@ +import { z } from 'zod' +import { CursorOperationError } from '@/lib/internal/cursor/errors' +import { + cursorOperationErrorMessage, + downloadCursorArtifact, +} from '@/lib/internal/cursor/operations' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const inputSchema = z.object({ + apiKey: z.string().min(1, 'API key is required'), + agentId: z.string().min(1, 'Agent ID is required'), + path: z.string().min(1, 'Artifact path is required'), +}) + +export const executeCursorTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if ( + request.toolId !== 'cursor_download_artifact' && + request.toolId !== 'cursor_download_artifact_v2' + ) { + return Response.json( + { success: false, error: `Unsupported Cursor tool: ${request.toolId}` }, + { status: 500 } + ) + } + const parsed = inputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + + try { + return Response.json( + await downloadCursorArtifact(parsed.data, { + requestId: request.requestId, + signal: request.signal, + }) + ) + } catch (error) { + request.signal?.throwIfAborted() + const status = error instanceof CursorOperationError ? error.status : 500 + return Response.json({ success: false, error: cursorOperationErrorMessage(error) }, { status }) + } +} diff --git a/apps/sim/lib/internal/cursor/operations.test.ts b/apps/sim/lib/internal/cursor/operations.test.ts new file mode 100644 index 00000000000..1105d9df52c --- /dev/null +++ b/apps/sim/lib/internal/cursor/operations.test.ts @@ -0,0 +1,57 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + secureFetchWithPinnedIP: vi.fn(), + validateUrlWithDNS: vi.fn(), +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + secureFetchWithPinnedIP: mocks.secureFetchWithPinnedIP, + validateUrlWithDNS: mocks.validateUrlWithDNS, +})) + +import { downloadCursorArtifact } from '@/lib/internal/cursor/operations' + +describe('downloadCursorArtifact', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.unstubAllGlobals() + mocks.validateUrlWithDNS.mockResolvedValue({ isValid: true, resolvedIP: '203.0.113.1' }) + mocks.secureFetchWithPinnedIP.mockResolvedValue( + new Response('artifact', { headers: { 'content-type': 'text/plain' } }) + ) + }) + + it('uses one metadata request and one DNS-pinned artifact download with cancellation', async () => { + const controller = new AbortController() + const fetchMock = vi + .fn() + .mockResolvedValue(Response.json({ url: 'https://download.example/artifact' })) + vi.stubGlobal('fetch', fetchMock) + + const result = await downloadCursorArtifact( + { apiKey: 'cursor-key', agentId: 'agent-1', path: '/src/index.ts' }, + { requestId: 'request-1', signal: controller.signal } + ) + + expect(fetchMock).toHaveBeenCalledOnce() + expect(fetchMock).toHaveBeenCalledWith( + expect.stringContaining('/agents/agent-1/artifacts/download'), + expect.objectContaining({ signal: controller.signal }) + ) + expect(mocks.secureFetchWithPinnedIP).toHaveBeenCalledWith( + 'https://download.example/artifact', + '203.0.113.1', + { signal: controller.signal } + ) + expect(result.output.file).toEqual({ + name: 'index.ts', + mimeType: 'text/plain', + data: Buffer.from('artifact').toString('base64'), + size: 8, + }) + }) +}) diff --git a/apps/sim/lib/internal/cursor/operations.ts b/apps/sim/lib/internal/cursor/operations.ts new file mode 100644 index 00000000000..2c1ce81b477 --- /dev/null +++ b/apps/sim/lib/internal/cursor/operations.ts @@ -0,0 +1,101 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { + readResponseJsonWithLimit, + readResponseTextWithLimit, +} from '@/lib/core/utils/stream-limits' +import { CursorOperationError } from '@/lib/internal/cursor/errors' +import type { DownloadArtifactParams } from '@/tools/cursor/types' + +const logger = createLogger('CursorOperations') +const MAX_CURSOR_METADATA_BYTES = 256 * 1024 + +interface CursorArtifactLocation { + url?: string + downloadUrl?: string + presignedUrl?: string +} + +export interface CursorOperationContext { + requestId: string + signal?: AbortSignal +} + +export async function downloadCursorArtifact( + input: DownloadArtifactParams, + context: CursorOperationContext +): Promise<{ + success: true + output: { file: { name: string; mimeType: string; data: string; size: number } } +}> { + context.signal?.throwIfAborted() + const authHeader = `Basic ${Buffer.from(`${input.apiKey}:`).toString('base64')}` + const artifactResponse = await fetch( + `https://api.cursor.com/v0/agents/${encodeURIComponent(input.agentId)}/artifacts/download?path=${encodeURIComponent(input.path)}`, + { + method: 'GET', + headers: { Authorization: authHeader }, + signal: context.signal, + } + ) + + if (!artifactResponse.ok) { + const errorText = await readResponseTextWithLimit(artifactResponse, { + maxBytes: MAX_CURSOR_METADATA_BYTES, + label: 'Cursor artifact error response', + signal: context.signal, + }).catch(() => '') + throw new CursorOperationError( + errorText || `Failed to get artifact URL (${artifactResponse.status})`, + artifactResponse.status + ) + } + + const artifactData = await readResponseJsonWithLimit(artifactResponse, { + maxBytes: MAX_CURSOR_METADATA_BYTES, + label: 'Cursor artifact metadata response', + signal: context.signal, + }) + const downloadUrl = artifactData.url || artifactData.downloadUrl || artifactData.presignedUrl + if (!downloadUrl) { + throw new CursorOperationError('No download URL returned for artifact', 400) + } + + const validation = await validateUrlWithDNS(downloadUrl, 'downloadUrl') + context.signal?.throwIfAborted() + if (!validation.isValid || !validation.resolvedIP) { + throw new CursorOperationError(validation.error || 'Invalid download URL', 400) + } + const downloadResponse = await secureFetchWithPinnedIP(downloadUrl, validation.resolvedIP, { + signal: context.signal, + }) + if (!downloadResponse.ok) { + throw new CursorOperationError( + `Failed to download artifact content (${downloadResponse.status}: ${downloadResponse.statusText})`, + downloadResponse.status + ) + } + + const fileBuffer = Buffer.from(await downloadResponse.arrayBuffer()) + context.signal?.throwIfAborted() + const file = { + name: input.path.split('/').pop() || 'artifact', + mimeType: downloadResponse.headers.get('content-type') || 'application/octet-stream', + data: fileBuffer.toString('base64'), + size: fileBuffer.length, + } + logger.info(`[${context.requestId}] Cursor artifact downloaded`, { + agentId: input.agentId, + path: input.path, + size: file.size, + }) + return { success: true, output: { file } } +} + +export function cursorOperationErrorMessage(error: unknown): string { + return getErrorMessage(error, 'Unknown error occurred') +} diff --git a/apps/sim/lib/internal/custom-tools/read-available-by-id-or-title.test.ts b/apps/sim/lib/internal/custom-tools/read-available-by-id-or-title.test.ts new file mode 100644 index 00000000000..8b459221de2 --- /dev/null +++ b/apps/sim/lib/internal/custom-tools/read-available-by-id-or-title.test.ts @@ -0,0 +1,180 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { CUSTOM_TOOL_DELEGATION_AUDIENCE } from '@/lib/custom-tools/application/authorization' +import type { ExecutionContext } from '@/executor/types' + +const { mocks } = vi.hoisted(() => ({ + mocks: { + createPrincipal: vi.fn(), + executeCopilot: vi.fn(), + readUseCase: { execute: vi.fn() }, + }, +})) + +vi.mock('@/lib/internal/principals/executor', () => ({ + createExecutorPrincipalFromExecutionContext: mocks.createPrincipal, +})) + +vi.mock('@/lib/custom-tools/application/use-cases', () => ({ + readAvailableCustomToolByIdOrTitleUseCase: mocks.readUseCase, +})) + +vi.mock('@/lib/copilot/application/execute-custom-tool-use-case', () => ({ + executeCopilotCustomToolUseCase: mocks.executeCopilot, +})) + +import { + readAvailableCustomToolByIdOrTitleAsCopilot, + readAvailableCustomToolByIdOrTitleAsExecutor, +} from '@/lib/internal/custom-tools/read-available-by-id-or-title' + +const principal = { + kind: 'delegated' as const, + serviceId: 'executor' as const, + subjectUserId: 'user-1', + workspaceId: 'canonical-workspace', + delegationId: 'delegation-1', + audience: CUSTOM_TOOL_DELEGATION_AUDIENCE, + issuedAt: new Date('2026-01-01T00:00:00Z'), + expiresAt: new Date('2027-01-01T00:00:00Z'), +} + +const tool = { + id: 'tool-1', + workspaceId: principal.workspaceId, + userId: principal.subjectUserId, + title: 'lookup_order', + schema: { type: 'function' }, + code: 'return 1', + createdAt: new Date('2026-01-01T00:00:00Z'), + updatedAt: new Date('2026-01-01T00:00:00Z'), +} + +function executionContext(abortSignal?: AbortSignal): ExecutionContext { + return { + workflowId: 'workflow-1', + workspaceId: 'untrusted-context-workspace', + userId: 'user-1', + blockStates: new Map(), + executedBlocks: new Set(), + blockLogs: [], + metadata: { duration: 0 }, + environmentVariables: {}, + decisions: { router: new Map(), condition: new Map() }, + completedLoops: new Set(), + activeExecutionPath: new Set(), + ...(abortSignal ? { abortSignal } : {}), + } +} + +describe('readAvailableCustomToolByIdOrTitleAsExecutor', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.createPrincipal.mockResolvedValue(principal) + mocks.readUseCase.execute.mockResolvedValue({ tool }) + mocks.executeCopilot.mockResolvedValue({ tool }) + }) + + it('constructs an executor principal and uses its canonical workspace', async () => { + const context = executionContext() + + await expect( + readAvailableCustomToolByIdOrTitleAsExecutor({ + context, + identifier: tool.id, + lookup: 'id', + }) + ).resolves.toEqual(tool) + + expect(mocks.createPrincipal).toHaveBeenCalledWith({ + context, + audience: CUSTOM_TOOL_DELEGATION_AUDIENCE, + }) + expect(mocks.readUseCase.execute).toHaveBeenCalledWith({ + principal, + input: { + workspaceId: principal.workspaceId, + identifier: tool.id, + lookup: 'id', + }, + }) + }) + + it('stops before principal construction when execution is already cancelled', async () => { + const controller = new AbortController() + controller.abort(new Error('cancelled')) + + await expect( + readAvailableCustomToolByIdOrTitleAsExecutor({ + context: executionContext(controller.signal), + identifier: tool.id, + lookup: 'id_or_title', + }) + ).rejects.toThrow('cancelled') + + expect(mocks.createPrincipal).not.toHaveBeenCalled() + expect(mocks.readUseCase.execute).not.toHaveBeenCalled() + }) +}) + +describe('readAvailableCustomToolByIdOrTitleAsCopilot', () => { + const context = { + userId: 'user-1', + workspaceId: 'canonical-workspace', + chatId: 'chat-1', + executionId: 'execution-1', + toolCallId: 'tool-call-1', + copilotToolExecution: true, + } as const + + beforeEach(() => { + vi.clearAllMocks() + mocks.executeCopilot.mockResolvedValue({ tool }) + }) + + it('enters the shared Copilot use case with canonical trusted scope', async () => { + await expect( + readAvailableCustomToolByIdOrTitleAsCopilot({ + context, + identifier: tool.id, + lookup: 'id_or_title', + }) + ).resolves.toEqual(tool) + + expect(mocks.executeCopilot).toHaveBeenCalledWith(context, mocks.readUseCase, { + workspaceId: context.workspaceId, + identifier: tool.id, + lookup: 'id_or_title', + }) + }) + + it('rejects forged Copilot authority before application execution', async () => { + await expect( + readAvailableCustomToolByIdOrTitleAsCopilot({ + context: { ...context, copilotToolExecution: false }, + identifier: tool.id, + lookup: 'id', + }) + ).rejects.toThrow('trusted Copilot execution context') + + expect(mocks.executeCopilot).not.toHaveBeenCalled() + }) + + it('stops before application execution when the caller is already cancelled', async () => { + const controller = new AbortController() + controller.abort(new Error('cancelled')) + + await expect( + readAvailableCustomToolByIdOrTitleAsCopilot({ + context, + identifier: tool.id, + lookup: 'id', + signal: controller.signal, + }) + ).rejects.toThrow('cancelled') + + expect(mocks.executeCopilot).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/custom-tools/read-available-by-id-or-title.ts b/apps/sim/lib/internal/custom-tools/read-available-by-id-or-title.ts new file mode 100644 index 00000000000..1fce57b1e89 --- /dev/null +++ b/apps/sim/lib/internal/custom-tools/read-available-by-id-or-title.ts @@ -0,0 +1,70 @@ +import { executeCopilotCustomToolUseCase } from '@/lib/copilot/application/execute-custom-tool-use-case' +import { + type CopilotExecutionContext, + requireTrustedCopilotExecutionContext, +} from '@/lib/copilot/auth/application-delegation' +import { CUSTOM_TOOL_DELEGATION_AUDIENCE } from '@/lib/custom-tools/application/authorization' +import { + type ReadAvailableCustomToolByIdOrTitleInput, + readAvailableCustomToolByIdOrTitleUseCase, +} from '@/lib/custom-tools/application/use-cases' +import { createExecutorPrincipalFromExecutionContext } from '@/lib/internal/principals/executor' +import type { ExecutionContext } from '@/executor/types' + +export interface ReadAvailableCustomToolByIdOrTitleAsExecutorInput { + context: ExecutionContext + identifier: string + lookup: ReadAvailableCustomToolByIdOrTitleInput['lookup'] +} + +export async function readAvailableCustomToolByIdOrTitleAsExecutor({ + context, + identifier, + lookup, +}: ReadAvailableCustomToolByIdOrTitleAsExecutorInput) { + context.abortSignal?.throwIfAborted() + const principal = await createExecutorPrincipalFromExecutionContext({ + context, + audience: CUSTOM_TOOL_DELEGATION_AUDIENCE, + }) + context.abortSignal?.throwIfAborted() + const { tool } = await readAvailableCustomToolByIdOrTitleUseCase.execute({ + principal, + input: { + workspaceId: principal.workspaceId, + identifier, + lookup, + }, + }) + context.abortSignal?.throwIfAborted() + return tool +} + +export interface ReadAvailableCustomToolByIdOrTitleAsCopilotInput { + context: CopilotExecutionContext | undefined + identifier: string + lookup: ReadAvailableCustomToolByIdOrTitleInput['lookup'] + signal?: AbortSignal +} + +/** Resolves a dynamic custom tool through the shared Copilot application boundary. */ +export async function readAvailableCustomToolByIdOrTitleAsCopilot({ + context, + identifier, + lookup, + signal, +}: ReadAvailableCustomToolByIdOrTitleAsCopilotInput) { + signal?.throwIfAborted() + const trustedContext = requireTrustedCopilotExecutionContext(context) + const { tool } = await executeCopilotCustomToolUseCase( + trustedContext, + readAvailableCustomToolByIdOrTitleUseCase, + { + workspaceId: trustedContext.workspaceId, + identifier, + lookup, + } + ) + signal?.throwIfAborted() + return tool +} diff --git a/apps/sim/lib/internal/daytona/errors.ts b/apps/sim/lib/internal/daytona/errors.ts new file mode 100644 index 00000000000..33bbe645779 --- /dev/null +++ b/apps/sim/lib/internal/daytona/errors.ts @@ -0,0 +1,9 @@ +export class DaytonaOperationError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'DaytonaOperationError' + } +} diff --git a/apps/sim/lib/internal/daytona/execute-tool.test.ts b/apps/sim/lib/internal/daytona/execute-tool.test.ts new file mode 100644 index 00000000000..b98584e78ac --- /dev/null +++ b/apps/sim/lib/internal/daytona/execute-tool.test.ts @@ -0,0 +1,90 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { DocCompileUserError } from '@/lib/copilot/tools/server/files/doc-compile-error' +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' + +const mocks = vi.hoisted(() => ({ uploadDaytonaFile: vi.fn() })) + +vi.mock('@/lib/internal/daytona/operations', () => ({ + uploadDaytonaFile: mocks.uploadDaytonaFile, +})) + +import { executeDaytonaTool } from '@/lib/internal/daytona/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +function request(overrides: Partial = {}): InternalToolOperationCall { + return { + toolId: 'daytona_upload_file', + input: { + apiKey: 'daytona-key', + sandboxId: 'sandbox-1', + destinationPath: '/tmp/', + fileContent: 'YQ==', + fileName: 'a.txt', + }, + headers: new Headers(), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeDaytonaTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.uploadDaytonaFile.mockResolvedValue({ + success: true, + output: { uploadedPath: '/tmp/a.txt', name: 'a.txt', size: 1 }, + }) + }) + + it('dispatches with trusted user identity and cancellation', async () => { + const controller = new AbortController() + const response = await executeDaytonaTool(request({ signal: controller.signal })) + + expect(response.status).toBe(200) + expect(mocks.uploadDaytonaFile).toHaveBeenCalledWith(expect.any(Object), { + userId: 'user-1', + requestId: 'request-1', + signal: controller.signal, + }) + }) + + it('preserves retryable generated-document errors', async () => { + mocks.uploadDaytonaFile.mockRejectedValueOnce( + new DocCompileUserError('still compiling', { pending: true }) + ) + + const response = await executeDaytonaTool(request()) + + expect(response.status).toBe(409) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'A document is still being generated. Wait for it to finish, then try again.', + }) + }) + + it('preserves typed file size errors instead of returning 500', async () => { + mocks.uploadDaytonaFile.mockRejectedValueOnce( + new PayloadSizeLimitError({ + label: 'Daytona upload', + maxBytes: 100, + observedBytes: 101, + }) + ) + + const response = await executeDaytonaTool(request()) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + success: false, + error: expect.stringContaining('Daytona upload'), + }) + }) +}) diff --git a/apps/sim/lib/internal/daytona/execute-tool.ts b/apps/sim/lib/internal/daytona/execute-tool.ts new file mode 100644 index 00000000000..ca0dff83fba --- /dev/null +++ b/apps/sim/lib/internal/daytona/execute-tool.ts @@ -0,0 +1,66 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { z } from 'zod' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { DaytonaOperationError } from '@/lib/internal/daytona/errors' +import { uploadDaytonaFile } from '@/lib/internal/daytona/operations' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' + +const inputSchema = z.object({ + apiKey: z.string().min(1), + sandboxId: z.string().min(1), + destinationPath: z.string().min(1), + file: RawFileInputSchema.optional().nullable(), + fileContent: z.string().nullish(), + fileName: z.string().nullish(), +}) + +export const executeDaytonaTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'daytona_upload_file') { + return Response.json( + { success: false, error: `Unsupported Daytona tool: ${request.toolId}` }, + { status: 500 } + ) + } + const userId = request.context.userId + if (!userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + const parsed = inputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + try { + return Response.json( + await uploadDaytonaFile( + { + ...parsed.data, + file: parsed.data.file ?? undefined, + fileContent: parsed.data.fileContent ?? undefined, + fileName: parsed.data.fileName ?? undefined, + }, + { + userId, + requestId: request.requestId, + signal: request.signal, + } + ) + ) + } catch (error) { + request.signal?.throwIfAborted() + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + const status = + error instanceof DaytonaOperationError + ? error.status + : isPayloadSizeLimitError(error) + ? 400 + : 500 + return Response.json( + { success: false, error: getErrorMessage(error, 'Unknown error') }, + { status } + ) + } +} diff --git a/apps/sim/lib/internal/daytona/operations.test.ts b/apps/sim/lib/internal/daytona/operations.test.ts new file mode 100644 index 00000000000..607f9c8ab7d --- /dev/null +++ b/apps/sim/lib/internal/daytona/operations.test.ts @@ -0,0 +1,62 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { DocCompileUserError } from '@/lib/copilot/tools/server/files/doc-compile-error' +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' + +const mocks = vi.hoisted(() => ({ + assertToolFileAccess: vi.fn(), + downloadServableFileFromStorage: vi.fn(), +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertToolFileAccess, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mocks.downloadServableFileFromStorage, +})) + +import { uploadDaytonaFile } from '@/lib/internal/daytona/operations' + +const input = { + apiKey: 'daytona-key', + sandboxId: 'sandbox-1', + destinationPath: '/tmp/', + file: { key: 'workspace/file.txt', name: 'file.txt', size: 4 }, +} + +describe('uploadDaytonaFile', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.assertToolFileAccess.mockResolvedValue(null) + }) + + it.each([ + new PayloadSizeLimitError({ + label: 'Daytona upload', + maxBytes: 100, + observedBytes: 101, + }), + new DocCompileUserError('still compiling', { pending: true }), + ])('preserves typed file errors from storage materialization', async (error) => { + const controller = new AbortController() + mocks.downloadServableFileFromStorage.mockRejectedValueOnce(error) + + await expect( + uploadDaytonaFile(input, { + userId: 'user-1', + requestId: 'request-1', + signal: controller.signal, + }) + ).rejects.toBe(error) + + expect(mocks.downloadServableFileFromStorage).toHaveBeenCalledWith( + expect.objectContaining({ key: 'workspace/file.txt' }), + 'request-1', + expect.anything(), + expect.objectContaining({ maxBytes: 100 * 1024 * 1024, signal: controller.signal }) + ) + }) +}) diff --git a/apps/sim/lib/internal/daytona/operations.ts b/apps/sim/lib/internal/daytona/operations.ts new file mode 100644 index 00000000000..8682b958374 --- /dev/null +++ b/apps/sim/lib/internal/daytona/operations.ts @@ -0,0 +1,121 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' +import { DaytonaOperationError } from '@/lib/internal/daytona/errors' +import { isDocNotReadyError } from '@/lib/uploads/utils/doc-not-ready' +import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' +import type { DaytonaUploadFileParams, DaytonaUploadFileResponse } from '@/tools/daytona/types' +import { daytonaToolboxUrl } from '@/tools/daytona/utils' + +const logger = createLogger('DaytonaOperations') +const MAX_UPLOAD_SIZE_BYTES = 100 * 1024 * 1024 +const MAX_DAYTONA_ERROR_BYTES = 256 * 1024 + +export interface DaytonaOperationContext { + userId: string + requestId: string + signal?: AbortSignal +} + +export interface DaytonaUploadFileInput extends Omit { + file?: RawFileInput +} + +async function getDaytonaError(response: Response, signal?: AbortSignal): Promise { + try { + const body = await readResponseJsonWithLimit<{ message?: string | string[]; error?: string }>( + response, + { maxBytes: MAX_DAYTONA_ERROR_BYTES, label: 'Daytona error response', signal } + ) + if (typeof body.message === 'string') return body.message + if (Array.isArray(body.message)) return body.message.join(', ') + if (body.error) return body.error + } catch {} + return `Failed to upload file (status ${response.status})` +} + +export async function uploadDaytonaFile( + input: DaytonaUploadFileInput, + context: DaytonaOperationContext +): Promise { + context.signal?.throwIfAborted() + let fileBuffer: Buffer + let fileName: string + if (input.file) { + const userFile = processFilesToUserFiles([input.file], context.requestId, logger)[0] + if (!userFile) throw new DaytonaOperationError('Invalid file input', 400) + const denied = await assertToolFileAccess( + userFile.key, + context.userId, + context.requestId, + logger + ) + if (denied) throw new DaytonaOperationError('File not found', denied.status) + if (userFile.size > MAX_UPLOAD_SIZE_BYTES) { + const sizeMB = (userFile.size / (1024 * 1024)).toFixed(2) + throw new DaytonaOperationError(`File size (${sizeMB}MB) exceeds upload limit of 100MB`, 400) + } + try { + const servable = await downloadServableFileFromStorage(userFile, context.requestId, logger, { + maxBytes: MAX_UPLOAD_SIZE_BYTES, + signal: context.signal, + }) + fileBuffer = servable.buffer + } catch (error) { + if (isPayloadSizeLimitError(error) || isDocNotReadyError(error)) throw error + throw new DaytonaOperationError(getErrorMessage(error, 'Failed to download file'), 500) + } + fileName = input.fileName || userFile.name + } else if (input.fileContent) { + const estimatedSize = Math.floor((input.fileContent.length * 3) / 4) + if (estimatedSize > MAX_UPLOAD_SIZE_BYTES) { + const sizeMB = (estimatedSize / (1024 * 1024)).toFixed(2) + throw new DaytonaOperationError(`File size (${sizeMB}MB) exceeds upload limit of 100MB`, 400) + } + fileBuffer = Buffer.from(input.fileContent, 'base64') + fileName = input.fileName || 'file' + } else { + throw new DaytonaOperationError('File is required', 400) + } + context.signal?.throwIfAborted() + if (fileBuffer.length > MAX_UPLOAD_SIZE_BYTES) { + const sizeMB = (fileBuffer.length / (1024 * 1024)).toFixed(2) + throw new DaytonaOperationError(`File size (${sizeMB}MB) exceeds upload limit of 100MB`, 400) + } + const requestedPath = input.destinationPath.trim() + if (!requestedPath) throw new DaytonaOperationError('Destination path is required', 400) + const destinationPath = requestedPath.endsWith('/') + ? `${requestedPath}${fileName}` + : requestedPath + const form = new FormData() + form.append( + 'file', + new Blob([new Uint8Array(fileBuffer)], { type: 'application/octet-stream' }), + fileName + ) + const response = await fetch( + daytonaToolboxUrl( + input.sandboxId, + `/files/upload-v2?path=${encodeURIComponent(destinationPath)}` + ), + { + method: 'POST', + headers: { Authorization: `Bearer ${input.apiKey}` }, + body: form, + signal: context.signal, + } + ) + if (!response.ok) { + throw new DaytonaOperationError( + await getDaytonaError(response, context.signal), + response.status + ) + } + return { + success: true, + output: { uploadedPath: destinationPath, name: fileName, size: fileBuffer.length }, + } +} diff --git a/apps/sim/lib/internal/deployments/client.test.ts b/apps/sim/lib/internal/deployments/client.test.ts new file mode 100644 index 00000000000..a6dd3d243a3 --- /dev/null +++ b/apps/sim/lib/internal/deployments/client.test.ts @@ -0,0 +1,159 @@ +/** + * @vitest-environment node + */ +import type { DelegatedPrincipal } from '@sim/auth/principal' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + activate: vi.fn(), + deploy: vi.fn(), + getVersion: vi.fn(), + listVersions: vi.fn(), + undeploy: vi.fn(), +})) + +vi.mock('@/lib/workflows/application/deployments', () => ({ + activateWorkflowVersion: { execute: mocks.activate }, + deployWorkflow: { execute: mocks.deploy }, + undeployWorkflow: { execute: mocks.undeploy }, +})) + +vi.mock('@/lib/workflows/application/list-workflow-versions', () => ({ + listWorkflowVersions: { execute: mocks.listVersions }, +})) + +vi.mock('@/lib/workflows/application/read-workflow-version', () => ({ + readWorkflowVersion: { execute: mocks.getVersion }, +})) + +import { + deployWorkflowDeployment, + getWorkflowDeploymentVersion, + listWorkflowDeploymentVersions, + promoteWorkflowDeployment, + undeployWorkflowDeployment, +} from '@/lib/internal/deployments/client' + +const principal: DelegatedPrincipal = { + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'workspace-1', + delegationId: 'delegation-1', + audience: 'sim:workflows', + issuedAt: new Date('2026-01-01T00:00:00Z'), + expiresAt: new Date('2026-01-01T00:05:00Z'), +} + +const context = { principal, requestId: 'request-1' } + +describe('deployment application client', () => { + beforeEach(() => { + vi.clearAllMocks() + for (const execute of Object.values(mocks)) execute.mockResolvedValue({}) + }) + + it('uses the authorized deployment use cases for mutations', async () => { + await deployWorkflowDeployment( + { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + name: 'Release 4', + description: 'Fixes the agent prompt', + }, + context + ) + await undeployWorkflowDeployment( + { workflowId: 'workflow-1', workspaceId: 'workspace-1' }, + context + ) + await promoteWorkflowDeployment( + { workflowId: 'workflow-1', workspaceId: 'workspace-1', version: 3 }, + context + ) + + expect(mocks.deploy).toHaveBeenCalledWith({ + principal, + input: { + workflowId: 'workflow-1', + assertedWorkspaceId: 'workspace-1', + name: 'Release 4', + description: 'Fixes the agent prompt', + requestId: 'request-1', + idempotencyKey: 'request-1', + }, + }) + expect(mocks.undeploy).toHaveBeenCalledWith({ + principal, + input: { + workflowId: 'workflow-1', + assertedWorkspaceId: 'workspace-1', + requestId: 'request-1', + }, + }) + expect(mocks.activate).toHaveBeenCalledWith({ + principal, + input: { + workflowId: 'workflow-1', + assertedWorkspaceId: 'workspace-1', + version: 3, + transition: 'activate', + requestId: 'request-1', + idempotencyKey: 'request-1', + }, + }) + }) + + it('uses bounded and credential-sanitizing application reads', async () => { + await listWorkflowDeploymentVersions( + { workflowId: 'workflow-1', workspaceId: 'workspace-1' }, + context + ) + await getWorkflowDeploymentVersion( + { workflowId: 'workflow-1', workspaceId: 'workspace-1', version: 3 }, + context + ) + + expect(mocks.listVersions).toHaveBeenCalledWith({ + principal, + input: { workflowId: 'workflow-1', assertedWorkspaceId: 'workspace-1' }, + }) + expect(mocks.getVersion).toHaveBeenCalledWith({ + principal, + input: { + workflowId: 'workflow-1', + assertedWorkspaceId: 'workspace-1', + version: 3, + }, + }) + }) + + it('does not start application work after cancellation', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + deployWorkflowDeployment( + { workflowId: 'workflow-1', workspaceId: 'workspace-1' }, + { ...context, signal: controller.signal } + ) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mocks.deploy).not.toHaveBeenCalled() + }) + + it('returns a committed mutation result when cancellation arrives after the use case succeeds', async () => { + const controller = new AbortController() + const committed = { activeDeployment: { id: 'deployment-1' } } + mocks.deploy.mockImplementationOnce(async () => { + controller.abort(new DOMException('cancelled', 'AbortError')) + return committed + }) + + await expect( + deployWorkflowDeployment( + { workflowId: 'workflow-1', workspaceId: 'workspace-1' }, + { ...context, signal: controller.signal } + ) + ).resolves.toBe(committed) + }) +}) diff --git a/apps/sim/lib/internal/deployments/client.ts b/apps/sim/lib/internal/deployments/client.ts new file mode 100644 index 00000000000..8051584b0c3 --- /dev/null +++ b/apps/sim/lib/internal/deployments/client.ts @@ -0,0 +1,112 @@ +import type { DelegatedPrincipal } from '@sim/auth/principal' +import type { + DeploymentsDeployBody, + DeploymentsGetVersionQuery, + DeploymentsListVersionsQuery, + DeploymentsPromoteBody, + DeploymentsUndeployBody, +} from '@/lib/internal/deployments/input' +import { + activateWorkflowVersion, + deployWorkflow, + undeployWorkflow, +} from '@/lib/workflows/application/deployments' +import { listWorkflowVersions } from '@/lib/workflows/application/list-workflow-versions' +import { readWorkflowVersion } from '@/lib/workflows/application/read-workflow-version' + +export interface DeploymentApplicationClientContext { + principal: DelegatedPrincipal + requestId: string + signal?: AbortSignal +} + +function assertNotAborted(signal?: AbortSignal): void { + signal?.throwIfAborted() +} + +export async function deployWorkflowDeployment( + input: DeploymentsDeployBody, + context: DeploymentApplicationClientContext +) { + assertNotAborted(context.signal) + const result = await deployWorkflow.execute({ + principal: context.principal, + input: { + workflowId: input.workflowId, + assertedWorkspaceId: input.workspaceId, + name: input.name, + description: input.description ?? undefined, + requestId: context.requestId, + idempotencyKey: context.requestId, + }, + }) + return result +} + +export async function undeployWorkflowDeployment( + input: DeploymentsUndeployBody, + context: DeploymentApplicationClientContext +) { + assertNotAborted(context.signal) + const result = await undeployWorkflow.execute({ + principal: context.principal, + input: { + workflowId: input.workflowId, + assertedWorkspaceId: input.workspaceId, + requestId: context.requestId, + }, + }) + return result +} + +export async function promoteWorkflowDeployment( + input: DeploymentsPromoteBody, + context: DeploymentApplicationClientContext +) { + assertNotAborted(context.signal) + const result = await activateWorkflowVersion.execute({ + principal: context.principal, + input: { + workflowId: input.workflowId, + assertedWorkspaceId: input.workspaceId, + version: input.version, + transition: 'activate', + requestId: context.requestId, + idempotencyKey: context.requestId, + }, + }) + return result +} + +export async function listWorkflowDeploymentVersions( + input: DeploymentsListVersionsQuery, + context: DeploymentApplicationClientContext +) { + assertNotAborted(context.signal) + const result = await listWorkflowVersions.execute({ + principal: context.principal, + input: { + workflowId: input.workflowId, + assertedWorkspaceId: input.workspaceId, + }, + }) + assertNotAborted(context.signal) + return result +} + +export async function getWorkflowDeploymentVersion( + input: DeploymentsGetVersionQuery, + context: DeploymentApplicationClientContext +) { + assertNotAborted(context.signal) + const result = await readWorkflowVersion.execute({ + principal: context.principal, + input: { + workflowId: input.workflowId, + assertedWorkspaceId: input.workspaceId, + version: input.version, + }, + }) + assertNotAborted(context.signal) + return result +} diff --git a/apps/sim/lib/internal/deployments/execute-tool.test.ts b/apps/sim/lib/internal/deployments/execute-tool.test.ts new file mode 100644 index 00000000000..d1dcd08c242 --- /dev/null +++ b/apps/sim/lib/internal/deployments/execute-tool.test.ts @@ -0,0 +1,174 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + createPrincipal: vi.fn(), + deploy: vi.fn(), + getVersion: vi.fn(), + listVersions: vi.fn(), + promote: vi.fn(), + undeploy: vi.fn(), +})) + +vi.mock('@/lib/internal/principals/executor', () => ({ + createExecutorPrincipalFromExecutionContext: mocks.createPrincipal, +})) + +vi.mock('@/lib/internal/deployments/operations', () => ({ + executeDeploymentsDeploy: mocks.deploy, + executeDeploymentsGetVersion: mocks.getVersion, + executeDeploymentsListVersions: mocks.listVersions, + executeDeploymentsPromote: mocks.promote, + executeDeploymentsUndeploy: mocks.undeploy, +})) + +import { DelegatedWorkspaceAuthorizationError } from '@/lib/core/application' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { executeDeploymentsTool } from '@/lib/internal/deployments/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' +import { WORKFLOW_DELEGATION_AUDIENCE } from '@/lib/workflows/application/authorization' + +const INPUTS = { + deployments_deploy: { workflowId: 'workflow-1', name: 'Release 4' }, + deployments_undeploy: { workflowId: 'workflow-1' }, + deployments_promote: { workflowId: 'workflow-1', version: 4 }, + deployments_list_versions: { workflowId: 'workflow-1' }, + deployments_get_version: { workflowId: 'workflow-1', version: 4 }, +} as const + +const DISPATCH = { + deployments_deploy: mocks.deploy, + deployments_undeploy: mocks.undeploy, + deployments_promote: mocks.promote, + deployments_list_versions: mocks.listVersions, + deployments_get_version: mocks.getVersion, +} as const + +function request( + toolId: keyof typeof INPUTS, + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId, + input: INPUTS[toolId], + headers: new Headers(), + context: { + ...createExecutionContext({ workflowId: 'origin-workflow' }), + executionId: 'execution-1', + userId: 'user-1', + workspaceId: 'workspace-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeDeploymentsTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.createPrincipal.mockResolvedValue({ + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'workspace-1', + }) + for (const operation of Object.values(DISPATCH)) { + operation.mockResolvedValue({ success: true, output: { ok: true } }) + } + }) + + it.each(Object.keys(INPUTS) as Array)( + 'binds trusted workspace scope and dispatches %s', + async (toolId) => { + const executionRequest = request(toolId, { + input: { ...INPUTS[toolId], workspaceId: 'workspace-attacker' }, + }) + const response = await executeDeploymentsTool(executionRequest) + + expect(response.status).toBe(200) + expect(mocks.createPrincipal).toHaveBeenCalledWith({ + context: executionRequest.context, + audience: WORKFLOW_DELEGATION_AUDIENCE, + }) + expect(DISPATCH[toolId]).toHaveBeenCalledWith( + { ...INPUTS[toolId], workspaceId: 'workspace-1' }, + expect.objectContaining({ requestId: 'request-1' }) + ) + } + ) + + it('rejects missing trusted workspace scope before principal construction', async () => { + const response = await executeDeploymentsTool( + request('deployments_deploy', { + context: { + ...createExecutionContext({ workflowId: 'origin-workflow' }), + userId: 'user-1', + }, + }) + ) + + expect(response.status).toBe(401) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Authentication required', + }) + expect(mocks.createPrincipal).not.toHaveBeenCalled() + expect(mocks.deploy).not.toHaveBeenCalled() + }) + + it('preserves canonical validation and classified error status', async () => { + const invalid = await executeDeploymentsTool( + request('deployments_promote', { input: { workflowId: 'workflow-1' } }) + ) + expect(invalid.status).toBe(400) + expect(mocks.promote).not.toHaveBeenCalled() + + mocks.promote.mockRejectedValueOnce( + new OrchestrationError('not_found', 'Deployment version not found') + ) + const missing = await executeDeploymentsTool(request('deployments_promote')) + expect(missing.status).toBe(404) + await expect(missing.json()).resolves.toEqual({ + success: false, + error: 'Deployment version not found', + }) + }) + + it('conceals cross-workspace deployment targets as not found', async () => { + mocks.deploy.mockRejectedValueOnce(new DelegatedWorkspaceAuthorizationError()) + + const response = await executeDeploymentsTool(request('deployments_deploy')) + + expect(response.status).toBe(404) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Workflow not found in this workspace', + }) + }) + + it('propagates cancellation before principal or application work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeDeploymentsTool(request('deployments_deploy', { signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mocks.createPrincipal).not.toHaveBeenCalled() + expect(mocks.deploy).not.toHaveBeenCalled() + }) + + it('propagates cancellation that arrives during application work', async () => { + const controller = new AbortController() + mocks.deploy.mockImplementationOnce(async () => { + controller.abort(new DOMException('cancelled', 'AbortError')) + return { success: true } + }) + + await expect( + executeDeploymentsTool(request('deployments_deploy', { signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) +}) diff --git a/apps/sim/lib/internal/deployments/execute-tool.ts b/apps/sim/lib/internal/deployments/execute-tool.ts new file mode 100644 index 00000000000..13f70b35f68 --- /dev/null +++ b/apps/sim/lib/internal/deployments/execute-tool.ts @@ -0,0 +1,145 @@ +import { createLogger } from '@sim/logger' +import { isPlainRecord } from '@sim/utils/object' +import type { ZodError, ZodType } from 'zod' +import { getValidationErrorMessage } from '@/lib/api/server' +import { concealCrossTenantResourceError } from '@/lib/api/server/routes' +import { InvalidInternalDelegationBindingError } from '@/lib/auth/internal-delegation' +import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' +import { + deploymentsDeployBodySchema, + deploymentsGetVersionQuerySchema, + deploymentsListVersionsQuerySchema, + deploymentsPromoteBodySchema, + deploymentsUndeployBodySchema, +} from '@/lib/internal/deployments/input' +import { + executeDeploymentsDeploy, + executeDeploymentsGetVersion, + executeDeploymentsListVersions, + executeDeploymentsPromote, + executeDeploymentsUndeploy, +} from '@/lib/internal/deployments/operations' +import { createExecutorPrincipalFromExecutionContext } from '@/lib/internal/principals/executor' +import type { + InternalToolOperationCall, + InternalToolOperationHandler, +} from '@/lib/internal/tool-operations/types' +import { WORKFLOW_DELEGATION_AUDIENCE } from '@/lib/workflows/application/authorization' + +const logger = createLogger('DeploymentsInternalOperation') + +const FAILURE_MESSAGES: Record = { + deployments_deploy: 'Failed to deploy workflow', + deployments_undeploy: 'Failed to undeploy workflow', + deployments_promote: 'Failed to promote deployment version', + deployments_list_versions: 'Failed to list deployment versions', + deployments_get_version: 'Failed to get deployment version', +} + +function validationResponse(error: ZodError): Response { + return Response.json( + { success: false, error: getValidationErrorMessage(error, 'Invalid request data') }, + { status: 400 } + ) +} + +function parseInput(schema: ZodType, request: InternalToolOperationCall) { + const parsed = schema.safeParse({ + ...(isPlainRecord(request.input) ? request.input : {}), + workspaceId: request.context.workspaceId, + }) + return parsed.success ? parsed.data : validationResponse(parsed.error) +} + +function errorResponse(request: InternalToolOperationCall, error: unknown): Response { + if ( + error instanceof InvalidInternalDelegationBindingError || + (error instanceof Error && error.message === 'Authentication required') + ) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + + const classified = asOrchestrationError( + concealCrossTenantResourceError(error, 'Workflow not found in this workspace') + ) + if (classified) { + return Response.json( + { success: false, error: classified.message }, + { status: statusForOrchestrationError(classified.code) } + ) + } + + const message = FAILURE_MESSAGES[request.toolId] ?? 'Deployment operation failed' + logger.error(message, { + error, + requestId: request.requestId, + toolId: request.toolId, + }) + return Response.json({ success: false, error: message }, { status: 500 }) +} + +async function dispatchDeploymentTool( + request: InternalToolOperationCall +): Promise { + const workspaceId = request.context.workspaceId + if (!workspaceId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + + const principal = await createExecutorPrincipalFromExecutionContext({ + context: request.context, + audience: WORKFLOW_DELEGATION_AUDIENCE, + }) + const context = { + principal, + requestId: request.requestId, + signal: request.signal, + } + + switch (request.toolId) { + case 'deployments_deploy': { + const input = parseInput(deploymentsDeployBodySchema, request) + return input instanceof Response ? input : executeDeploymentsDeploy(input, context) + } + case 'deployments_undeploy': { + const input = parseInput(deploymentsUndeployBodySchema, request) + return input instanceof Response ? input : executeDeploymentsUndeploy(input, context) + } + case 'deployments_promote': { + const input = parseInput(deploymentsPromoteBodySchema, request) + return input instanceof Response ? input : executeDeploymentsPromote(input, context) + } + case 'deployments_list_versions': { + const input = parseInput(deploymentsListVersionsQuerySchema, request) + return input instanceof Response ? input : executeDeploymentsListVersions(input, context) + } + case 'deployments_get_version': { + const input = parseInput(deploymentsGetVersionQuerySchema, request) + return input instanceof Response ? input : executeDeploymentsGetVersion(input, context) + } + default: + return Response.json( + { success: false, error: `Unsupported Deployments tool: ${request.toolId}` }, + { status: 500 } + ) + } +} + +export const executeDeploymentsTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!Object.hasOwn(FAILURE_MESSAGES, request.toolId)) { + return Response.json( + { success: false, error: `Unsupported Deployments tool: ${request.toolId}` }, + { status: 500 } + ) + } + + try { + const result = await dispatchDeploymentTool(request) + request.signal?.throwIfAborted() + return result instanceof Response ? result : Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + return errorResponse(request, error) + } +} diff --git a/apps/sim/lib/internal/deployments/input.ts b/apps/sim/lib/internal/deployments/input.ts new file mode 100644 index 00000000000..9b01554c361 --- /dev/null +++ b/apps/sim/lib/internal/deployments/input.ts @@ -0,0 +1,49 @@ +import { z } from 'zod' +import { deploymentVersionMetadataFieldsSchema } from '@/lib/api/contracts/deployments' +import { workflowIdSchema, workspaceIdSchema } from '@/lib/api/contracts/primitives' + +/** Bounded to the Postgres `integer` range of `workflow_deployment_version.version`. */ +const versionSchema = z + .number() + .int('Version must be an integer') + .min(1, 'Version must be a positive integer') + .max(2147483647, 'Version is out of range') + +export const deploymentsDeployBodySchema = z.object({ + workflowId: workflowIdSchema, + workspaceId: workspaceIdSchema, + name: deploymentVersionMetadataFieldsSchema.shape.name, + description: deploymentVersionMetadataFieldsSchema.shape.description, +}) + +export type DeploymentsDeployBody = z.output + +export const deploymentsUndeployBodySchema = z.object({ + workflowId: workflowIdSchema, + workspaceId: workspaceIdSchema, +}) + +export type DeploymentsUndeployBody = z.output + +export const deploymentsPromoteBodySchema = z.object({ + workflowId: workflowIdSchema, + workspaceId: workspaceIdSchema, + version: versionSchema, +}) + +export type DeploymentsPromoteBody = z.output + +export const deploymentsListVersionsQuerySchema = z.object({ + workflowId: workflowIdSchema, + workspaceId: workspaceIdSchema, +}) + +export type DeploymentsListVersionsQuery = z.output + +export const deploymentsGetVersionQuerySchema = z.object({ + workflowId: workflowIdSchema, + workspaceId: workspaceIdSchema, + version: z.coerce.number().pipe(versionSchema), +}) + +export type DeploymentsGetVersionQuery = z.output diff --git a/apps/sim/lib/internal/deployments/operations.test.ts b/apps/sim/lib/internal/deployments/operations.test.ts new file mode 100644 index 00000000000..2f94d3bdd8e --- /dev/null +++ b/apps/sim/lib/internal/deployments/operations.test.ts @@ -0,0 +1,149 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + deploy: vi.fn(), + getVersion: vi.fn(), + listVersions: vi.fn(), + promote: vi.fn(), + undeploy: vi.fn(), +})) + +vi.mock('@/lib/internal/deployments/client', () => ({ + deployWorkflowDeployment: mocks.deploy, + getWorkflowDeploymentVersion: mocks.getVersion, + listWorkflowDeploymentVersions: mocks.listVersions, + promoteWorkflowDeployment: mocks.promote, + undeployWorkflowDeployment: mocks.undeploy, +})) + +import { + executeDeploymentsDeploy, + executeDeploymentsGetVersion, + executeDeploymentsListVersions, + executeDeploymentsPromote, + executeDeploymentsUndeploy, +} from '@/lib/internal/deployments/operations' + +const context = { + principal: { + kind: 'delegated' as const, + serviceId: 'executor' as const, + subjectUserId: 'user-1', + workspaceId: 'workspace-1', + delegationId: 'delegation-1', + audience: 'sim:workflows', + issuedAt: new Date('2026-01-01T00:00:00Z'), + expiresAt: new Date('2026-01-01T00:05:00Z'), + }, + requestId: 'request-1', +} + +describe('deployment tool operations', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('preserves deploy and promote result envelopes', async () => { + const activeDeployment = { deploymentVersionId: 'version-3', version: 3 } + mocks.deploy.mockResolvedValue({ + deployedAt: new Date('2026-06-12T00:00:00Z'), + version: 3, + activeDeployment, + warnings: ['schedule sync pending'], + }) + mocks.promote.mockResolvedValue({ + deployedAt: new Date('2026-06-13T00:00:00Z'), + activeDeployment, + }) + + await expect( + executeDeploymentsDeploy({ workflowId: 'workflow-1', workspaceId: 'workspace-1' }, context) + ).resolves.toEqual({ + success: true, + output: { + workflowId: 'workflow-1', + isDeployed: true, + deployedAt: '2026-06-12T00:00:00.000Z', + version: 3, + activeDeployment, + latestDeploymentAttempt: undefined, + warnings: ['schedule sync pending'], + }, + }) + await expect( + executeDeploymentsPromote( + { workflowId: 'workflow-1', workspaceId: 'workspace-1', version: 3 }, + context + ) + ).resolves.toEqual({ + success: true, + output: { + workflowId: 'workflow-1', + isDeployed: true, + deployedAt: '2026-06-13T00:00:00.000Z', + version: 3, + activeDeployment, + latestDeploymentAttempt: undefined, + warnings: [], + }, + }) + }) + + it('preserves undeploy, list, and sanitized-version envelopes', async () => { + const versions = [{ id: 'version-3', version: 3 }] + const state = { blocks: {}, edges: [] } + mocks.undeploy.mockResolvedValue({ warnings: [] }) + mocks.listVersions.mockResolvedValue({ versions }) + mocks.getVersion.mockResolvedValue({ + version: { + version: 3, + name: 'Release 3', + description: null, + isActive: false, + createdAt: '2026-06-12T00:00:00.000Z', + state, + }, + }) + + await expect( + executeDeploymentsUndeploy({ workflowId: 'workflow-1', workspaceId: 'workspace-1' }, context) + ).resolves.toEqual({ + success: true, + output: { + workflowId: 'workflow-1', + isDeployed: false, + deployedAt: null, + warnings: [], + }, + }) + await expect( + executeDeploymentsListVersions( + { workflowId: 'workflow-1', workspaceId: 'workspace-1' }, + context + ) + ).resolves.toEqual({ + success: true, + output: { workflowId: 'workflow-1', versions }, + }) + await expect( + executeDeploymentsGetVersion( + { workflowId: 'workflow-1', workspaceId: 'workspace-1', version: 3 }, + context + ) + ).resolves.toEqual({ + success: true, + output: { + workflowId: 'workflow-1', + version: 3, + name: 'Release 3', + description: null, + isActive: false, + createdAt: '2026-06-12T00:00:00.000Z', + deployedState: state, + }, + }) + }) +}) diff --git a/apps/sim/lib/internal/deployments/operations.ts b/apps/sim/lib/internal/deployments/operations.ts new file mode 100644 index 00000000000..311bfc881fc --- /dev/null +++ b/apps/sim/lib/internal/deployments/operations.ts @@ -0,0 +1,103 @@ +import { + type DeploymentApplicationClientContext, + deployWorkflowDeployment, + getWorkflowDeploymentVersion, + listWorkflowDeploymentVersions, + promoteWorkflowDeployment, + undeployWorkflowDeployment, +} from '@/lib/internal/deployments/client' +import type { + DeploymentsDeployBody, + DeploymentsGetVersionQuery, + DeploymentsListVersionsQuery, + DeploymentsPromoteBody, + DeploymentsUndeployBody, +} from '@/lib/internal/deployments/input' + +function serializeDeploymentTimestamp(value?: Date): string | null { + return value?.toISOString() ?? null +} + +export async function executeDeploymentsDeploy( + input: DeploymentsDeployBody, + context: DeploymentApplicationClientContext +) { + const result = await deployWorkflowDeployment(input, context) + return { + success: true, + output: { + workflowId: input.workflowId, + isDeployed: Boolean(result.activeDeployment), + deployedAt: serializeDeploymentTimestamp(result.deployedAt), + version: result.version, + activeDeployment: result.activeDeployment, + latestDeploymentAttempt: result.latestDeploymentAttempt, + warnings: result.warnings ?? [], + }, + } +} + +export async function executeDeploymentsUndeploy( + input: DeploymentsUndeployBody, + context: DeploymentApplicationClientContext +) { + const result = await undeployWorkflowDeployment(input, context) + return { + success: true, + output: { + workflowId: input.workflowId, + isDeployed: false, + deployedAt: null, + warnings: result.warnings ?? [], + }, + } +} + +export async function executeDeploymentsPromote( + input: DeploymentsPromoteBody, + context: DeploymentApplicationClientContext +) { + const result = await promoteWorkflowDeployment(input, context) + return { + success: true, + output: { + workflowId: input.workflowId, + isDeployed: Boolean(result.activeDeployment), + deployedAt: serializeDeploymentTimestamp(result.deployedAt), + version: input.version, + activeDeployment: result.activeDeployment, + latestDeploymentAttempt: result.latestDeploymentAttempt, + warnings: result.warnings ?? [], + }, + } +} + +export async function executeDeploymentsListVersions( + input: DeploymentsListVersionsQuery, + context: DeploymentApplicationClientContext +) { + const { versions } = await listWorkflowDeploymentVersions(input, context) + return { + success: true, + output: { workflowId: input.workflowId, versions }, + } +} + +export async function executeDeploymentsGetVersion( + input: DeploymentsGetVersionQuery, + context: DeploymentApplicationClientContext +) { + const { version } = await getWorkflowDeploymentVersion(input, context) + return { + success: true, + output: { + workflowId: input.workflowId, + version: version.version, + name: version.name, + description: version.description, + isActive: version.isActive, + createdAt: version.createdAt, + deployedState: version.state, + }, + } +} diff --git a/apps/sim/lib/internal/discord/client.test.ts b/apps/sim/lib/internal/discord/client.test.ts new file mode 100644 index 00000000000..e0fecc2f939 --- /dev/null +++ b/apps/sim/lib/internal/discord/client.test.ts @@ -0,0 +1,27 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ readResponseJsonWithLimit: vi.fn() })) + +vi.mock('@/lib/core/utils/stream-limits', () => ({ + readResponseJsonWithLimit: mocks.readResponseJsonWithLimit, +})) + +import { sendDiscordMessage } from '@/lib/internal/discord/client' + +describe('sendDiscordMessage', () => { + it('does not swallow cancellation while reading the provider response', async () => { + const controller = new AbortController() + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(null, { status: 200 }))) + mocks.readResponseJsonWithLimit.mockImplementationOnce(async () => { + controller.abort(new DOMException('cancelled', 'AbortError')) + throw controller.signal.reason + }) + + await expect( + sendDiscordMessage('token', '123', '{}', 'json', controller.signal) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) +}) diff --git a/apps/sim/lib/internal/discord/client.ts b/apps/sim/lib/internal/discord/client.ts new file mode 100644 index 00000000000..df86a4a1e15 --- /dev/null +++ b/apps/sim/lib/internal/discord/client.ts @@ -0,0 +1,45 @@ +import { isRecordLike } from '@sim/utils/object' +import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' +import { DiscordOperationError } from '@/lib/internal/discord/errors' + +export async function sendDiscordMessage( + botToken: string, + channelId: string, + body: BodyInit, + contentType: 'json' | 'multipart', + signal?: AbortSignal +): Promise> { + signal?.throwIfAborted() + const response = await fetch(`https://discord.com/api/v10/channels/${channelId}/messages`, { + method: 'POST', + headers: { + Authorization: `Bot ${botToken}`, + ...(contentType === 'json' ? { 'Content-Type': 'application/json' } : {}), + }, + body, + signal, + }) + let data: unknown + try { + data = await readResponseJsonWithLimit(response, { + maxBytes: MAX_JSON_API_RESPONSE_BYTES, + label: 'Discord message response', + signal, + }) + } catch (error) { + signal?.throwIfAborted() + data = null + } + if (!response.ok) { + const message = isRecordLike(data) && typeof data.message === 'string' ? data.message : null + throw new DiscordOperationError( + message || + (contentType === 'multipart' + ? 'Failed to send message with files' + : 'Failed to send message'), + response.status + ) + } + return isRecordLike(data) ? data : {} +} diff --git a/apps/sim/lib/internal/discord/errors.ts b/apps/sim/lib/internal/discord/errors.ts new file mode 100644 index 00000000000..838dcd75dc2 --- /dev/null +++ b/apps/sim/lib/internal/discord/errors.ts @@ -0,0 +1,10 @@ +export class DiscordOperationError extends Error { + constructor( + message: string, + readonly status: number, + readonly body: Record = { success: false, error: message } + ) { + super(message) + this.name = 'DiscordOperationError' + } +} diff --git a/apps/sim/lib/internal/discord/execute-tool.ts b/apps/sim/lib/internal/discord/execute-tool.ts new file mode 100644 index 00000000000..023a0be5a79 --- /dev/null +++ b/apps/sim/lib/internal/discord/execute-tool.ts @@ -0,0 +1,67 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { DiscordOperationError } from '@/lib/internal/discord/errors' +import { executeDiscordSendMessage } from '@/lib/internal/discord/operations' +import { discordSendMessageInputSchema } from '@/lib/internal/discord/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const logger = createLogger('DiscordToolExecution') + +function inputSizeError(input: unknown): Response | null { + let serialized: string + try { + serialized = JSON.stringify(input) ?? '' + } catch { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + return Buffer.byteLength(serialized) > DEFAULT_MAX_JSON_BODY_BYTES + ? Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + : null +} + +export const executeDiscordTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'discord_send_message') { + return Response.json( + { success: false, error: `Unsupported Discord tool: ${request.toolId}` }, + { status: 500 } + ) + } + const userId = request.context.userId + if (!userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + const sizeError = inputSizeError(request.input) + if (sizeError) return sizeError + const parsed = discordSendMessageInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { success: false, error: getValidationErrorMessage(parsed.error, 'Invalid request data') }, + { status: 400 } + ) + } + try { + const result = await executeDiscordSendMessage(parsed.data, { + requestId: request.requestId, + signal: request.signal, + userId, + }) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof DiscordOperationError) { + return Response.json(error.body, { status: error.status }) + } + const message = getErrorMessage(error, 'Unknown error occurred') + logger.error('Discord message send failed', { error: message, requestId: request.requestId }) + return Response.json({ success: false, error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/discord/operations.test.ts b/apps/sim/lib/internal/discord/operations.test.ts new file mode 100644 index 00000000000..6dfac7443b6 --- /dev/null +++ b/apps/sim/lib/internal/discord/operations.test.ts @@ -0,0 +1,122 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const fileMocks = vi.hoisted(() => ({ + assertToolFileAccess: vi.fn(), + downloadServableFilesWithinBudget: vi.fn(), + processFilesToUserFiles: vi.fn(), +})) + +const discordMocks = vi.hoisted(() => ({ + sendDiscordMessage: vi.fn(), +})) + +vi.mock('@/lib/internal/discord/client', () => ({ + sendDiscordMessage: discordMocks.sendDiscordMessage, +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: fileMocks.assertToolFileAccess, +})) + +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + processFilesToUserFiles: fileMocks.processFilesToUserFiles, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFilesWithinBudget: fileMocks.downloadServableFilesWithinBudget, +})) + +import { DiscordOperationError } from '@/lib/internal/discord/errors' +import { executeDiscordSendMessage } from '@/lib/internal/discord/operations' + +describe('executeDiscordSendMessage', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.unstubAllGlobals() + }) + + it('preserves provider errors when every supplied file is filtered out', async () => { + fileMocks.processFilesToUserFiles.mockReturnValue([]) + discordMocks.sendDiscordMessage.mockRejectedValue( + new DiscordOperationError('Missing Access', 403) + ) + + await expect( + executeDiscordSendMessage( + { + botToken: 'bot-token', + channelId: '123', + content: 'hello', + files: [{ key: 'workspace/file.txt', name: 'file.txt', size: 4 }], + }, + { requestId: 'request-1', userId: 'user-1' } + ) + ).rejects.toEqual(new DiscordOperationError('Missing Access', 403)) + }) + + it('returns a committed text message when cancellation arrives after the send', async () => { + const controller = new AbortController() + discordMocks.sendDiscordMessage.mockImplementation(async () => { + controller.abort() + return { id: 'message-1', content: 'hello' } + }) + + await expect( + executeDiscordSendMessage( + { botToken: 'bot-token', channelId: '123', content: 'hello' }, + { + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + } + ) + ).resolves.toEqual({ + success: true, + output: { + data: { id: 'message-1', content: 'hello' }, + message: 'hello', + }, + }) + }) + + it('returns a committed multipart message when cancellation arrives after the send', async () => { + const controller = new AbortController() + fileMocks.processFilesToUserFiles.mockReturnValue([ + { key: 'workspace/file.txt', name: 'file.txt', size: 4, type: 'text/plain' }, + ]) + fileMocks.assertToolFileAccess.mockResolvedValue(null) + fileMocks.downloadServableFilesWithinBudget.mockResolvedValue([ + { buffer: Buffer.from('file'), contentType: 'text/plain' }, + ]) + discordMocks.sendDiscordMessage.mockImplementation(async () => { + controller.abort() + return { id: 'message-1', content: 'hello' } + }) + + await expect( + executeDiscordSendMessage( + { + botToken: 'bot-token', + channelId: '123', + content: 'hello', + files: [{ key: 'workspace/file.txt', name: 'file.txt', size: 4 }], + }, + { + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + } + ) + ).resolves.toMatchObject({ + success: true, + output: { + data: { id: 'message-1', content: 'hello' }, + fileCount: 1, + message: 'hello', + }, + }) + }) +}) diff --git a/apps/sim/lib/internal/discord/operations.ts b/apps/sim/lib/internal/discord/operations.ts new file mode 100644 index 00000000000..00a5d2211eb --- /dev/null +++ b/apps/sim/lib/internal/discord/operations.ts @@ -0,0 +1,115 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { isRecordLike } from '@sim/utils/object' +import { validateNumericId } from '@/lib/core/security/input-validation' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { sendDiscordMessage } from '@/lib/internal/discord/client' +import { DiscordOperationError } from '@/lib/internal/discord/errors' +import type { DiscordSendMessageInput } from '@/lib/internal/discord/schema' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/doc-not-ready' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +const logger = createLogger('DiscordOperations') + +export interface DiscordOperationContext { + requestId: string + signal?: AbortSignal + userId: string +} + +async function deniedBody(response: Response): Promise> { + const body: unknown = await response.json().catch(() => null) + return isRecordLike(body) ? body : { success: false, error: 'File not found' } +} + +async function textMessage(input: DiscordSendMessageInput, signal?: AbortSignal) { + const data = await sendDiscordMessage( + input.botToken, + input.channelId, + JSON.stringify({ content: input.content || '' }), + 'json', + signal + ) + return { + success: true, + output: { message: typeof data.content === 'string' ? data.content : undefined, data }, + } +} + +export async function executeDiscordSendMessage( + input: DiscordSendMessageInput, + context: DiscordOperationContext +) { + context.signal?.throwIfAborted() + const channelIdValidation = validateNumericId(input.channelId, 'channelId') + if (!channelIdValidation.isValid) { + throw new DiscordOperationError(channelIdValidation.error || 'Invalid channelId', 400) + } + if (!input.files || input.files.length === 0) return textMessage(input, context.signal) + + const userFiles = processFilesToUserFiles(input.files, context.requestId, logger) + if (userFiles.length === 0) return textMessage(input, context.signal) + for (const file of userFiles) { + context.signal?.throwIfAborted() + const denied = await assertToolFileAccess(file.key, context.userId, context.requestId, logger) + context.signal?.throwIfAborted() + if (denied) { + throw new DiscordOperationError('File not found', denied.status, await deniedBody(denied)) + } + } + + let resolved: Awaited> + try { + resolved = await downloadServableFilesWithinBudget(userFiles, context.requestId, logger, { + totalMaxBytes: MAX_BUFFERED_TRANSFER_BYTES, + label: 'Total attachment size', + signal: context.signal, + }) + } catch (error) { + context.signal?.throwIfAborted() + if (isDocNotReadyError(error)) { + throw new DiscordOperationError(docNotReadyMessage(), 409) + } + const message = `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}` + throw new DiscordOperationError(message, isPayloadSizeLimitError(error) ? 413 : 500) + } + + const formData = new FormData() + formData.append('payload_json', JSON.stringify({ content: input.content || '' })) + const files = userFiles.map((file, index) => { + const downloaded = resolved[index] + if (!downloaded) + throw new DiscordOperationError('Failed to download attachment: Missing file data', 500) + const mimeType = downloaded.contentType || file.type || 'application/octet-stream' + formData.append( + `files[${index}]`, + new Blob([new Uint8Array(downloaded.buffer)], { type: mimeType }), + file.name + ) + return { + name: file.name, + mimeType, + data: downloaded.buffer.toString('base64'), + size: downloaded.buffer.length, + } + }) + const data = await sendDiscordMessage( + input.botToken, + input.channelId, + formData, + 'multipart', + context.signal + ) + return { + success: true, + output: { + message: typeof data.content === 'string' ? data.content : undefined, + data, + fileCount: userFiles.length, + files, + }, + } +} diff --git a/apps/sim/lib/internal/discord/schema.ts b/apps/sim/lib/internal/discord/schema.ts new file mode 100644 index 00000000000..79656f3068e --- /dev/null +++ b/apps/sim/lib/internal/discord/schema.ts @@ -0,0 +1,11 @@ +import { z } from 'zod' +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' + +export const discordSendMessageInputSchema = z.object({ + botToken: z.string().min(1, 'Bot token is required'), + channelId: z.string().min(1, 'Channel ID is required'), + content: z.string().optional().nullable(), + files: RawFileInputArraySchema.optional().nullable(), +}) + +export type DiscordSendMessageInput = z.output diff --git a/apps/sim/lib/internal/docusign/client.test.ts b/apps/sim/lib/internal/docusign/client.test.ts new file mode 100644 index 00000000000..42f21599e8e --- /dev/null +++ b/apps/sim/lib/internal/docusign/client.test.ts @@ -0,0 +1,91 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { DocuSignClient } from '@/lib/internal/docusign/client' + +function accountResponse(): Response { + return Response.json({ + accounts: [ + { + is_default: true, + account_id: 'account-1', + base_uri: 'https://demo.docusign.net', + }, + ], + }) +} + +describe('DocuSignClient', () => { + const fetchMock = vi.fn() + + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', fetchMock) + }) + + afterEach(() => vi.unstubAllGlobals()) + + it('resolves the default account and forwards OAuth credentials', async () => { + fetchMock + .mockResolvedValueOnce(accountResponse()) + .mockResolvedValueOnce(Response.json({ envelopeId: 'envelope-1' })) + const controller = new AbortController() + const client = await DocuSignClient.create('access-token', controller.signal) + await client.json('/envelopes/envelope-1', {}, 'Envelope', 'Failed', controller.signal) + + expect(fetchMock.mock.calls[0]?.[1]).toMatchObject({ + headers: { Authorization: 'Bearer access-token' }, + }) + expect(fetchMock).toHaveBeenLastCalledWith( + 'https://demo.docusign.net/restapi/v2.1/accounts/account-1/envelopes/envelope-1', + expect.objectContaining({ + headers: expect.objectContaining({ Authorization: 'Bearer access-token' }), + }) + ) + }) + + it('preserves provider status and error messages', async () => { + fetchMock + .mockResolvedValueOnce(accountResponse()) + .mockResolvedValueOnce(Response.json({ message: 'Envelope not found' }, { status: 404 })) + const client = await DocuSignClient.create('access-token') + + await expect( + client.json('/envelopes/missing', {}, 'Envelope', 'Failed to get envelope') + ).rejects.toMatchObject({ + status: 404, + body: { success: false, error: 'Envelope not found' }, + }) + }) + + it('caps binary document downloads before materializing them', async () => { + let cancelled = false + fetchMock.mockResolvedValueOnce(accountResponse()).mockResolvedValueOnce( + new Response( + new ReadableStream({ + cancel: () => { + cancelled = true + }, + }), + { headers: { 'Content-Length': String(25 * 1024 * 1024 + 1) } } + ) + ) + const client = await DocuSignClient.create('access-token') + + await expect(client.document('envelope-1', 'combined')).rejects.toMatchObject({ + name: 'PayloadSizeLimitError', + }) + expect(cancelled).toBe(true) + }) + + it('does not start provider work after cancellation', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect(DocuSignClient.create('access-token', controller.signal)).rejects.toMatchObject({ + name: 'AbortError', + }) + expect(fetchMock).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/docusign/client.ts b/apps/sim/lib/internal/docusign/client.ts new file mode 100644 index 00000000000..3e8894c5950 --- /dev/null +++ b/apps/sim/lib/internal/docusign/client.ts @@ -0,0 +1,155 @@ +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + readResponseJsonWithLimit, + readResponseTextWithLimit, + readResponseToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' +import { DocuSignOperationError } from '@/lib/internal/docusign/errors' +import { getDocusignOAuthUrl } from '@/lib/oauth/docusign' + +const MAX_DOCUSIGN_JSON_BYTES = 2 * 1024 * 1024 +export const MAX_DOCUSIGN_DOCUMENT_BYTES = 25 * 1024 * 1024 +const DOCUSIGN_FETCH_TIMEOUT_MS = 30_000 + +export type DocuSignJson = Record + +function providerError(data: DocuSignJson, fallback: string): string { + return ( + (typeof data.message === 'string' && data.message) || + (typeof data.errorCode === 'string' && data.errorCode) || + fallback + ) +} + +async function fetchDocusign( + input: string, + init: RequestInit = {}, + parentSignal?: AbortSignal +): Promise { + parentSignal?.throwIfAborted() + const controller = new AbortController() + const timeout = setTimeout( + () => controller.abort(new Error('DocuSign request timed out')), + DOCUSIGN_FETCH_TIMEOUT_MS + ) + const abort = () => controller.abort(parentSignal?.reason ?? new Error('Request aborted')) + parentSignal?.addEventListener('abort', abort, { once: true }) + try { + return await fetch(input, { ...init, signal: controller.signal }) + } finally { + clearTimeout(timeout) + parentSignal?.removeEventListener('abort', abort) + } +} + +async function readJson( + response: Response, + label: string, + signal?: AbortSignal +): Promise { + return readResponseJsonWithLimit(response, { + maxBytes: MAX_DOCUSIGN_JSON_BYTES, + label, + signal, + }) +} + +export class DocuSignClient { + private constructor( + private readonly accessToken: string, + private readonly apiBase: string + ) {} + + static async create(accessToken: string, signal?: AbortSignal): Promise { + const response = await fetchDocusign( + getDocusignOAuthUrl('/oauth/userinfo'), + { headers: { Authorization: `Bearer ${accessToken}` } }, + signal + ) + if (!response.ok) { + await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'DocuSign account error response', + signal, + }).catch(() => '') + throw new DocuSignOperationError( + `Failed to resolve DocuSign account: ${response.status}`, + 500 + ) + } + const data = await readJson(response, 'DocuSign account response', signal) + const accounts = Array.isArray(data.accounts) ? data.accounts : [] + const account = + accounts.find( + (candidate) => candidate && typeof candidate === 'object' && candidate.is_default === true + ) ?? accounts[0] + if (!account || typeof account !== 'object') { + throw new DocuSignOperationError('No DocuSign accounts found for this user', 500) + } + const baseUri = typeof account.base_uri === 'string' ? account.base_uri : undefined + const accountId = typeof account.account_id === 'string' ? account.account_id : undefined + if (!baseUri) throw new DocuSignOperationError('DocuSign account is missing base_uri', 500) + if (!accountId) throw new DocuSignOperationError('DocuSign account is missing account_id', 500) + return new DocuSignClient(accessToken, `${baseUri}/restapi/v2.1/accounts/${accountId}`) + } + + async json( + path: string, + init: RequestInit, + label: string, + fallback: string, + signal?: AbortSignal + ): Promise { + const response = await fetchDocusign( + `${this.apiBase}${path}`, + { + ...init, + headers: { + Authorization: `Bearer ${this.accessToken}`, + 'Content-Type': 'application/json', + ...init.headers, + }, + }, + signal + ) + const data = await readJson(response, label, signal) + if (!response.ok) { + const error = providerError(data, fallback) + throw new DocuSignOperationError(error, response.status) + } + return data + } + + async document( + envelopeId: string, + documentId: string, + signal?: AbortSignal + ): Promise<{ buffer: Buffer; contentType: string; fileName: string }> { + const response = await fetchDocusign( + `${this.apiBase}/envelopes/${envelopeId}/documents/${documentId}`, + { headers: { Authorization: `Bearer ${this.accessToken}` } }, + signal + ) + if (!response.ok) { + const details = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'DocuSign document error response', + signal, + }).catch(() => '') + throw new DocuSignOperationError( + `Failed to download document: ${response.status} ${details}`, + response.status + ) + } + const contentType = response.headers.get('content-type') || 'application/pdf' + const contentDisposition = response.headers.get('content-disposition') || '' + const match = contentDisposition.match(/filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/) + const fileName = match ? match[1].replace(/['"]/g, '') : `document-${documentId}.pdf` + const buffer = await readResponseToBufferWithLimit(response, { + maxBytes: MAX_DOCUSIGN_DOCUMENT_BYTES, + label: 'DocuSign document download', + signal, + }) + return { buffer, contentType, fileName } + } +} diff --git a/apps/sim/lib/internal/docusign/errors.ts b/apps/sim/lib/internal/docusign/errors.ts new file mode 100644 index 00000000000..de26eca38e8 --- /dev/null +++ b/apps/sim/lib/internal/docusign/errors.ts @@ -0,0 +1,10 @@ +export class DocuSignOperationError extends Error { + constructor( + message: string, + readonly status: number, + readonly body: Record = { success: false, error: message } + ) { + super(message) + this.name = 'DocuSignOperationError' + } +} diff --git a/apps/sim/lib/internal/docusign/execute-tool.test.ts b/apps/sim/lib/internal/docusign/execute-tool.test.ts new file mode 100644 index 00000000000..97bca0effad --- /dev/null +++ b/apps/sim/lib/internal/docusign/execute-tool.test.ts @@ -0,0 +1,134 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operations = vi.hoisted(() => ({ + executeDocuSignCreateFromTemplate: vi.fn(), + executeDocuSignDownloadDocument: vi.fn(), + executeDocuSignGetEnvelope: vi.fn(), + executeDocuSignListEnvelopes: vi.fn(), + executeDocuSignListRecipients: vi.fn(), + executeDocuSignListTemplates: vi.fn(), + executeDocuSignSendEnvelope: vi.fn(), + executeDocuSignVoidEnvelope: vi.fn(), +})) + +vi.mock('@/lib/internal/docusign/operations', () => operations) + +import { DocuSignOperationError } from '@/lib/internal/docusign/errors' +import { executeDocuSignTool } from '@/lib/internal/docusign/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const AUTH = { accessToken: 'access-token' } + +function request(overrides: Partial = {}): InternalToolOperationCall { + return { + toolId: 'docusign_list_templates', + input: AUTH, + headers: new Headers(), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + userId: 'user-1', + executionId: 'execution-1', + }, + requestId: 'request-1', + ...overrides, + } as InternalToolOperationCall +} + +const CASES = [ + [ + 'docusign_create_from_template', + { ...AUTH, templateId: 'template-1', templateRoles: '[]' }, + operations.executeDocuSignCreateFromTemplate, + ], + [ + 'docusign_download_document', + { ...AUTH, envelopeId: 'envelope-1' }, + operations.executeDocuSignDownloadDocument, + ], + [ + 'docusign_get_envelope', + { ...AUTH, envelopeId: 'envelope-1' }, + operations.executeDocuSignGetEnvelope, + ], + ['docusign_list_envelopes', AUTH, operations.executeDocuSignListEnvelopes], + [ + 'docusign_list_recipients', + { ...AUTH, envelopeId: 'envelope-1' }, + operations.executeDocuSignListRecipients, + ], + ['docusign_list_templates', AUTH, operations.executeDocuSignListTemplates], + [ + 'docusign_send_envelope', + { ...AUTH, emailSubject: 'Sign', signerEmail: 'a@example.com', signerName: 'A' }, + operations.executeDocuSignSendEnvelope, + ], + [ + 'docusign_void_envelope', + { ...AUTH, envelopeId: 'envelope-1', voidedReason: 'Cancelled' }, + operations.executeDocuSignVoidEnvelope, + ], +] as const + +describe('executeDocuSignTool', () => { + beforeEach(() => vi.clearAllMocks()) + + it.each(CASES)( + 'dispatches %s with trusted execution context', + async (toolId, input, operation) => { + const controller = new AbortController() + operation.mockResolvedValue({ toolId }) + const response = await executeDocuSignTool( + request({ toolId, input, signal: controller.signal }) + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ toolId }) + expect(operation).toHaveBeenCalledWith(input, { + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }) + } + ) + + it('authenticates before validating input', async () => { + const response = await executeDocuSignTool( + request({ input: null, context: createExecutionContext({ workflowId: 'workflow-1' }) }) + ) + expect(response.status).toBe(401) + await expect(response.json()).resolves.toEqual({ success: false, error: 'Unauthorized' }) + }) + + it('returns route-compatible validation errors', async () => { + const response = await executeDocuSignTool(request({ input: {} })) + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ success: false }) + expect(operations.executeDocuSignListTemplates).not.toHaveBeenCalled() + }) + + it('preserves typed operation status and body', async () => { + operations.executeDocuSignListTemplates.mockRejectedValue( + new DocuSignOperationError('Rate limited', 429) + ) + const response = await executeDocuSignTool(request()) + expect(response.status).toBe(429) + await expect(response.json()).resolves.toEqual({ success: false, error: 'Rate limited' }) + }) + + it('stops before provider work after cancellation', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + await expect(executeDocuSignTool(request({ signal: controller.signal }))).rejects.toMatchObject( + { name: 'AbortError' } + ) + expect(operations.executeDocuSignListTemplates).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/docusign/execute-tool.ts b/apps/sim/lib/internal/docusign/execute-tool.ts new file mode 100644 index 00000000000..5ed0366b6dd --- /dev/null +++ b/apps/sim/lib/internal/docusign/execute-tool.ts @@ -0,0 +1,197 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { z } from 'zod' +import { getValidationErrorMessage } from '@/lib/api/server' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { DocuSignOperationError } from '@/lib/internal/docusign/errors' +import { + type DocuSignOperationContext, + executeDocuSignCreateFromTemplate, + executeDocuSignDownloadDocument, + executeDocuSignGetEnvelope, + executeDocuSignListEnvelopes, + executeDocuSignListRecipients, + executeDocuSignListTemplates, + executeDocuSignSendEnvelope, + executeDocuSignVoidEnvelope, +} from '@/lib/internal/docusign/operations' +import type { + InternalToolOperationCall, + InternalToolOperationHandler, +} from '@/lib/internal/tool-operations/types' + +const auth = { accessToken: z.string().min(1, 'Access token is required') } +const schemas = { + docusign_create_from_template: z.object({ + ...auth, + templateId: z.string().min(1), + emailSubject: z.string().optional(), + emailBody: z.string().optional(), + templateRoles: z.string().optional().default(''), + status: z.string().optional(), + }), + docusign_download_document: z.object({ + ...auth, + envelopeId: z.string().min(1), + documentId: z.string().optional(), + }), + docusign_get_envelope: z.object({ ...auth, envelopeId: z.string().min(1) }), + docusign_list_envelopes: z.object({ + ...auth, + fromDate: z.string().optional(), + toDate: z.string().optional(), + envelopeStatus: z.string().optional(), + searchText: z.string().optional(), + count: z.string().optional(), + }), + docusign_list_recipients: z.object({ ...auth, envelopeId: z.string().min(1) }), + docusign_list_templates: z.object({ + ...auth, + searchText: z.string().optional(), + count: z.string().optional(), + }), + docusign_send_envelope: z.object({ + ...auth, + emailSubject: z.string().min(1), + emailBody: z.string().optional(), + signerEmail: z.string().min(1), + signerName: z.string().min(1), + ccEmail: z.string().optional(), + ccName: z.string().optional(), + file: z.unknown().optional(), + status: z.string().optional(), + }), + docusign_void_envelope: z.object({ + ...auth, + envelopeId: z.string().min(1), + voidedReason: z.string().min(1), + }), +} as const + +type DocuSignToolId = keyof typeof schemas + +function isDocuSignToolId(value: string): value is DocuSignToolId { + return Object.hasOwn(schemas, value) +} + +function requiredInputError(toolId: DocuSignToolId, input: unknown): string | undefined { + if (!input || typeof input !== 'object' || Array.isArray(input)) return undefined + const value = input as Record + if (toolId === 'docusign_send_envelope') { + if (!value.signerEmail || !value.signerName || !value.emailSubject) { + return 'signerEmail, signerName, and emailSubject are required' + } + } + if (toolId === 'docusign_create_from_template' && !value.templateId) { + return 'templateId is required' + } + if ( + (toolId === 'docusign_download_document' || + toolId === 'docusign_get_envelope' || + toolId === 'docusign_list_recipients' || + toolId === 'docusign_void_envelope') && + !value.envelopeId + ) { + return 'envelopeId is required' + } + if (toolId === 'docusign_void_envelope' && !value.voidedReason) { + return 'voidedReason is required' + } + return undefined +} + +async function executeOperation( + schema: z.ZodType, + request: InternalToolOperationCall, + execute: (input: I, context: DocuSignOperationContext) => Promise +): Promise { + request.signal?.throwIfAborted() + const requiredError = requiredInputError(request.toolId as DocuSignToolId, request.input) + if (requiredError) { + return Response.json({ success: false, error: requiredError }, { status: 400 }) + } + const parsed = schema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { + success: false, + error: getValidationErrorMessage(parsed.error, 'Invalid request data'), + }, + { status: 400 } + ) + } + const userId = request.context.userId + if (!userId) return Response.json({ success: false, error: 'Unauthorized' }, { status: 401 }) + const context: DocuSignOperationContext = { + requestId: request.requestId, + signal: request.signal, + userId, + workspaceId: request.context.workspaceId, + workflowId: request.context.workflowId, + executionId: request.context.executionId, + } + try { + const result = await execute(parsed.data, context) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof DocuSignOperationError) { + return Response.json(error.body, { status: error.status }) + } + return Response.json( + { success: false, error: getErrorMessage(error, 'Internal server error') }, + { status: isPayloadSizeLimitError(error) ? 413 : 500 } + ) + } +} + +export const executeDocuSignTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!request.context.userId) { + return Response.json({ success: false, error: 'Unauthorized' }, { status: 401 }) + } + if (!isDocuSignToolId(request.toolId)) { + return Response.json( + { success: false, error: `Unsupported DocuSign tool: ${request.toolId}` }, + { status: 500 } + ) + } + switch (request.toolId) { + case 'docusign_create_from_template': + return executeOperation( + schemas.docusign_create_from_template, + request, + executeDocuSignCreateFromTemplate + ) + case 'docusign_download_document': + return executeOperation( + schemas.docusign_download_document, + request, + executeDocuSignDownloadDocument + ) + case 'docusign_get_envelope': + return executeOperation(schemas.docusign_get_envelope, request, executeDocuSignGetEnvelope) + case 'docusign_list_envelopes': + return executeOperation( + schemas.docusign_list_envelopes, + request, + executeDocuSignListEnvelopes + ) + case 'docusign_list_recipients': + return executeOperation( + schemas.docusign_list_recipients, + request, + executeDocuSignListRecipients + ) + case 'docusign_list_templates': + return executeOperation( + schemas.docusign_list_templates, + request, + executeDocuSignListTemplates + ) + case 'docusign_send_envelope': + return executeOperation(schemas.docusign_send_envelope, request, executeDocuSignSendEnvelope) + case 'docusign_void_envelope': + return executeOperation(schemas.docusign_void_envelope, request, executeDocuSignVoidEnvelope) + } +} diff --git a/apps/sim/lib/internal/docusign/operations.test.ts b/apps/sim/lib/internal/docusign/operations.test.ts new file mode 100644 index 00000000000..79c9d1bc35f --- /dev/null +++ b/apps/sim/lib/internal/docusign/operations.test.ts @@ -0,0 +1,186 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + create: vi.fn(), + json: vi.fn(), + document: vi.fn(), + assertToolFileAccess: vi.fn(), + downloadServableFileFromStorage: vi.fn(), + processFilesToUserFiles: vi.fn(), + fileParse: vi.fn(), + uploadExecutionFile: vi.fn(), + uploadCopilotFile: vi.fn(), +})) + +vi.mock('@/lib/internal/docusign/client', () => ({ + MAX_DOCUSIGN_DOCUMENT_BYTES: 25 * 1024 * 1024, + DocuSignClient: class { + static create = mocks.create + }, +})) +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertToolFileAccess, +})) +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mocks.downloadServableFileFromStorage, +})) +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + processFilesToUserFiles: mocks.processFilesToUserFiles, +})) +vi.mock('@/lib/uploads/utils/file-schemas', () => ({ + FileInputSchema: { parse: mocks.fileParse }, +})) +vi.mock('@/lib/uploads/contexts/execution', () => ({ + uploadExecutionFile: mocks.uploadExecutionFile, +})) +vi.mock('@/lib/uploads/contexts/copilot', () => ({ + uploadCopilotFile: mocks.uploadCopilotFile, +})) + +import { DocuSignOperationError } from '@/lib/internal/docusign/errors' +import { + executeDocuSignCreateFromTemplate, + executeDocuSignDownloadDocument, + executeDocuSignListEnvelopes, + executeDocuSignSendEnvelope, +} from '@/lib/internal/docusign/operations' + +const CONTEXT = { + requestId: 'request-1', + userId: 'user-1', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', +} + +describe('DocuSign operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.create.mockResolvedValue({ json: mocks.json, document: mocks.document }) + mocks.json.mockResolvedValue({ envelopeId: 'envelope-1', status: 'sent' }) + mocks.assertToolFileAccess.mockResolvedValue(null) + mocks.fileParse.mockReturnValue({ id: 'file-1' }) + mocks.processFilesToUserFiles.mockReturnValue([ + { + id: 'file-1', + key: 'workspace/file-1', + name: 'contract.pdf', + size: 8, + type: 'application/pdf', + }, + ]) + mocks.downloadServableFileFromStorage.mockResolvedValue({ buffer: Buffer.from('contract') }) + mocks.document.mockResolvedValue({ + buffer: Buffer.from('signed'), + contentType: 'application/pdf', + fileName: 'signed.pdf', + }) + mocks.uploadExecutionFile.mockResolvedValue({ id: 'output-1', name: 'signed.pdf' }) + }) + + it('authorizes and bounds file input before sending an envelope', async () => { + const controller = new AbortController() + await executeDocuSignSendEnvelope( + { + accessToken: 'access-token', + emailSubject: 'Sign', + signerEmail: 'a@example.com', + signerName: 'A', + file: { id: 'file-1' }, + }, + { ...CONTEXT, signal: controller.signal } + ) + + expect(mocks.assertToolFileAccess).toHaveBeenCalledWith( + 'workspace/file-1', + 'user-1', + 'request-1', + expect.anything() + ) + expect(mocks.downloadServableFileFromStorage).toHaveBeenCalledWith( + expect.objectContaining({ key: 'workspace/file-1' }), + 'request-1', + expect.anything(), + { maxBytes: 25 * 1024 * 1024, signal: controller.signal } + ) + const request = mocks.json.mock.calls[0]?.[1] + const body = JSON.parse(String(request.body)) + expect(body.documents[0]).toMatchObject({ + documentBase64: Buffer.from('contract').toString('base64'), + name: 'contract.pdf', + }) + }) + + it('does not download or contact DocuSign when file access is denied', async () => { + mocks.assertToolFileAccess.mockResolvedValue(new Response(null, { status: 404 })) + + await expect( + executeDocuSignSendEnvelope( + { + accessToken: 'access-token', + emailSubject: 'Sign', + signerEmail: 'a@example.com', + signerName: 'A', + file: { id: 'file-1' }, + }, + CONTEXT + ) + ).rejects.toEqual(new DocuSignOperationError('File not found', 404)) + expect(mocks.downloadServableFileFromStorage).not.toHaveBeenCalled() + expect(mocks.create).not.toHaveBeenCalled() + }) + + it('stores downloads in the trusted execution scope', async () => { + const result = await executeDocuSignDownloadDocument( + { accessToken: 'access-token', envelopeId: 'envelope-1' }, + CONTEXT + ) + + expect(mocks.uploadExecutionFile).toHaveBeenCalledWith( + { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }, + Buffer.from('signed'), + 'signed.pdf', + 'application/pdf', + 'user-1' + ) + expect(result).toMatchObject({ + file: { id: 'output-1' }, + base64Content: Buffer.from('signed').toString('base64'), + }) + expect(mocks.uploadCopilotFile).not.toHaveBeenCalled() + }) + + it('preserves single-page list filters without accumulating pages', async () => { + await executeDocuSignListEnvelopes( + { + accessToken: 'access-token', + fromDate: '2026-01-01', + toDate: '2026-02-01', + count: '25', + }, + CONTEXT + ) + + expect(mocks.json.mock.calls[0]?.[0]).toBe( + '/envelopes?from_date=2026-01-01&to_date=2026-02-01&count=25' + ) + expect(mocks.json).toHaveBeenCalledOnce() + }) + + it('rejects malformed template roles before provider work', async () => { + await expect( + executeDocuSignCreateFromTemplate( + { accessToken: 'access-token', templateId: 'template-1', templateRoles: '{' }, + CONTEXT + ) + ).rejects.toEqual(new DocuSignOperationError('Invalid JSON for templateRoles', 400)) + expect(mocks.create).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/docusign/operations.ts b/apps/sim/lib/internal/docusign/operations.ts new file mode 100644 index 00000000000..5098ce81b30 --- /dev/null +++ b/apps/sim/lib/internal/docusign/operations.ts @@ -0,0 +1,304 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { assertKnownSizeWithinLimit, isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { DocuSignClient, MAX_DOCUSIGN_DOCUMENT_BYTES } from '@/lib/internal/docusign/client' +import { DocuSignOperationError } from '@/lib/internal/docusign/errors' +import { uploadCopilotFile } from '@/lib/uploads/contexts/copilot' +import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' +import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/doc-not-ready' +import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' +import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' +import type { + DocuSignCreateFromTemplateParams, + DocuSignDownloadDocumentParams, + DocuSignGetEnvelopeParams, + DocuSignListEnvelopesParams, + DocuSignListRecipientsParams, + DocuSignListTemplatesParams, + DocuSignSendEnvelopeParams, + DocuSignVoidEnvelopeParams, +} from '@/tools/docusign/types' + +const logger = createLogger('DocuSignOperations') +const MAX_LEGACY_INLINE_DOCUMENT_BYTES = 7 * 1024 * 1024 + +export interface DocuSignOperationContext { + requestId: string + signal?: AbortSignal + userId: string + workspaceId?: string + workflowId?: string + executionId?: string +} + +function jsonBody(data: Record): RequestInit { + return { method: 'POST', body: JSON.stringify(data) } +} + +async function client(accessToken: string, signal?: AbortSignal): Promise { + return DocuSignClient.create(accessToken, signal) +} + +export async function executeDocuSignCreateFromTemplate( + input: DocuSignCreateFromTemplateParams, + context: DocuSignOperationContext +) { + let templateRoles: unknown[] = [] + if (input.templateRoles) { + try { + const parsed: unknown = JSON.parse(input.templateRoles) + templateRoles = Array.isArray(parsed) ? parsed : [] + } catch { + throw new DocuSignOperationError('Invalid JSON for templateRoles', 400) + } + } + const body: Record = { + templateId: input.templateId, + status: input.status || 'sent', + templateRoles, + } + if (input.emailSubject) body.emailSubject = input.emailSubject + if (input.emailBody) body.emailBlurb = input.emailBody + const provider = await client(input.accessToken, context.signal) + return provider.json( + '/envelopes', + jsonBody(body), + 'DocuSign create from template response', + 'Failed to create envelope from template', + context.signal + ) +} + +export async function executeDocuSignSendEnvelope( + input: DocuSignSendEnvelopeParams, + context: DocuSignOperationContext +) { + let documentBase64 = '' + let documentName = 'document.pdf' + if (input.file) { + try { + const parsed = FileInputSchema.parse(input.file) + const files = processFilesToUserFiles([parsed as RawFileInput], context.requestId, logger) + const file = files[0] + if (file) { + const denied = await assertToolFileAccess( + file.key, + context.userId, + context.requestId, + logger + ) + if (denied) throw new DocuSignOperationError('File not found', denied.status) + if (file.size > MAX_DOCUSIGN_DOCUMENT_BYTES) { + throw new DocuSignOperationError('Document is too large to send through DocuSign', 413) + } + const { buffer } = await downloadServableFileFromStorage(file, context.requestId, logger, { + maxBytes: MAX_DOCUSIGN_DOCUMENT_BYTES, + signal: context.signal, + }) + assertKnownSizeWithinLimit(buffer.length, MAX_DOCUSIGN_DOCUMENT_BYTES, 'DocuSign document') + documentBase64 = buffer.toString('base64') + documentName = file.name + } + } catch (error) { + context.signal?.throwIfAborted() + if (error instanceof DocuSignOperationError) throw error + if (isDocNotReadyError(error)) { + throw new DocuSignOperationError(docNotReadyMessage(), 409) + } + if (isPayloadSizeLimitError(error)) { + throw new DocuSignOperationError( + getErrorMessage(error, 'Document is too large to send through DocuSign'), + 413 + ) + } + throw new DocuSignOperationError('Failed to process uploaded file', 400) + } + } + + const status = input.status || 'sent' + if (!documentBase64 && status === 'sent') { + throw new DocuSignOperationError('A document file is required to send an envelope', 400) + } + const envelope: Record = { + emailSubject: input.emailSubject, + status, + recipients: { + signers: [ + { + email: input.signerEmail, + name: input.signerName, + recipientId: '1', + routingOrder: '1', + tabs: { + signHereTabs: [ + { + anchorString: '/sig1/', + anchorUnits: 'pixels', + anchorXOffset: '0', + anchorYOffset: '0', + }, + ], + dateSignedTabs: [ + { + anchorString: '/date1/', + anchorUnits: 'pixels', + anchorXOffset: '0', + anchorYOffset: '0', + }, + ], + }, + }, + ], + carbonCopies: input.ccEmail + ? [ + { + email: input.ccEmail, + name: input.ccName || input.ccEmail, + recipientId: '2', + routingOrder: '2', + }, + ] + : [], + }, + } + if (input.emailBody) envelope.emailBlurb = input.emailBody + if (documentBase64) { + envelope.documents = [ + { + documentBase64, + name: documentName, + fileExtension: documentName.split('.').pop() || 'pdf', + documentId: '1', + }, + ] + } + const provider = await client(input.accessToken, context.signal) + return provider.json( + '/envelopes', + jsonBody(envelope), + 'DocuSign send envelope response', + 'Failed to send envelope', + context.signal + ) +} + +export async function executeDocuSignGetEnvelope( + input: DocuSignGetEnvelopeParams, + context: DocuSignOperationContext +) { + const provider = await client(input.accessToken, context.signal) + return provider.json( + `/envelopes/${input.envelopeId.trim()}?include=recipients,documents`, + {}, + 'DocuSign envelope response', + 'Failed to get envelope', + context.signal + ) +} + +export async function executeDocuSignListEnvelopes( + input: DocuSignListEnvelopesParams, + context: DocuSignOperationContext +) { + const query = new URLSearchParams() + if (input.fromDate) query.append('from_date', input.fromDate) + else { + const thirtyDaysAgo = new Date() + thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30) + query.append('from_date', thirtyDaysAgo.toISOString()) + } + if (input.toDate) query.append('to_date', input.toDate) + if (input.envelopeStatus) query.append('status', input.envelopeStatus) + if (input.searchText) query.append('search_text', input.searchText) + if (input.count) query.append('count', input.count) + const provider = await client(input.accessToken, context.signal) + return provider.json( + `/envelopes?${query}`, + {}, + 'DocuSign envelope list response', + 'Failed to list envelopes', + context.signal + ) +} + +export async function executeDocuSignVoidEnvelope( + input: DocuSignVoidEnvelopeParams, + context: DocuSignOperationContext +) { + const provider = await client(input.accessToken, context.signal) + await provider.json( + `/envelopes/${input.envelopeId.trim()}`, + { method: 'PUT', body: JSON.stringify({ status: 'voided', voidedReason: input.voidedReason }) }, + 'DocuSign void envelope response', + 'Failed to void envelope', + context.signal + ) + return { envelopeId: input.envelopeId, status: 'voided' as const } +} + +export async function executeDocuSignDownloadDocument( + input: DocuSignDownloadDocumentParams, + context: DocuSignOperationContext +) { + const documentId = input.documentId || 'combined' + const provider = await client(input.accessToken, context.signal) + const { buffer, contentType, fileName } = await provider.document( + input.envelopeId.trim(), + documentId, + context.signal + ) + context.signal?.throwIfAborted() + const legacy = + buffer.length <= MAX_LEGACY_INLINE_DOCUMENT_BYTES + ? { base64Content: buffer.toString('base64') } + : {} + const file = + context.workspaceId && context.workflowId && context.executionId + ? await uploadExecutionFile( + { + workspaceId: context.workspaceId, + workflowId: context.workflowId, + executionId: context.executionId, + }, + buffer, + fileName, + contentType, + context.userId + ) + : await uploadCopilotFile({ buffer, fileName, contentType, userId: context.userId }) + return { file, mimeType: contentType, fileName, ...legacy } +} + +export async function executeDocuSignListTemplates( + input: DocuSignListTemplatesParams, + context: DocuSignOperationContext +) { + const query = new URLSearchParams() + if (input.searchText) query.append('search_text', input.searchText) + if (input.count) query.append('count', input.count) + const suffix = query.size ? `?${query}` : '' + const provider = await client(input.accessToken, context.signal) + return provider.json( + `/templates${suffix}`, + {}, + 'DocuSign template list response', + 'Failed to list templates', + context.signal + ) +} + +export async function executeDocuSignListRecipients( + input: DocuSignListRecipientsParams, + context: DocuSignOperationContext +) { + const provider = await client(input.accessToken, context.signal) + return provider.json( + `/envelopes/${input.envelopeId.trim()}/recipients`, + {}, + 'DocuSign recipients response', + 'Failed to list recipients', + context.signal + ) +} diff --git a/apps/sim/lib/internal/dropbox/client.test.ts b/apps/sim/lib/internal/dropbox/client.test.ts new file mode 100644 index 00000000000..3941ef9f6fc --- /dev/null +++ b/apps/sim/lib/internal/dropbox/client.test.ts @@ -0,0 +1,55 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ fetch: vi.fn() })) + +import { DropboxClient, DropboxUploadError } from '@/lib/internal/dropbox/client' + +describe('DropboxClient', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mocks.fetch) + }) + + it('preserves Dropbox content upload headers, flags, bytes, and output', async () => { + const controller = new AbortController() + const metadata = { id: 'dropbox-1', name: 'file.pdf', path_display: '/Reports/file.pdf' } + mocks.fetch.mockResolvedValue(Response.json(metadata)) + const buffer = Buffer.from('file') + const output = await new DropboxClient('token', controller.signal).upload( + '/Reports/file.pdf', + buffer, + { mode: 'overwrite', autorename: true, mute: true } + ) + + const [url, init] = mocks.fetch.mock.calls[0] as [string, RequestInit] + expect(url).toBe('https://content.dropboxapi.com/2/files/upload') + expect(init.method).toBe('POST') + expect(init.signal).toBe(controller.signal) + expect(init.body).toEqual(new Uint8Array(buffer)) + const headers = init.headers as Record + expect(headers.Authorization).toBe('Bearer token') + expect(headers['Content-Type']).toBe('application/octet-stream') + expect(JSON.parse(headers['Dropbox-API-Arg'])).toEqual({ + path: '/Reports/file.pdf', + mode: 'overwrite', + autorename: true, + mute: true, + }) + expect(output).toEqual(metadata) + }) + + it('preserves Dropbox provider status and error summary', async () => { + mocks.fetch.mockResolvedValue( + Response.json({ error_summary: 'path/not_found/' }, { status: 409 }) + ) + const error = await new DropboxClient('token') + .upload('/missing/file.pdf', Buffer.from('file'), {}) + .catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(DropboxUploadError) + expect(error).toMatchObject({ message: 'path/not_found/', status: 409 }) + }) +}) diff --git a/apps/sim/lib/internal/dropbox/client.ts b/apps/sim/lib/internal/dropbox/client.ts new file mode 100644 index 00000000000..5e0f66d6a0a --- /dev/null +++ b/apps/sim/lib/internal/dropbox/client.ts @@ -0,0 +1,69 @@ +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + readResponseJsonWithLimit, +} from '@/lib/core/utils/stream-limits' +import { httpHeaderSafeJson } from '@/lib/core/utils/validation' + +interface DropboxErrorBody { + error_summary?: string + error?: { message?: string } +} + +export class DropboxUploadError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'DropboxUploadError' + } +} + +export class DropboxClient { + constructor( + private readonly accessToken: string, + private readonly signal?: AbortSignal + ) {} + + async upload( + path: string, + buffer: Buffer, + options: { + mode?: 'add' | 'overwrite' | null + autorename?: boolean | null + mute?: boolean | null + } + ): Promise> { + this.signal?.throwIfAborted() + const response = await fetch('https://content.dropboxapi.com/2/files/upload', { + method: 'POST', + headers: { + Authorization: `Bearer ${this.accessToken}`, + 'Content-Type': 'application/octet-stream', + 'Dropbox-API-Arg': httpHeaderSafeJson({ + path, + mode: options.mode || 'add', + autorename: options.autorename ?? false, + mute: options.mute ?? false, + }), + }, + body: new Uint8Array(buffer), + signal: this.signal, + }) + const data = await readResponseJsonWithLimit & DropboxErrorBody>( + response, + { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Dropbox file upload response', + signal: this.signal, + } + ) + if (!response.ok) { + throw new DropboxUploadError( + data.error_summary || data.error?.message || 'Failed to upload file', + response.status + ) + } + return data + } +} diff --git a/apps/sim/lib/internal/dropbox/execute-tool.test.ts b/apps/sim/lib/internal/dropbox/execute-tool.test.ts new file mode 100644 index 00000000000..b573e386e73 --- /dev/null +++ b/apps/sim/lib/internal/dropbox/execute-tool.test.ts @@ -0,0 +1,58 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ upload: vi.fn() })) + +vi.mock('@/lib/internal/dropbox/operations', () => ({ executeDropboxUpload: mocks.upload })) + +import { executeDropboxTool } from '@/lib/internal/dropbox/execute-tool' +import { dropboxUploadTool } from '@/tools/dropbox/upload' + +const file = { key: 'uploads/file.pdf', name: 'file.pdf', size: 5 } + +describe('executeDropboxTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.upload.mockResolvedValue(Response.json({ success: true, output: {} })) + }) + + it('dispatches normalized input through the trusted operation', async () => { + const input = { accessToken: 'token', path: '/Reports/file.pdf', file } + await executeDropboxTool({ + toolId: 'dropbox_upload', + input, + headers: new Headers(), + context: { userId: 'user-1' }, + requestId: 'request-1', + }) + expect(mocks.upload).toHaveBeenCalledWith( + input, + expect.objectContaining({ userId: 'user-1', requestId: 'request-1' }) + ) + }) + + it('uses operation-only metadata and keeps OAuth, legacy content, and private options hidden', () => { + expect(dropboxUploadTool).not.toHaveProperty('request') + const params = { + accessToken: 'private-token', + path: ' /Reports/ ', + file, + fileContent: 'private-base64', + fileName: 'file.pdf', + mode: 'overwrite' as const, + autorename: true, + mute: true, + } + expect(dropboxUploadTool.operation.modelInput?.select?.(params)).toEqual({ + path: ' /Reports/ ', + file, + fileName: 'file.pdf', + }) + expect(dropboxUploadTool.operation.input(params)).toEqual({ + ...params, + path: '/Reports/', + }) + }) +}) diff --git a/apps/sim/lib/internal/dropbox/execute-tool.ts b/apps/sim/lib/internal/dropbox/execute-tool.ts new file mode 100644 index 00000000000..c1a548f44cc --- /dev/null +++ b/apps/sim/lib/internal/dropbox/execute-tool.ts @@ -0,0 +1,55 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { executeDropboxUpload } from '@/lib/internal/dropbox/operations' +import { dropboxUploadInputSchema } from '@/lib/internal/dropbox/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +export const executeDropboxTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + const userId = request.context.userId + if (!userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + if (request.toolId !== 'dropbox_upload') { + return Response.json( + { success: false, error: `Unsupported Dropbox tool: ${request.toolId}` }, + { status: 500 } + ) + } + let serializedInput: string + try { + serializedInput = JSON.stringify(request.input) ?? '' + } catch { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + if (Buffer.byteLength(serializedInput, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + success: false, + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + const parsed = dropboxUploadInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { success: false, error: getValidationErrorMessage(parsed.error, 'Invalid request data') }, + { status: 400 } + ) + } + try { + return await executeDropboxUpload(parsed.data, { + userId, + requestId: request.requestId, + signal: request.signal, + }) + } catch (error) { + request.signal?.throwIfAborted() + return Response.json( + { success: false, error: getErrorMessage(error, 'Unknown error') }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/dropbox/operations.test.ts b/apps/sim/lib/internal/dropbox/operations.test.ts new file mode 100644 index 00000000000..2461a860fd6 --- /dev/null +++ b/apps/sim/lib/internal/dropbox/operations.test.ts @@ -0,0 +1,124 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + clientConstructed: vi.fn(), + upload: vi.fn(), + processFiles: vi.fn(), + downloadStorage: vi.fn(), + assertAccess: vi.fn(), +})) + +vi.mock('@/lib/internal/dropbox/client', () => { + class DropboxUploadError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + } + } + class DropboxClient { + constructor(token: string, signal?: AbortSignal) { + mocks.clientConstructed(token, signal) + } + + upload = mocks.upload + } + return { DropboxClient, DropboxUploadError } +}) +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + processFilesToUserFiles: mocks.processFiles, +})) +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mocks.downloadStorage, +})) +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertAccess, +})) + +import { executeDropboxUpload } from '@/lib/internal/dropbox/operations' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' + +const rawFile = { key: 'uploads/file.pdf', name: 'file.pdf', size: 4 } +const userFile = { ...rawFile, type: 'application/pdf' } + +describe('executeDropboxUpload', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.processFiles.mockReturnValue([userFile]) + mocks.assertAccess.mockResolvedValue(null) + mocks.downloadStorage.mockResolvedValue({ buffer: Buffer.from('file') }) + mocks.upload.mockResolvedValue({ id: 'dropbox-1', name: 'file.pdf' }) + }) + + it('authorizes provenance, appends a folder filename, and carries cancellation', async () => { + const controller = new AbortController() + const input = { + accessToken: 'token', + path: '/Reports/', + file: rawFile, + fileName: 'renamed.pdf', + mode: 'overwrite' as const, + autorename: true, + mute: true, + } + const response = await executeDropboxUpload(input, { + userId: 'user-1', + requestId: 'request-1', + signal: controller.signal, + }) + + expect(mocks.assertAccess).toHaveBeenCalledWith( + userFile.key, + 'user-1', + 'request-1', + expect.anything() + ) + expect(mocks.downloadStorage).toHaveBeenCalledWith(userFile, 'request-1', expect.anything(), { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + signal: controller.signal, + }) + expect(mocks.clientConstructed).toHaveBeenCalledWith('token', controller.signal) + expect(mocks.upload).toHaveBeenCalledWith('/Reports/renamed.pdf', Buffer.from('file'), input) + expect(await response.json()).toEqual({ + success: true, + output: { file: { id: 'dropbox-1', name: 'file.pdf' } }, + }) + }) + + it('does not load bytes for an unauthorized file', async () => { + mocks.assertAccess.mockResolvedValue( + Response.json({ success: false, error: 'File not found' }, { status: 404 }) + ) + const response = await executeDropboxUpload( + { accessToken: 'token', path: '/file.pdf', file: rawFile }, + { userId: 'user-1', requestId: 'request-1' } + ) + + expect(response.status).toBe(404) + expect(mocks.downloadStorage).not.toHaveBeenCalled() + expect(mocks.upload).not.toHaveBeenCalled() + }) + + it('preserves legacy base64 and filename behavior for folder paths', async () => { + await executeDropboxUpload( + { + accessToken: 'token', + path: '/Legacy/', + fileContent: Buffer.from('legacy').toString('base64'), + fileName: 'legacy.txt', + }, + { userId: 'user-1', requestId: 'request-1' } + ) + + expect(mocks.assertAccess).not.toHaveBeenCalled() + expect(mocks.upload).toHaveBeenCalledWith( + '/Legacy/legacy.txt', + Buffer.from('legacy'), + expect.objectContaining({ path: '/Legacy/' }) + ) + }) +}) diff --git a/apps/sim/lib/internal/dropbox/operations.ts b/apps/sim/lib/internal/dropbox/operations.ts new file mode 100644 index 00000000000..c97286bb51b --- /dev/null +++ b/apps/sim/lib/internal/dropbox/operations.ts @@ -0,0 +1,93 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { assertKnownSizeWithinLimit, isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { DropboxClient, DropboxUploadError } from '@/lib/internal/dropbox/client' +import type { DropboxUploadInput } from '@/lib/internal/dropbox/schema' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +const logger = createLogger('DropboxOperations') + +export interface DropboxOperationContext { + userId: string + requestId: string + signal?: AbortSignal +} + +function failureResponse(error: string, status: number): Response { + return Response.json({ success: false, error }, { status }) +} + +export async function executeDropboxUpload( + input: DropboxUploadInput, + context: DropboxOperationContext +): Promise { + context.signal?.throwIfAborted() + let buffer: Buffer + let fileName: string + + if (input.file) { + if (typeof input.file === 'string') return failureResponse('Invalid file input', 400) + const userFile = processFilesToUserFiles([input.file], context.requestId, logger)[0] + if (!userFile) return failureResponse('Invalid file input', 400) + const denied = await assertToolFileAccess( + userFile.key, + context.userId, + context.requestId, + logger + ) + context.signal?.throwIfAborted() + if (denied) return denied + try { + const resolved = await downloadServableFileFromStorage(userFile, context.requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + signal: context.signal, + }) + buffer = resolved.buffer + } catch (error) { + context.signal?.throwIfAborted() + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + return failureResponse( + getErrorMessage(error, 'Failed to download file'), + isPayloadSizeLimitError(error) ? 413 : 500 + ) + } + fileName = input.fileName || userFile.name + } else if (input.fileContent) { + buffer = Buffer.from(input.fileContent, 'base64') + try { + assertKnownSizeWithinLimit(buffer.length, MAX_BUFFERED_TRANSFER_BYTES, 'Dropbox upload file') + } catch (error) { + return failureResponse( + getErrorMessage(error, 'Failed to decode file'), + isPayloadSizeLimitError(error) ? 413 : 500 + ) + } + fileName = input.fileName || 'file' + } else { + return failureResponse('File is required', 400) + } + + const finalPath = input.path.endsWith('/') ? `${input.path}${fileName}` : input.path + try { + const file = await new DropboxClient(input.accessToken, context.signal).upload( + finalPath, + buffer, + input + ) + context.signal?.throwIfAborted() + return Response.json({ success: true, output: { file } }) + } catch (error) { + context.signal?.throwIfAborted() + if (error instanceof DropboxUploadError) return failureResponse(error.message, error.status) + logger.error('Unexpected Dropbox upload error', { + error: getErrorMessage(error), + requestId: context.requestId, + }) + return failureResponse(getErrorMessage(error, 'Unknown error'), 500) + } +} diff --git a/apps/sim/lib/internal/dropbox/schema.ts b/apps/sim/lib/internal/dropbox/schema.ts new file mode 100644 index 00000000000..db415c953e8 --- /dev/null +++ b/apps/sim/lib/internal/dropbox/schema.ts @@ -0,0 +1,15 @@ +import { z } from 'zod' +import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' + +export const dropboxUploadInputSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + path: z.string().trim().min(1, 'Destination path is required'), + file: FileInputSchema.optional().nullable(), + fileContent: z.string().optional().nullable(), + fileName: z.string().optional().nullable(), + mode: z.enum(['add', 'overwrite']).optional().nullable(), + autorename: z.boolean().optional().nullable(), + mute: z.boolean().optional().nullable(), +}) + +export type DropboxUploadInput = z.output diff --git a/apps/sim/lib/internal/dynamodb/client.test.ts b/apps/sim/lib/internal/dynamodb/client.test.ts new file mode 100644 index 00000000000..66d5fdc240e --- /dev/null +++ b/apps/sim/lib/internal/dynamodb/client.test.ts @@ -0,0 +1,24 @@ +/** + * @vitest-environment node + */ +import type { DynamoDBClient } from '@aws-sdk/client-dynamodb' +import { describe, expect, it, vi } from 'vitest' +import { listTables } from '@/lib/internal/dynamodb/client' + +describe('DynamoDB client operations', () => { + it('passes cancellation to every paginated list-tables request', async () => { + const controller = new AbortController() + const send = vi + .fn() + .mockResolvedValueOnce({ TableNames: ['table-a'], LastEvaluatedTableName: 'table-a' }) + .mockResolvedValueOnce({ TableNames: ['table-b'] }) + const client = { send } as unknown as DynamoDBClient + + await expect(listTables(client, controller.signal)).resolves.toEqual({ + tables: ['table-a', 'table-b'], + }) + expect(send).toHaveBeenCalledTimes(2) + expect(send.mock.calls[0]?.[1]).toEqual({ abortSignal: controller.signal }) + expect(send.mock.calls[1]?.[1]).toEqual({ abortSignal: controller.signal }) + }) +}) diff --git a/apps/sim/lib/internal/dynamodb/client.ts b/apps/sim/lib/internal/dynamodb/client.ts new file mode 100644 index 00000000000..76ee3ceef9c --- /dev/null +++ b/apps/sim/lib/internal/dynamodb/client.ts @@ -0,0 +1,318 @@ +import { DescribeTableCommand, DynamoDBClient, ListTablesCommand } from '@aws-sdk/client-dynamodb' +import { + DeleteCommand, + DynamoDBDocumentClient, + GetCommand, + PutCommand, + QueryCommand, + ScanCommand, + UpdateCommand, +} from '@aws-sdk/lib-dynamodb' +import type { DynamoDBConnectionConfig, DynamoDBTableSchema } from '@/tools/dynamodb/types' + +export function createDynamoDBClient(config: DynamoDBConnectionConfig): DynamoDBDocumentClient { + const client = new DynamoDBClient({ + region: config.region, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + }) + + return DynamoDBDocumentClient.from(client, { + marshallOptions: { + removeUndefinedValues: true, + convertEmptyValues: false, + }, + unmarshallOptions: { + wrapNumbers: false, + }, + }) +} + +export async function getItem( + client: DynamoDBDocumentClient, + tableName: string, + key: Record, + consistentRead?: boolean, + signal?: AbortSignal +): Promise<{ item: Record | null }> { + const command = new GetCommand({ + TableName: tableName, + Key: key, + ConsistentRead: consistentRead, + }) + + const response = await client.send(command, { abortSignal: signal }) + return { + item: (response.Item as Record) || null, + } +} + +export async function putItem( + client: DynamoDBDocumentClient, + tableName: string, + item: Record, + options?: { + conditionExpression?: string + expressionAttributeNames?: Record + expressionAttributeValues?: Record + }, + signal?: AbortSignal +): Promise<{ success: boolean }> { + const command = new PutCommand({ + TableName: tableName, + Item: item, + ...(options?.conditionExpression && { ConditionExpression: options.conditionExpression }), + ...(options?.expressionAttributeNames && { + ExpressionAttributeNames: options.expressionAttributeNames, + }), + ...(options?.expressionAttributeValues && { + ExpressionAttributeValues: options.expressionAttributeValues, + }), + }) + + await client.send(command, { abortSignal: signal }) + return { success: true } +} + +export async function queryItems( + client: DynamoDBDocumentClient, + tableName: string, + keyConditionExpression: string, + options?: { + filterExpression?: string + expressionAttributeNames?: Record + expressionAttributeValues?: Record + indexName?: string + limit?: number + exclusiveStartKey?: Record + scanIndexForward?: boolean + }, + signal?: AbortSignal +): Promise<{ + items: Record[] + count: number + lastEvaluatedKey?: Record +}> { + const command = new QueryCommand({ + TableName: tableName, + KeyConditionExpression: keyConditionExpression, + ...(options?.filterExpression && { FilterExpression: options.filterExpression }), + ...(options?.expressionAttributeNames && { + ExpressionAttributeNames: options.expressionAttributeNames, + }), + ...(options?.expressionAttributeValues && { + ExpressionAttributeValues: options.expressionAttributeValues, + }), + ...(options?.indexName && { IndexName: options.indexName }), + ...(options?.limit && { Limit: options.limit }), + ...(options?.exclusiveStartKey && { ExclusiveStartKey: options.exclusiveStartKey }), + ...(options?.scanIndexForward !== undefined && { ScanIndexForward: options.scanIndexForward }), + }) + + const response = await client.send(command, { abortSignal: signal }) + return { + items: (response.Items as Record[]) || [], + count: response.Count || 0, + lastEvaluatedKey: response.LastEvaluatedKey as Record | undefined, + } +} + +export async function scanItems( + client: DynamoDBDocumentClient, + tableName: string, + options?: { + filterExpression?: string + projectionExpression?: string + expressionAttributeNames?: Record + expressionAttributeValues?: Record + limit?: number + exclusiveStartKey?: Record + }, + signal?: AbortSignal +): Promise<{ + items: Record[] + count: number + lastEvaluatedKey?: Record +}> { + const command = new ScanCommand({ + TableName: tableName, + ...(options?.filterExpression && { FilterExpression: options.filterExpression }), + ...(options?.projectionExpression && { ProjectionExpression: options.projectionExpression }), + ...(options?.expressionAttributeNames && { + ExpressionAttributeNames: options.expressionAttributeNames, + }), + ...(options?.expressionAttributeValues && { + ExpressionAttributeValues: options.expressionAttributeValues, + }), + ...(options?.limit && { Limit: options.limit }), + ...(options?.exclusiveStartKey && { ExclusiveStartKey: options.exclusiveStartKey }), + }) + + const response = await client.send(command, { abortSignal: signal }) + return { + items: (response.Items as Record[]) || [], + count: response.Count || 0, + lastEvaluatedKey: response.LastEvaluatedKey as Record | undefined, + } +} + +export async function updateItem( + client: DynamoDBDocumentClient, + tableName: string, + key: Record, + updateExpression: string, + options?: { + expressionAttributeNames?: Record + expressionAttributeValues?: Record + conditionExpression?: string + }, + signal?: AbortSignal +): Promise<{ attributes: Record | null }> { + const command = new UpdateCommand({ + TableName: tableName, + Key: key, + UpdateExpression: updateExpression, + ...(options?.expressionAttributeNames && { + ExpressionAttributeNames: options.expressionAttributeNames, + }), + ...(options?.expressionAttributeValues && { + ExpressionAttributeValues: options.expressionAttributeValues, + }), + ...(options?.conditionExpression && { ConditionExpression: options.conditionExpression }), + ReturnValues: 'ALL_NEW', + }) + + const response = await client.send(command, { abortSignal: signal }) + return { + attributes: (response.Attributes as Record) || null, + } +} + +export async function deleteItem( + client: DynamoDBDocumentClient, + tableName: string, + key: Record, + options?: { + conditionExpression?: string + expressionAttributeNames?: Record + expressionAttributeValues?: Record + }, + signal?: AbortSignal +): Promise<{ success: boolean }> { + const command = new DeleteCommand({ + TableName: tableName, + Key: key, + ...(options?.conditionExpression && { ConditionExpression: options.conditionExpression }), + ...(options?.expressionAttributeNames && { + ExpressionAttributeNames: options.expressionAttributeNames, + }), + ...(options?.expressionAttributeValues && { + ExpressionAttributeValues: options.expressionAttributeValues, + }), + }) + + await client.send(command, { abortSignal: signal }) + return { success: true } +} + +/** + * Creates a raw DynamoDB client for operations that don't require DocumentClient + */ +export function createRawDynamoDBClient(config: DynamoDBConnectionConfig): DynamoDBClient { + return new DynamoDBClient({ + region: config.region, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + }) +} + +/** + * Lists all DynamoDB tables in the configured region + */ +export async function listTables( + client: DynamoDBClient, + signal?: AbortSignal +): Promise<{ tables: string[] }> { + const tables: string[] = [] + let exclusiveStartTableName: string | undefined + + do { + const command = new ListTablesCommand({ + ExclusiveStartTableName: exclusiveStartTableName, + }) + + const response = await client.send(command, { abortSignal: signal }) + if (response.TableNames) { + tables.push(...response.TableNames) + } + exclusiveStartTableName = response.LastEvaluatedTableName + } while (exclusiveStartTableName) + + return { tables } +} + +/** + * Describes a specific DynamoDB table and returns its schema information + */ +export async function describeTable( + client: DynamoDBClient, + tableName: string, + signal?: AbortSignal +): Promise<{ tableDetails: DynamoDBTableSchema }> { + const command = new DescribeTableCommand({ + TableName: tableName, + }) + + const response = await client.send(command, { abortSignal: signal }) + const table = response.Table + + if (!table) { + throw new Error(`Table '${tableName}' not found`) + } + + const tableDetails: DynamoDBTableSchema = { + tableName: table.TableName || tableName, + tableStatus: table.TableStatus || 'UNKNOWN', + keySchema: + table.KeySchema?.map((key) => ({ + attributeName: key.AttributeName || '', + keyType: (key.KeyType as 'HASH' | 'RANGE') || 'HASH', + })) || [], + attributeDefinitions: + table.AttributeDefinitions?.map((attr) => ({ + attributeName: attr.AttributeName || '', + attributeType: (attr.AttributeType as 'S' | 'N' | 'B') || 'S', + })) || [], + globalSecondaryIndexes: + table.GlobalSecondaryIndexes?.map((gsi) => ({ + indexName: gsi.IndexName || '', + keySchema: + gsi.KeySchema?.map((key) => ({ + attributeName: key.AttributeName || '', + keyType: (key.KeyType as 'HASH' | 'RANGE') || 'HASH', + })) || [], + projectionType: gsi.Projection?.ProjectionType || 'ALL', + indexStatus: gsi.IndexStatus || 'UNKNOWN', + })) || [], + localSecondaryIndexes: + table.LocalSecondaryIndexes?.map((lsi) => ({ + indexName: lsi.IndexName || '', + keySchema: + lsi.KeySchema?.map((key) => ({ + attributeName: key.AttributeName || '', + keyType: (key.KeyType as 'HASH' | 'RANGE') || 'HASH', + })) || [], + projectionType: lsi.Projection?.ProjectionType || 'ALL', + indexStatus: 'ACTIVE', + })) || [], + itemCount: Number(table.ItemCount) || 0, + tableSizeBytes: Number(table.TableSizeBytes) || 0, + billingMode: table.BillingModeSummary?.BillingMode || 'PROVISIONED', + } + + return { tableDetails } +} diff --git a/apps/sim/lib/internal/dynamodb/execute-tool.test.ts b/apps/sim/lib/internal/dynamodb/execute-tool.test.ts new file mode 100644 index 00000000000..8133acf2459 --- /dev/null +++ b/apps/sim/lib/internal/dynamodb/execute-tool.test.ts @@ -0,0 +1,157 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mockOperations = vi.hoisted(() => ({ + executeDynamodbDelete: vi.fn(), + executeDynamodbGet: vi.fn(), + executeDynamodbIntrospect: vi.fn(), + executeDynamodbPut: vi.fn(), + executeDynamodbQuery: vi.fn(), + executeDynamodbScan: vi.fn(), + executeDynamodbUpdate: vi.fn(), +})) + +vi.mock('@/lib/internal/dynamodb/operations', () => mockOperations) + +import { executeDynamodbTool } from '@/lib/internal/dynamodb/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const CONNECTION = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', +} + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'dynamodb_get', + input: { ...CONNECTION, tableName: 'test-table', key: { id: 'item-1' } }, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + userId: 'user-1', + metadata: {}, + }, + requestId: 'request-1', + ...overrides, + } +} + +const TOOL_CASES = [ + { + toolId: 'dynamodb_delete', + input: { ...CONNECTION, tableName: 'test-table', key: { id: 'item-1' } }, + operation: mockOperations.executeDynamodbDelete, + }, + { + toolId: 'dynamodb_get', + input: { ...CONNECTION, tableName: 'test-table', key: { id: 'item-1' } }, + operation: mockOperations.executeDynamodbGet, + }, + { + toolId: 'dynamodb_introspect', + input: { ...CONNECTION, tableName: 'test-table' }, + operation: mockOperations.executeDynamodbIntrospect, + }, + { + toolId: 'dynamodb_put', + input: { ...CONNECTION, tableName: 'test-table', item: { id: 'item-1' } }, + operation: mockOperations.executeDynamodbPut, + }, + { + toolId: 'dynamodb_query', + input: { + ...CONNECTION, + tableName: 'test-table', + keyConditionExpression: '#id = :id', + }, + operation: mockOperations.executeDynamodbQuery, + }, + { + toolId: 'dynamodb_scan', + input: { ...CONNECTION, tableName: 'test-table' }, + operation: mockOperations.executeDynamodbScan, + }, + { + toolId: 'dynamodb_update', + input: { + ...CONNECTION, + tableName: 'test-table', + key: { id: 'item-1' }, + updateExpression: 'SET #name = :name', + }, + operation: mockOperations.executeDynamodbUpdate, + }, +] as const + +describe('executeDynamodbTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it.each(TOOL_CASES)('validates and dispatches $toolId', async ({ toolId, input, operation }) => { + const controller = new AbortController() + operation.mockResolvedValue({ toolId }) + + const response = await executeDynamodbTool( + createRequest({ toolId, input, signal: controller.signal }) + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ toolId }) + expect(operation).toHaveBeenCalledWith(input, controller.signal) + }) + + it('returns the route-compatible validation envelope before provider work', async () => { + const response = await executeDynamodbTool( + createRequest({ input: { ...CONNECTION, tableName: 'test-table', key: {} } }) + ) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(mockOperations.executeDynamodbGet).not.toHaveBeenCalled() + }) + + it('preserves the unprefixed provider error envelope', async () => { + mockOperations.executeDynamodbGet.mockRejectedValue(new Error('AWS rejected credentials')) + + const response = await executeDynamodbTool(createRequest()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ error: 'AWS rejected credentials' }) + }) + + it('preserves the introspection provider error envelope', async () => { + mockOperations.executeDynamodbIntrospect.mockRejectedValue(new Error('table unavailable')) + + const response = await executeDynamodbTool( + createRequest({ + toolId: 'dynamodb_introspect', + input: CONNECTION, + }) + ) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'DynamoDB introspection failed: table unavailable', + }) + }) + + it('propagates cancellation without starting provider work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeDynamodbTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mockOperations.executeDynamodbGet).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/dynamodb/execute-tool.ts b/apps/sim/lib/internal/dynamodb/execute-tool.ts new file mode 100644 index 00000000000..f1c5edb7bef --- /dev/null +++ b/apps/sim/lib/internal/dynamodb/execute-tool.ts @@ -0,0 +1,118 @@ +import { toError } from '@sim/utils/errors' +import type { AnyApiRouteContract, ContractBody } from '@/lib/api/contracts' +import { awsDynamodbDeleteContract } from '@/lib/api/contracts/tools/aws/dynamodb-delete' +import { awsDynamodbGetContract } from '@/lib/api/contracts/tools/aws/dynamodb-get' +import { awsDynamodbIntrospectContract } from '@/lib/api/contracts/tools/aws/dynamodb-introspect' +import { awsDynamodbPutContract } from '@/lib/api/contracts/tools/aws/dynamodb-put' +import { awsDynamodbQueryContract } from '@/lib/api/contracts/tools/aws/dynamodb-query' +import { awsDynamodbScanContract } from '@/lib/api/contracts/tools/aws/dynamodb-scan' +import { awsDynamodbUpdateContract } from '@/lib/api/contracts/tools/aws/dynamodb-update' +import { + executeDynamodbDelete, + executeDynamodbGet, + executeDynamodbIntrospect, + executeDynamodbPut, + executeDynamodbQuery, + executeDynamodbScan, + executeDynamodbUpdate, +} from '@/lib/internal/dynamodb/operations' +import { parseInternalToolInput } from '@/lib/internal/tool-operations/parse-input' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +interface DynamoDbErrorPolicy { + fallback: string + prefix?: string +} + +async function executeOperation( + contract: C, + input: unknown, + execute: (input: ContractBody, signal?: AbortSignal) => Promise, + errorPolicy: DynamoDbErrorPolicy, + signal?: AbortSignal +): Promise { + const parsed = parseInternalToolInput(contract, input) + if (!parsed.success) return parsed.response + + try { + const result = await execute(parsed.data, signal) + signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + signal?.throwIfAborted() + const message = toError(error).message || errorPolicy.fallback + return Response.json( + { error: errorPolicy.prefix ? `${errorPolicy.prefix}: ${message}` : message }, + { status: 500 } + ) + } +} + +export const executeDynamodbTool: InternalToolOperationHandler = async ({ + toolId, + input, + signal, +}) => { + signal?.throwIfAborted() + + switch (toolId) { + case 'dynamodb_delete': + return executeOperation( + awsDynamodbDeleteContract, + input, + executeDynamodbDelete, + { fallback: 'DynamoDB delete failed' }, + signal + ) + case 'dynamodb_get': + return executeOperation( + awsDynamodbGetContract, + input, + executeDynamodbGet, + { fallback: 'DynamoDB get failed' }, + signal + ) + case 'dynamodb_introspect': + return executeOperation( + awsDynamodbIntrospectContract, + input, + executeDynamodbIntrospect, + { fallback: 'Unknown error occurred', prefix: 'DynamoDB introspection failed' }, + signal + ) + case 'dynamodb_put': + return executeOperation( + awsDynamodbPutContract, + input, + executeDynamodbPut, + { fallback: 'DynamoDB put failed' }, + signal + ) + case 'dynamodb_query': + return executeOperation( + awsDynamodbQueryContract, + input, + executeDynamodbQuery, + { fallback: 'DynamoDB query failed' }, + signal + ) + case 'dynamodb_scan': + return executeOperation( + awsDynamodbScanContract, + input, + executeDynamodbScan, + { fallback: 'DynamoDB scan failed' }, + signal + ) + case 'dynamodb_update': + return executeOperation( + awsDynamodbUpdateContract, + input, + executeDynamodbUpdate, + { fallback: 'DynamoDB update failed' }, + signal + ) + default: + return Response.json({ error: `Unsupported DynamoDB tool: ${toolId}` }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/dynamodb/operations.test.ts b/apps/sim/lib/internal/dynamodb/operations.test.ts new file mode 100644 index 00000000000..11ef0f41b78 --- /dev/null +++ b/apps/sim/lib/internal/dynamodb/operations.test.ts @@ -0,0 +1,104 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + createDocumentClient: vi.fn(), + createRawClient: vi.fn(), + describeTable: vi.fn(), + documentDestroy: vi.fn(), + getItem: vi.fn(), + listTables: vi.fn(), + rawDestroy: vi.fn(), +})) + +vi.mock('@/lib/internal/dynamodb/client', () => ({ + createDynamoDBClient: mocks.createDocumentClient, + createRawDynamoDBClient: mocks.createRawClient, + deleteItem: vi.fn(), + describeTable: mocks.describeTable, + getItem: mocks.getItem, + listTables: mocks.listTables, + putItem: vi.fn(), + queryItems: vi.fn(), + scanItems: vi.fn(), + updateItem: vi.fn(), +})) + +import { executeDynamodbGet, executeDynamodbIntrospect } from '@/lib/internal/dynamodb/operations' + +const CONNECTION = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', +} + +describe('DynamoDB operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.createDocumentClient.mockReturnValue({ destroy: mocks.documentDestroy }) + mocks.createRawClient.mockReturnValue({ destroy: mocks.rawDestroy }) + }) + + it('forwards cancellation and destroys the document client after success', async () => { + const controller = new AbortController() + mocks.getItem.mockResolvedValue({ item: { id: 'item-1' } }) + + await expect( + executeDynamodbGet( + { ...CONNECTION, tableName: 'test-table', key: { id: 'item-1' }, consistentRead: true }, + controller.signal + ) + ).resolves.toEqual({ message: 'Item retrieved successfully', item: { id: 'item-1' } }) + expect(mocks.getItem).toHaveBeenCalledWith( + { destroy: mocks.documentDestroy }, + 'test-table', + { id: 'item-1' }, + true, + controller.signal + ) + expect(mocks.documentDestroy).toHaveBeenCalledOnce() + }) + + it('destroys the document client when provider execution fails', async () => { + mocks.getItem.mockRejectedValue(new Error('provider failure')) + + await expect( + executeDynamodbGet({ ...CONNECTION, tableName: 'test-table', key: { id: 'item-1' } }) + ).rejects.toThrow('provider failure') + expect(mocks.documentDestroy).toHaveBeenCalledOnce() + }) + + it('forwards cancellation across introspection and destroys the raw client', async () => { + const controller = new AbortController() + const tableDetails = { + tableName: 'test-table', + tableStatus: 'ACTIVE', + keySchema: [], + attributeDefinitions: [], + globalSecondaryIndexes: [], + localSecondaryIndexes: [], + itemCount: 0, + tableSizeBytes: 0, + billingMode: 'PAY_PER_REQUEST', + } + mocks.listTables.mockResolvedValue({ tables: ['test-table'] }) + mocks.describeTable.mockResolvedValue({ tableDetails }) + + await expect( + executeDynamodbIntrospect({ ...CONNECTION, tableName: 'test-table' }, controller.signal) + ).resolves.toEqual({ + message: "Table 'test-table' described successfully.", + tables: ['test-table'], + tableDetails, + }) + expect(mocks.listTables).toHaveBeenCalledWith({ destroy: mocks.rawDestroy }, controller.signal) + expect(mocks.describeTable).toHaveBeenCalledWith( + { destroy: mocks.rawDestroy }, + 'test-table', + controller.signal + ) + expect(mocks.rawDestroy).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/internal/dynamodb/operations.ts b/apps/sim/lib/internal/dynamodb/operations.ts new file mode 100644 index 00000000000..58fcd8e95fb --- /dev/null +++ b/apps/sim/lib/internal/dynamodb/operations.ts @@ -0,0 +1,190 @@ +import type { AwsDynamodbDeleteBody } from '@/lib/api/contracts/tools/aws/dynamodb-delete' +import type { AwsDynamodbGetBody } from '@/lib/api/contracts/tools/aws/dynamodb-get' +import type { AwsDynamodbIntrospectBody } from '@/lib/api/contracts/tools/aws/dynamodb-introspect' +import type { AwsDynamodbPutBody } from '@/lib/api/contracts/tools/aws/dynamodb-put' +import type { AwsDynamodbQueryBody } from '@/lib/api/contracts/tools/aws/dynamodb-query' +import type { AwsDynamodbScanBody } from '@/lib/api/contracts/tools/aws/dynamodb-scan' +import type { AwsDynamodbUpdateBody } from '@/lib/api/contracts/tools/aws/dynamodb-update' +import { + createDynamoDBClient, + createRawDynamoDBClient, + deleteItem, + describeTable, + getItem, + listTables, + putItem, + queryItems, + scanItems, + updateItem, +} from '@/lib/internal/dynamodb/client' + +export async function executeDynamodbGet(input: AwsDynamodbGetBody, signal?: AbortSignal) { + signal?.throwIfAborted() + const client = createDynamoDBClient(input) + try { + const result = await getItem(client, input.tableName, input.key, input.consistentRead, signal) + signal?.throwIfAborted() + return { + message: result.item ? 'Item retrieved successfully' : 'Item not found', + item: result.item, + } + } finally { + client.destroy() + } +} + +export async function executeDynamodbPut(input: AwsDynamodbPutBody, signal?: AbortSignal) { + signal?.throwIfAborted() + const client = createDynamoDBClient(input) + try { + await putItem( + client, + input.tableName, + input.item, + { + conditionExpression: input.conditionExpression, + expressionAttributeNames: input.expressionAttributeNames, + expressionAttributeValues: input.expressionAttributeValues, + }, + signal + ) + signal?.throwIfAborted() + return { message: 'Item created successfully', item: input.item } + } finally { + client.destroy() + } +} + +export async function executeDynamodbQuery(input: AwsDynamodbQueryBody, signal?: AbortSignal) { + signal?.throwIfAborted() + const client = createDynamoDBClient(input) + try { + const result = await queryItems( + client, + input.tableName, + input.keyConditionExpression, + { + filterExpression: input.filterExpression, + expressionAttributeNames: input.expressionAttributeNames, + expressionAttributeValues: input.expressionAttributeValues, + indexName: input.indexName, + limit: input.limit, + exclusiveStartKey: input.exclusiveStartKey, + scanIndexForward: input.scanIndexForward, + }, + signal + ) + signal?.throwIfAborted() + return { + message: `Query returned ${result.count} items`, + items: result.items, + count: result.count, + ...(result.lastEvaluatedKey && { lastEvaluatedKey: result.lastEvaluatedKey }), + } + } finally { + client.destroy() + } +} + +export async function executeDynamodbScan(input: AwsDynamodbScanBody, signal?: AbortSignal) { + signal?.throwIfAborted() + const client = createDynamoDBClient(input) + try { + const result = await scanItems( + client, + input.tableName, + { + filterExpression: input.filterExpression, + projectionExpression: input.projectionExpression, + expressionAttributeNames: input.expressionAttributeNames, + expressionAttributeValues: input.expressionAttributeValues, + limit: input.limit, + exclusiveStartKey: input.exclusiveStartKey, + }, + signal + ) + signal?.throwIfAborted() + return { + message: `Scan returned ${result.count} items`, + items: result.items, + count: result.count, + ...(result.lastEvaluatedKey && { lastEvaluatedKey: result.lastEvaluatedKey }), + } + } finally { + client.destroy() + } +} + +export async function executeDynamodbUpdate(input: AwsDynamodbUpdateBody, signal?: AbortSignal) { + signal?.throwIfAborted() + const client = createDynamoDBClient(input) + try { + const result = await updateItem( + client, + input.tableName, + input.key, + input.updateExpression, + { + expressionAttributeNames: input.expressionAttributeNames, + expressionAttributeValues: input.expressionAttributeValues, + conditionExpression: input.conditionExpression, + }, + signal + ) + signal?.throwIfAborted() + return { message: 'Item updated successfully', item: result.attributes } + } finally { + client.destroy() + } +} + +export async function executeDynamodbDelete(input: AwsDynamodbDeleteBody, signal?: AbortSignal) { + signal?.throwIfAborted() + const client = createDynamoDBClient(input) + try { + await deleteItem( + client, + input.tableName, + input.key, + { + conditionExpression: input.conditionExpression, + expressionAttributeNames: input.expressionAttributeNames, + expressionAttributeValues: input.expressionAttributeValues, + }, + signal + ) + signal?.throwIfAborted() + return { message: 'Item deleted successfully' } + } finally { + client.destroy() + } +} + +export async function executeDynamodbIntrospect( + input: AwsDynamodbIntrospectBody, + signal?: AbortSignal +) { + signal?.throwIfAborted() + const client = createRawDynamoDBClient(input) + try { + const { tables } = await listTables(client, signal) + + if (input.tableName) { + const { tableDetails } = await describeTable(client, input.tableName, signal) + signal?.throwIfAborted() + return { + message: `Table '${input.tableName}' described successfully.`, + tables, + tableDetails, + } + } + + signal?.throwIfAborted() + return { + message: `Found ${tables.length} table(s) in region '${input.region}'.`, + tables, + } + } finally { + client.destroy() + } +} diff --git a/apps/sim/lib/internal/elevenlabs/client.test.ts b/apps/sim/lib/internal/elevenlabs/client.test.ts new file mode 100644 index 00000000000..b55e22fc61b --- /dev/null +++ b/apps/sim/lib/internal/elevenlabs/client.test.ts @@ -0,0 +1,151 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + generateElevenLabsAudio, + MAX_ELEVENLABS_AUDIO_BYTES, +} from '@/lib/internal/elevenlabs/client' +import { ElevenLabsOperationError } from '@/lib/internal/elevenlabs/errors' + +describe('ElevenLabs audio client', () => { + beforeEach(() => vi.restoreAllMocks()) + + it('sends the exact sound generation payload and bounds the audio response', async () => { + const fetchMock = vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response(new Uint8Array([1, 2, 3]), { + status: 200, + headers: { 'content-type': 'audio/mpeg' }, + }) + ) + + await expect( + generateElevenLabsAudio({ + operation: 'sound_effects', + input: { + apiKey: 'secret', + text: 'A soft chime', + modelId: 'eleven_text_to_sound_v2', + durationSeconds: 2, + promptInfluence: 0.4, + loop: true, + }, + }) + ).resolves.toEqual(Buffer.from([1, 2, 3])) + + const [url, init] = fetchMock.mock.calls[0] + expect(url).toBe('https://api.elevenlabs.io/v1/sound-generation') + expect(init).toMatchObject({ + method: 'POST', + headers: { + Accept: 'audio/mpeg', + 'Content-Type': 'application/json', + 'xi-api-key': 'secret', + }, + body: JSON.stringify({ + text: 'A soft chime', + model_id: 'eleven_text_to_sound_v2', + duration_seconds: 2, + prompt_influence: 0.4, + loop: true, + }), + }) + expect(init?.signal).toBeInstanceOf(AbortSignal) + }) + + it('sends source audio and speech conversion fields as multipart data', async () => { + const fetchMock = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValue(new Response(new Uint8Array([1]), { status: 200 })) + + await generateElevenLabsAudio({ + operation: 'speech_to_speech', + input: { + apiKey: 'secret', + voiceId: 'voice-1', + modelId: 'eleven_english_sts_v2', + removeBackgroundNoise: true, + }, + source: { + buffer: Buffer.from('audio'), + fileName: 'source.wav', + mimeType: 'audio/wav', + }, + }) + + const [url, init] = fetchMock.mock.calls[0] + expect(url).toBe('https://api.elevenlabs.io/v1/speech-to-speech/voice-1') + const body = init?.body as FormData + expect(body.get('model_id')).toBe('eleven_english_sts_v2') + expect(body.get('remove_background_noise')).toBe('true') + expect(body.get('audio')).toBeInstanceOf(File) + }) + + it('preserves provider status errors without materializing an unbounded body', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response('denied', { status: 401, statusText: 'Unauthorized' }) + ) + + await expect( + generateElevenLabsAudio({ + operation: 'sound_effects', + input: { apiKey: 'bad', text: 'sound' }, + }) + ).rejects.toEqual( + new ElevenLabsOperationError('ElevenLabs request failed: 401 Unauthorized', 401) + ) + }) + + it('preserves cancellation that occurs while draining a provider error', async () => { + const controller = new AbortController() + const reason = new DOMException('cancelled', 'AbortError') + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response( + new ReadableStream({ + pull(stream) { + controller.abort(reason) + stream.close() + }, + }), + { status: 500, statusText: 'Internal Server Error' } + ) + ) + + await expect( + generateElevenLabsAudio( + { operation: 'sound_effects', input: { apiKey: 'bad', text: 'sound' } }, + controller.signal + ) + ).rejects.toBe(reason) + }) + + it('rejects oversized audio from content length before buffering it', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response(new Uint8Array([1]), { + status: 200, + headers: { 'content-length': String(MAX_ELEVENLABS_AUDIO_BYTES + 1) }, + }) + ) + + await expect( + generateElevenLabsAudio({ + operation: 'sound_effects', + input: { apiKey: 'secret', text: 'sound' }, + }) + ).rejects.toMatchObject({ name: 'PayloadSizeLimitError' }) + }) + + it('does no provider work after cancellation', async () => { + const fetchMock = vi.spyOn(globalThis, 'fetch') + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + generateElevenLabsAudio( + { operation: 'sound_effects', input: { apiKey: 'secret', text: 'sound' } }, + controller.signal + ) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(fetchMock).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/elevenlabs/client.ts b/apps/sim/lib/internal/elevenlabs/client.ts new file mode 100644 index 00000000000..9fc0e862db8 --- /dev/null +++ b/apps/sim/lib/internal/elevenlabs/client.ts @@ -0,0 +1,114 @@ +import { + createTimeoutAbortController, + DEFAULT_EXECUTION_TIMEOUT_MS, +} from '@/lib/core/execution-limits' +import { consumeOrCancelBody, readResponseToBufferWithLimit } from '@/lib/core/utils/stream-limits' +import { ElevenLabsOperationError } from '@/lib/internal/elevenlabs/errors' +import type { + ElevenLabsAudioIsolationInput, + ElevenLabsSoundEffectsInput, + ElevenLabsSpeechToSpeechInput, +} from '@/lib/internal/elevenlabs/schema' + +const BASE_URL = 'https://api.elevenlabs.io/v1' +export const MAX_ELEVENLABS_AUDIO_BYTES = 25 * 1024 * 1024 + +export interface ElevenLabsSourceAudio { + buffer: Buffer + fileName: string + mimeType: string +} + +type GenerateElevenLabsAudioArgs = + | { operation: 'sound_effects'; input: ElevenLabsSoundEffectsInput; source?: never } + | { + operation: 'speech_to_speech' + input: ElevenLabsSpeechToSpeechInput + source: ElevenLabsSourceAudio + } + | { + operation: 'audio_isolation' + input: ElevenLabsAudioIsolationInput + source: ElevenLabsSourceAudio + } + +function buildRequest(args: GenerateElevenLabsAudioArgs): { url: string; init: RequestInit } { + const headers: Record = { + 'xi-api-key': args.input.apiKey, + Accept: 'audio/mpeg', + } + + if (args.operation === 'sound_effects') { + const body: Record = { text: args.input.text } + if (args.input.modelId) body.model_id = args.input.modelId + if (args.input.durationSeconds !== undefined) { + body.duration_seconds = args.input.durationSeconds + } + if (args.input.promptInfluence !== undefined) { + body.prompt_influence = args.input.promptInfluence + } + if (args.input.loop !== undefined) body.loop = args.input.loop + return { + url: `${BASE_URL}/sound-generation`, + init: { + method: 'POST', + headers: { ...headers, 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }, + } + } + + const formData = new FormData() + formData.append( + 'audio', + new Blob([new Uint8Array(args.source.buffer)], { type: args.source.mimeType }), + args.source.fileName + ) + + if (args.operation === 'speech_to_speech') { + if (args.input.modelId) formData.append('model_id', args.input.modelId) + if (args.input.removeBackgroundNoise !== undefined) { + formData.append('remove_background_noise', String(args.input.removeBackgroundNoise)) + } + return { + url: `${BASE_URL}/speech-to-speech/${args.input.voiceId}`, + init: { method: 'POST', headers, body: formData }, + } + } + + return { + url: `${BASE_URL}/audio-isolation`, + init: { method: 'POST', headers, body: formData }, + } +} + +export async function generateElevenLabsAudio( + args: GenerateElevenLabsAudioArgs, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const timeout = createTimeoutAbortController(DEFAULT_EXECUTION_TIMEOUT_MS, signal) + try { + const { url, init } = buildRequest(args) + const response = await fetch(url, { ...init, signal: timeout.signal }) + timeout.signal.throwIfAborted() + if (!response.ok) { + await consumeOrCancelBody(response) + timeout.signal.throwIfAborted() + throw new ElevenLabsOperationError( + `ElevenLabs request failed: ${response.status} ${response.statusText}`, + response.status + ) + } + + const buffer = await readResponseToBufferWithLimit(response, { + maxBytes: MAX_ELEVENLABS_AUDIO_BYTES, + label: `ElevenLabs ${args.operation} response`, + signal: timeout.signal, + }) + if (buffer.length === 0) throw new ElevenLabsOperationError('Empty audio received', 422) + return buffer + } finally { + timeout.cleanup() + } +} diff --git a/apps/sim/lib/internal/elevenlabs/errors.ts b/apps/sim/lib/internal/elevenlabs/errors.ts new file mode 100644 index 00000000000..f105885d8c1 --- /dev/null +++ b/apps/sim/lib/internal/elevenlabs/errors.ts @@ -0,0 +1,10 @@ +export class ElevenLabsOperationError extends Error { + constructor( + message: string, + readonly status: number, + readonly body: Record = { error: message } + ) { + super(message) + this.name = 'ElevenLabsOperationError' + } +} diff --git a/apps/sim/lib/internal/elevenlabs/execute-tool.test.ts b/apps/sim/lib/internal/elevenlabs/execute-tool.test.ts new file mode 100644 index 00000000000..ad11afb34ba --- /dev/null +++ b/apps/sim/lib/internal/elevenlabs/execute-tool.test.ts @@ -0,0 +1,108 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operations = vi.hoisted(() => ({ + executeElevenLabsAudioIsolation: vi.fn(), + executeElevenLabsSoundEffects: vi.fn(), + executeElevenLabsSpeechToSpeech: vi.fn(), +})) + +vi.mock('@/lib/internal/elevenlabs/operations', () => operations) + +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { ElevenLabsOperationError } from '@/lib/internal/elevenlabs/errors' +import { executeElevenLabsTool } from '@/lib/internal/elevenlabs/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +function toolRequest(overrides: Partial = {}) { + return { + toolId: 'elevenlabs_sound_effects', + input: { apiKey: 'secret', text: 'sound' }, + headers: new Headers(), + context: { ...createExecutionContext({ workflowId: 'workflow-1' }), userId: 'user-1' }, + requestId: 'request-1', + ...overrides, + } as InternalToolOperationCall +} + +const CASES = [ + [ + 'elevenlabs_sound_effects', + { apiKey: 'secret', text: 'sound' }, + operations.executeElevenLabsSoundEffects, + ], + [ + 'elevenlabs_speech_to_speech', + { apiKey: 'secret', voiceId: 'voice-1' }, + operations.executeElevenLabsSpeechToSpeech, + ], + ['elevenlabs_audio_isolation', { apiKey: 'secret' }, operations.executeElevenLabsAudioIsolation], +] as const + +describe('executeElevenLabsTool', () => { + beforeEach(() => { + vi.clearAllMocks() + for (const [, , operation] of CASES) operation.mockResolvedValue({ audioUrl: '/audio.mp3' }) + }) + + it.each(CASES)( + 'dispatches %s to the authoritative operation', + async (toolId, input, operation) => { + const response = await executeElevenLabsTool(toolRequest({ toolId, input })) + + expect(response.status).toBe(200) + expect(operation).toHaveBeenCalledWith( + input, + expect.objectContaining({ requestId: 'request-1', userId: 'user-1' }) + ) + } + ) + + it('authenticates before parsing operation input', async () => { + const response = await executeElevenLabsTool( + toolRequest({ + input: null, + context: createExecutionContext({ workflowId: 'workflow-1' }), + }) + ) + + expect(response.status).toBe(401) + await expect(response.json()).resolves.toEqual({ error: 'Unauthorized' }) + expect(operations.executeElevenLabsSoundEffects).not.toHaveBeenCalled() + }) + + it('preserves the legacy required-field validation message', async () => { + const response = await executeElevenLabsTool(toolRequest({ input: {} })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ + error: 'Missing required fields: operation and apiKey', + }) + }) + + it('preserves the route input byte ceiling', async () => { + const response = await executeElevenLabsTool( + toolRequest({ input: { apiKey: 'secret', text: 'x'.repeat(DEFAULT_MAX_JSON_BODY_BYTES) } }) + ) + + expect(response.status).toBe(413) + await expect(response.json()).resolves.toEqual({ + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }) + expect(operations.executeElevenLabsSoundEffects).not.toHaveBeenCalled() + }) + + it('projects typed operation failures without changing the envelope', async () => { + operations.executeElevenLabsSoundEffects.mockRejectedValueOnce( + new ElevenLabsOperationError('text is required', 400) + ) + + const response = await executeElevenLabsTool(toolRequest()) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ error: 'text is required' }) + }) +}) diff --git a/apps/sim/lib/internal/elevenlabs/execute-tool.ts b/apps/sim/lib/internal/elevenlabs/execute-tool.ts new file mode 100644 index 00000000000..13558ee091a --- /dev/null +++ b/apps/sim/lib/internal/elevenlabs/execute-tool.ts @@ -0,0 +1,116 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { z } from 'zod' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { ElevenLabsOperationError } from '@/lib/internal/elevenlabs/errors' +import { + type ElevenLabsOperationContext, + executeElevenLabsAudioIsolation, + executeElevenLabsSoundEffects, + executeElevenLabsSpeechToSpeech, +} from '@/lib/internal/elevenlabs/operations' +import { + elevenLabsAudioIsolationInputSchema, + elevenLabsSoundEffectsInputSchema, + elevenLabsSpeechToSpeechInputSchema, +} from '@/lib/internal/elevenlabs/schema' +import type { + InternalToolOperationCall, + InternalToolOperationHandler, +} from '@/lib/internal/tool-operations/types' + +const logger = createLogger('ElevenLabsToolExecution') + +async function executeOperation( + schema: z.ZodType, + request: InternalToolOperationCall, + execute: (input: Input, context: ElevenLabsOperationContext) => Promise +): Promise { + request.signal?.throwIfAborted() + let serializedInput: string + try { + serializedInput = JSON.stringify(request.input) ?? '' + } catch { + return Response.json({ error: 'Invalid request data' }, { status: 400 }) + } + if (Buffer.byteLength(serializedInput, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + const parsed = schema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { error: getValidationErrorMessage(parsed.error, 'Missing required parameters') }, + { status: 400 } + ) + } + const userId = request.context.userId + if (!userId) return Response.json({ error: 'Unauthorized' }, { status: 401 }) + + try { + const result = await execute(parsed.data, { + headers: request.headers, + requestId: request.requestId, + signal: request.signal, + userId, + workspaceId: request.context.workspaceId, + workflowId: request.context.workflowId, + executionId: request.context.executionId, + }) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof ElevenLabsOperationError) { + return Response.json(error.body, { status: error.status }) + } + const message = getErrorMessage(error, 'Unknown error') + logger.error('ElevenLabs operation failed', { + error: message, + requestId: request.requestId, + toolId: request.toolId, + }) + return Response.json( + { error: `Internal Server Error: ${message}` }, + { status: isPayloadSizeLimitError(error) ? 413 : 500 } + ) + } +} + +export const executeElevenLabsTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!request.context.userId) { + return Response.json({ error: 'Unauthorized' }, { status: 401 }) + } + switch (request.toolId) { + case 'elevenlabs_sound_effects': + return executeOperation( + elevenLabsSoundEffectsInputSchema, + request, + executeElevenLabsSoundEffects + ) + case 'elevenlabs_speech_to_speech': + return executeOperation( + elevenLabsSpeechToSpeechInputSchema, + request, + executeElevenLabsSpeechToSpeech + ) + case 'elevenlabs_audio_isolation': + return executeOperation( + elevenLabsAudioIsolationInputSchema, + request, + executeElevenLabsAudioIsolation + ) + default: + return Response.json( + { error: `Unsupported ElevenLabs tool: ${request.toolId}` }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/elevenlabs/operations.test.ts b/apps/sim/lib/internal/elevenlabs/operations.test.ts new file mode 100644 index 00000000000..1a206e69d5b --- /dev/null +++ b/apps/sim/lib/internal/elevenlabs/operations.test.ts @@ -0,0 +1,170 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { PRIVATE_MODEL_INPUT_PROVENANCE_HEADER } from '@/lib/execution/model-input-provenance' +import { + RESOLVED_SECRET_PROVENANCE_FIELD, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, +} from '@/lib/execution/private-tool-metadata' + +const mocks = vi.hoisted(() => ({ + assertToolFileAccess: vi.fn(), + downloadFileFromStorage: vi.fn(), + generateElevenLabsAudio: vi.fn(), + isModelSafeWorkspaceFileKey: vi.fn(), + storageUpload: vi.fn(), + uploadExecutionFile: vi.fn(), +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertToolFileAccess, +})) +vi.mock('@/lib/internal/elevenlabs/client', () => ({ + generateElevenLabsAudio: mocks.generateElevenLabsAudio, +})) +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadFileFromStorage: mocks.downloadFileFromStorage, +})) +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({ + isModelSafeWorkspaceFileKey: mocks.isModelSafeWorkspaceFileKey, + MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE: + 'File cannot be sent to a model because its secret provenance is unavailable', +})) +vi.mock('@/lib/uploads', () => ({ + StorageService: { uploadFile: mocks.storageUpload }, +})) +vi.mock('@/lib/uploads/contexts/execution', () => ({ + uploadExecutionFile: mocks.uploadExecutionFile, +})) +vi.mock('@/lib/core/utils/urls', () => ({ + getBaseUrl: () => 'https://sim.example.com', +})) + +import { + executeElevenLabsAudioIsolation, + executeElevenLabsSoundEffects, + executeElevenLabsSpeechToSpeech, +} from '@/lib/internal/elevenlabs/operations' + +const audioFile = { + id: 'file-1', + name: 'audio.wav', + size: 5, + type: 'audio/wav', + key: 'workspace/workspace-1/audio.wav', +} + +const context = { + headers: new Headers(), + requestId: 'request-1', + userId: 'user-1', +} + +describe('ElevenLabs operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.assertToolFileAccess.mockResolvedValue(null) + mocks.downloadFileFromStorage.mockResolvedValue(Buffer.from('source-audio')) + mocks.generateElevenLabsAudio.mockResolvedValue(Buffer.from('result-audio')) + mocks.isModelSafeWorkspaceFileKey.mockResolvedValue(true) + mocks.storageUpload.mockResolvedValue({ path: '/generated.mp3', size: 12 }) + mocks.uploadExecutionFile.mockResolvedValue({ + ...audioFile, + name: 'generated.mp3', + url: 'https://storage.example.com/generated.mp3', + }) + }) + + it('keeps headerless sound-effect execution compatible', async () => { + await expect( + executeElevenLabsSoundEffects({ apiKey: 'secret', text: 'A soft chime' }, context) + ).resolves.toEqual({ audioUrl: 'https://sim.example.com/generated.mp3', size: 12 }) + + expect(mocks.generateElevenLabsAudio).toHaveBeenCalledWith( + { operation: 'sound_effects', input: { apiKey: 'secret', text: 'A soft chime' } }, + undefined + ) + }) + + it('rejects incomplete private provenance before reading audio bytes', async () => { + const headers = new Headers({ + [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1, + }) + + await expect( + executeElevenLabsAudioIsolation( + { + apiKey: 'secret', + audioFile, + [RESOLVED_SECRET_PROVENANCE_FIELD]: { + version: 1, + complete: false, + entries: [], + }, + }, + { ...context, headers } + ) + ).rejects.toMatchObject({ + status: 400, + body: { error: 'Model input provenance is unavailable' }, + }) + expect(mocks.downloadFileFromStorage).not.toHaveBeenCalled() + expect(mocks.generateElevenLabsAudio).not.toHaveBeenCalled() + }) + + it('rejects unsafe tracked audio before reading or sending it', async () => { + mocks.isModelSafeWorkspaceFileKey.mockResolvedValue(false) + + await expect( + executeElevenLabsAudioIsolation({ apiKey: 'secret', audioFile }, context) + ).rejects.toMatchObject({ + status: 400, + body: { + error: 'File cannot be sent to a model because its secret provenance is unavailable', + }, + }) + expect(mocks.downloadFileFromStorage).not.toHaveBeenCalled() + expect(mocks.generateElevenLabsAudio).not.toHaveBeenCalled() + }) + + it('uses only trusted execution scope when storing the result', async () => { + const executionContext = { + ...context, + workspaceId: 'trusted-workspace', + workflowId: 'trusted-workflow', + executionId: 'trusted-execution', + } + const input = { + apiKey: 'secret', + audioFile, + workspaceId: 'spoofed-workspace', + workflowId: 'spoofed-workflow', + executionId: 'spoofed-execution', + } + + await executeElevenLabsAudioIsolation(input, executionContext) + + expect(mocks.uploadExecutionFile).toHaveBeenCalledWith( + { + workspaceId: 'trusted-workspace', + workflowId: 'trusted-workflow', + executionId: 'trusted-execution', + }, + Buffer.from('result-audio'), + expect.stringMatching(/^elevenlabs-audio_isolation-\d+\.mp3$/), + 'audio/mpeg', + 'user-1' + ) + }) + + it('preserves file authorization before speech voice validation', async () => { + mocks.assertToolFileAccess.mockResolvedValue( + Response.json({ success: false, error: 'File not found' }, { status: 404 }) + ) + + await expect( + executeElevenLabsSpeechToSpeech({ apiKey: 'secret', audioFile }, context) + ).rejects.toMatchObject({ status: 404 }) + }) +}) diff --git a/apps/sim/lib/internal/elevenlabs/operations.ts b/apps/sim/lib/internal/elevenlabs/operations.ts new file mode 100644 index 00000000000..a059270a493 --- /dev/null +++ b/apps/sim/lib/internal/elevenlabs/operations.ts @@ -0,0 +1,165 @@ +import { createLogger } from '@sim/logger' +import { validateAlphanumericId } from '@/lib/core/security/input-validation' +import { getBaseUrl } from '@/lib/core/utils/urls' +import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' +import { + type ElevenLabsSourceAudio, + generateElevenLabsAudio, +} from '@/lib/internal/elevenlabs/client' +import { ElevenLabsOperationError } from '@/lib/internal/elevenlabs/errors' +import type { + ElevenLabsAudioIsolationInput, + ElevenLabsSoundEffectsInput, + ElevenLabsSpeechToSpeechInput, +} from '@/lib/internal/elevenlabs/schema' +import { StorageService } from '@/lib/uploads' +import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' +import { + isModelSafeWorkspaceFileKey, + MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, +} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils' +import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' +import type { UserFile } from '@/executor/types' + +const logger = createLogger('ElevenLabsOperations') + +export interface ElevenLabsOperationContext { + headers: Headers + requestId: string + signal?: AbortSignal + userId: string + workspaceId?: string + workflowId?: string + executionId?: string +} + +export interface ElevenLabsAudioResult { + audioUrl: string + audioFile?: UserFile + size?: number +} + +function validatePrivateModelInput( + input: ElevenLabsSpeechToSpeechInput | ElevenLabsAudioIsolationInput, + context: ElevenLabsOperationContext +): void { + const provenance = validateOpaqueModelInputProvenance({ + headers: context.headers, + payload: input, + isInternalRequest: true, + }) + if (!provenance.success) { + throw new ElevenLabsOperationError(provenance.error, provenance.status) + } +} + +async function loadSourceAudio( + file: UserFile, + context: ElevenLabsOperationContext +): Promise { + context.signal?.throwIfAborted() + const denied = await assertToolFileAccess(file.key, context.userId, context.requestId, logger) + context.signal?.throwIfAborted() + if (denied) { + throw new ElevenLabsOperationError( + 'File not found', + denied.status, + (await denied.json()) as Record + ) + } + if (!(await isModelSafeWorkspaceFileKey(file.key))) { + throw new ElevenLabsOperationError(MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, 400) + } + context.signal?.throwIfAborted() + const buffer = await downloadFileFromStorage(file, context.requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) + context.signal?.throwIfAborted() + const extension = file.name.split('.').pop()?.toLowerCase() || '' + return { + buffer, + fileName: file.name, + mimeType: file.type || getMimeTypeFromExtension(extension), + } +} + +async function storeAudio( + operation: 'sound_effects' | 'speech_to_speech' | 'audio_isolation', + buffer: Buffer, + context: ElevenLabsOperationContext +): Promise { + context.signal?.throwIfAborted() + const fileName = `elevenlabs-${operation}-${Date.now()}.mp3` + if (context.workspaceId && context.workflowId && context.executionId) { + const audioFile = await uploadExecutionFile( + { + workspaceId: context.workspaceId, + workflowId: context.workflowId, + executionId: context.executionId, + }, + buffer, + fileName, + 'audio/mpeg', + context.userId + ) + context.signal?.throwIfAborted() + return { audioFile, audioUrl: audioFile.url } + } + + const file = await StorageService.uploadFile({ + file: buffer, + fileName, + contentType: 'audio/mpeg', + context: 'copilot', + }) + context.signal?.throwIfAborted() + return { audioUrl: `${getBaseUrl()}${file.path}`, size: file.size } +} + +export async function executeElevenLabsSoundEffects( + input: ElevenLabsSoundEffectsInput, + context: ElevenLabsOperationContext +): Promise { + if (!input.text) throw new ElevenLabsOperationError('text is required', 400) + const buffer = await generateElevenLabsAudio( + { operation: 'sound_effects', input }, + context.signal + ) + return storeAudio('sound_effects', buffer, context) +} + +export async function executeElevenLabsSpeechToSpeech( + input: ElevenLabsSpeechToSpeechInput, + context: ElevenLabsOperationContext +): Promise { + validatePrivateModelInput(input, context) + if (!input.audioFile) throw new ElevenLabsOperationError('audioFile is required', 400) + const source = await loadSourceAudio(input.audioFile, context) + if (!input.voiceId) throw new ElevenLabsOperationError('voiceId is required', 400) + const validation = validateAlphanumericId(input.voiceId, 'voiceId', 255) + if (!validation.isValid) { + throw new ElevenLabsOperationError(validation.error || 'Invalid voiceId', 400) + } + const buffer = await generateElevenLabsAudio( + { operation: 'speech_to_speech', input, source }, + context.signal + ) + return storeAudio('speech_to_speech', buffer, context) +} + +export async function executeElevenLabsAudioIsolation( + input: ElevenLabsAudioIsolationInput, + context: ElevenLabsOperationContext +): Promise { + validatePrivateModelInput(input, context) + if (!input.audioFile) throw new ElevenLabsOperationError('audioFile is required', 400) + const source = await loadSourceAudio(input.audioFile, context) + const buffer = await generateElevenLabsAudio( + { operation: 'audio_isolation', input, source }, + context.signal + ) + return storeAudio('audio_isolation', buffer, context) +} diff --git a/apps/sim/lib/internal/elevenlabs/schema.ts b/apps/sim/lib/internal/elevenlabs/schema.ts new file mode 100644 index 00000000000..d38b178d3d9 --- /dev/null +++ b/apps/sim/lib/internal/elevenlabs/schema.ts @@ -0,0 +1,51 @@ +import { z } from 'zod' +import { resolvedSecretTraceProvenanceSchema, userFileSchema } from '@/lib/api/contracts/primitives' +import { toolBooleanSchema } from '@/lib/api/contracts/tools/media/shared' +import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' + +const MISSING_FIELDS_ERROR = 'Missing required fields: operation and apiKey' + +const apiKeySchema = z.string({ error: MISSING_FIELDS_ERROR }).min(1, MISSING_FIELDS_ERROR) + +const audioFileSchema = userFileSchema.extend({ + type: z.string().optional().default(''), +}) + +const privateProvenance = { + [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), +} + +export const elevenLabsSoundEffectsInputSchema = z + .object({ + apiKey: apiKeySchema, + text: z.string().optional(), + modelId: z.string().optional(), + durationSeconds: z.coerce.number().min(0.5).max(30).optional(), + promptInfluence: z.coerce.number().min(0).max(1).optional(), + loop: toolBooleanSchema.optional(), + ...privateProvenance, + }) + .passthrough() + +export const elevenLabsSpeechToSpeechInputSchema = z + .object({ + apiKey: apiKeySchema, + voiceId: z.string().optional(), + audioFile: audioFileSchema.optional(), + modelId: z.string().optional(), + removeBackgroundNoise: toolBooleanSchema.optional(), + ...privateProvenance, + }) + .passthrough() + +export const elevenLabsAudioIsolationInputSchema = z + .object({ + apiKey: apiKeySchema, + audioFile: audioFileSchema.optional(), + ...privateProvenance, + }) + .passthrough() + +export type ElevenLabsSoundEffectsInput = z.output +export type ElevenLabsSpeechToSpeechInput = z.output +export type ElevenLabsAudioIsolationInput = z.output diff --git a/apps/sim/lib/internal/embeddings/execute-tool.test.ts b/apps/sim/lib/internal/embeddings/execute-tool.test.ts new file mode 100644 index 00000000000..66524f6d630 --- /dev/null +++ b/apps/sim/lib/internal/embeddings/execute-tool.test.ts @@ -0,0 +1,78 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mockExecuteEmbedding = vi.hoisted(() => vi.fn()) + +vi.mock('@/lib/internal/embeddings/operations', () => ({ + executeEmbedding: mockExecuteEmbedding, +})) + +import { executeEmbeddingsTool } from '@/lib/internal/embeddings/execute-tool' + +function request(input: unknown, overrides: Record = {}) { + return { + toolId: 'embeddings_openai', + input, + headers: new Headers(), + context: { userId: 'user-1' }, + requestId: 'request-1', + ...overrides, + } as never +} + +describe('executeEmbeddingsTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mockExecuteEmbedding.mockResolvedValue(Response.json({ success: true })) + }) + + it('rejects a provider that does not match the declared tool', async () => { + const response = await executeEmbeddingsTool( + request({ provider: 'gemini', apiKey: 'key', input: 'hello' }) + ) + + expect(response.status).toBe(400) + expect(mockExecuteEmbedding).not.toHaveBeenCalled() + }) + + it('supports the legacy OpenAI alias through the same operation', async () => { + const response = await executeEmbeddingsTool( + request( + { provider: 'openai', apiKey: 'key', input: '' }, + { toolId: 'openai_embeddings' } + ) + ) + + expect(response.status).toBe(200) + expect(mockExecuteEmbedding).toHaveBeenCalledWith( + expect.objectContaining({ provider: 'openai', input: '' }), + expect.objectContaining({ signal: undefined }) + ) + }) + + it.each([null, '', ' '])( + 'treats dimensions sentinel %j as the native default', + async (value) => { + const response = await executeEmbeddingsTool( + request({ provider: 'openai', apiKey: 'key', input: 'hello', dimensions: value }) + ) + + expect(response.status).toBe(200) + expect(mockExecuteEmbedding).toHaveBeenCalledWith( + expect.objectContaining({ dimensions: undefined }), + expect.any(Object) + ) + } + ) + + it('still rejects explicit zero dimensions', async () => { + const response = await executeEmbeddingsTool( + request({ provider: 'openai', apiKey: 'key', input: 'hello', dimensions: 0 }) + ) + + expect(response.status).toBe(400) + expect(mockExecuteEmbedding).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/embeddings/execute-tool.ts b/apps/sim/lib/internal/embeddings/execute-tool.ts new file mode 100644 index 00000000000..ca400dbf556 --- /dev/null +++ b/apps/sim/lib/internal/embeddings/execute-tool.ts @@ -0,0 +1,57 @@ +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { executeEmbedding } from '@/lib/internal/embeddings/operations' +import { type EmbeddingProvider, embeddingsInputSchema } from '@/lib/internal/embeddings/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const PROVIDERS_BY_TOOL_ID: Record = { + openai_embeddings: 'openai', + embeddings_openai: 'openai', + embeddings_openrouter: 'openrouter', + embeddings_gemini: 'gemini', + embeddings_cohere: 'cohere', + embeddings_mistral: 'mistral', +} + +export const executeEmbeddingsTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!request.context.userId) { + return Response.json({ success: false, error: 'Unauthorized' }, { status: 401 }) + } + const expectedProvider = PROVIDERS_BY_TOOL_ID[request.toolId] + if (!expectedProvider) { + return Response.json( + { success: false, error: `Unsupported embeddings tool: ${request.toolId}` }, + { status: 500 } + ) + } + let serializedInput: string + try { + serializedInput = JSON.stringify(request.input) ?? '' + } catch { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + if (Buffer.byteLength(serializedInput, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + success: false, + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + const parsed = embeddingsInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { success: false, error: getValidationErrorMessage(parsed.error, 'Invalid request data') }, + { status: 400 } + ) + } + if (parsed.data.provider !== expectedProvider) { + return Response.json( + { success: false, error: `Provider must be ${expectedProvider} for ${request.toolId}` }, + { status: 400 } + ) + } + return executeEmbedding(parsed.data, { signal: request.signal }) +} diff --git a/apps/sim/lib/internal/embeddings/operations.test.ts b/apps/sim/lib/internal/embeddings/operations.test.ts new file mode 100644 index 00000000000..66a5c795427 --- /dev/null +++ b/apps/sim/lib/internal/embeddings/operations.test.ts @@ -0,0 +1,55 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + embed: vi.fn(), + embedOpenRouter: vi.fn(), +})) + +vi.mock('@/lib/embeddings', () => ({ + DEFAULT_MODEL_BY_PROVIDER: { openai: 'text-embedding-3-small' }, + DEFAULT_OPENROUTER_EMBEDDING_MODEL: 'openrouter/openai/text-embedding-3-small', + EmbeddingOutputLimitError: class EmbeddingOutputLimitError extends Error {}, + embed: mocks.embed, + embedOpenRouter: mocks.embedOpenRouter, + findEmbeddingModelInfo: vi.fn(), + resolveDimensions: vi.fn(), +})) + +vi.mock('@/lib/embeddings/openrouter-model-catalog.server', () => ({ + getOpenRouterEmbeddingModelMetadata: vi.fn(), + OpenRouterEmbeddingModelNotFoundError: class OpenRouterEmbeddingModelNotFoundError extends Error {}, +})) + +import { executeEmbedding } from '@/lib/internal/embeddings/operations' +import { MAX_EMBEDDING_INPUTS, MAX_EMBEDDING_TOTAL_CHARS } from '@/lib/internal/embeddings/schema' + +const baseInput = { + provider: 'openai' as const, + apiKey: 'key', + model: 'text-embedding-3-small', +} + +describe('embedding operation admission limits', () => { + it('rejects a JSON-encoded array after expansion when it exceeds the item cap', async () => { + const input = JSON.stringify(Array.from({ length: MAX_EMBEDDING_INPUTS + 1 }, () => 'x')) + const response = await executeEmbedding({ ...baseInput, input }, {}) + + expect(response.status).toBe(400) + expect((await response.json()).error).toContain(`${MAX_EMBEDDING_INPUTS}`) + expect(mocks.embed).not.toHaveBeenCalled() + }) + + it('rejects aggregate input characters before provider dispatch', async () => { + const response = await executeEmbedding( + { ...baseInput, input: 'x'.repeat(MAX_EMBEDDING_TOTAL_CHARS + 1) }, + {} + ) + + expect(response.status).toBe(400) + expect((await response.json()).error).toContain(`${MAX_EMBEDDING_TOTAL_CHARS}`) + expect(mocks.embed).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/embeddings/operations.ts b/apps/sim/lib/internal/embeddings/operations.ts new file mode 100644 index 00000000000..8154b1da469 --- /dev/null +++ b/apps/sim/lib/internal/embeddings/operations.ts @@ -0,0 +1,164 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { + DEFAULT_MODEL_BY_PROVIDER, + DEFAULT_OPENROUTER_EMBEDDING_MODEL, + EmbeddingOutputLimitError, + embed, + embedOpenRouter, + findEmbeddingModelInfo, + resolveDimensions, +} from '@/lib/embeddings' +import { + getOpenRouterEmbeddingModelMetadata, + type OpenRouterEmbeddingModelMetadata, + OpenRouterEmbeddingModelNotFoundError, +} from '@/lib/embeddings/openrouter-model-catalog.server' +import { normalizeOpenRouterEmbeddingModelId } from '@/lib/embeddings/openrouter-models' +import type { EmbedResult } from '@/lib/embeddings/types' +import { + type EmbeddingsInput, + MAX_EMBEDDING_INPUTS, + MAX_EMBEDDING_TOTAL_CHARS, +} from '@/lib/internal/embeddings/schema' + +const logger = createLogger('EmbeddingOperations') + +export interface EmbeddingOperationContext { + signal?: AbortSignal +} + +function failureResponse(error: string, status: number): Response { + return Response.json({ success: false, error }, { status }) +} + +export function normalizeEmbeddingInput(input: string | string[]): string[] { + if (Array.isArray(input)) return input + if (/^\s*\[/.test(input)) { + try { + const parsed: unknown = JSON.parse(input) + if (Array.isArray(parsed) && parsed.every((entry) => typeof entry === 'string')) return parsed + } catch {} + } + return [input] +} + +export async function executeEmbedding( + input: EmbeddingsInput, + context: EmbeddingOperationContext +): Promise { + context.signal?.throwIfAborted() + const { provider, apiKey, model, taskType, dimensions } = input + const texts = normalizeEmbeddingInput(input.input) + if (texts.length === 0) return failureResponse('input must contain at least one text', 400) + if (texts.length > MAX_EMBEDDING_INPUTS) { + return failureResponse( + `input cannot exceed ${MAX_EMBEDDING_INPUTS} texts, received ${texts.length}`, + 400 + ) + } + const totalChars = texts.reduce((sum, text) => sum + text.length, 0) + if (totalChars > MAX_EMBEDDING_TOTAL_CHARS) { + return failureResponse( + `Input is too large: ${totalChars} characters exceeds the ${MAX_EMBEDDING_TOTAL_CHARS} limit`, + 400 + ) + } + if (texts.some((text) => !/\S/.test(text))) { + return failureResponse('input entries cannot be empty', 400) + } + + let resolvedModel: string + let openRouterModelMetadata: OpenRouterEmbeddingModelMetadata | undefined + if (provider === 'openrouter') { + try { + resolvedModel = normalizeOpenRouterEmbeddingModelId( + model || DEFAULT_OPENROUTER_EMBEDDING_MODEL + ) + } catch (error) { + return failureResponse(getErrorMessage(error, 'Invalid OpenRouter embedding model'), 400) + } + try { + openRouterModelMetadata = await getOpenRouterEmbeddingModelMetadata( + resolvedModel, + context.signal + ) + } catch (error) { + context.signal?.throwIfAborted() + const notFound = error instanceof OpenRouterEmbeddingModelNotFoundError + return failureResponse( + getErrorMessage( + error, + notFound + ? 'Unsupported OpenRouter embedding model' + : 'Failed to load OpenRouter embedding model metadata' + ), + notFound ? 400 : 502 + ) + } + } else { + resolvedModel = model || DEFAULT_MODEL_BY_PROVIDER[provider] + } + + if (provider !== 'openrouter') { + const info = findEmbeddingModelInfo(resolvedModel) + if (!info) return failureResponse(`Unsupported embedding model: ${resolvedModel}`, 400) + if (info.provider !== provider) { + return failureResponse( + `Model ${resolvedModel} belongs to ${info.provider}, not ${provider}`, + 400 + ) + } + try { + resolveDimensions(info, dimensions) + } catch (error) { + return failureResponse(getErrorMessage(error, 'Invalid dimensions'), 400) + } + } + + logger.info(`Embedding ${texts.length} input(s) with ${provider}/${resolvedModel}`) + try { + let result: EmbedResult + if (provider === 'openrouter') { + if (!openRouterModelMetadata) { + throw new Error('Failed to load OpenRouter embedding model metadata') + } + result = await embedOpenRouter(texts, { + model: resolvedModel, + dimensions, + apiKey, + maxInputTokens: openRouterModelMetadata.maxInputTokens, + projectInputs: null, + signal: context.signal, + }) + } else { + result = await embed(texts, { + model: resolvedModel, + taskType, + dimensions, + apiKey, + projectInputs: null, + signal: context.signal, + }) + } + context.signal?.throwIfAborted() + return Response.json({ + success: true, + embeddings: result.embeddings, + model: result.modelName, + provider, + dimensions: result.dimensions, + usage: { prompt_tokens: result.totalTokens, total_tokens: result.totalTokens }, + __embeddingTokens: result.totalTokens, + }) + } catch (error) { + context.signal?.throwIfAborted() + const message = getErrorMessage(error, 'Embedding generation failed') + if (error instanceof EmbeddingOutputLimitError) { + logger.warn('Embedding output exceeds safe limit', { error: message }) + return failureResponse(message, 413) + } + logger.error('Embedding generation failed', { error: message }) + return failureResponse(message, 502) + } +} diff --git a/apps/sim/lib/internal/embeddings/schema.ts b/apps/sim/lib/internal/embeddings/schema.ts new file mode 100644 index 00000000000..ab37064c629 --- /dev/null +++ b/apps/sim/lib/internal/embeddings/schema.ts @@ -0,0 +1,72 @@ +import { z } from 'zod' +import type { EmbeddingCatalogProvider, EmbeddingTaskType } from '@/lib/embeddings/types' + +type EmbeddingToolProvider = EmbeddingCatalogProvider | 'openrouter' + +export const embeddingProviders = [ + 'openai', + 'openrouter', + 'gemini', + 'cohere', + 'mistral', +] as const satisfies readonly EmbeddingToolProvider[] + +export const embeddingTaskTypes = [ + 'document', + 'query', + 'similarity', + 'classification', + 'clustering', +] as const satisfies readonly EmbeddingTaskType[] + +export const MAX_EMBEDDING_INPUTS = 1000 +export const MAX_EMBEDDING_TOTAL_CHARS = 1_000_000 + +const commonShape = { + model: z.string().min(1, 'model cannot be empty').optional(), + input: z.union( + [ + z.string().min(1, 'input cannot be empty'), + z + .array(z.string().min(1, 'input entries cannot be empty')) + .min(1, 'input must contain at least one text') + .max(MAX_EMBEDDING_INPUTS, `input cannot exceed ${MAX_EMBEDDING_INPUTS} texts`), + ], + { error: 'Missing required field: input' } + ), + taskType: z.enum(embeddingTaskTypes).optional(), + dimensions: z.preprocess( + (value) => + value === null || (typeof value === 'string' && value.trim() === '') ? undefined : value, + z.coerce + .number() + .int('dimensions must be an integer') + .min(1, 'dimensions must be at least 1') + .max(4096, 'dimensions cannot exceed 4096') + .optional() + ), +} + +const catalogProviders = [ + 'openai', + 'gemini', + 'cohere', + 'mistral', +] as const satisfies readonly EmbeddingCatalogProvider[] + +export const embeddingsInputSchema = z.discriminatedUnion('provider', [ + z.object({ + ...commonShape, + provider: z.enum(catalogProviders), + apiKey: z.string({ error: 'apiKey is required' }).min(1, 'apiKey cannot be empty'), + }), + z.object({ + ...commonShape, + provider: z.literal('openrouter'), + apiKey: z.string({ error: 'apiKey is required' }).min(1, 'apiKey cannot be empty'), + }), +]) + +export type EmbeddingsInput = z.output +export type EmbeddingProvider = (typeof embeddingProviders)[number] +export type EmbeddingTaskTypeName = (typeof embeddingTaskTypes)[number] diff --git a/apps/sim/lib/internal/enrichment/execute-tool.test.ts b/apps/sim/lib/internal/enrichment/execute-tool.test.ts new file mode 100644 index 00000000000..8b3dde2b338 --- /dev/null +++ b/apps/sim/lib/internal/enrichment/execute-tool.test.ts @@ -0,0 +1,66 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mockExecuteEnrichment = vi.hoisted(() => vi.fn()) + +vi.mock('@/lib/internal/enrichment/operations', () => ({ + executeEnrichment: mockExecuteEnrichment, +})) + +import { executeEnrichmentTool } from '@/lib/internal/enrichment/execute-tool' + +describe('executeEnrichmentTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mockExecuteEnrichment.mockResolvedValue(Response.json({ matched: false })) + }) + + it('uses trusted workspace scope and preserves mapped model inputs', async () => { + const traceRegistry = new Map() + const response = await executeEnrichmentTool({ + toolId: 'enrichment_run', + input: { + enrichmentId: 'work-email', + inputs: { company: '', secret: '{{SECRET_NAME}}' }, + workspaceId: 'attacker-workspace', + }, + headers: new Headers(), + context: { + userId: 'user-1', + workspaceId: 'trusted-workspace', + resolvedSecretTraceRegistry: traceRegistry, + }, + requestId: 'request-1', + } as never) + + expect(response.status).toBe(200) + expect(mockExecuteEnrichment).toHaveBeenCalledWith( + { + enrichmentId: 'work-email', + inputs: { company: '', secret: '{{SECRET_NAME}}' }, + }, + expect.objectContaining({ + workspaceId: 'trusted-workspace', + resolvedSecretTraceRegistry: traceRegistry, + }) + ) + }) + + it('rejects more than 100 mapped fields before running a provider', async () => { + const inputs = Object.fromEntries( + Array.from({ length: 101 }, (_, index) => [`f${index}`, index]) + ) + const response = await executeEnrichmentTool({ + toolId: 'enrichment_run', + input: { enrichmentId: 'work-email', inputs }, + headers: new Headers(), + context: { userId: 'user-1', workspaceId: 'workspace-1' }, + requestId: 'request-1', + } as never) + + expect(response.status).toBe(400) + expect(mockExecuteEnrichment).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/enrichment/execute-tool.ts b/apps/sim/lib/internal/enrichment/execute-tool.ts new file mode 100644 index 00000000000..0d353357566 --- /dev/null +++ b/apps/sim/lib/internal/enrichment/execute-tool.ts @@ -0,0 +1,47 @@ +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { executeEnrichment } from '@/lib/internal/enrichment/operations' +import { enrichmentInputSchema } from '@/lib/internal/enrichment/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +export const executeEnrichmentTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!request.context.userId || !request.context.workspaceId) { + return Response.json({ error: 'Unauthorized' }, { status: 401 }) + } + if (request.toolId !== 'enrichment_run') { + return Response.json( + { error: `Unsupported enrichment tool: ${request.toolId}` }, + { status: 500 } + ) + } + + let serializedInput: string + try { + serializedInput = JSON.stringify(request.input) ?? '' + } catch { + return Response.json({ error: 'Invalid request' }, { status: 400 }) + } + if (Buffer.byteLength(serializedInput, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + + const parsed = enrichmentInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { error: getValidationErrorMessage(parsed.error, 'Invalid request') }, + { status: 400 } + ) + } + + return executeEnrichment(parsed.data, { + workspaceId: request.context.workspaceId, + signal: request.signal, + resolvedSecretTraceRegistry: request.context.resolvedSecretTraceRegistry, + }) +} diff --git a/apps/sim/lib/internal/enrichment/operations.ts b/apps/sim/lib/internal/enrichment/operations.ts new file mode 100644 index 00000000000..ee9f9a1f97d --- /dev/null +++ b/apps/sim/lib/internal/enrichment/operations.ts @@ -0,0 +1,34 @@ +import { createLogger } from '@sim/logger' +import type { EnrichmentInput } from '@/lib/internal/enrichment/schema' +import { getEnrichment } from '@/enrichments/registry' +import { runEnrichment } from '@/enrichments/run' +import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' + +const logger = createLogger('EnrichmentOperations') + +export interface EnrichmentOperationContext { + workspaceId: string + signal?: AbortSignal + resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry +} + +export async function executeEnrichment( + input: EnrichmentInput, + context: EnrichmentOperationContext +): Promise { + context.signal?.throwIfAborted() + const enrichment = getEnrichment(input.enrichmentId) + if (!enrichment) { + return Response.json({ error: `Unknown enrichment "${input.enrichmentId}"` }, { status: 400 }) + } + + const { result, cost, error, provider } = await runEnrichment(enrichment, input.inputs, { + workspaceId: context.workspaceId, + signal: context.signal, + resolvedSecretTraceRegistry: context.resolvedSecretTraceRegistry, + }) + context.signal?.throwIfAborted() + const matched = Object.keys(result).length > 0 + logger.info('Enrichment block run', { enrichmentId: input.enrichmentId, matched, provider }) + return Response.json({ matched, result, cost, error, provider }) +} diff --git a/apps/sim/lib/internal/enrichment/schema.ts b/apps/sim/lib/internal/enrichment/schema.ts new file mode 100644 index 00000000000..e39bdc573aa --- /dev/null +++ b/apps/sim/lib/internal/enrichment/schema.ts @@ -0,0 +1,16 @@ +import { z } from 'zod' + +export const MAX_ENRICHMENT_INPUT_FIELDS = 100 + +export const enrichmentInputSchema = z.object({ + enrichmentId: z.string().min(1, 'enrichmentId is required'), + inputs: z + .record(z.string(), z.unknown()) + .default({}) + .refine( + (inputs) => Object.keys(inputs).length <= MAX_ENRICHMENT_INPUT_FIELDS, + `inputs cannot exceed ${MAX_ENRICHMENT_INPUT_FIELDS} fields` + ), +}) + +export type EnrichmentInput = z.output diff --git a/apps/sim/lib/internal/extend/client.test.ts b/apps/sim/lib/internal/extend/client.test.ts new file mode 100644 index 00000000000..ea22ebb50a3 --- /dev/null +++ b/apps/sim/lib/internal/extend/client.test.ts @@ -0,0 +1,122 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + loggerError: vi.fn(), + secureFetch: vi.fn(), + validateUrl: vi.fn(), +})) + +vi.mock('@sim/logger', () => ({ + createLogger: () => ({ error: mocks.loggerError }), +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + MAX_JSON_API_RESPONSE_BYTES: 10 * 1024 * 1024, + secureFetchWithPinnedIP: mocks.secureFetch, + validateUrlWithDNS: mocks.validateUrl, +})) + +import { submitExtendParse } from '@/lib/internal/extend/client' +import { ExtendOperationError } from '@/lib/internal/extend/errors' + +describe('submitExtendParse', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.validateUrl.mockResolvedValue({ isValid: true, resolvedIP: '203.0.113.1' }) + }) + + it('preserves provider status when its error body exceeds the diagnostic cap', async () => { + let cancelled = false + mocks.secureFetch.mockResolvedValue( + new Response( + new ReadableStream({ + cancel: () => { + cancelled = true + }, + }), + { + status: 429, + statusText: 'Too Many Requests', + headers: { 'content-length': String(64 * 1024 + 1) }, + } + ) + ) + + await expect(submitExtendParse('key', { file: {} })).rejects.toEqual( + new ExtendOperationError(429, { + success: false, + error: 'Extend API error: Too Many Requests', + }) + ) + expect(cancelled).toBe(true) + expect(mocks.secureFetch).toHaveBeenCalledWith( + 'https://api.extend.ai/parse', + '203.0.113.1', + expect.objectContaining({ maxResponseBytes: 10 * 1024 * 1024 }) + ) + }) + + it('preserves a bounded provider error message and status', async () => { + mocks.secureFetch.mockResolvedValue( + Response.json( + { + message: 'Document format is unsupported', + diagnostic: 'https://storage.example.com/file?signature=private', + }, + { status: 422 } + ) + ) + + await expect(submitExtendParse('key', { file: {} })).rejects.toEqual( + new ExtendOperationError(422, { + success: false, + error: 'Document format is unsupported', + }) + ) + expect(mocks.loggerError).toHaveBeenCalledWith('Extend API error', { status: 422 }) + expect(JSON.stringify(mocks.loggerError.mock.calls)).not.toContain('signature=private') + }) + + it('rejects malformed successful provider responses', async () => { + mocks.secureFetch.mockResolvedValue(Response.json([])) + + await expect(submitExtendParse('key', { file: {} })).rejects.toEqual( + new ExtendOperationError(502, { + success: false, + error: 'Extend API returned an invalid response', + }) + ) + }) + + it('maps transport failures to a provider-unavailable response without exposing details', async () => { + mocks.secureFetch.mockRejectedValue(new Error('TLS handshake exposed private details')) + + await expect(submitExtendParse('key', { file: {} })).rejects.toEqual( + new ExtendOperationError(502, { + success: false, + error: 'Failed to reach Extend API', + }) + ) + expect(mocks.loggerError).toHaveBeenCalledWith('Extend API request failed', { + errorName: 'Error', + }) + expect(JSON.stringify(mocks.loggerError.mock.calls)).not.toContain('private details') + }) + + it('preserves caller cancellation when the pinned request rejects', async () => { + const controller = new AbortController() + const cancellation = new DOMException('cancelled', 'AbortError') + mocks.secureFetch.mockImplementation(async () => { + controller.abort(cancellation) + throw cancellation + }) + + await expect(submitExtendParse('key', { file: {} }, controller.signal)).rejects.toBe( + cancellation + ) + expect(mocks.loggerError).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/extend/client.ts b/apps/sim/lib/internal/extend/client.ts new file mode 100644 index 00000000000..df11e20eca3 --- /dev/null +++ b/apps/sim/lib/internal/extend/client.ts @@ -0,0 +1,91 @@ +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { isPlainRecord } from '@sim/utils/object' +import { + MAX_JSON_API_RESPONSE_BYTES, + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + readResponseTextWithLimit, +} from '@/lib/core/utils/stream-limits' +import { ExtendOperationError } from '@/lib/internal/extend/errors' + +const logger = createLogger('ExtendClient') +const EXTEND_ENDPOINT = 'https://api.extend.ai/parse' + +export async function submitExtendParse( + apiKey: string, + body: Record, + signal?: AbortSignal +): Promise> { + signal?.throwIfAborted() + const validation = await validateUrlWithDNS(EXTEND_ENDPOINT, 'Extend API URL') + signal?.throwIfAborted() + if (!validation.isValid || !validation.resolvedIP) { + throw new ExtendOperationError(502, { success: false, error: 'Failed to reach Extend API' }) + } + + let response: Awaited> + try { + response = await secureFetchWithPinnedIP(EXTEND_ENDPOINT, validation.resolvedIP, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + Authorization: `Bearer ${apiKey}`, + 'x-extend-api-version': '2025-04-21', + }, + body: JSON.stringify(body), + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + signal, + }) + } catch (error) { + signal?.throwIfAborted() + logger.error('Extend API request failed', { errorName: toError(error).name }) + throw new ExtendOperationError(502, { success: false, error: 'Failed to reach Extend API' }) + } + signal?.throwIfAborted() + + if (!response.ok) { + let diagnostic = '' + try { + diagnostic = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Extend API error response', + signal, + }) + } catch { + signal?.throwIfAborted() + } + logger.error('Extend API error', { status: response.status }) + let clientError = `Extend API error: ${response.statusText || response.status}` + try { + const parsed: unknown = JSON.parse(diagnostic) + if (isPlainRecord(parsed)) { + const detail = parsed.message ?? parsed.error + if (typeof detail === 'string') clientError = detail + } + } catch {} + throw new ExtendOperationError(response.status, { success: false, error: clientError }) + } + + let output: unknown + try { + output = await response.json() + } catch { + signal?.throwIfAborted() + throw new ExtendOperationError(502, { + success: false, + error: 'Extend API returned an invalid response', + }) + } + if (!isPlainRecord(output)) { + throw new ExtendOperationError(502, { + success: false, + error: 'Extend API returned an invalid response', + }) + } + return output +} diff --git a/apps/sim/lib/internal/extend/errors.ts b/apps/sim/lib/internal/extend/errors.ts new file mode 100644 index 00000000000..657d7d7b8ff --- /dev/null +++ b/apps/sim/lib/internal/extend/errors.ts @@ -0,0 +1,9 @@ +export class ExtendOperationError extends Error { + constructor( + readonly status: number, + readonly body: unknown + ) { + super('Extend operation failed') + this.name = 'ExtendOperationError' + } +} diff --git a/apps/sim/lib/internal/extend/execute-tool.test.ts b/apps/sim/lib/internal/extend/execute-tool.test.ts new file mode 100644 index 00000000000..c0d3e640ff6 --- /dev/null +++ b/apps/sim/lib/internal/extend/execute-tool.test.ts @@ -0,0 +1,38 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operation = vi.hoisted(() => vi.fn()) +vi.mock('@/lib/internal/extend/operations', () => ({ executeExtendParse: operation })) + +import { executeExtendTool } from '@/lib/internal/extend/execute-tool' + +describe('executeExtendTool', () => { + beforeEach(() => { + vi.clearAllMocks() + operation.mockResolvedValue({ success: true, output: { id: 'parse-1' } }) + }) + + it('dispatches both canonical IDs and rejects unsupported IDs', async () => { + for (const toolId of ['extend_parser', 'extend_parser_v2']) { + const response = await executeExtendTool({ + toolId, + input: { apiKey: 'key', filePath: 'https://example.com/file.pdf' }, + headers: new Headers(), + context: { ...createExecutionContext(), userId: 'user-1' }, + requestId: 'request-1', + }) + expect(response.status).toBe(200) + } + const unsupported = await executeExtendTool({ + toolId: 'extend_unknown', + input: {}, + headers: new Headers(), + context: createExecutionContext(), + requestId: 'request-1', + }) + expect(unsupported.status).toBe(500) + }) +}) diff --git a/apps/sim/lib/internal/extend/execute-tool.ts b/apps/sim/lib/internal/extend/execute-tool.ts new file mode 100644 index 00000000000..bce0b77ad86 --- /dev/null +++ b/apps/sim/lib/internal/extend/execute-tool.ts @@ -0,0 +1,71 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { z } from 'zod' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { ExtendOperationError } from '@/lib/internal/extend/errors' +import { extendParseInputSchema } from '@/lib/internal/extend/input' +import { executeExtendParse } from '@/lib/internal/extend/operations' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const logger = createLogger('ExtendToolExecution') + +function validationResponse(error: z.ZodError): Response { + return Response.json( + { + success: false, + error: error.issues[0]?.message || 'Invalid request data', + details: error.issues, + }, + { status: 400 } + ) +} + +export const executeExtendTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!['extend_parser', 'extend_parser_v2'].includes(request.toolId)) { + return Response.json( + { success: false, error: `Unsupported Extend tool: ${request.toolId}` }, + { status: 500 } + ) + } + let serialized: string + try { + serialized = JSON.stringify(request.input) ?? '' + } catch { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + if (Buffer.byteLength(serialized, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + success: false, + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + const parsed = extendParseInputSchema.safeParse(request.input) + if (!parsed.success) return validationResponse(parsed.error) + + try { + const result = await executeExtendParse(parsed.data, { + headers: request.headers, + requestId: request.requestId, + signal: request.signal, + userId: request.context.userId, + }) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof ExtendOperationError) { + return Response.json(error.body, { status: error.status }) + } + const message = getErrorMessage(error, 'Internal server error') + logger.error('Extend operation failed', { + error: message, + requestId: request.requestId, + toolId: request.toolId, + }) + return Response.json({ success: false, error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/extend/input.ts b/apps/sim/lib/internal/extend/input.ts new file mode 100644 index 00000000000..e48f9b1dedd --- /dev/null +++ b/apps/sim/lib/internal/extend/input.ts @@ -0,0 +1,16 @@ +import { z } from 'zod' +import { resolvedSecretTraceProvenanceSchema } from '@/lib/api/contracts/primitives' +import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' + +export const extendParseInputSchema = z.object({ + apiKey: z.string().min(1, 'API key is required'), + filePath: z.string().optional(), + file: RawFileInputSchema.optional(), + outputFormat: z.enum(['markdown', 'spatial']).optional(), + chunking: z.enum(['page', 'document', 'section']).optional(), + engine: z.enum(['parse_performance', 'parse_light']).optional(), + [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), +}) + +export type ExtendParseInput = z.infer diff --git a/apps/sim/lib/internal/extend/operations.test.ts b/apps/sim/lib/internal/extend/operations.test.ts new file mode 100644 index 00000000000..0f28a7825c2 --- /dev/null +++ b/apps/sim/lib/internal/extend/operations.test.ts @@ -0,0 +1,73 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + resolveFileInputToUrl: vi.fn(), + submitExtendParse: vi.fn(), + validateProvenance: vi.fn(), +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + resolveFileInputToUrl: mocks.resolveFileInputToUrl, +})) +vi.mock('@/lib/internal/extend/client', () => ({ submitExtendParse: mocks.submitExtendParse })) +vi.mock('@/lib/execution/model-input-provenance', () => ({ + validateOpaqueModelInputProvenance: mocks.validateProvenance, +})) + +import { executeExtendParse } from '@/lib/internal/extend/operations' + +describe('executeExtendParse', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.validateProvenance.mockReturnValue({ success: true }) + mocks.resolveFileInputToUrl.mockResolvedValue({ fileUrl: 'https://example.com/file.pdf' }) + mocks.submitExtendParse.mockResolvedValue({ id: 'parse-1', page_count: 3 }) + }) + + it('preserves provider configuration and response projection', async () => { + const controller = new AbortController() + await expect( + executeExtendParse( + { + apiKey: 'key', + filePath: 'https://example.com/file.pdf', + outputFormat: 'markdown', + chunking: 'section', + engine: 'parse_light', + }, + { + headers: new Headers(), + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + } + ) + ).resolves.toEqual({ + success: true, + output: { + id: 'parse-1', + status: 'PROCESSED', + chunks: [], + blocks: [], + pageCount: 3, + creditsUsed: null, + }, + }) + + expect(mocks.submitExtendParse).toHaveBeenCalledWith( + 'key', + { + file: { fileUrl: 'https://example.com/file.pdf' }, + config: { + target: 'markdown', + chunkingStrategy: { type: 'section' }, + engine: 'parse_light', + }, + }, + controller.signal + ) + }) +}) diff --git a/apps/sim/lib/internal/extend/operations.ts b/apps/sim/lib/internal/extend/operations.ts new file mode 100644 index 00000000000..0532c4634f1 --- /dev/null +++ b/apps/sim/lib/internal/extend/operations.ts @@ -0,0 +1,76 @@ +import { createLogger } from '@sim/logger' +import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' +import { submitExtendParse } from '@/lib/internal/extend/client' +import { ExtendOperationError } from '@/lib/internal/extend/errors' +import type { ExtendParseInput } from '@/lib/internal/extend/input' +import { resolveFileInputToUrl } from '@/lib/uploads/utils/file-utils.server' + +const logger = createLogger('ExtendOperations') + +export interface ExtendOperationContext { + headers: Headers + requestId: string + signal?: AbortSignal + userId?: string +} + +export async function executeExtendParse( + input: ExtendParseInput, + context: ExtendOperationContext +): Promise<{ success: true; output: Record }> { + context.signal?.throwIfAborted() + if (!context.userId) { + throw new ExtendOperationError(401, { success: false, error: 'Unauthorized' }) + } + const provenance = validateOpaqueModelInputProvenance({ + headers: context.headers, + payload: input, + isInternalRequest: true, + }) + if (!provenance.success) { + throw new ExtendOperationError(provenance.status, { + success: false, + error: provenance.error, + }) + } + + const resolution = await resolveFileInputToUrl({ + file: input.file, + filePath: input.filePath, + userId: context.userId, + requestId: context.requestId, + logger, + modelEgress: true, + }) + context.signal?.throwIfAborted() + if (resolution.error) { + throw new ExtendOperationError(resolution.error.status, { + success: false, + error: resolution.error.message, + }) + } + if (!resolution.fileUrl) { + throw new ExtendOperationError(400, { success: false, error: 'File input is required' }) + } + + const body: Record = { file: { fileUrl: resolution.fileUrl } } + const config: Record = {} + if (input.outputFormat) config.target = input.outputFormat + if (input.chunking) config.chunkingStrategy = { type: input.chunking } + if (input.engine) config.engine = input.engine + if (Object.keys(config).length) body.config = config + + const data = await submitExtendParse(input.apiKey, body, context.signal) + context.signal?.throwIfAborted() + return { + success: true, + output: { + id: data.id ?? null, + status: data.status ?? 'PROCESSED', + chunks: data.chunks ?? [], + blocks: data.blocks ?? [], + pageCount: data.pageCount ?? data.page_count ?? null, + creditsUsed: data.creditsUsed ?? data.credits_used ?? null, + }, + } +} diff --git a/apps/sim/lib/internal/file/execute-tool.test.ts b/apps/sim/lib/internal/file/execute-tool.test.ts new file mode 100644 index 00000000000..9079b2b8aaa --- /dev/null +++ b/apps/sim/lib/internal/file/execute-tool.test.ts @@ -0,0 +1,282 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' + +const mocks = vi.hoisted(() => ({ + createPrincipal: vi.fn(), + executeManage: vi.fn(), + executeParser: vi.fn(), +})) + +vi.mock('@/lib/internal/principals/executor', () => ({ + createExecutorPrincipalFromExecutionContext: mocks.createPrincipal, +})) + +vi.mock('@/lib/internal/file/operations', () => ({ + executeFileManageOperation: mocks.executeManage, +})) + +vi.mock('@/lib/internal/file/parser', () => ({ + executeFileParserOperation: mocks.executeParser, +})) + +import { executeFileTool } from '@/lib/internal/file/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' +import { WORKSPACE_FILES_DELEGATION_AUDIENCE } from '@/lib/workspace-files/application/authorization' + +const MANAGE_INPUTS = { + file_append: { operation: 'append', fileName: 'notes.txt', content: 'next' }, + file_compress: { operation: 'compress', fileId: 'file-1' }, + file_decompress: { operation: 'decompress', fileId: 'file-1' }, + file_get: { operation: 'get', fileId: 'file-1' }, + file_get_content: { operation: 'content', fileId: 'file-1' }, + file_manage_sharing: { operation: 'manage_sharing', fileId: 'file-1', isActive: false }, + file_read: { operation: 'read', fileId: 'file-1' }, + file_write: { operation: 'write', fileName: 'notes.txt', content: 'hello' }, +} as const + +const PARSER_TOOL_IDS = ['file_fetch', 'file_parser', 'file_parser_v2', 'file_parser_v3'] as const + +const BILLING_ATTRIBUTION = { + actorUserId: 'user-1', + workspaceId: 'workspace-1', + organizationId: null, + billedAccountUserId: 'workspace-owner', + billingEntity: { type: 'user', id: 'workspace-owner' }, + billingPeriod: { + start: '2026-08-01T00:00:00.000Z', + end: '2026-09-01T00:00:00.000Z', + }, + payerSubscription: null, +} satisfies BillingAttributionSnapshot + +function request( + toolId: string, + input: unknown, + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId, + input, + headers: new Headers(), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + executionId: 'execution-1', + userId: 'user-1', + workspaceId: 'workspace-1', + billingAttribution: BILLING_ATTRIBUTION, + executorDelegationOrigin: { + subjectUserId: 'user-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + currentWorkflow: { workflowId: 'workflow-1', mode: 'draft' }, + }, + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeFileTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.createPrincipal.mockResolvedValue({ + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'workspace-1', + }) + mocks.executeManage.mockResolvedValue(Response.json({ success: true })) + mocks.executeParser.mockResolvedValue(Response.json({ success: true })) + }) + + it.each(Object.entries(MANAGE_INPUTS))('validates and dispatches %s', async (toolId, input) => { + const response = await executeFileTool(request(toolId, input)) + + expect(response.status).toBe(200) + expect(mocks.executeManage).toHaveBeenCalledWith( + expect.objectContaining(input), + expect.objectContaining({ + workspaceId: 'workspace-1', + attributedUserId: 'user-1', + fileAccessUserId: 'user-1', + requestId: 'request-1', + }) + ) + expect(mocks.executeParser).not.toHaveBeenCalled() + }) + + it.each(PARSER_TOOL_IDS)('dispatches %s with trusted execution scope', async (toolId) => { + const response = await executeFileTool( + request(toolId, { filePath: 'https://example.com/report.txt', fileType: '' }) + ) + + expect(response.status).toBe(200) + expect(mocks.executeParser).toHaveBeenCalledWith( + expect.objectContaining({ filePath: 'https://example.com/report.txt' }), + expect.objectContaining({ + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + attributedUserId: 'user-1', + fileAccessUserId: 'user-1', + }) + ) + expect(mocks.executeManage).not.toHaveBeenCalled() + }) + + it('constructs the executor principal from trusted context', async () => { + const executionRequest = request('file_get', MANAGE_INPUTS.file_get) + + await executeFileTool(executionRequest) + + expect(mocks.createPrincipal).toHaveBeenCalledWith({ + context: executionRequest.context, + audience: WORKSPACE_FILES_DELEGATION_AUDIENCE, + }) + }) + + it('uses the delegation origin as the file authorization subject in child workflows', async () => { + mocks.createPrincipal.mockResolvedValueOnce({ + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'invoking-user', + workspaceId: 'workspace-1', + }) + await executeFileTool( + request('file_get', MANAGE_INPUTS.file_get, { + context: { + ...createExecutionContext({ workflowId: 'workflow-child' }), + executionId: 'execution-child', + userId: 'workflow-owner', + workspaceId: 'workspace-1', + executorDelegationOrigin: { + subjectUserId: 'invoking-user', + workflowId: 'workflow-parent', + executionId: 'execution-parent', + }, + }, + }) + ) + + expect(mocks.executeManage).toHaveBeenCalledWith( + expect.objectContaining(MANAGE_INPUTS.file_get), + expect.objectContaining({ + attributedUserId: 'invoking-user', + fileAccessUserId: 'invoking-user', + }) + ) + }) + + it('uses compatibility attribution without replacing an actorless deployed principal', async () => { + const principal = { + kind: 'delegated' as const, + serviceId: 'executor' as const, + workspaceId: 'workspace-1', + delegationId: 'delegation-1', + audience: WORKSPACE_FILES_DELEGATION_AUDIENCE, + issuedAt: new Date(), + expiresAt: new Date(Date.now() + 60_000), + delegationContext: { + kind: 'workflow_execution' as const, + workflowId: 'workflow-1', + executionId: 'execution-1', + principal: { + kind: 'system' as const, + serviceId: 'schedule' as const, + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + }, + currentWorkflow: { + workflowId: 'workflow-1', + mode: 'deployment' as const, + deploymentVersionId: 'deployment-1', + }, + }, + } + mocks.createPrincipal.mockResolvedValueOnce(principal) + + await executeFileTool( + request('file_decompress', MANAGE_INPUTS.file_decompress, { + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + executionId: 'execution-1', + userId: 'legacy-actor', + workspaceId: 'workspace-1', + billingAttribution: BILLING_ATTRIBUTION, + executorDelegationOrigin: { + workflowId: 'workflow-1', + executionId: 'execution-1', + principal: principal.delegationContext.principal, + currentWorkflow: principal.delegationContext.currentWorkflow, + }, + }, + }) + ) + + expect(mocks.executeManage).toHaveBeenCalledWith( + expect.objectContaining(MANAGE_INPUTS.file_decompress), + expect.objectContaining({ + principal, + attributedUserId: 'workspace-owner', + fileAccessUserId: undefined, + workspaceId: 'workspace-1', + }) + ) + }) + + it('rejects missing trusted identity during principal construction', async () => { + const response = await executeFileTool( + request('file_get', MANAGE_INPUTS.file_get, { + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + userId: undefined, + executorDelegationOrigin: undefined, + }, + }) + ) + + expect(response.status).toBe(401) + expect(mocks.createPrincipal).not.toHaveBeenCalled() + expect(mocks.executeManage).not.toHaveBeenCalled() + }) + + it('returns canonical validation errors before operation work', async () => { + const response = await executeFileTool(request('file_write', { operation: 'write' })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(mocks.executeManage).not.toHaveBeenCalled() + }) + + it('propagates cancellation before principal or operation work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeFileTool(request('file_get', MANAGE_INPUTS.file_get, { signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mocks.createPrincipal).not.toHaveBeenCalled() + expect(mocks.executeManage).not.toHaveBeenCalled() + }) + + it('propagates cancellation that arrives while operation work is running', async () => { + const controller = new AbortController() + mocks.executeManage.mockImplementationOnce(async () => { + controller.abort(new DOMException('cancelled', 'AbortError')) + return Response.json({ success: true }) + }) + + await expect( + executeFileTool(request('file_get', MANAGE_INPUTS.file_get, { signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) +}) diff --git a/apps/sim/lib/internal/file/execute-tool.ts b/apps/sim/lib/internal/file/execute-tool.ts new file mode 100644 index 00000000000..b709572e237 --- /dev/null +++ b/apps/sim/lib/internal/file/execute-tool.ts @@ -0,0 +1,122 @@ +import { + PrincipalSubjectUserRequiredError, + resolvePrincipalAttribution, + resolvePrincipalSubject, +} from '@sim/auth/principal' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { fileParseContract } from '@/lib/api/contracts/storage-transfer' +import { fileManageContract } from '@/lib/api/contracts/tools/file' +import { InvalidInternalDelegationBindingError } from '@/lib/auth/internal-delegation' +import { executeFileManageOperation } from '@/lib/internal/file/operations' +import { executeFileParserOperation } from '@/lib/internal/file/parser' +import { createExecutorPrincipalFromExecutionContext } from '@/lib/internal/principals/executor' +import { parseInternalToolInput } from '@/lib/internal/tool-operations/parse-input' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { WORKSPACE_FILES_DELEGATION_AUDIENCE } from '@/lib/workspace-files/application/authorization' + +const logger = createLogger('FileToolExecution') + +const FILE_MANAGE_TOOL_IDS = new Set([ + 'file_append', + 'file_compress', + 'file_decompress', + 'file_get', + 'file_get_content', + 'file_manage_sharing', + 'file_fetch', + 'file_parser', + 'file_parser_v2', + 'file_parser_v3', + 'file_read', + 'file_write', +]) + +export const executeFileTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!FILE_MANAGE_TOOL_IDS.has(request.toolId)) { + return Response.json( + { success: false, error: `Unsupported File tool: ${request.toolId}` }, + { status: 500 } + ) + } + + const workspaceId = request.context.workspaceId + if (!workspaceId || !request.context.executorDelegationOrigin) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + + const isParserTool = + request.toolId === 'file_fetch' || + request.toolId === 'file_parser' || + request.toolId === 'file_parser_v2' || + request.toolId === 'file_parser_v3' + const parserInput = isParserTool ? parseInternalToolInput(fileParseContract, request.input) : null + if (parserInput && !parserInput.success) return parserInput.response + const manageInput = isParserTool + ? null + : parseInternalToolInput(fileManageContract, request.input) + if (manageInput && !manageInput.success) return manageInput.response + try { + const principal = await createExecutorPrincipalFromExecutionContext({ + context: request.context, + audience: WORKSPACE_FILES_DELEGATION_AUDIENCE, + }) + const { attributedUserId } = resolvePrincipalAttribution(principal, { + workspaceBillingOwnerUserId: request.context.billingAttribution?.billedAccountUserId, + }) + const subject = resolvePrincipalSubject(principal) + const fileAccessUserId = subject?.kind === 'sim_user' ? subject.userId : undefined + request.signal?.throwIfAborted() + let response: Response + if (parserInput) { + response = await executeFileParserOperation(parserInput.data, { + principal, + workspaceId, + workflowId: request.context.workflowId, + executionId: request.context.executionId, + attributedUserId, + fileAccessUserId, + largeValueExecutionIds: request.context.largeValueExecutionIds, + fileKeys: request.context.fileKeys, + allowLargeValueWorkflowScope: request.context.allowLargeValueWorkflowScope, + requestId: request.requestId, + signal: request.signal, + }) + } else { + if (!manageInput) throw new Error('File tool dispatch input is unavailable') + response = await executeFileManageOperation(manageInput.data, { + principal, + workspaceId, + attributedUserId, + fileAccessUserId, + workflowId: request.context.workflowId, + executionId: request.context.executionId, + largeValueExecutionIds: request.context.largeValueExecutionIds, + fileKeys: request.context.fileKeys, + allowLargeValueWorkflowScope: request.context.allowLargeValueWorkflowScope, + headers: request.headers, + requestId: request.requestId, + signal: request.signal, + }) + } + request.signal?.throwIfAborted() + return response + } catch (error) { + request.signal?.throwIfAborted() + if ( + error instanceof InvalidInternalDelegationBindingError || + error instanceof PrincipalSubjectUserRequiredError || + (error instanceof Error && error.message === 'Authentication required') + ) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + const message = getErrorMessage(error, 'Unknown error') + logger.error('File operation dispatch failed', { + error: message, + requestId: request.requestId, + toolId: request.toolId, + }) + return Response.json({ success: false, error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/file/operations.test.ts b/apps/sim/lib/internal/file/operations.test.ts new file mode 100644 index 00000000000..57413837212 --- /dev/null +++ b/apps/sim/lib/internal/file/operations.test.ts @@ -0,0 +1,965 @@ +/** + * @vitest-environment node + */ +import { createMockRequest, hybridAuthMockFns } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { MAX_FOLDER_PATH_SEGMENTS } from '@/lib/folders/paths' + +const { + mockAssertActiveWorkspaceAccess, + mockDownloadServableFileFromStorage, + mockDownloadFileFromStorage, + mockDecompressArchiveBufferToWorkspaceFiles, + mockEnsureWorkspaceFileFolderPath, + mockFetchWorkspaceFileBuffer, + mockGetBoundWorkspaceFileSecretProvenance, + mockLoadActiveWorkspaceContext, + mockLoadActiveWorkspaceFileContext, + mockMoveWorkspaceFileItems, + mockResolveEffectiveWorkspacePermission, + mockGetFileMetadataByKey, + mockGetWorkspaceFile, + mockVerifyFileAccess, + mockResolveWorkspaceFileReference, + mockUpdateWorkspaceFileContent, + mockUploadWorkspaceFile, +} = vi.hoisted(() => ({ + mockAssertActiveWorkspaceAccess: vi.fn(), + mockDownloadServableFileFromStorage: vi.fn(), + mockDownloadFileFromStorage: vi.fn(), + mockDecompressArchiveBufferToWorkspaceFiles: vi.fn(), + mockEnsureWorkspaceFileFolderPath: vi.fn(), + mockFetchWorkspaceFileBuffer: vi.fn(), + mockGetBoundWorkspaceFileSecretProvenance: vi.fn(), + mockLoadActiveWorkspaceContext: vi.fn(), + mockLoadActiveWorkspaceFileContext: vi.fn(), + mockMoveWorkspaceFileItems: vi.fn(), + mockResolveEffectiveWorkspacePermission: vi.fn(), + mockGetFileMetadataByKey: vi.fn(), + mockGetWorkspaceFile: vi.fn(), + mockVerifyFileAccess: vi.fn(), + mockResolveWorkspaceFileReference: vi.fn(), + mockUpdateWorkspaceFileContent: vi.fn(), + mockUploadWorkspaceFile: vi.fn(), +})) + +vi.mock('@/lib/uploads/archive', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + decompressArchiveBufferToWorkspaceFiles: (...args: unknown[]) => + mockDecompressArchiveBufferToWorkspaceFiles(...args), + } +}) + +vi.mock('@/lib/file-parsers', () => ({ + isSupportedFileType: vi.fn(() => false), + parseBuffer: vi.fn(), +})) + +vi.mock('@sim/audit', () => ({ + AuditAction: { FILE_UPLOADED: 'file_uploaded', FILE_UPDATED: 'file_updated' }, + AuditResourceType: { FILE: 'file' }, + recordAudit: vi.fn(), +})) + +vi.mock('@/lib/realtime/notify', () => ({ + notifyWorkspaceFilesChanged: vi.fn(async () => undefined), +})) + +vi.mock('@/lib/public-shares/share-manager', () => ({ + getShareForResource: vi.fn().mockResolvedValue(null), + getSharesForResources: vi.fn().mockResolvedValue(new Map()), + ShareValidationError: class ShareValidationError extends Error {}, +})) + +vi.mock('@sim/platform-authz/workspace', () => ({ + permissionSatisfies: (permission: string | null, required: string) => + permission === 'admin' || + permission === required || + (permission === 'write' && required === 'read'), + resolveEffectiveWorkspacePermission: (...args: unknown[]) => + mockResolveEffectiveWorkspacePermission(...args), +})) + +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ + fetchWorkspaceFileBuffer: (...args: unknown[]) => mockFetchWorkspaceFileBuffer(...args), + getWorkspaceFile: (...args: unknown[]) => mockGetWorkspaceFile(...args), + loadActiveWorkspaceContext: (...args: unknown[]) => mockLoadActiveWorkspaceContext(...args), + loadActiveWorkspaceFileContext: (...args: unknown[]) => + mockLoadActiveWorkspaceFileContext(...args), + resolveWorkspaceFileReference: (...args: unknown[]) => mockResolveWorkspaceFileReference(...args), + updateWorkspaceFileContent: (...args: unknown[]) => mockUpdateWorkspaceFileContent(...args), + uploadWorkspaceFile: (...args: unknown[]) => mockUploadWorkspaceFile(...args), +})) + +vi.mock('@/lib/uploads/contexts/workspace', () => ({ + FileConflictError: class FileConflictError extends Error {}, + ContentVersionConflictError: class ContentVersionConflictError extends Error {}, + fetchWorkspaceFileBuffer: (...args: unknown[]) => mockFetchWorkspaceFileBuffer(...args), + getWorkspaceFile: (...args: unknown[]) => mockGetWorkspaceFile(...args), + loadActiveWorkspaceContext: (...args: unknown[]) => mockLoadActiveWorkspaceContext(...args), + updateWorkspaceFileContent: (...args: unknown[]) => mockUpdateWorkspaceFileContent(...args), + uploadWorkspaceFile: (...args: unknown[]) => mockUploadWorkspaceFile(...args), +})) + +vi.mock('@/lib/workspace-files/application/workspace-file-folders', () => ({ + ensureWorkspaceFileFolderPathOperation: { + execute: (...args: unknown[]) => mockEnsureWorkspaceFileFolderPath(...args), + }, +})) + +vi.mock('@/lib/workspace-files/application/move-workspace-file-items', () => ({ + moveWorkspaceFileItemsOperation: { + execute: (...args: unknown[]) => mockMoveWorkspaceFileItems(...args), + }, +})) + +vi.mock('@/lib/core/config/redis', () => ({ + acquireLock: vi.fn(async () => true), + releaseLock: vi.fn(async () => undefined), +})) + +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({ + EXACT_EMPTY_WORKSPACE_FILE_SECRET_PROVENANCE: { status: 'exact', entries: [] }, + getBoundWorkspaceFileSecretProvenance: (...args: unknown[]) => + mockGetBoundWorkspaceFileSecretProvenance(...args), + mergeWorkspaceFileSecretProvenance: ( + ...provenances: Array< + | { status: 'exact'; entries: Array<{ name: string; encryptedValue: string }> } + | { + status: 'unknown' + } + > + ) => + provenances.some((provenance) => provenance.status === 'unknown') + ? { status: 'unknown' } + : { + status: 'exact', + entries: provenances.flatMap((provenance) => + provenance.status === 'exact' ? provenance.entries : [] + ), + }, +})) + +vi.mock('@/lib/uploads/server/metadata', () => ({ + getFileMetadataByKey: (...args: unknown[]) => mockGetFileMetadataByKey(...args), +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadFileFromStorage: (...args: unknown[]) => mockDownloadFileFromStorage(...args), + downloadServableFileFromStorage: (...args: unknown[]) => + mockDownloadServableFileFromStorage(...args), +})) + +vi.mock('@/lib/workspaces/permissions/utils', () => ({ + assertActiveWorkspaceAccess: (...args: unknown[]) => mockAssertActiveWorkspaceAccess(...args), + getUserEntityPermissions: vi.fn(), + isWorkspaceAccessDeniedError: vi.fn(() => false), +})) + +vi.mock('@/app/api/files/authorization', () => ({ + verifyFileAccess: (...args: unknown[]) => mockVerifyFileAccess(...args), +})) + +import { fileManageBodySchema } from '@/lib/api/contracts/tools/file' +import { executeFileManageOperation } from '@/lib/internal/file/operations' +import { createWorkspaceFileDelegatedPrincipal } from '@/lib/workspace-files/application/delegated-principal' + +async function POST(request: Request): Promise { + const parsed = fileManageBodySchema.safeParse(await request.json()) + if (!parsed.success) { + return Response.json( + { success: false, error: parsed.error.issues[0]?.message ?? 'Invalid request data' }, + { status: 400 } + ) + } + const workspaceId = parsed.data.workspaceId || 'workspace-1' + return executeFileManageOperation(parsed.data, { + principal: createWorkspaceFileDelegatedPrincipal({ + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId, + delegationId: 'test-file-operation', + }), + workspaceId, + attributedUserId: 'user-1', + fileAccessUserId: 'user-1', + workflowId: 'workflow-1', + headers: request.headers, + requestId: 'request-1', + signal: request.signal, + }) +} + +const PRIVATE_REQUEST_HEADER = { + 'x-sim-request-private-tool-metadata': 'resolved-secret-provenance-v1', +} +const PRIVATE_SECRET_PROVENANCE_HEADER = { + 'x-sim-private-secret-provenance': 'private-secret-provenance-bundle-v1', +} +const CONTENT_UPDATED_AT = new Date('2026-08-04T00:00:00.000Z') + +function workspaceFile(id: string, ownerUserId = 'user-1') { + return { + id, + workspaceId: 'workspace-1', + name: `${id}.txt`, + key: `workspace/workspace-1/${id}.txt`, + path: `/api/files/serve/${id}`, + size: id.length, + type: 'text/plain', + uploadedBy: ownerUserId, + uploadedAt: CONTENT_UPDATED_AT, + updatedAt: CONTENT_UPDATED_AT, + contentUpdatedAt: CONTENT_UPDATED_AT, + } +} + +function actorlessDeploymentPrincipal(workspaceId = 'workspace-1') { + return { + kind: 'delegated' as const, + serviceId: 'executor' as const, + workspaceId, + delegationId: 'delegation-1', + audience: 'sim:workspace-files', + issuedAt: new Date(Date.now() - 1_000), + expiresAt: new Date(Date.now() + 60_000), + delegationContext: { + kind: 'workflow_execution' as const, + workflowId: 'workflow-1', + executionId: 'execution-1', + principal: { + kind: 'system' as const, + serviceId: 'schedule' as const, + workspaceId, + workflowId: 'workflow-1', + }, + currentWorkflow: { + workflowId: 'workflow-1', + mode: 'deployment' as const, + deploymentVersionId: 'deployment-1', + }, + }, + } +} + +describe('file manage operations', () => { + beforeEach(() => { + vi.clearAllMocks() + hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ + success: true, + userId: 'user-1', + authType: 'internal_jwt', + }) + mockAssertActiveWorkspaceAccess.mockResolvedValue(undefined) + mockResolveEffectiveWorkspacePermission.mockResolvedValue('write') + mockVerifyFileAccess.mockResolvedValue(true) + mockGetWorkspaceFile.mockImplementation(async (_workspaceId: string, fileId: string) => + workspaceFile(fileId) + ) + mockLoadActiveWorkspaceContext.mockResolvedValue({ + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + billedAccountUserId: 'user-1', + }) + mockLoadActiveWorkspaceFileContext.mockImplementation(async (fileId: string) => ({ + fileId, + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + billedAccountUserId: 'user-1', + })) + mockEnsureWorkspaceFileFolderPath.mockImplementation( + async ({ input }: { input: { pathSegments: string[] } }) => ({ + folderId: input.pathSegments.length === 0 ? null : 'folder-1', + createdFolderIds: [], + }) + ) + mockDownloadServableFileFromStorage.mockImplementation(async (file: { name: string }) => ({ + buffer: Buffer.from(`content:${file.name}`), + })) + mockFetchWorkspaceFileBuffer.mockResolvedValue(Buffer.from('before')) + mockUpdateWorkspaceFileContent.mockResolvedValue({ file: workspaceFile('file-1') }) + mockMoveWorkspaceFileItems.mockResolvedValue({ moved: 1 }) + mockUploadWorkspaceFile.mockResolvedValue({ + id: 'new-file', + name: 'new.txt', + key: 'workspace/workspace-1/new.txt', + url: '/api/files/serve/new-file', + }) + }) + + it('returns a scoped, deduplicated union of exact canonical file provenance', async () => { + mockGetBoundWorkspaceFileSecretProvenance.mockImplementation( + async (_workspaceId: string, identity: { fileId: string }) => + identity.fileId === 'file-1' + ? { + status: 'exact', + entries: [ + { name: 'TOKEN', encryptedValue: 'encrypted-token' }, + { name: 'ALPHA', encryptedValue: 'encrypted-alpha' }, + ], + } + : { + status: 'exact', + entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], + } + ) + + const response = await POST( + createMockRequest( + 'POST', + { operation: 'content', workspaceId: 'workspace-1', fileId: ['file-1', 'file-2'] }, + PRIVATE_REQUEST_HEADER + ) + ) + + expect(response.status).toBe(200) + expect(response.headers.get('x-sim-private-tool-metadata')).toBe( + 'resolved-secret-provenance-v1' + ) + await expect(response.json()).resolves.toEqual({ + success: true, + data: { contents: ['content:file-1.txt', 'content:file-2.txt'] }, + __resolvedSecretTraceProvenance: { + version: 1, + complete: true, + entries: [ + { name: 'ALPHA', encryptedValue: 'encrypted-alpha' }, + { name: 'TOKEN', encryptedValue: 'encrypted-token' }, + ], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }, + }) + }) + + it('stores exact causal provenance from a different user in the actor workspace', async () => { + const response = await POST( + createMockRequest( + 'POST', + { + operation: 'write', + workspaceId: 'workspace-1', + fileName: 'new.txt', + content: 'secret-value', + __privateSecretProvenance: { + version: 1, + complete: true, + selections: [ + { + key: 'content', + provenance: { + version: 1, + complete: true, + entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], + scope: { userId: 'workflow-owner', workspaceId: 'workspace-1' }, + }, + }, + ], + }, + }, + PRIVATE_SECRET_PROVENANCE_HEADER + ) + ) + + expect(response.status).toBe(200) + expect(mockUploadWorkspaceFile).toHaveBeenCalledWith( + 'workspace-1', + 'user-1', + Buffer.from('secret-value'), + 'new.txt', + 'text/plain', + { + exactName: false, + folderId: null, + folderPath: undefined, + secretProvenance: { + status: 'exact', + entries: [ + { + name: 'TOKEN', + encryptedValue: 'encrypted-token', + sourceUserId: 'workflow-owner', + sourceWorkspaceId: 'workspace-1', + }, + ], + }, + } + ) + }) + + it('rejects file-write provenance from another workspace', async () => { + const response = await POST( + createMockRequest( + 'POST', + { + operation: 'write', + workspaceId: 'workspace-1', + fileName: 'new.txt', + content: 'secret-value', + __privateSecretProvenance: { + version: 1, + complete: true, + selections: [ + { + key: 'content', + provenance: { + version: 1, + complete: true, + entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], + scope: { userId: 'workflow-owner', workspaceId: 'workspace-2' }, + }, + }, + ], + }, + }, + PRIVATE_SECRET_PROVENANCE_HEADER + ) + ) + + expect(response.status).toBe(400) + expect(mockUploadWorkspaceFile).not.toHaveBeenCalled() + }) + + it('preserves existing file-path behavior when a filename was resolved from a secret', async () => { + const response = await POST( + createMockRequest( + 'POST', + { + operation: 'write', + workspaceId: 'workspace-1', + fileName: 'Reports & Plans/2026/secret-value.txt', + content: 'ordinary text', + __privateSecretProvenance: { + version: 1, + complete: true, + selections: [ + { + key: 'content', + provenance: { + version: 1, + complete: true, + entries: [], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }, + }, + ], + }, + }, + PRIVATE_SECRET_PROVENANCE_HEADER + ) + ) + + expect(response.status).toBe(200) + expect(mockEnsureWorkspaceFileFolderPath).toHaveBeenCalledWith( + expect.objectContaining({ + principal: expect.objectContaining({ kind: 'delegated', subjectUserId: 'user-1' }), + input: { workspaceId: 'workspace-1', pathSegments: ['Reports & Plans', '2026'] }, + }) + ) + expect(mockUploadWorkspaceFile).toHaveBeenCalledWith( + 'workspace-1', + 'user-1', + Buffer.from('ordinary text'), + 'secret-value.txt', + 'text/plain', + { + exactName: false, + folderId: 'folder-1', + folderPath: undefined, + secretProvenance: { status: 'exact', entries: [] }, + } + ) + }) + + it('keeps a headerless file write on the legacy untracked path', async () => { + const response = await POST( + createMockRequest('POST', { + operation: 'write', + workspaceId: 'workspace-1', + fileName: 'new.txt', + content: 'ordinary text', + }) + ) + expect(response.status).toBe(200) + expect(mockUploadWorkspaceFile).toHaveBeenCalledWith( + 'workspace-1', + 'user-1', + Buffer.from('ordinary text'), + 'new.txt', + 'text/plain', + { + exactName: false, + folderId: null, + folderPath: undefined, + secretProvenance: { status: 'exact', entries: [] }, + } + ) + }) + + it.each([ + ['Reports & Plans/2026', '/Reports%20%26%20Plans/2026'], + ['', '/'], + ])('moves files to the canonical folder path for %j', async (targetFolder, expectedPath) => { + const response = await POST( + createMockRequest('POST', { + operation: 'move', + workspaceId: 'workspace-1', + fileId: 'file-1', + targetFolder, + }) + ) + + expect(response.status).toBe(200) + expect(mockMoveWorkspaceFileItems).toHaveBeenCalledWith( + expect.objectContaining({ + input: { + workspaceId: 'workspace-1', + fileIds: ['file-1'], + targetFolderPath: expectedPath, + }, + }) + ) + }) + + it('returns 400 before moving when the target folder path exceeds canonical limits', async () => { + const response = await POST( + createMockRequest('POST', { + operation: 'move', + workspaceId: 'workspace-1', + fileId: 'file-1', + targetFolder: Array.from( + { length: MAX_FOLDER_PATH_SEGMENTS + 1 }, + (_, index) => `folder-${index}` + ).join('/'), + }) + ) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + success: false, + error: `Folder paths cannot exceed ${MAX_FOLDER_PATH_SEGMENTS} segments`, + }) + expect(mockMoveWorkspaceFileItems).not.toHaveBeenCalled() + }) + + it('persists an authenticated file write with unavailable lineage as unknown', async () => { + const response = await POST( + createMockRequest( + 'POST', + { + operation: 'write', + workspaceId: 'workspace-1', + fileName: 'new.txt', + content: 'possibly secret', + __privateSecretProvenance: { + version: 1, + complete: false, + selections: [], + }, + }, + PRIVATE_SECRET_PROVENANCE_HEADER + ) + ) + + expect(response.status).toBe(200) + expect(mockUploadWorkspaceFile).toHaveBeenCalledWith( + 'workspace-1', + 'user-1', + Buffer.from('possibly secret'), + 'new.txt', + 'text/plain', + { + exactName: false, + folderId: null, + folderPath: undefined, + secretProvenance: { status: 'unknown' }, + } + ) + }) + + it('atomically binds append provenance to the exact predecessor version', async () => { + const existing = workspaceFile('file-1') + mockResolveWorkspaceFileReference.mockResolvedValue(existing) + mockGetBoundWorkspaceFileSecretProvenance.mockResolvedValue({ + status: 'exact', + entries: [ + { + name: 'OLD', + encryptedValue: 'encrypted-old', + sourceUserId: 'user-1', + sourceWorkspaceId: 'workspace-1', + }, + ], + }) + + const response = await POST( + createMockRequest( + 'POST', + { + operation: 'append', + workspaceId: 'workspace-1', + fileName: 'file-1.txt', + content: 'secret-value', + __privateSecretProvenance: { + version: 1, + complete: true, + selections: [ + { + key: 'content', + provenance: { + version: 1, + complete: true, + entries: [{ name: 'NEW', encryptedValue: 'encrypted-new' }], + scope: { userId: 'user-1', workspaceId: 'workspace-1' }, + }, + }, + ], + }, + }, + PRIVATE_SECRET_PROVENANCE_HEADER + ) + ) + + expect(response.status).toBe(200) + expect(mockUpdateWorkspaceFileContent).toHaveBeenCalledWith( + 'workspace-1', + 'file-1', + 'user-1', + Buffer.from('beforesecret-value'), + undefined, + { + expectedUpdatedAt: CONTENT_UPDATED_AT, + secretProvenancePolicy: { + mode: 'replace', + provenance: { + status: 'exact', + entries: [ + { + name: 'OLD', + encryptedValue: 'encrypted-old', + sourceUserId: 'user-1', + sourceWorkspaceId: 'workspace-1', + }, + { + name: 'NEW', + encryptedValue: 'encrypted-new', + sourceUserId: 'user-1', + sourceWorkspaceId: 'workspace-1', + }, + ], + }, + }, + } + ) + }) + + it('preserves the prior classification for a legacy headerless append', async () => { + const existing = workspaceFile('file-1') + mockResolveWorkspaceFileReference.mockResolvedValue(existing) + mockGetBoundWorkspaceFileSecretProvenance.mockResolvedValue({ + status: 'exact', + entries: [{ name: 'OLD', encryptedValue: 'encrypted-old' }], + }) + + const response = await POST( + createMockRequest('POST', { + operation: 'append', + workspaceId: 'workspace-1', + fileName: 'file-1.txt', + content: 'ordinary text', + }) + ) + + expect(response.status).toBe(200) + expect(mockUpdateWorkspaceFileContent).toHaveBeenCalledWith( + 'workspace-1', + 'file-1', + 'user-1', + Buffer.from('beforeordinary text'), + undefined, + { + expectedUpdatedAt: CONTENT_UPDATED_AT, + secretProvenancePolicy: { mode: 'preserve' }, + } + ) + }) + + it('carries the union of source provenance into a compressed archive', async () => { + mockGetWorkspaceFile.mockResolvedValue(workspaceFile('file-1')) + mockGetBoundWorkspaceFileSecretProvenance.mockResolvedValue({ + status: 'exact', + entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], + }) + + const response = await POST( + createMockRequest('POST', { + operation: 'compress', + workspaceId: 'workspace-1', + fileId: 'file-1', + archiveName: 'bundle', + }) + ) + + expect(response.status).toBe(200) + expect(Buffer.isBuffer(mockUploadWorkspaceFile.mock.calls[0]?.[2])).toBe(true) + expect(mockUploadWorkspaceFile).toHaveBeenCalledWith( + 'workspace-1', + 'user-1', + expect.anything(), + 'bundle.zip', + 'application/zip', + expect.objectContaining({ + folderId: null, + secretProvenance: { + status: 'exact', + entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], + }, + }) + ) + }) + + it('passes secret-bearing archive provenance to the decompressor', async () => { + const archiveBuffer = Buffer.from('archive-bytes') + mockDownloadFileFromStorage.mockResolvedValue(archiveBuffer) + mockGetWorkspaceFile.mockResolvedValue({ + ...workspaceFile('archive'), + name: 'archive.zip', + type: 'application/zip', + }) + mockGetBoundWorkspaceFileSecretProvenance.mockResolvedValue({ + status: 'exact', + entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], + }) + mockDecompressArchiveBufferToWorkspaceFiles.mockResolvedValue({ + extracted: [ + { + id: 'new-file', + name: 'child.txt', + key: 'workspace/workspace-1/child.txt', + url: '/api/files/serve/new-file', + size: 12, + type: 'text/plain', + context: 'workspace', + }, + ], + skipped: 0, + skippedUnsafePaths: [], + }) + + const response = await POST( + createMockRequest('POST', { + operation: 'decompress', + workspaceId: 'workspace-1', + fileId: 'archive', + }) + ) + + expect(response.status).toBe(200) + expect(mockDownloadFileFromStorage).toHaveBeenCalledTimes(1) + expect(mockDecompressArchiveBufferToWorkspaceFiles).toHaveBeenCalledWith( + archiveBuffer, + expect.objectContaining({ + workspaceId: 'workspace-1', + principal: expect.objectContaining({ kind: 'delegated', subjectUserId: 'user-1' }), + secretProvenance: { + status: 'exact', + entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], + }, + }) + ) + }) + + it('decompresses a canonical workspace archive for an actorless deployed execution', async () => { + const archiveBuffer = Buffer.from('archive-bytes') + const principal = actorlessDeploymentPrincipal() + mockDownloadFileFromStorage.mockResolvedValue(archiveBuffer) + mockGetWorkspaceFile.mockResolvedValue({ + ...workspaceFile('archive'), + name: 'archive.zip', + type: 'application/zip', + }) + mockGetBoundWorkspaceFileSecretProvenance.mockResolvedValue({ + status: 'exact', + entries: [], + }) + mockDecompressArchiveBufferToWorkspaceFiles.mockResolvedValue({ + extracted: [ + { + ...workspaceFile('child'), + url: '/api/files/serve/child', + context: 'workspace', + }, + ], + skipped: 0, + skippedUnsafePaths: [], + }) + + const response = await executeFileManageOperation( + fileManageBodySchema.parse({ + operation: 'decompress', + workspaceId: 'workspace-1', + fileId: 'archive', + }), + { + principal, + workspaceId: 'workspace-1', + attributedUserId: 'workspace-owner', + workflowId: 'workflow-1', + executionId: 'execution-1', + headers: new Headers(), + requestId: 'request-actorless', + } + ) + + expect(response.status).toBe(200) + expect(mockResolveEffectiveWorkspacePermission).not.toHaveBeenCalled() + expect(mockGetWorkspaceFile).toHaveBeenCalledWith('workspace-1', 'archive', { + throwOnError: true, + }) + expect(mockDecompressArchiveBufferToWorkspaceFiles).toHaveBeenCalledWith( + archiveBuffer, + expect.objectContaining({ principal, workspaceId: 'workspace-1' }) + ) + }) + + it('rejects an actorless deployment principal bound to a different workspace', async () => { + const response = await executeFileManageOperation( + fileManageBodySchema.parse({ + operation: 'decompress', + workspaceId: 'workspace-1', + fileId: 'archive', + }), + { + principal: actorlessDeploymentPrincipal('workspace-2'), + workspaceId: 'workspace-1', + attributedUserId: 'workspace-owner', + workflowId: 'workflow-1', + executionId: 'execution-1', + headers: new Headers(), + requestId: 'request-cross-workspace', + } + ) + + expect(response.status).toBe(403) + expect(mockGetWorkspaceFile).not.toHaveBeenCalled() + expect(mockDownloadFileFromStorage).not.toHaveBeenCalled() + expect(mockDecompressArchiveBufferToWorkspaceFiles).not.toHaveBeenCalled() + }) + + it('omits source scope when canonical files have different owners', async () => { + mockGetWorkspaceFile.mockImplementation(async (_workspaceId: string, fileId: string) => + workspaceFile(fileId, fileId === 'file-1' ? 'user-1' : 'user-2') + ) + mockGetBoundWorkspaceFileSecretProvenance.mockResolvedValue({ + status: 'exact', + entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], + }) + + const response = await POST( + createMockRequest( + 'POST', + { operation: 'content', workspaceId: 'workspace-1', fileId: ['file-1', 'file-2'] }, + PRIVATE_REQUEST_HEADER + ) + ) + const body = await response.json() + + expect(body.__resolvedSecretTraceProvenance).toEqual({ + version: 1, + complete: true, + entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], + }) + }) + + it('returns incomplete provenance for an input that cannot bind to a canonical file row', async () => { + mockResolveWorkspaceFileReference.mockResolvedValue(null) + + const response = await POST( + createMockRequest( + 'POST', + { + operation: 'content', + workspaceId: 'workspace-1', + fileInput: { + key: 'workspace/workspace-1/unbound.txt', + name: 'unbound.txt', + type: 'text/plain', + size: 7, + }, + }, + PRIVATE_REQUEST_HEADER + ) + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + __resolvedSecretTraceProvenance: { version: 1, complete: false, entries: [] }, + }) + expect(mockGetBoundWorkspaceFileSecretProvenance).not.toHaveBeenCalled() + }) + + it('keeps a normal not-found error while returning a valid private envelope', async () => { + mockGetWorkspaceFile.mockResolvedValue(null) + + const response = await POST( + createMockRequest( + 'POST', + { operation: 'content', workspaceId: 'workspace-1', fileId: 'missing-file' }, + PRIVATE_REQUEST_HEADER + ) + ) + + expect(response.status).toBe(404) + expect(response.headers.get('x-sim-private-tool-metadata')).toBe( + 'resolved-secret-provenance-v1' + ) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'File not found: "missing-file"', + __resolvedSecretTraceProvenance: { version: 1, complete: true, entries: [] }, + }) + }) + + it('does not add private transport fields when provenance was not requested', async () => { + mockGetWorkspaceFile.mockResolvedValue(workspaceFile('file-1')) + + const response = await POST( + createMockRequest('POST', { + operation: 'content', + workspaceId: 'workspace-1', + fileId: 'file-1', + }) + ) + + expect(response.headers.get('x-sim-private-tool-metadata')).toBeNull() + const body = await response.json() + expect(body).not.toHaveProperty('__resolvedSecretTraceProvenance') + expect(mockGetBoundWorkspaceFileSecretProvenance).not.toHaveBeenCalled() + }) + + it('never uses query.userId as the authorization identity', async () => { + mockGetWorkspaceFile.mockResolvedValue(workspaceFile('file-1')) + + const response = await POST( + createMockRequest( + 'POST', + { operation: 'get', workspaceId: 'workspace-1', fileId: 'file-1' }, + {}, + 'http://localhost:3000/api/tools/file/manage?userId=attacker' + ) + ) + + expect(response.status).toBe(200) + expect(mockResolveEffectiveWorkspacePermission).toHaveBeenCalledWith( + 'user-1', + 'workspace-1', + null, + undefined, + { forUpdate: undefined } + ) + }) +}) diff --git a/apps/sim/lib/internal/file/operations.ts b/apps/sim/lib/internal/file/operations.ts new file mode 100644 index 00000000000..190592076f8 --- /dev/null +++ b/apps/sim/lib/internal/file/operations.ts @@ -0,0 +1,1334 @@ +import { Buffer, isUtf8 } from 'buffer' +import type { Principal } from '@sim/auth/principal' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { generateShortId } from '@sim/utils/id' +import { isRecordLike } from '@sim/utils/object' +import JSZip from 'jszip' +import type { ContractBody } from '@/lib/api/contracts' +import type { fileManageContract } from '@/lib/api/contracts/tools/file' +import { splitWorkspaceFilePath } from '@/lib/copilot/tools/server/files/workspace-file' +import { acquireLock, releaseLock } from '@/lib/core/config/redis' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { ensureAbsoluteUrl } from '@/lib/core/utils/urls' +import { durableSecretProvenanceFromPrivateBundle } from '@/lib/execution/durable-secret-provenance' +import { + inspectPrivateSecretProvenanceRequest, + isPrivateSecretProvenanceBundleV1, +} from '@/lib/execution/model-input-provenance' +import { assertUserFileContentAccess } from '@/lib/execution/payloads/materialization.server' +import { + PRIVATE_TOOL_METADATA_RESPONSE_HEADER, + RESOLVED_SECRET_PROVENANCE_FIELD, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, + requestsPrivateToolMetadata, +} from '@/lib/execution/private-tool-metadata' +import { isSupportedFileType, parseBuffer } from '@/lib/file-parsers' +import { buildFolderPath } from '@/lib/folders/paths' +import { ShareValidationError } from '@/lib/public-shares/share-manager' +import { + ArchiveError, + type DecompressResult, + decompressArchiveBufferToWorkspaceFiles, + MAX_ARCHIVE_BYTES, + statusForArchiveError, +} from '@/lib/uploads/archive' +import type { getWorkspaceFile } from '@/lib/uploads/contexts/workspace/workspace-file-manager' +import { + mergeWorkspaceFileSecretProvenance, + type WorkspaceFileSecretProvenance, + type WorkspaceFileSecretProvenanceIdentity, +} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { + getFileExtension, + getMimeTypeFromExtension, + inferContextFromKey, +} from '@/lib/uploads/utils/file-utils' +import { + downloadFileFromStorage, + downloadServableFileFromStorage, +} from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' +import { buildZipEntryPaths } from '@/lib/uploads/zip-entry-path' +import { + admitCreateWorkspaceFile, + createWorkspaceFile, + createWorkspaceFileFromBuffer, +} from '@/lib/workspace-files/application/create-workspace-file' +import { moveWorkspaceFileItemsOperation } from '@/lib/workspace-files/application/move-workspace-file-items' +import { fileOperations } from '@/lib/workspace-files/application/operations' +import { readWorkspaceFileContent } from '@/lib/workspace-files/application/read-workspace-file-content' +import { readWorkspaceFileMetadata } from '@/lib/workspace-files/application/read-workspace-file-metadata' +import { downloadWorkspaceFileRecord } from '@/lib/workspace-files/application/read-workspace-file-record' +import { readWorkspaceFileSecretProvenance } from '@/lib/workspace-files/application/read-workspace-file-secret-provenance' +import { resolveWorkspaceFileReference } from '@/lib/workspace-files/application/resolve-workspace-file-reference' +import { + getWorkspaceFileShare, + updateWorkspaceFileShare, +} from '@/lib/workspace-files/application/share-workspace-file' +import { updateWorkspaceFileContent } from '@/lib/workspace-files/application/update-workspace-file-content' +import { ensureWorkspaceFileFolderPathOperation } from '@/lib/workspace-files/application/workspace-file-folders' +import { MAX_WORKSPACE_FILE_CONTENT_BYTES } from '@/lib/workspace-files/orchestration' +import { isWorkspaceAccessDeniedError } from '@/lib/workspaces/permissions/utils' +import type { UserFile } from '@/executor/types' +import { + ResolvedSecretTraceProvenanceAccumulator, + type ResolvedSecretTraceProvenanceV1, + type ResolvedSecretTraceScopeV1, +} from '@/executor/utils/resolved-secret-trace-registry' + +const logger = createLogger('FileManageAPI') + +export type FileManageOperationInput = ContractBody + +export interface FileManageOperationContext { + principal: Principal + workspaceId: string + attributedUserId: string + fileAccessUserId?: string + workflowId: string + executionId?: string + largeValueExecutionIds?: string[] + fileKeys?: string[] + allowLargeValueWorkflowScope?: boolean + headers: Headers + requestId: string + signal?: AbortSignal +} + +async function assertOperationFileAccess( + file: Pick, + context: FileManageOperationContext +): Promise { + try { + await assertUserFileContentAccess(file, { + principal: context.principal, + workspaceId: context.workspaceId, + workflowId: context.workflowId, + executionId: context.executionId, + largeValueExecutionIds: context.largeValueExecutionIds, + fileKeys: context.fileKeys, + allowLargeValueWorkflowScope: context.allowLargeValueWorkflowScope, + userId: context.fileAccessUserId, + requestId: context.requestId, + logger, + }) + return null + } catch { + logger.warn('File access denied', { key: file.key, requestId: context.requestId }) + return Response.json({ success: false, error: 'File not found' }, { status: 404 }) + } +} + +const workspaceFileToUserFile = (file: Awaited>) => { + if (!file) return null + + return { + id: file.id, + name: file.name, + url: ensureAbsoluteUrl(file.path), + size: file.size, + type: file.type, + key: file.key, + context: 'workspace' as const, + } +} + +const fileInputToUserFile = (fileInput: unknown) => { + if (!fileInput || typeof fileInput !== 'object' || Array.isArray(fileInput)) return null + + const record = fileInput as Record + const id = + typeof record.id === 'string' + ? record.id.trim() + : typeof record.fileId === 'string' + ? record.fileId.trim() + : '' + + // Objects with ids are resolved through workspace metadata. This fallback is for + // picker/upload values that only carry storage fields. + if (id) return null + + const key = typeof record.key === 'string' ? record.key.trim() : '' + const path = typeof record.path === 'string' ? record.path.trim() : '' + const url = typeof record.url === 'string' ? record.url.trim() : '' + const fileUrl = + url || path || (key ? `/api/files/serve/${encodeURIComponent(key)}?context=workspace` : '') + + if (!fileUrl && !key) return null + + return { + id: key || fileUrl, + name: + typeof record.name === 'string' && record.name.trim() ? record.name.trim() : 'workspace-file', + url: fileUrl ? ensureAbsoluteUrl(fileUrl) : '', + size: typeof record.size === 'number' ? record.size : 0, + type: + typeof record.type === 'string' && record.type.trim() + ? record.type.trim() + : 'application/octet-stream', + key, + context: inferContextFromKey(key), + } +} + +const normalizeFileIdList = (value: unknown): string[] => { + if (typeof value === 'string') { + const trimmed = value.trim() + if (!trimmed) return [] + + try { + return normalizeFileIdList(JSON.parse(trimmed)) + } catch { + return [trimmed] + } + } + + if (!Array.isArray(value)) return [] + + return value + .map((item) => (typeof item === 'string' ? item.trim() : '')) + .filter((id) => id.length > 0) +} + +const extractUserFilesFromInput = (fileInput: unknown) => { + const inputs = Array.isArray(fileInput) ? fileInput : fileInput ? [fileInput] : [] + return inputs + .map((input) => fileInputToUserFile(input)) + .filter((file): file is NonNullable> => Boolean(file)) +} + +const extractFileIdsFromInput = (fileInput: unknown): string[] => { + const inputs = Array.isArray(fileInput) ? fileInput : fileInput ? [fileInput] : [] + + return inputs + .flatMap((input) => { + if (typeof input === 'string') return normalizeFileIdList(input) + if (input && typeof input === 'object') { + const record = input as Record + if (typeof record.id === 'string') return normalizeFileIdList(record.id) + if (typeof record.fileId === 'string') return normalizeFileIdList(record.fileId) + } + return [] + }) + .filter((id) => id.length > 0) +} + +/** Per-file download cap for the content operation. Aligned with the durable large-value ceiling. */ +const MAX_GET_CONTENT_FILE_BYTES = 64 * 1024 * 1024 +/** Combined extracted-text cap so the content array stays within the large-value-ref ceiling. */ +const MAX_GET_CONTENT_TOTAL_BYTES = 64 * 1024 * 1024 + +/** Per-file download cap for the compress operation. */ +const MAX_COMPRESS_FILE_BYTES = 100 * 1024 * 1024 +/** Combined input cap for the compress operation to bound in-memory archiving. */ +const MAX_COMPRESS_TOTAL_BYTES = 100 * 1024 * 1024 + +/** Ensure an archive name ends with a single `.zip` extension. */ +const ensureZipExtension = (name: string): string => + name.toLowerCase().endsWith('.zip') ? name : `${name}.zip` + +/** Strip the trailing extension from a file name (e.g., "report.pdf" -> "report"). */ +const stripExtension = (name: string): string => { + const dot = name.lastIndexOf('.') + return dot > 0 ? name.slice(0, dot) : name +} + +/** + * Reduce an arbitrary name to a safe, flat file name: takes the final path + * segment, drops directory and traversal components, and falls back when the + * result would be empty or a dot segment. Used for the compress archive name so + * untrusted input cannot introduce nested or zip-slip-style paths. + */ +const toFlatFileName = (name: string, fallback: string): string => { + const leaf = name.replace(/\\/g, '/').split('/').pop()?.trim() + if (!leaf || leaf === '.' || leaf === '..') return fallback + return leaf +} + +/** A file bound for a compress archive, paired with the workspace folder it lives in. */ +interface ArchiveEntry { + file: UserFile + folderPath: string | null +} + +const isLikelyTextBuffer = (buffer: Buffer): boolean => isUtf8(buffer) && !buffer.includes(0) + +/** + * Download a stored file and extract its text content. Parseable types (PDF, DOCX, + * CSV, etc.) go through the shared file-parsers; other UTF-8 files are returned as + * raw text; binary files yield a short placeholder rather than corrupt bytes. + */ +const extractUserFileTextContent = async ( + userFile: UserFile, + requestId: string +): Promise => { + const { buffer } = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_GET_CONTENT_FILE_BYTES, + }) + + const extension = getFileExtension(userFile.name) + if (extension && isSupportedFileType(extension)) { + try { + const result = await parseBuffer(buffer, extension) + return result.content ?? '' + } catch (error) { + logger.warn('Falling back to raw text after parser failure', { + name: userFile.name, + error: getErrorMessage(error, 'Unknown error'), + }) + } + } + + if (isLikelyTextBuffer(buffer)) { + return buffer.toString('utf-8') + } + + return `[Binary file: ${userFile.name} (${userFile.type || 'application/octet-stream'}, ${buffer.length} bytes). Cannot extract text content.]` +} + +interface FileContentSource { + file: UserFile + identity?: WorkspaceFileSecretProvenanceIdentity + ownerUserId?: string +} + +async function bindSelectedContentFile( + principal: Principal, + workspaceId: string, + file: UserFile +): Promise { + if (!file.key || file.context !== 'workspace') return { file } + + let metadata: Awaited> + try { + metadata = await resolveWorkspaceFileReference({ + principal, + operation: fileOperations.readContent, + workspaceId, + reference: file.key, + }) + } catch (error) { + if (error instanceof OrchestrationError && error.code === 'not_found') return { file } + throw error + } + if (!metadata) return { file } + + return { + file, + identity: { fileId: metadata.id, key: metadata.key, context: 'workspace' }, + ownerUserId: metadata.uploadedBy, + } +} + +async function getFileContentProvenance( + principal: Principal, + workspaceId: string, + sources: readonly FileContentSource[] +): Promise { + const ownerIds = new Set( + sources + .map((source) => source.ownerUserId) + .filter((ownerUserId): ownerUserId is string => Boolean(ownerUserId)) + ) + const ownerUserId = ownerIds.size === 1 ? ownerIds.values().next().value : undefined + const scope: ResolvedSecretTraceScopeV1 | undefined = ownerUserId + ? { userId: ownerUserId, workspaceId } + : undefined + const accumulator = new ResolvedSecretTraceProvenanceAccumulator(scope) + + for (const source of sources) { + if (!source.identity || !source.ownerUserId) { + accumulator.markIncomplete('file-source-unidentified') + continue + } + const { provenance } = await readWorkspaceFileSecretProvenance.execute({ + principal, + input: { fileId: source.identity.fileId, assertedWorkspaceId: workspaceId }, + }) + /** + * `unrecorded` is a more specific `unknown`, and this accumulator has not opted into the + * workspace file surface's policy, so it latches exactly as it did before. + */ + if (provenance.status !== 'exact') { + accumulator.markIncomplete('workspace-file-provenance-unknown') + continue + } + accumulator.record({ + version: 1, + complete: true, + entries: [...provenance.entries], + ...(scope ? { scope } : {}), + }) + } + + return accumulator.exportProvenance() +} + +type FileMutationProvenanceResolution = + | { + success: true + provenanceBySelection?: ReadonlyMap + } + | { success: false; error: string } + +/** Authenticates exact, causally selected file-mutation provenance from an internal caller. */ +function resolveFileMutationSecretProvenance(options: { + headers: Headers + payload: unknown + userId: string + workspaceId: string + selectionKeys: readonly string[] +}): FileMutationProvenanceResolution { + const inspection = inspectPrivateSecretProvenanceRequest(options.headers, options.payload) + if (inspection.status === 'unsupported') return { success: true } + if (inspection.status !== 'verified' || !isPrivateSecretProvenanceBundleV1(inspection.value)) { + return { success: false, error: 'Invalid file secret provenance' } + } + + const provenanceBySelection = new Map() + if (!inspection.value.complete) { + for (const selectionKey of options.selectionKeys) { + provenanceBySelection.set(selectionKey, { status: 'unknown' }) + } + return { success: true, provenanceBySelection } + } + if (inspection.value.selections.length !== options.selectionKeys.length) { + return { success: false, error: 'Invalid file secret provenance' } + } + + const destinationScope = { userId: options.userId, workspaceId: options.workspaceId } + for (const selectionKey of options.selectionKeys) { + const provenance = durableSecretProvenanceFromPrivateBundle( + inspection.value, + selectionKey, + destinationScope + ) + if (!provenance) { + return { success: false, error: 'Invalid file secret provenance' } + } + if (provenance.status === 'unknown') { + provenanceBySelection.set(selectionKey, provenance) + continue + } + if (provenance.entries.some((entry) => !entry.name || !entry.sourceUserId)) { + return { success: false, error: 'Invalid file secret provenance' } + } + provenanceBySelection.set(selectionKey, { + status: 'exact', + entries: provenance.entries.map((entry) => ({ + name: entry.name as string, + encryptedValue: entry.encryptedValue, + sourceUserId: entry.sourceUserId as string, + ...(entry.sourceWorkspaceId ? { sourceWorkspaceId: entry.sourceWorkspaceId } : {}), + })), + }) + } + return { success: true, provenanceBySelection } +} + +type FileWriteProvenanceResolution = + | { success: true; contentProvenance?: WorkspaceFileSecretProvenance } + | { success: false; error: string } + +/** Resolves file-content provenance before any folder or file mutation. */ +function resolveFileWriteSecretProvenance(options: { + headers: Headers + payload: unknown + userId: string + workspaceId: string +}): FileWriteProvenanceResolution { + const resolution = resolveFileMutationSecretProvenance({ + ...options, + selectionKeys: ['content'], + }) + if (!resolution.success || !resolution.provenanceBySelection) return resolution + const content = resolution.provenanceBySelection.get('content') + if (!content) { + return { success: false, error: 'Invalid file secret provenance' } + } + return { success: true, contentProvenance: content } +} + +async function deriveWorkspaceFileSecretProvenance(options: { + principal: Principal + workspaceId: string + targetOwnerUserId: string + sources: readonly FileContentSource[] +}): Promise { + const provenances: WorkspaceFileSecretProvenance[] = [] + for (const source of options.sources) { + if (!source.identity || !source.ownerUserId) return { status: 'unknown' } + const { provenance } = await readWorkspaceFileSecretProvenance.execute({ + principal: options.principal, + input: { fileId: source.identity.fileId, assertedWorkspaceId: options.workspaceId }, + }) + if ( + provenance.status === 'exact' && + provenance.entries.length > 0 && + source.ownerUserId !== options.targetOwnerUserId + ) { + return { status: 'unknown' } + } + provenances.push(provenance) + } + return mergeWorkspaceFileSecretProvenance(...provenances) +} + +function fileContentJsonResponse( + body: Record, + includePrivateProvenance: boolean, + init?: ResponseInit, + provenance: ResolvedSecretTraceProvenanceV1 = { version: 1, complete: true, entries: [] } +): Response { + if (!includePrivateProvenance) return Response.json(body, init) + + const headers = new Headers(init?.headers) + headers.delete('content-length') + headers.set(PRIVATE_TOOL_METADATA_RESPONSE_HEADER, RESOLVED_SECRET_PROVENANCE_METADATA_V1) + return Response.json( + { ...body, [RESOLVED_SECRET_PROVENANCE_FIELD]: provenance }, + { ...init, headers } + ) +} + +export async function executeFileManageOperation( + body: FileManageOperationInput, + context: FileManageOperationContext +): Promise { + const { attributedUserId: userId, headers, principal, requestId, signal, workspaceId } = context + signal?.throwIfAborted() + if (body.workspaceId && body.workspaceId !== workspaceId) { + return Response.json({ success: false, error: 'Workspace access denied' }, { status: 403 }) + } + const includePrivateContentProvenance = + body.operation === 'content' && + requestsPrivateToolMetadata(headers, RESOLVED_SECRET_PROVENANCE_METADATA_V1) + const contentResponse = ( + responseBody: Record, + init?: ResponseInit, + provenance?: ResolvedSecretTraceProvenanceV1 + ) => fileContentJsonResponse(responseBody, includePrivateContentProvenance, init, provenance) + + try { + switch (body.operation) { + case 'get': { + const { fileId, fileInput } = body + const selectedFileId = + fileId || + (isRecordLike(fileInput) + ? (() => { + const obj = fileInput as Record + return typeof obj.id === 'string' + ? obj.id + : typeof obj.fileId === 'string' + ? obj.fileId + : '' + })() + : '') + + if (!selectedFileId) { + return Response.json({ success: false, error: 'File is required' }, { status: 400 }) + } + + let file: Awaited> + try { + file = ( + await readWorkspaceFileMetadata.execute({ + principal, + input: { fileId: selectedFileId, assertedWorkspaceId: workspaceId }, + }) + ).file + } catch (error) { + if (error instanceof OrchestrationError && error.code === 'not_found') { + return Response.json( + { success: false, error: `File not found: "${selectedFileId}"` }, + { status: 404 } + ) + } + throw error + } + + logger.info('File retrieved', { + fileId: file.id, + name: file.name, + }) + + return Response.json({ + success: true, + data: { + file: workspaceFileToUserFile(file), + }, + }) + } + + case 'read': { + const { fileId, fileInput } = body + const selectedFileIds = Array.isArray(fileId) + ? fileId.map((id) => id.trim()).filter(Boolean) + : fileId + ? normalizeFileIdList(fileId) + : extractFileIdsFromInput(fileInput) + const selectedInputFiles = fileId ? [] : extractUserFilesFromInput(fileInput) + + if (selectedFileIds.length === 0 && selectedInputFiles.length === 0) { + return Response.json({ success: false, error: 'File is required' }, { status: 400 }) + } + + const files = [] as Array>>> + for (const id of selectedFileIds) { + signal?.throwIfAborted() + try { + files.push( + ( + await readWorkspaceFileMetadata.execute({ + principal, + input: { fileId: id, assertedWorkspaceId: workspaceId }, + }) + ).file + ) + } catch (error) { + if (error instanceof OrchestrationError && error.code === 'not_found') { + return Response.json( + { success: false, error: `File not found: "${id}"` }, + { status: 404 } + ) + } + throw error + } + } + + const shares = new Map( + await Promise.all( + files.map( + async (file) => + [ + file.id, + ( + await getWorkspaceFileShare.execute({ + principal, + input: { fileId: file.id, assertedWorkspaceId: workspaceId }, + }) + ).share, + ] as const + ) + ) + ) + const privateReadShare = () => ({ + visibility: 'private' as const, + url: null, + allowedEmails: [] as string[], + }) + const toReadShare = (fileId: string) => { + const share = shares.get(fileId) + if (!share || !share.isActive) return privateReadShare() + return { + visibility: share.authType, + url: share.url, + allowedEmails: share.allowedEmails, + } + } + const canonicalUserFiles = files + .map((file) => workspaceFileToUserFile(file)) + .filter((file): file is NonNullable> => + Boolean(file) + ) + .map((file) => ({ ...file, share: toReadShare(file.id) })) + const userFiles = [ + ...canonicalUserFiles, + ...selectedInputFiles.map((file) => ({ ...file, share: privateReadShare() })), + ] + + logger.info('Files retrieved', { + count: userFiles.length, + fileIds: userFiles.map((file) => file.id), + }) + + return Response.json({ + success: true, + data: { + file: userFiles[0], + files: userFiles, + }, + }) + } + + case 'content': { + const { fileId, fileInput } = body + const selectedFileIds = Array.isArray(fileId) + ? fileId.map((id) => id.trim()).filter(Boolean) + : fileId + ? normalizeFileIdList(fileId) + : extractFileIdsFromInput(fileInput) + const selectedInputFiles = fileId ? [] : extractUserFilesFromInput(fileInput) + + if (selectedFileIds.length === 0 && selectedInputFiles.length === 0) { + return contentResponse({ success: false, error: 'File is required' }, { status: 400 }) + } + + const workspaceFiles = [] as Array< + NonNullable>> + > + for (const id of selectedFileIds) { + signal?.throwIfAborted() + try { + workspaceFiles.push( + ( + await readWorkspaceFileMetadata.execute({ + principal, + input: { fileId: id, assertedWorkspaceId: workspaceId }, + }) + ).file + ) + } catch (error) { + if (error instanceof OrchestrationError && error.code === 'not_found') { + return contentResponse( + { success: false, error: `File not found: "${id}"` }, + { status: 404 } + ) + } + throw error + } + } + + const canonicalSources: FileContentSource[] = workspaceFiles.flatMap((file) => { + const userFile = workspaceFileToUserFile(file) + if (!file || !userFile) return [] + return [ + { + file: userFile, + identity: { fileId: file.id, key: file.key, context: 'workspace' }, + ownerUserId: file.uploadedBy, + }, + ] + }) + const selectedSources = await Promise.all( + selectedInputFiles.map((file) => bindSelectedContentFile(principal, workspaceId, file)) + ) + const sources = canonicalSources.concat(selectedSources) + + const contents: string[] = [] + let totalBytes = 0 + for (const source of sources) { + signal?.throwIfAborted() + const denied = source.identity + ? null + : await assertOperationFileAccess(source.file, context) + if (denied) { + const deniedBody = (await denied.clone().json()) as Record + return contentResponse(deniedBody, { + status: denied.status, + statusText: denied.statusText, + headers: denied.headers, + }) + } + + const content = await extractUserFileTextContent(source.file, requestId) + totalBytes += Buffer.byteLength(content, 'utf8') + if (totalBytes > MAX_GET_CONTENT_TOTAL_BYTES) { + return contentResponse( + { + success: false, + error: `Combined file content is too large to return safely. Maximum is ${ + MAX_GET_CONTENT_TOTAL_BYTES / (1024 * 1024) + } MB.`, + }, + { status: 413 } + ) + } + contents.push(content) + } + + logger.info('File content extracted', { count: contents.length }) + const provenance = includePrivateContentProvenance + ? await getFileContentProvenance(principal, workspaceId, sources) + : undefined + + return contentResponse({ success: true, data: { contents } }, undefined, provenance) + } + + case 'write': { + const { fileName, content, contentType } = body + signal?.throwIfAborted() + const provenanceResolution = resolveFileWriteSecretProvenance({ + headers, + payload: body, + userId, + workspaceId, + }) + if (!provenanceResolution.success) { + return Response.json( + { success: false, error: provenanceResolution.error }, + { status: 400 } + ) + } + const { folderSegments, leafName } = splitWorkspaceFilePath(fileName) + await admitCreateWorkspaceFile(principal, workspaceId) + const { folderId } = await ensureWorkspaceFileFolderPathOperation.execute({ + principal, + input: { workspaceId, pathSegments: folderSegments }, + }) + const mimeType = contentType || getMimeTypeFromExtension(getFileExtension(leafName)) + const result = await createWorkspaceFile.execute({ + principal, + input: { + workspaceId, + name: leafName, + contentType: mimeType, + content: content ?? '', + encoding: 'utf-8', + folderId, + exactName: false, + ...(provenanceResolution.contentProvenance + ? { secretProvenance: provenanceResolution.contentProvenance } + : {}), + }, + }) + const fileBuffer = Buffer.from(content ?? '', 'utf-8') + + logger.info('File created', { + fileId: result.file.id, + name: fileName, + size: fileBuffer.length, + }) + + return Response.json({ + success: true, + data: { + id: result.file.id, + name: result.file.name, + size: fileBuffer.length, + url: ensureAbsoluteUrl(result.file.url ?? result.file.path), + }, + }) + } + + case 'move': { + const { fileId, targetFolder } = body + signal?.throwIfAborted() + const pathSegments = targetFolder.trim() + ? targetFolder + .trim() + .split('/') + .map((s) => s.trim()) + .filter(Boolean) + : [] + let targetFolderPath: string + try { + targetFolderPath = buildFolderPath(pathSegments) + } catch (error) { + throw new OrchestrationError('validation', getErrorMessage(error)) + } + await moveWorkspaceFileItemsOperation.execute({ + principal, + input: { + workspaceId, + fileIds: [fileId], + targetFolderPath, + }, + }) + logger.info('File moved', { fileId, targetFolder: targetFolder || '(root)' }) + return Response.json({ + success: true, + data: { fileId, targetFolder: targetFolder || '(root)' }, + }) + } + + case 'manage_sharing': { + const { fileId, fileInput, isActive, authType, password, allowedEmails } = body + signal?.throwIfAborted() + + // Resolve the canonical file id. The basic file picker provides an object + // with a storage `key` but no id, so map the key to the workspace file row. + let resolvedFileId = typeof fileId === 'string' ? fileId : undefined + if (!resolvedFileId && fileInput) { + const single = Array.isArray(fileInput) ? fileInput[0] : fileInput + if (single && typeof single === 'object') { + const record = single as Record + if (typeof record.id === 'string' && record.id) resolvedFileId = record.id + else if (typeof record.fileId === 'string' && record.fileId) + resolvedFileId = record.fileId + else if (typeof record.key === 'string' && record.key) { + resolvedFileId = ( + await resolveWorkspaceFileReference({ + principal, + operation: fileOperations.updateShare, + workspaceId, + reference: record.key, + }) + ).id + } + } + } + if (!resolvedFileId) { + return Response.json( + { success: false, error: 'A valid file is required to manage sharing' }, + { status: 400 } + ) + } + + const share = ( + await updateWorkspaceFileShare.execute({ + principal, + input: { + fileId: resolvedFileId, + assertedWorkspaceId: workspaceId, + isActive, + authType, + password, + allowedEmails, + }, + }) + ).share + + logger.info('File sharing updated', { + fileId: resolvedFileId, + isActive, + authType: share.authType, + }) + + // A disabled link doesn't resolve, so don't hand back a dead URL. + const responseShare = share.isActive ? share : { ...share, url: '' } + return Response.json({ success: true, data: { share: responseShare } }) + } + + case 'append': { + const { fileName, content } = body + signal?.throwIfAborted() + + const existing = await resolveWorkspaceFileReference({ + principal, + operation: fileOperations.updateContent, + workspaceId, + reference: fileName, + }) + + const lockKey = `file-append:${workspaceId}:${existing.id}` + const lockValue = `${Date.now()}-${generateShortId()}` + const acquired = await acquireLock(lockKey, lockValue, 30) + if (!acquired) { + return Response.json( + { success: false, error: 'File is busy, please retry' }, + { status: 409 } + ) + } + + try { + if (!existing.contentUpdatedAt) { + throw new Error('File content version is unavailable') + } + const { provenance: existingProvenance } = + await readWorkspaceFileSecretProvenance.execute({ + principal, + input: { fileId: existing.id, assertedWorkspaceId: workspaceId }, + }) + const appendedResolution = resolveFileMutationSecretProvenance({ + headers, + payload: body, + userId, + workspaceId, + selectionKeys: ['content'], + }) + if (!appendedResolution.success) { + return Response.json( + { success: false, error: appendedResolution.error }, + { status: 400 } + ) + } + const appendedProvenance = appendedResolution.provenanceBySelection?.get('content') + const secretProvenance = + appendedProvenance?.status === 'exact' && + appendedProvenance.entries.length > 0 && + existing.uploadedBy !== userId + ? { status: 'unknown' as const } + : appendedProvenance + ? mergeWorkspaceFileSecretProvenance(existingProvenance, appendedProvenance) + : undefined + const { content: existingBuffer } = await readWorkspaceFileContent.execute({ + principal, + input: { + fileId: existing.id, + assertedWorkspaceId: workspaceId, + maxBytes: MAX_WORKSPACE_FILE_CONTENT_BYTES, + }, + }) + const finalContent = existingBuffer.toString('utf-8') + content + const fileBuffer = Buffer.from(finalContent, 'utf-8') + await updateWorkspaceFileContent.execute({ + principal, + input: { + fileId: existing.id, + assertedWorkspaceId: workspaceId, + content: finalContent, + encoding: 'utf-8', + expectedUpdatedAt: existing.contentUpdatedAt ?? undefined, + provenanceMode: secretProvenance ? undefined : 'preserve', + ...(secretProvenance ? { secretProvenance } : {}), + }, + }) + + logger.info('File appended', { + fileId: existing.id, + name: existing.name, + size: fileBuffer.length, + }) + + return Response.json({ + success: true, + data: { + id: existing.id, + name: existing.name, + size: fileBuffer.length, + url: ensureAbsoluteUrl(existing.path), + }, + }) + } finally { + await releaseLock(lockKey, lockValue) + } + } + + case 'compress': { + const { fileId, fileInput, archiveName } = body + const selectedFileIds = Array.isArray(fileId) + ? fileId.map((id) => id.trim()).filter(Boolean) + : fileId + ? normalizeFileIdList(fileId) + : extractFileIdsFromInput(fileInput) + const selectedInputFiles = fileId ? [] : extractUserFilesFromInput(fileInput) + + if (selectedFileIds.length === 0 && selectedInputFiles.length === 0) { + return Response.json({ success: false, error: 'File is required' }, { status: 400 }) + } + await admitCreateWorkspaceFile(principal, workspaceId) + + const workspaceFiles = [] as Array< + NonNullable>> + > + for (const id of selectedFileIds) { + signal?.throwIfAborted() + try { + workspaceFiles.push( + ( + await downloadWorkspaceFileRecord.execute({ + principal, + input: { fileId: id, assertedWorkspaceId: workspaceId }, + }) + ).file + ) + } catch (error) { + if (error instanceof OrchestrationError && error.code === 'not_found') { + return Response.json( + { success: false, error: `File not found: "${id}"` }, + { status: 404 } + ) + } + throw error + } + } + + const workspaceEntries: ArchiveEntry[] = workspaceFiles.flatMap((file) => { + const userFile = workspaceFileToUserFile(file) + return userFile ? [{ file: userFile, folderPath: file?.folderPath ?? null }] : [] + }) + + // Picker/upload values carry no workspace folder, so they archive at the root. + const archiveEntries = workspaceEntries.concat( + selectedInputFiles.map((file) => ({ file, folderPath: null })) + ) + const userFiles: UserFile[] = archiveEntries.map((entry) => entry.file) + const canonicalArchiveSources: FileContentSource[] = workspaceFiles.flatMap((file) => { + const userFile = workspaceFileToUserFile(file) + if (!file || !userFile) return [] + return [ + { + file: userFile, + identity: { fileId: file.id, key: file.key, context: 'workspace' }, + ownerUserId: file.uploadedBy, + }, + ] + }) + const selectedArchiveSources = await Promise.all( + selectedInputFiles.map((file) => bindSelectedContentFile(principal, workspaceId, file)) + ) + const archiveSources = canonicalArchiveSources.concat(selectedArchiveSources) + const archiveProvenance = await deriveWorkspaceFileSecretProvenance({ + principal, + workspaceId, + targetOwnerUserId: userId, + sources: archiveSources, + }) + + // Mirror the workspace folder layout, dropping the ancestor chain the whole + // selection shares so archiving one folder does not nest it under its parents. + const entryPaths = buildZipEntryPaths( + archiveEntries.map((entry) => ({ name: entry.file.name, folderPath: entry.folderPath })), + { rebaseOnCommonFolder: true } + ) + + const zip = new JSZip() + let totalBytes = 0 + for (const [index, userFile] of userFiles.entries()) { + signal?.throwIfAborted() + const denied = archiveSources[index]?.identity + ? null + : await assertOperationFileAccess(userFile, context) + if (denied) return denied + + // Generated docs store their generation source, not the rendered binary, so + // the archive must carry the servable bytes instead of the raw source text. + // A still-compiling artifact throws, and the handler's catch turns that into + // the shared 409 via `docNotReadyResponse`. + const { buffer } = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_COMPRESS_FILE_BYTES, + }) + totalBytes += buffer.length + if (totalBytes > MAX_COMPRESS_TOTAL_BYTES) { + return Response.json( + { + success: false, + error: `Combined input is too large to compress. Maximum is ${ + MAX_COMPRESS_TOTAL_BYTES / (1024 * 1024) + } MB.`, + }, + { status: 413 } + ) + } + zip.file(entryPaths[index], buffer) + } + + const zipBuffer = await zip.generateAsync({ + type: 'nodebuffer', + compression: 'DEFLATE', + compressionOptions: { level: 6 }, + }) + signal?.throwIfAborted() + + const requestedName = typeof archiveName === 'string' ? archiveName.trim() : '' + const baseName = requestedName + ? toFlatFileName(requestedName, 'archive') + : userFiles.length === 1 + ? stripExtension(toFlatFileName(userFiles[0].name, 'archive')) + : 'archive' + const leafName = ensureZipExtension(baseName) + const result = await createWorkspaceFileFromBuffer.execute({ + principal, + input: { + workspaceId, + name: leafName, + contentType: 'application/zip', + content: zipBuffer, + folderId: null, + exactName: false, + secretProvenance: archiveProvenance, + }, + }) + + const compressedFile: UserFile = { + ...result.file, + url: ensureAbsoluteUrl(result.file.url ?? result.file.path), + size: zipBuffer.length, + } + + logger.info('Files compressed', { + fileId: result.file.id, + name: result.file.name, + fileCount: userFiles.length, + size: zipBuffer.length, + }) + + return Response.json({ + success: true, + data: { + id: compressedFile.id, + name: compressedFile.name, + size: compressedFile.size, + url: compressedFile.url, + files: [compressedFile], + }, + }) + } + + case 'decompress': { + const { fileId, fileInput } = body + const selectedFileIds = fileId ? [fileId] : extractFileIdsFromInput(fileInput) + const selectedInputFiles = fileId ? [] : extractUserFilesFromInput(fileInput) + + if (selectedFileIds.length === 0 && selectedInputFiles.length === 0) { + return Response.json({ success: false, error: 'File is required' }, { status: 400 }) + } + if (selectedFileIds.length + selectedInputFiles.length > 1) { + return Response.json( + { success: false, error: 'Decompress accepts a single .zip archive at a time' }, + { status: 400 } + ) + } + await admitCreateWorkspaceFile(principal, workspaceId) + + const workspaceFiles = [] as Array< + NonNullable>> + > + for (const id of selectedFileIds) { + signal?.throwIfAborted() + try { + workspaceFiles.push( + ( + await downloadWorkspaceFileRecord.execute({ + principal, + input: { fileId: id, assertedWorkspaceId: workspaceId }, + }) + ).file + ) + } catch (error) { + if (error instanceof OrchestrationError && error.code === 'not_found') { + return Response.json( + { success: false, error: `File not found: "${id}"` }, + { status: 404 } + ) + } + throw error + } + } + + const archive = [ + ...workspaceFiles + .map((file) => workspaceFileToUserFile(file)) + .filter((file): file is NonNullable> => + Boolean(file) + ), + ...selectedInputFiles, + ][0] + + if (!archive) { + return Response.json({ success: false, error: 'File is required' }, { status: 400 }) + } + + const canonicalArchiveSource: FileContentSource[] = workspaceFiles.flatMap((file) => { + const userFile = workspaceFileToUserFile(file) + if (!file || !userFile) return [] + return [ + { + file: userFile, + identity: { fileId: file.id, key: file.key, context: 'workspace' }, + ownerUserId: file.uploadedBy, + }, + ] + }) + const selectedArchiveSource = await Promise.all( + selectedInputFiles.map((file) => bindSelectedContentFile(principal, workspaceId, file)) + ) + const archiveSource = canonicalArchiveSource.concat(selectedArchiveSource)[0] + if (!archiveSource?.identity) { + const denied = await assertOperationFileAccess(archive, context) + if (denied) return denied + } + const archiveProvenance = await deriveWorkspaceFileSecretProvenance({ + principal, + workspaceId, + targetOwnerUserId: userId, + sources: archiveSource ? [archiveSource] : [], + }) + + const archiveBuffer = await downloadFileFromStorage(archive, requestId, logger, { + maxBytes: MAX_ARCHIVE_BYTES, + }) + signal?.throwIfAborted() + + let result: DecompressResult + try { + result = await decompressArchiveBufferToWorkspaceFiles(archiveBuffer, { + workspaceId, + principal, + secretProvenance: archiveProvenance, + signal, + }) + } catch (archiveError) { + if (archiveError instanceof ArchiveError) { + // The error message is single-sourced in ArchiveError (caps included); + // only the HTTP status is mapped here. + const status = statusForArchiveError(archiveError) + return Response.json( + { success: false, error: `"${archive.name}": ${archiveError.message}` }, + { status } + ) + } + throw archiveError + } + + if (result.extracted.length === 0) { + return Response.json( + { success: false, error: `No files could be extracted from "${archive.name}".` }, + { status: 422 } + ) + } + + const extractedFiles = result.extracted.map((file) => ({ + ...file, + url: ensureAbsoluteUrl(file.url), + })) + + if (result.skippedUnsafePaths.length > 0) { + logger.warn('Skipped unsafe archive entries', { + fileId: archive.id, + name: archive.name, + entryNames: result.skippedUnsafePaths, + }) + } + + logger.info('Archive decompressed', { + fileId: archive.id, + name: archive.name, + extractedCount: extractedFiles.length, + skippedCount: result.skipped, + }) + + return Response.json({ + success: true, + data: { + files: extractedFiles, + }, + }) + } + } + } catch (error) { + if (isWorkspaceAccessDeniedError(error)) { + return contentResponse({ success: false, error: 'Workspace access denied' }, { status: 403 }) + } + if (error instanceof OrchestrationError) { + const status = + error.code === 'forbidden' + ? 403 + : error.code === 'not_found' + ? 404 + : error.code === 'conflict' + ? 409 + : error.code === 'payload_too_large' + ? 413 + : error.code === 'validation' + ? 400 + : 500 + return contentResponse({ success: false, error: error.message }, { status }) + } + const notReady = docNotReadyResponse(error) + if (notReady) { + if (!includePrivateContentProvenance) return notReady + const notReadyBody = (await notReady.clone().json()) as Record + return contentResponse(notReadyBody, { + status: notReady.status, + statusText: notReady.statusText, + headers: notReady.headers, + }) + } + // A file over its per-file cap is a size rejection, not a fault. Rendered + // documents can cross it even when the stored source was well under. + if (isPayloadSizeLimitError(error)) { + return contentResponse({ success: false, error: error.message }, { status: 413 }) + } + if (error instanceof ShareValidationError) { + return contentResponse({ success: false, error: error.message }, { status: 400 }) + } + const message = getErrorMessage(error, 'Unknown error') + logger.error('File operation failed', { operation: body.operation, error: message }) + return contentResponse({ success: false, error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/file/parser.test.ts b/apps/sim/lib/internal/file/parser.test.ts new file mode 100644 index 00000000000..0b69bfe7332 --- /dev/null +++ b/apps/sim/lib/internal/file/parser.test.ts @@ -0,0 +1,997 @@ +/** + * Tests for the direct file parser operation. + * + * @vitest-environment node + */ +import { + authMockFns, + createMockRequest, + hybridAuthMockFns, + inputValidationMock, + inputValidationMockFns, + permissionsMock, + permissionsMockFns, + storageServiceMock, + storageServiceMockFns, +} from '@sim/testing' +import { NextRequest } from 'next/server' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { FileParserError } from '@/lib/file-parsers/errors' + +const { + mockVerifyFileAccess, + mockVerifyWorkspaceFileAccess, + mockGetStorageProvider, + mockIsUsingCloudStorage, + mockIsSupportedFileType, + mockParseFile, + mockParseBuffer, + mockFsAccess, + mockFsStat, + mockFsReadFile, + mockFsWriteFile, + mockJoin, + actualPath, + mockUploadWorkspaceFile, + mockReadWorkspaceFileNameByKey, +} = vi.hoisted(() => { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const actualPath = require('path') as typeof import('path') + return { + mockVerifyFileAccess: vi.fn().mockResolvedValue(true), + mockVerifyWorkspaceFileAccess: vi.fn().mockResolvedValue(true), + mockGetStorageProvider: vi.fn().mockReturnValue('s3'), + mockIsUsingCloudStorage: vi.fn().mockReturnValue(true), + mockIsSupportedFileType: vi.fn().mockReturnValue(true), + mockParseFile: vi.fn().mockResolvedValue({ + content: 'parsed content', + metadata: { pageCount: 1 }, + }), + mockParseBuffer: vi.fn().mockResolvedValue({ + content: 'parsed buffer content', + metadata: { pageCount: 1 }, + }), + mockFsAccess: vi.fn().mockResolvedValue(undefined), + mockFsStat: vi.fn().mockImplementation(() => ({ isFile: () => true, size: 17 })), + mockFsReadFile: vi.fn().mockResolvedValue(Buffer.from('test file content')), + mockFsWriteFile: vi.fn().mockResolvedValue(undefined), + mockJoin: vi.fn((...args: string[]): string => { + if (args[0] === '/test/uploads') { + return `/test/uploads/${args[args.length - 1]}` + } + return actualPath.join(...args) + }), + actualPath, + mockUploadWorkspaceFile: vi + .fn() + .mockImplementation( + async (workspaceId: string, _userId: string, _buffer: Buffer, fileName: string) => ({ + id: 'wf_test', + name: fileName, + size: 0, + type: 'application/octet-stream', + url: `/api/files/serve/${workspaceId}/${fileName}`, + key: `${workspaceId}/${fileName}`, + context: 'workspace', + }) + ), + mockReadWorkspaceFileNameByKey: vi.fn(), + } +}) + +vi.mock('@/lib/execution/payloads/materialization.server', () => ({ + assertUserFileContentAccess: async (file: { key: string }) => { + if (!(await mockVerifyFileAccess(file.key))) throw new Error('File not found') + }, +})) + +vi.mock('@/lib/uploads', () => ({ + getStorageProvider: mockGetStorageProvider, + isUsingCloudStorage: mockIsUsingCloudStorage, + StorageService: storageServiceMock, +})) + +vi.mock('@/lib/file-parsers', () => ({ + isSupportedFileType: mockIsSupportedFileType, + parseFile: mockParseFile, + parseBuffer: mockParseBuffer, +})) + +vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) + +vi.mock('path', () => ({ + default: actualPath, + ...actualPath, + join: mockJoin, + basename: actualPath.basename, + extname: actualPath.extname, +})) + +vi.mock('@/lib/uploads/core/setup.server', () => ({ + UPLOAD_DIR_SERVER: '/test/uploads', +})) + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + +vi.mock('@/lib/core/utils/logging', () => ({ + sanitizeUrlForLog: vi.fn((url: string) => url), +})) + +vi.mock('@/lib/uploads/contexts/execution', () => ({ + uploadExecutionFile: vi.fn(), +})) + +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ + uploadWorkspaceFile: mockUploadWorkspaceFile, +})) + +vi.mock('@/lib/uploads/server/metadata', () => ({ + getFileMetadataByKey: vi.fn(), +})) + +vi.mock('@/lib/workspace-files/application/read-workspace-file-name-by-key', () => ({ + readWorkspaceFileNameByKey: { execute: mockReadWorkspaceFileNameByKey }, +})) + +vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock) + +vi.mock('fs/promises', () => ({ + default: { + access: mockFsAccess, + stat: mockFsStat, + readFile: mockFsReadFile, + writeFile: mockFsWriteFile, + }, + access: mockFsAccess, + stat: mockFsStat, + readFile: mockFsReadFile, + writeFile: mockFsWriteFile, +})) + +import { fileParseBodySchema } from '@/lib/api/contracts/storage-transfer' +import { executeFileParserOperation } from '@/lib/internal/file/parser' +import { createWorkspaceFileDelegatedPrincipal } from '@/lib/workspace-files/application/delegated-principal' + +async function POST(request: NextRequest): Promise { + const parsed = fileParseBodySchema.safeParse(await request.json()) + if (!parsed.success) { + const message = parsed.error.issues[0]?.message ?? 'Invalid request data' + return Response.json( + { success: false, error: message, filePath: '' }, + { status: message.includes('At most 10 files') ? 413 : 400 } + ) + } + return executeFileParserOperation(parsed.data, { + principal: createWorkspaceFileDelegatedPrincipal({ + serviceId: 'executor', + subjectUserId: 'test-user-id', + workspaceId: parsed.data.workspaceId || 'workspace-id', + delegationId: 'test-file-parser', + }), + workspaceId: parsed.data.workspaceId || 'workspace-id', + workflowId: parsed.data.workflowId || 'workflow-id', + executionId: parsed.data.executionId || 'execution-id', + attributedUserId: 'test-user-id', + fileAccessUserId: 'test-user-id', + signal: request.signal, + }) +} + +function setupFileApiMocks( + options: { + authenticated?: boolean + storageProvider?: 's3' | 'blob' | 'local' + cloudEnabled?: boolean + } = {} +) { + const { authenticated = true, storageProvider = 's3', cloudEnabled = true } = options + + if (authenticated) { + authMockFns.mockGetSession.mockResolvedValue({ + user: { id: 'test-user-id', email: 'test@example.com' }, + }) + } else { + authMockFns.mockGetSession.mockResolvedValue(null) + } + + hybridAuthMockFns.mockCheckInternalAuth.mockResolvedValue({ + success: authenticated, + userId: authenticated ? 'test-user-id' : undefined, + error: authenticated ? undefined : 'Unauthorized', + }) + + hybridAuthMockFns.mockCheckHybridAuth.mockResolvedValue({ + success: authenticated, + userId: authenticated ? 'test-user-id' : undefined, + error: authenticated ? undefined : 'Unauthorized', + }) + + hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ + success: authenticated, + userId: authenticated ? 'test-user-id' : undefined, + error: authenticated ? undefined : 'Unauthorized', + }) + + mockGetStorageProvider.mockReturnValue(storageProvider) + mockIsUsingCloudStorage.mockReturnValue(cloudEnabled) +} + +describe('file parser operation', () => { + beforeEach(() => { + vi.clearAllMocks() + + setupFileApiMocks({ + authenticated: true, + }) + + permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue({ canView: true }) + storageServiceMockFns.mockHasCloudStorage.mockReturnValue(true) + storageServiceMockFns.mockDownloadFile.mockResolvedValue(Buffer.from('test file content')) + mockFsStat.mockResolvedValue({ isFile: () => true, size: 17 }) + mockFsReadFile.mockResolvedValue(Buffer.from('test file content')) + mockIsSupportedFileType.mockReturnValue(true) + mockUploadWorkspaceFile.mockClear() + mockReadWorkspaceFileNameByKey.mockResolvedValue({ name: null }) + mockParseFile.mockResolvedValue({ + content: 'parsed content', + metadata: { pageCount: 1 }, + }) + mockParseBuffer.mockResolvedValue({ + content: 'parsed buffer content', + metadata: { pageCount: 1 }, + }) + }) + + afterEach(() => { + vi.clearAllMocks() + }) + + it('should handle missing file path', async () => { + const req = createMockRequest('POST', {}) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(400) + expect(data).toHaveProperty('error', 'No file path provided') + }) + + it('should accept and process a local file', async () => { + setupFileApiMocks({ + cloudEnabled: false, + storageProvider: 'local', + authenticated: true, + }) + + const req = createMockRequest('POST', { + filePath: '/api/files/serve/test-file.txt', + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data).not.toBeNull() + + if (data.success === true) { + expect(data).toHaveProperty('output') + } else { + expect(data).toHaveProperty('error') + expect(typeof data.error).toBe('string') + } + }) + + it('should process S3 files', async () => { + setupFileApiMocks({ + cloudEnabled: true, + storageProvider: 's3', + authenticated: true, + }) + + const req = createMockRequest('POST', { + filePath: '/api/files/serve/s3/test-file.pdf', + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + + if (data.success === true) { + expect(data).toHaveProperty('output') + } else { + expect(data).toHaveProperty('error') + } + }) + + it('should keep known binary extensions as binary even when the bytes are valid UTF-8', async () => { + setupFileApiMocks({ + cloudEnabled: true, + storageProvider: 's3', + authenticated: true, + }) + mockIsSupportedFileType.mockReturnValue(false) + storageServiceMockFns.mockDownloadFile.mockResolvedValue(Buffer.from('valid utf8 bytes')) + + const req = createMockRequest('POST', { + filePath: '/api/files/serve/execution/workspace-1/workflow-1/execution-1/image.png', + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.success).toBe(true) + expect(data.output.content).toBe('[Binary PNG file - 16 bytes]') + }) + + it('should parse unknown extensions as text when the bytes look like UTF-8 text', async () => { + setupFileApiMocks({ + cloudEnabled: true, + storageProvider: 's3', + authenticated: true, + }) + mockIsSupportedFileType.mockReturnValue(false) + storageServiceMockFns.mockDownloadFile.mockResolvedValue(Buffer.from('plain text content')) + + const req = createMockRequest('POST', { + filePath: '/api/files/serve/execution/workspace-1/workflow-1/execution-1/readme.customtext', + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.success).toBe(true) + expect(data.output.content).toBe('plain text content') + }) + + it('should reject parser complexity limits instead of returning raw text', async () => { + setupFileApiMocks({ + cloudEnabled: true, + storageProvider: 's3', + authenticated: true, + }) + storageServiceMockFns.mockDownloadFile.mockResolvedValue(Buffer.from('{"value":true}')) + mockParseBuffer.mockRejectedValueOnce( + new FileParserError('complexity_limit', 'JSON document exceeds the complexity limit') + ) + + const req = createMockRequest('POST', { + filePath: '/api/files/serve/execution/workspace-1/workflow-1/execution-1/data.json', + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.success).toBe(false) + expect(data.error).toContain('complexity limit') + expect(data).not.toHaveProperty('output') + }) + + it('should handle multiple files', async () => { + setupFileApiMocks({ + cloudEnabled: false, + storageProvider: 'local', + authenticated: true, + }) + + const req = createMockRequest('POST', { + filePath: ['/api/files/serve/file1.txt', '/api/files/serve/file2.txt'], + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data).toHaveProperty('success') + expect(data).toHaveProperty('results') + expect(Array.isArray(data.results)).toBe(true) + expect(data.results).toHaveLength(2) + }) + + it('should keep the multi-file download cap independent from the remaining parsed-output cap', async () => { + inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ + isValid: true, + resolvedIP: '203.0.113.10', + }) + inputValidationMockFns.mockSecureFetchWithPinnedIP + .mockResolvedValueOnce( + new Response('file content', { + status: 200, + headers: { 'content-type': 'text/plain' }, + }) + ) + .mockResolvedValueOnce( + new Response('second file content', { + status: 200, + headers: { + 'content-length': String(20 * 1024 * 1024), + 'content-type': 'text/plain', + }, + }) + ) + + const fourMbContent = 'a'.repeat(4 * 1024 * 1024) + mockParseBuffer + .mockResolvedValueOnce({ + content: fourMbContent, + metadata: { pageCount: 1 }, + }) + .mockResolvedValueOnce({ + content: 'second file', + metadata: { pageCount: 1 }, + }) + + const req = createMockRequest('POST', { + filePath: ['https://example.com/file1.txt', 'https://example.com/file2.txt'], + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.results).toHaveLength(2) + expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenNthCalledWith( + 1, + 'https://example.com/file1.txt', + '203.0.113.10', + expect.objectContaining({ maxResponseBytes: 100 * 1024 * 1024 }) + ) + expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenNthCalledWith( + 2, + 'https://example.com/file2.txt', + '203.0.113.10', + expect.objectContaining({ maxResponseBytes: 100 * 1024 * 1024 }) + ) + }) + + it('should never dedup external URL fetches by path filename — two URLs sharing image.png both download', async () => { + inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ + isValid: true, + resolvedIP: '203.0.113.10', + }) + inputValidationMockFns.mockSecureFetchWithPinnedIP + .mockResolvedValueOnce( + new Response('first image bytes', { + status: 200, + headers: { 'content-type': 'image/png' }, + }) + ) + .mockResolvedValueOnce( + new Response('second image bytes — different content', { + status: 200, + headers: { 'content-type': 'image/png' }, + }) + ) + mockIsSupportedFileType.mockReturnValue(false) + permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('write') + + const req = createMockRequest('POST', { + filePath: [ + 'https://files.slack.com/files-pri/T07-FAAA/download/image.png', + 'https://files.slack.com/files-pri/T07-FBBB/download/image.png', + ], + workspaceId: 'workspace-id', + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.results).toHaveLength(2) + expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenCalledTimes(2) + expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenNthCalledWith( + 1, + 'https://files.slack.com/files-pri/T07-FAAA/download/image.png', + '203.0.113.10', + expect.any(Object) + ) + expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenNthCalledWith( + 2, + 'https://files.slack.com/files-pri/T07-FBBB/download/image.png', + '203.0.113.10', + expect.any(Object) + ) + expect(mockUploadWorkspaceFile).toHaveBeenCalledTimes(2) + expect(storageServiceMockFns.mockDownloadFile).not.toHaveBeenCalled() + }) + + it('should stop multi-file parsing once the combined parsed output is too large', async () => { + inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ + isValid: true, + resolvedIP: '203.0.113.10', + }) + inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValue( + new Response('file content', { + status: 200, + headers: { 'content-type': 'text/plain' }, + }) + ) + + mockParseBuffer.mockResolvedValueOnce({ + content: 'a'.repeat(5 * 1024 * 1024 + 1), + metadata: { pageCount: 1 }, + }) + + const req = createMockRequest('POST', { + filePath: ['https://example.com/file1.txt', 'https://example.com/file2.txt'], + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(413) + expect(data.success).toBe(false) + expect(data.error).toContain('too large') + expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenCalledTimes(1) + }) + + it('should include successful multi-file parse results when a later file exceeds the cap', async () => { + inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ + isValid: true, + resolvedIP: '203.0.113.10', + }) + inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValue( + new Response('file content', { + status: 200, + headers: { 'content-type': 'text/plain' }, + }) + ) + + mockParseBuffer + .mockResolvedValueOnce({ + content: 'first file', + metadata: { pageCount: 1 }, + }) + .mockResolvedValueOnce({ + content: 'a'.repeat(5 * 1024 * 1024), + metadata: { pageCount: 1 }, + }) + + const req = createMockRequest('POST', { + filePath: ['https://example.com/file1.txt', 'https://example.com/file2.txt'], + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.success).toBe(true) + expect(data.error).toContain('too large') + expect(data.results).toHaveLength(1) + expect(data.results[0].output.content).toBe('first file') + expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenCalledTimes(2) + }) + + it('should pass custom headers when fetching external URLs', async () => { + inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ + isValid: true, + resolvedIP: '203.0.113.10', + }) + inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValue( + new Response('private file content', { + status: 200, + headers: { 'content-type': 'text/plain' }, + }) + ) + + const headers = { Authorization: 'Bearer xoxb-test-token' } + const req = createMockRequest('POST', { + filePath: 'https://files.slack.com/files-pri/T000-F000/download/report.txt', + headers, + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.success).toBe(true) + expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenCalledWith( + 'https://files.slack.com/files-pri/T000-F000/download/report.txt', + '203.0.113.10', + expect.objectContaining({ + timeout: 30000, + headers, + }) + ) + }) + + it('should reject oversized external downloads before reading the body', async () => { + inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ + isValid: true, + resolvedIP: '203.0.113.10', + }) + inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValue( + new Response('oversized', { + status: 200, + headers: { 'content-length': '104857601', 'content-type': 'text/plain' }, + }) + ) + + const req = createMockRequest('POST', { + filePath: 'https://example.com/large.txt', + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.success).toBe(false) + expect(data.error).toContain('too large') + expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenCalledWith( + 'https://example.com/large.txt', + '203.0.113.10', + expect.objectContaining({ + maxResponseBytes: 104857600, + }) + ) + }) + + it('should reject oversized local files before materializing them', async () => { + setupFileApiMocks({ + cloudEnabled: false, + storageProvider: 'local', + authenticated: true, + }) + mockFsStat.mockResolvedValue({ isFile: () => true, size: 104857601 }) + + const req = createMockRequest('POST', { + filePath: 'workspace/large.txt', + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data.success).toBe(false) + expect(data.error).toContain('too large') + expect(mockFsReadFile).not.toHaveBeenCalled() + }) + + it('should process execution file URLs with context query param', async () => { + setupFileApiMocks({ + cloudEnabled: true, + storageProvider: 's3', + authenticated: true, + }) + + const req = createMockRequest('POST', { + filePath: + '/api/files/serve/s3/6vzIweweXAS1pJ1mMSrr9Flh6paJpHAx/79dac297-5ebb-410b-b135-cc594dfcb361/c36afbb0-af50-42b0-9b23-5dae2d9384e8/Confirmation.pdf?context=execution', + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + + if (data.success === true) { + expect(data).toHaveProperty('output') + } else { + expect(data).toHaveProperty('error') + } + }) + + it('should process workspace file URLs with context query param', async () => { + setupFileApiMocks({ + cloudEnabled: true, + storageProvider: 's3', + authenticated: true, + }) + + const req = createMockRequest('POST', { + filePath: + '/api/files/serve/s3/fa8e96e6-7482-4e3c-a0e8-ea083b28af55-be56ca4f-83c2-4559-a6a4-e25eb4ab8ee2_1761691045516-1ie5q86-Confirmation.pdf?context=workspace', + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + + if (data.success === true) { + expect(data).toHaveProperty('output') + } else { + expect(data).toHaveProperty('error') + } + }) + + it('should handle S3 access errors gracefully', async () => { + setupFileApiMocks({ + cloudEnabled: true, + storageProvider: 's3', + authenticated: true, + }) + + storageServiceMockFns.mockDownloadFile.mockRejectedValue(new Error('Access denied')) + storageServiceMockFns.mockHasCloudStorage.mockReturnValue(true) + + const req = new NextRequest('http://localhost:3000/api/files/parse', { + method: 'POST', + body: JSON.stringify({ + filePath: '/api/files/serve/s3/test-file.txt', + }), + }) + + const response = await POST(req) + const data = await response.json() + + expect(data).toBeDefined() + expect(typeof data).toBe('object') + }) + + it('should handle access errors gracefully', async () => { + setupFileApiMocks({ + cloudEnabled: false, + storageProvider: 'local', + authenticated: true, + }) + + mockFsAccess.mockRejectedValue(new Error('ENOENT: no such file')) + + const req = createMockRequest('POST', { + filePath: 'nonexistent.txt', + }) + + const response = await POST(req) + const data = await response.json() + + expect(response.status).toBe(200) + expect(data).toHaveProperty('success') + expect(data).toHaveProperty('error') + }) +}) + +describe('Files Parse API - Path Traversal Security', () => { + beforeEach(() => { + vi.clearAllMocks() + setupFileApiMocks({ + authenticated: true, + }) + permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue({ canView: true }) + }) + + describe('Path Traversal Prevention', () => { + it('should reject path traversal attempts with .. segments', async () => { + const maliciousRequests = [ + '../../../etc/passwd', + '/api/files/serve/../../../etc/passwd', + '/api/files/serve/../../app.js', + '/api/files/serve/../.env', + 'uploads/../../../etc/hosts', + ] + + for (const maliciousPath of maliciousRequests) { + const request = new NextRequest('http://localhost:3000/api/files/parse', { + method: 'POST', + body: JSON.stringify({ + filePath: maliciousPath, + }), + }) + + const response = await POST(request) + const result = await response.json() + + expect(result.success).toBe(false) + expect(result.error).toMatch( + /Access denied|Invalid path|Path outside allowed directory|Unauthorized/ + ) + } + }) + + it('should reject paths with tilde characters', async () => { + const maliciousPaths = [ + '~/../../etc/passwd', + '/api/files/serve/~/secret.txt', + '~root/.ssh/id_rsa', + ] + + for (const maliciousPath of maliciousPaths) { + const request = new NextRequest('http://localhost:3000/api/files/parse', { + method: 'POST', + body: JSON.stringify({ + filePath: maliciousPath, + }), + }) + + const response = await POST(request) + const result = await response.json() + + expect(result.success).toBe(false) + expect(result.error).toMatch(/Access denied|Invalid path|Unauthorized/) + } + }) + + it('should reject absolute paths outside upload directory', async () => { + const maliciousPaths = [ + '/etc/passwd', + '/root/.bashrc', + '/app/.env', + '/var/log/auth.log', + 'C:\\Windows\\System32\\drivers\\etc\\hosts', + ] + + for (const maliciousPath of maliciousPaths) { + const request = new NextRequest('http://localhost:3000/api/files/parse', { + method: 'POST', + body: JSON.stringify({ + filePath: maliciousPath, + }), + }) + + const response = await POST(request) + const result = await response.json() + + expect(result.success).toBe(false) + expect(result.error).toMatch(/Access denied|Path outside allowed directory|Unauthorized/) + } + }) + + it('should allow valid paths within upload directory', async () => { + const validPaths = [ + '/api/files/serve/document.txt', + '/api/files/serve/folder/file.pdf', + '/api/files/serve/subfolder/image.png', + ] + + for (const validPath of validPaths) { + const request = new NextRequest('http://localhost:3000/api/files/parse', { + method: 'POST', + body: JSON.stringify({ + filePath: validPath, + }), + }) + + const response = await POST(request) + const result = await response.json() + + if (result.error) { + expect(result.error).not.toMatch( + /Access denied|Path outside allowed directory|Invalid path/ + ) + } + } + }) + + it('should not treat .. inside external URLs as path traversal', async () => { + inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ + isValid: true, + resolvedIP: '203.0.113.10', + }) + inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValue( + new Response('slack file content', { + status: 200, + headers: { 'content-type': 'text/plain' }, + }) + ) + permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('write') + + // Slack truncates long titles with a literal ellipsis, so the slug contains `..` + const slackUrl = + 'https://files.slack.com/files-pri/T08-F0B/_other__no_invitation_messages_get_sent_-_sim_on_railway...txt' + + const request = new NextRequest('http://localhost:3000/api/files/parse', { + method: 'POST', + body: JSON.stringify({ filePath: slackUrl, workspaceId: 'workspace-id' }), + }) + + const response = await POST(request) + const result = await response.json() + + expect(result.success).toBe(true) + // The URL reaching the pinned fetch proves it passed validation and routed + // to external-URL handling rather than being rejected as a local path. + expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenCalledWith( + slackUrl, + '203.0.113.10', + expect.any(Object) + ) + }) + + it('should still reject traversal in https URLs that look like internal serve URLs', async () => { + inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ + isValid: true, + resolvedIP: '203.0.113.10', + }) + inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValue( + new Response('should never be fetched', { status: 200 }) + ) + + // Absolute https URL containing `/api/files/serve/` matches isInternalFileUrl and would + // route to handleCloudFile — so it must keep traversal protection, not be waved through + // as an external URL. + const request = new NextRequest('http://localhost:3000/api/files/parse', { + method: 'POST', + body: JSON.stringify({ + filePath: 'https://attacker.com/api/files/serve/../../../etc/passwd', + }), + }) + + const response = await POST(request) + const result = await response.json() + + expect(result.success).toBe(false) + expect(result.error).toMatch(/Access denied: path traversal detected/) + expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() + }) + + it('should handle encoded path traversal attempts', async () => { + const encodedMaliciousPaths = [ + '/api/files/serve/%2e%2e%2f%2e%2e%2fetc%2fpasswd', // ../../../etc/passwd + '/api/files/serve/..%2f..%2f..%2fetc%2fpasswd', + '/api/files/serve/%2e%2e/%2e%2e/etc/passwd', + ] + + for (const maliciousPath of encodedMaliciousPaths) { + const request = new NextRequest('http://localhost:3000/api/files/parse', { + method: 'POST', + body: JSON.stringify({ + filePath: decodeURIComponent(maliciousPath), + }), + }) + + const response = await POST(request) + const result = await response.json() + + expect(result.success).toBe(false) + expect(result.error).toMatch( + /Access denied|Invalid path|Path outside allowed directory|Unauthorized/ + ) + } + }) + + it('should handle null byte injection attempts', async () => { + const nullBytePaths = [ + '/api/files/serve/file.txt\0../../etc/passwd', + 'file.txt\0/etc/passwd', + '/api/files/serve/document.pdf\0/var/log/auth.log', + ] + + for (const maliciousPath of nullBytePaths) { + const request = new NextRequest('http://localhost:3000/api/files/parse', { + method: 'POST', + body: JSON.stringify({ + filePath: maliciousPath, + }), + }) + + const response = await POST(request) + const result = await response.json() + + expect(result.success).toBe(false) + } + }) + }) + + describe('Edge Cases', () => { + it('should handle empty file paths', async () => { + const request = new NextRequest('http://localhost:3000/api/files/parse', { + method: 'POST', + body: JSON.stringify({ + filePath: '', + }), + }) + + const response = await POST(request) + const result = await response.json() + + expect(response.status).toBe(400) + expect(result.error).toBe('No file path provided') + }) + + it('should handle missing filePath parameter', async () => { + const request = new NextRequest('http://localhost:3000/api/files/parse', { + method: 'POST', + body: JSON.stringify({}), + }) + + const response = await POST(request) + const result = await response.json() + + expect(response.status).toBe(400) + expect(result.error).toBe('No file path provided') + }) + }) +}) diff --git a/apps/sim/lib/internal/file/parser.ts b/apps/sim/lib/internal/file/parser.ts new file mode 100644 index 00000000000..20be1fd610d --- /dev/null +++ b/apps/sim/lib/internal/file/parser.ts @@ -0,0 +1,1200 @@ +import { Buffer, isUtf8 } from 'buffer' +import { createHash } from 'crypto' +import fsPromises from 'fs/promises' +import path from 'path' +import type { Principal } from '@sim/auth/principal' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { generateShortId } from '@sim/utils/id' +import binaryExtensionsList from 'binary-extensions' +import type { ContractBody } from '@/lib/api/contracts' +import type { fileParseContract } from '@/lib/api/contracts/storage-transfer' +import { sanitizeUrlForLog } from '@/lib/core/utils/logging' +import { assertKnownSizeWithinLimit, isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { + assertUserFileContentAccess, + type ExecutionMaterializationContext, +} from '@/lib/execution/payloads/materialization.server' +import { isSupportedFileType, parseFile } from '@/lib/file-parsers' +import { isFileParserError } from '@/lib/file-parsers/errors' +import { isUsingCloudStorage, StorageService } from '@/lib/uploads' +import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' +import { + ExternalUrlValidationError, + fetchExternalUrlToWorkspace, +} from '@/lib/uploads/contexts/workspace' +import { UPLOAD_DIR_SERVER } from '@/lib/uploads/core/setup.server' +import { isWorkspaceScopedContext } from '@/lib/uploads/shared/types' +import { + extractCleanFilename, + extractStorageKey, + extractWorkspaceIdFromExecutionKey, + getMimeTypeFromExtension, + getViewerUrl, + inferContextFromKey, + isInternalFileUrl, +} from '@/lib/uploads/utils/file-utils' +import { readWorkspaceFileNameByKey } from '@/lib/workspace-files/application/read-workspace-file-name-by-key' +import type { UserFile } from '@/executor/types' +import '@/lib/uploads/core/setup.server' + +const logger = createLogger('FilesParseAPI') + +const MAX_DOWNLOAD_SIZE_BYTES = 100 * 1024 * 1024 // 100 MB +const DOWNLOAD_TIMEOUT_MS = 30000 // 30 seconds +const MAX_FILE_REFERENCE_LENGTH = 4096 +const MAX_MULTI_FILE_PARSE_OUTPUT_BYTES = 5 * 1024 * 1024 +const BINARY_EXTENSIONS = new Set(binaryExtensionsList) + +function isLikelyTextBuffer(fileBuffer: Buffer): boolean { + return isUtf8(fileBuffer) && !fileBuffer.includes(0) +} + +interface ExecutionContext { + workspaceId: string + workflowId: string + executionId: string +} + +export type FileParserOperationInput = ContractBody + +export interface FileParserOperationContext { + principal: Principal + workspaceId: string + workflowId: string + executionId?: string + attributedUserId: string + fileAccessUserId?: string + largeValueExecutionIds?: string[] + fileKeys?: string[] + allowLargeValueWorkflowScope?: boolean + requestId?: string + signal?: AbortSignal +} + +type FileReadAccessContext = ExecutionMaterializationContext & { + principal: Principal + workspaceId: string + workflowId: string +} + +interface ParseResult { + success: boolean + content?: string + error?: string + filePath: string + originalName?: string // Original filename from database (for workspace files) + viewerUrl?: string | null // Viewer URL for the file if available + userFile?: UserFile // UserFile object for the raw file + metadata?: { + fileType: string + size: number + hash: string + processingTime: number + } +} + +function getContentBytes(content: unknown): number { + return typeof content === 'string' ? Buffer.byteLength(content, 'utf8') : 0 +} + +export async function executeFileParserOperation( + input: FileParserOperationInput, + context: FileParserOperationContext +): Promise { + const startTime = Date.now() + + try { + context.signal?.throwIfAborted() + const { filePath, fileType, headers } = input + if (input.workspaceId && input.workspaceId !== context.workspaceId) { + return Response.json({ success: false, error: 'Workspace access denied' }, { status: 403 }) + } + if (input.workflowId && input.workflowId !== context.workflowId) { + return Response.json({ success: false, error: 'Workflow access denied' }, { status: 403 }) + } + if (input.executionId && input.executionId !== context.executionId) { + return Response.json({ success: false, error: 'Execution access denied' }, { status: 403 }) + } + const { attributedUserId, workspaceId } = context + const fileReadAccess: FileReadAccessContext = { + principal: context.principal, + workspaceId, + workflowId: context.workflowId, + executionId: context.executionId, + largeValueExecutionIds: context.largeValueExecutionIds, + fileKeys: context.fileKeys, + allowLargeValueWorkflowScope: context.allowLargeValueWorkflowScope, + userId: context.fileAccessUserId, + requestId: context.requestId, + } + + if (!filePath || (typeof filePath === 'string' && filePath.trim() === '')) { + return Response.json({ success: false, error: 'No file path provided' }, { status: 400 }) + } + + const executionContext: ExecutionContext | undefined = context.executionId + ? { workspaceId, workflowId: context.workflowId, executionId: context.executionId } + : undefined + + logger.info('File parse request received:', { + filePath, + fileType, + workspaceId, + userId: attributedUserId, + hasExecutionContext: !!executionContext, + hasHeaders: Boolean(headers && Object.keys(headers).length > 0), + }) + + if (Array.isArray(filePath)) { + const results = [] + let totalOutputBytes = 0 + + for (const singlePath of filePath) { + context.signal?.throwIfAborted() + if (!singlePath || (typeof singlePath === 'string' && singlePath.trim() === '')) { + results.push({ + success: false, + error: 'Empty file path in array', + filePath: singlePath || '', + }) + continue + } + + const remainingOutputBytes = MAX_MULTI_FILE_PARSE_OUTPUT_BYTES - totalOutputBytes + if (remainingOutputBytes <= 0) { + return parsedOutputTooLargeResponse(results) + } + + const result = await parseFileSingle( + singlePath, + fileType, + workspaceId, + attributedUserId, + fileReadAccess, + context.principal, + executionContext, + headers, + context.signal, + MAX_DOWNLOAD_SIZE_BYTES, + remainingOutputBytes + ) + if (result.metadata) { + result.metadata.processingTime = Date.now() - startTime + } + + if (result.success) { + totalOutputBytes += getContentBytes(result.content) + if (totalOutputBytes > MAX_MULTI_FILE_PARSE_OUTPUT_BYTES) { + return parsedOutputTooLargeResponse(results) + } + + const displayName = + result.originalName || extractCleanFilename(result.filePath) || 'unknown' + results.push({ + success: true, + output: { + content: result.content, + name: displayName, + fileType: result.metadata?.fileType || 'application/octet-stream', + size: result.metadata?.size || 0, + binary: false, + file: result.userFile, + }, + filePath: result.filePath, + viewerUrl: result.viewerUrl, + }) + continue + } + + if (result.error?.startsWith('Parsed file output is too large')) { + return parsedOutputTooLargeResponse(results) + } + + results.push(result) + } + + return Response.json({ + success: true, + results, + }) + } + + const result = await parseFileSingle( + filePath, + fileType, + workspaceId, + attributedUserId, + fileReadAccess, + context.principal, + executionContext, + headers, + context.signal + ) + + if (result.metadata) { + result.metadata.processingTime = Date.now() - startTime + } + + if (result.success) { + const displayName = result.originalName || extractCleanFilename(result.filePath) || 'unknown' + return Response.json({ + success: true, + output: { + content: result.content, + name: displayName, + fileType: result.metadata?.fileType || 'application/octet-stream', + size: result.metadata?.size || 0, + binary: false, + file: result.userFile, + }, + filePath: result.filePath, + viewerUrl: result.viewerUrl, + }) + } + + return Response.json(result) + } catch (error) { + logger.error('Error in file parse API:', error) + return Response.json( + { + success: false, + error: getErrorMessage(error, 'Unknown error occurred'), + filePath: '', + }, + { status: 500 } + ) + } +} + +/** + * Parse a single file and return its content + */ +async function parseFileSingle( + filePath: string, + fileType: string, + workspaceId: string, + attributedUserId: string, + fileReadAccess: FileReadAccessContext, + principal: Principal, + executionContext?: ExecutionContext, + headers?: Record, + signal?: AbortSignal, + maxDownloadBytes = MAX_DOWNLOAD_SIZE_BYTES, + maxParsedOutputBytes?: number +): Promise { + logger.info('Parsing file:', filePath) + + if (!filePath || filePath.trim() === '') { + return { + success: false, + error: 'Empty file path provided', + filePath: filePath || '', + } + } + + const referenceValidation = validateFileReferenceShape(filePath) + if (!referenceValidation.isValid) { + return { + success: false, + error: referenceValidation.error || 'Invalid file reference', + filePath, + } + } + + const pathValidation = validateFilePath(filePath) + if (!pathValidation.isValid) { + return { + success: false, + error: pathValidation.error || 'Invalid path', + filePath, + } + } + + if (isInternalFileUrl(filePath)) { + return handleCloudFile( + filePath, + fileType, + attributedUserId, + fileReadAccess, + principal, + workspaceId, + executionContext, + signal, + maxDownloadBytes, + maxParsedOutputBytes + ) + } + + if (filePath.startsWith('http://') || filePath.startsWith('https://')) { + return handleExternalUrl( + filePath, + fileType, + workspaceId, + attributedUserId, + executionContext, + headers, + signal, + maxDownloadBytes, + maxParsedOutputBytes + ) + } + + if (isUsingCloudStorage()) { + return handleCloudFile( + filePath, + fileType, + attributedUserId, + fileReadAccess, + principal, + workspaceId, + executionContext, + signal, + maxDownloadBytes, + maxParsedOutputBytes + ) + } + + return handleLocalFile( + filePath, + fileType, + attributedUserId, + fileReadAccess, + executionContext, + signal, + maxDownloadBytes, + maxParsedOutputBytes + ) +} + +function validateFileReferenceShape(filePath: string): { isValid: boolean; error?: string } { + const trimmed = filePath.trim() + if ( + trimmed.startsWith('http://') || + trimmed.startsWith('https://') || + isInternalFileUrl(trimmed) + ) { + return { isValid: true } + } + + if (trimmed.startsWith('data:')) { + return { + isValid: false, + error: 'File input must be a URL or uploaded file reference, not inline file content', + } + } + + if (filePath.length > MAX_FILE_REFERENCE_LENGTH) { + return { + isValid: false, + error: 'File reference is too long; provide a file URL or upload the file instead', + } + } + + if (/[\x00-\x08\x0B\x0C\x0E-\x1F]/.test(filePath)) { + return { + isValid: false, + error: + 'File reference contains binary content; provide a file URL or upload the file instead', + } + } + + const newlineCount = filePath.match(/\r\n|\r|\n/g)?.length ?? 0 + if (newlineCount > 2) { + return { + isValid: false, + error: + 'File reference looks like inline file content; provide a file URL or upload the file instead', + } + } + + return { isValid: true } +} + +function parsedOutputTooLargeResponse(results?: unknown[]): Response { + const hasPartialResults = Boolean(results && results.length > 0) + return Response.json( + { + success: hasPartialResults, + error: `Parsed file output is too large to return safely. Maximum combined parsed output is ${prettySize( + MAX_MULTI_FILE_PARSE_OUTPUT_BYTES + )}.`, + ...(results && results.length > 0 ? { results } : {}), + }, + { status: hasPartialResults ? 200 : 413 } + ) +} + +function getParsedOutputTooLargeMessage(maxBytes: number): string { + return `Parsed file output is too large to return safely. Maximum parsed output is ${prettySize( + maxBytes + )}.` +} + +function assertParsedContentWithinLimit(content: string, maxBytes?: number): string { + if (maxBytes !== undefined) { + assertKnownSizeWithinLimit(Buffer.byteLength(content, 'utf8'), maxBytes, 'parsed file output') + } + return content +} + +/** + * Validate file path for security - prevents null byte injection and path traversal attacks. + * + * External URLs (`http`/`https`) are fetched over HTTP — with SSRF protection applied + * downstream in `fetchExternalUrlToWorkspace` (DNS resolution + private/reserved IP blocking) + * — and are never resolved against the filesystem, so `..`/`~` are legal URL content and must + * not be rejected. Providers such as Slack routinely emit slugs containing a literal `...`. + * + * Internal file URLs (`/api/files/serve/...`) ARE resolved to storage keys and filesystem + * paths via `extractStorageKey`, so they keep full traversal protection. The external + * short-circuit explicitly excludes them: `parseFileSingle` routes anything matching + * `isInternalFileUrl` to `handleCloudFile` (even an absolute `https://host/api/files/serve/...`), + * so such inputs must stay subject to the `..`/`~` checks rather than being waved through as + * external URLs. Only the leading-`/` "outside allowed directory" check is relaxed for them, + * since that prefix is expected. + */ +function validateFilePath(filePath: string): { isValid: boolean; error?: string } { + if (filePath.includes('\0')) { + return { isValid: false, error: 'Invalid path: null byte detected' } + } + + if ( + (filePath.startsWith('http://') || filePath.startsWith('https://')) && + !isInternalFileUrl(filePath) + ) { + return { isValid: true } + } + + if (filePath.includes('..')) { + return { isValid: false, error: 'Access denied: path traversal detected' } + } + + if (filePath.includes('~')) { + return { isValid: false, error: 'Invalid path: tilde character not allowed' } + } + + if (filePath.startsWith('/') && !isInternalFileUrl(filePath)) { + return { isValid: false, error: 'Path outside allowed directory' } + } + + if (/^[A-Za-z]:\\/.test(filePath)) { + return { isValid: false, error: 'Path outside allowed directory' } + } + + return { isValid: true } +} + +/** + * Handle external URL. + * + * Always fetches the URL fresh — there is no filename-based dedup. Distinct URLs + * commonly share a path tail (e.g. every Slack clipboard paste is `image.png`), + * so keying a cache by filename returns stale bytes. `fetchExternalUrlToWorkspace` + * delegates to `uploadWorkspaceFile`, which suffix-disambiguates collisions on save. + * + * Workspace save is skipped when the URL already points at our execution-files + * bucket (re-uploading our own bytes is wasteful and would generate `image (1).png` + * style aliases for files we already own). + */ +async function handleExternalUrl( + url: string, + fileType: string, + workspaceId: string, + userId: string, + executionContext?: ExecutionContext, + headers?: Record, + signal?: AbortSignal, + maxDownloadBytes = MAX_DOWNLOAD_SIZE_BYTES, + maxParsedOutputBytes?: number +): Promise { + try { + logger.info('Fetching external URL:', url) + + const { getStorageConfig, USE_S3_STORAGE, USE_BLOB_STORAGE, USE_GCS_STORAGE } = await import( + '@/lib/uploads/config' + ) + const executionConfig = getStorageConfig('execution') + + let isExecutionFile = false + try { + const parsedUrl = new URL(url) + + if (USE_S3_STORAGE && executionConfig.bucket) { + const bucketInHost = parsedUrl.hostname.startsWith(executionConfig.bucket) + const bucketInPath = parsedUrl.pathname.startsWith(`/${executionConfig.bucket}/`) + isExecutionFile = bucketInHost || bucketInPath + } else if (USE_BLOB_STORAGE && executionConfig.containerName) { + isExecutionFile = url.includes(`/${executionConfig.containerName}/`) + } else if (USE_GCS_STORAGE && executionConfig.bucket) { + const bucketInHost = parsedUrl.hostname.startsWith(`${executionConfig.bucket}.`) + const bucketInPath = parsedUrl.pathname.startsWith(`/${executionConfig.bucket}/`) + isExecutionFile = bucketInHost || bucketInPath + } + } catch (error) { + logger.warn('Failed to parse URL for execution file check:', error) + isExecutionFile = false + } + + const { filename, buffer, mimeType } = await fetchExternalUrlToWorkspace({ + url, + userId, + workspaceId: workspaceId || undefined, + saveToWorkspace: Boolean(workspaceId) && !isExecutionFile, + headers, + signal, + maxDownloadBytes, + timeoutMs: DOWNLOAD_TIMEOUT_MS, + }) + const extension = path.extname(filename).toLowerCase().substring(1) + + logger.info(`Downloaded file from URL: ${url}, size: ${buffer.length} bytes`) + + let userFile: UserFile | undefined + if (executionContext) { + try { + userFile = await uploadExecutionFile(executionContext, buffer, filename, mimeType, userId) + logger.info(`Stored file in execution storage: ${filename}`, { key: userFile.key }) + } catch (uploadError) { + logger.warn('Failed to store file in execution storage:', uploadError) + } + } + + let parseResult: ParseResult + if (extension === 'pdf') { + parseResult = await handlePdfBuffer(buffer, filename, fileType, url, maxParsedOutputBytes) + } else if (extension === 'csv') { + parseResult = await handleCsvBuffer(buffer, filename, fileType, url, maxParsedOutputBytes) + } else if (isSupportedFileType(extension)) { + parseResult = await handleGenericTextBuffer( + buffer, + filename, + extension, + fileType, + url, + maxParsedOutputBytes + ) + } else { + parseResult = handleGenericBuffer(buffer, filename, extension, fileType, maxParsedOutputBytes) + } + + // Attach userFile to the result + if (userFile) { + parseResult.userFile = userFile + } + + return parseResult + } catch (error) { + logger.error(`Error handling external URL ${sanitizeUrlForLog(url)}:`, error) + if (isPayloadSizeLimitError(error)) { + logger.warn('Rejected oversized external file parse payload', { + maxBytes: error.maxBytes, + observedBytes: error.observedBytes, + label: error.label, + url: sanitizeUrlForLog(url), + }) + return { + success: false, + error: + error.label === 'parsed file output' + ? getParsedOutputTooLargeMessage(error.maxBytes) + : `File is too large to parse safely. Maximum supported download size is ${prettySize( + error.maxBytes + )}.`, + filePath: url, + } + } + + if (error instanceof ExternalUrlValidationError) { + logger.warn(`Blocked external URL request: ${error.message}`) + return { + success: false, + error: error.message, + filePath: url, + } + } + + return { + success: false, + error: `Error fetching URL: ${(error as Error).message}`, + filePath: url, + } + } +} + +/** + * Handle file stored in cloud storage + * If executionContext is provided and file is not already from execution storage, + * copies the file to execution storage and returns UserFile + */ +async function handleCloudFile( + filePath: string, + fileType: string, + attributedUserId: string, + fileReadAccess: FileReadAccessContext, + principal: Principal, + workspaceId: string, + executionContext?: ExecutionContext, + signal?: AbortSignal, + maxDownloadBytes = MAX_DOWNLOAD_SIZE_BYTES, + maxParsedOutputBytes?: number +): Promise { + try { + signal?.throwIfAborted() + const cloudKey = extractStorageKey(filePath) + + logger.info('Extracted cloud key:', cloudKey) + + const context = inferContextFromKey(cloudKey) + + try { + await assertUserFileContentAccess({ key: cloudKey, context }, fileReadAccess) + } catch { + logger.warn('Unauthorized cloud file parse attempt', { key: cloudKey, context }) + return { + success: false, + error: 'File not found', + filePath, + } + } + + let originalFilename: string | undefined + // Not filtered to `context = 'workspace'`: a chat attachment carries the same key + // prefix and has an `originalName` worth recovering too, and without it the parse + // result is labelled with the raw storage segment. Access was authorized above; + // this only recovers a display name. + if (isWorkspaceScopedContext(context)) { + try { + const { name } = await readWorkspaceFileNameByKey.execute({ + principal, + input: { workspaceId, key: cloudKey }, + }) + if (name) { + originalFilename = name + logger.debug(`Found original filename for workspace file: ${originalFilename}`) + } + } catch (dbError) { + logger.debug(`Failed to lookup original filename for ${cloudKey}:`, dbError) + } + } + + const fileBuffer = await StorageService.downloadFile({ + key: cloudKey, + context, + maxBytes: maxDownloadBytes, + }) + signal?.throwIfAborted() + logger.info( + `Downloaded file from ${context} storage: ${cloudKey}, size: ${fileBuffer.length} bytes` + ) + + const filename = originalFilename || cloudKey.split('/').pop() || cloudKey + const extension = path.extname(filename).toLowerCase().substring(1) + const mimeType = getMimeTypeFromExtension(extension) + + const normalizedFilePath = `/api/files/serve/${encodeURIComponent(cloudKey)}?context=${context}` + let workspaceIdFromKey: string | undefined + + if (context === 'execution') { + workspaceIdFromKey = extractWorkspaceIdFromExecutionKey(cloudKey) || undefined + } else if (context === 'workspace') { + const segments = cloudKey.split('/') + if (segments.length >= 2 && /^[a-f0-9-]{36}$/.test(segments[0])) { + workspaceIdFromKey = segments[0] + } + } + + const viewerUrl = getViewerUrl(cloudKey, workspaceIdFromKey) + + // Store file in execution storage if executionContext is provided + let userFile: UserFile | undefined + + if (executionContext) { + signal?.throwIfAborted() + // If file is already from execution context, create UserFile reference without re-uploading + if (context === 'execution') { + userFile = { + id: `file_${Date.now()}_${generateShortId(7)}`, + name: filename, + url: normalizedFilePath, + size: fileBuffer.length, + type: mimeType, + key: cloudKey, + context: 'execution', + } + logger.info(`Created UserFile reference for existing execution file: ${filename}`) + } else { + // Copy from workspace/other storage to execution storage + try { + userFile = await uploadExecutionFile( + executionContext, + fileBuffer, + filename, + mimeType, + attributedUserId + ) + logger.info(`Copied file to execution storage: ${filename}`, { key: userFile.key }) + } catch (uploadError) { + logger.warn(`Failed to copy file to execution storage:`, uploadError) + } + } + } + + let parseResult: ParseResult + if (extension === 'pdf') { + parseResult = await handlePdfBuffer( + fileBuffer, + filename, + fileType, + normalizedFilePath, + maxParsedOutputBytes + ) + } else if (extension === 'csv') { + parseResult = await handleCsvBuffer( + fileBuffer, + filename, + fileType, + normalizedFilePath, + maxParsedOutputBytes + ) + } else if (isSupportedFileType(extension)) { + parseResult = await handleGenericTextBuffer( + fileBuffer, + filename, + extension, + fileType, + normalizedFilePath, + maxParsedOutputBytes + ) + } else { + parseResult = handleGenericBuffer( + fileBuffer, + filename, + extension, + fileType, + maxParsedOutputBytes + ) + parseResult.filePath = normalizedFilePath + } + + if (originalFilename) { + parseResult.originalName = originalFilename + } + + parseResult.viewerUrl = viewerUrl + + // Attach userFile to the result + if (userFile) { + parseResult.userFile = userFile + } + + signal?.throwIfAborted() + + return parseResult + } catch (error) { + logger.error(`Error handling cloud file ${filePath}:`, error) + + const errorMessage = (error as Error).message + if (isPayloadSizeLimitError(error)) { + logger.warn('Rejected oversized cloud file parse payload', { + maxBytes: error.maxBytes, + observedBytes: error.observedBytes, + label: error.label, + filePath, + }) + return { + success: false, + error: + error.label === 'parsed file output' + ? getParsedOutputTooLargeMessage(error.maxBytes) + : `File is too large to parse safely. Maximum supported download size is ${prettySize( + error.maxBytes + )}.`, + filePath, + } + } + + if (errorMessage.includes('Access denied') || errorMessage.includes('Forbidden')) { + throw new Error(`Error accessing file from cloud storage: ${errorMessage}`) + } + + return { + success: false, + error: `Error accessing file from cloud storage: ${errorMessage}`, + filePath, + } + } +} + +/** + * Handle local file + */ +async function handleLocalFile( + filePath: string, + fileType: string, + attributedUserId: string, + fileReadAccess: FileReadAccessContext, + executionContext?: ExecutionContext, + signal?: AbortSignal, + maxDownloadBytes = MAX_DOWNLOAD_SIZE_BYTES, + maxParsedOutputBytes?: number +): Promise { + try { + signal?.throwIfAborted() + const storageKey = isInternalFileUrl(filePath) ? extractStorageKey(filePath) : filePath + const filename = storageKey.split('/').pop() || storageKey + + const context = inferContextFromKey(storageKey) + try { + await assertUserFileContentAccess({ key: storageKey, context }, fileReadAccess) + } catch { + logger.warn('Unauthorized local file parse attempt', { filename }) + return { + success: false, + error: 'File not found', + filePath, + } + } + + const fullPath = path.join(UPLOAD_DIR_SERVER, storageKey) + + logger.info('Processing local file:', fullPath) + + try { + await fsPromises.access(fullPath) + } catch { + throw new Error(`File not found: ${filename}`) + } + + const stats = await fsPromises.stat(fullPath) + assertKnownSizeWithinLimit(stats.size, maxDownloadBytes, 'local file') + + const result = await parseFile(fullPath) + const content = assertParsedContentWithinLimit(result.content, maxParsedOutputBytes) + const fileBuffer = await fsPromises.readFile(fullPath) + signal?.throwIfAborted() + const hash = createHash('md5').update(fileBuffer).digest('hex') + + const extension = path.extname(filename).toLowerCase().substring(1) + const mimeType = fileType || getMimeTypeFromExtension(extension) + + // Store file in execution storage if executionContext is provided + let userFile: UserFile | undefined + if (executionContext) { + signal?.throwIfAborted() + try { + userFile = await uploadExecutionFile( + executionContext, + fileBuffer, + filename, + mimeType, + attributedUserId + ) + logger.info(`Stored local file in execution storage: ${filename}`, { key: userFile.key }) + } catch (uploadError) { + logger.warn(`Failed to store local file in execution storage:`, uploadError) + } + } + + signal?.throwIfAborted() + return { + success: true, + content, + filePath, + userFile, + metadata: { + fileType: mimeType, + size: stats.size, + hash, + processingTime: 0, + }, + } + } catch (error) { + logger.error(`Error handling local file ${filePath}:`, error) + if (isPayloadSizeLimitError(error)) { + logger.warn('Rejected oversized local file parse payload', { + maxBytes: error.maxBytes, + observedBytes: error.observedBytes, + label: error.label, + filePath, + }) + return { + success: false, + error: + error.label === 'parsed file output' + ? getParsedOutputTooLargeMessage(error.maxBytes) + : `File is too large to parse safely. Maximum supported local file size is ${prettySize( + error.maxBytes + )}.`, + filePath, + } + } + + return { + success: false, + error: `Error processing local file: ${(error as Error).message}`, + filePath, + } + } +} + +/** + * Handle a PDF buffer directly in memory + */ +async function handlePdfBuffer( + fileBuffer: Buffer, + filename: string, + fileType?: string, + originalPath?: string, + maxParsedOutputBytes?: number +): Promise { + try { + logger.info(`Parsing PDF in memory: ${filename}`) + + const result = await parseBufferAsPdf(fileBuffer) + + const content = + result.content || + createPdfFallbackMessage(result.metadata?.pageCount || 0, fileBuffer.length, originalPath) + const limitedContent = assertParsedContentWithinLimit(content, maxParsedOutputBytes) + + return { + success: true, + content: limitedContent, + filePath: originalPath || filename, + metadata: { + fileType: fileType || 'application/pdf', + size: fileBuffer.length, + hash: createHash('md5').update(fileBuffer).digest('hex'), + processingTime: 0, + }, + } + } catch (error) { + if (isPayloadSizeLimitError(error)) throw error + + logger.error('Failed to parse PDF in memory:', error) + + const content = createPdfFailureMessage( + 0, + fileBuffer.length, + originalPath || filename, + (error as Error).message + ) + + return { + success: true, + content, + filePath: originalPath || filename, + metadata: { + fileType: fileType || 'application/pdf', + size: fileBuffer.length, + hash: createHash('md5').update(fileBuffer).digest('hex'), + processingTime: 0, + }, + } + } +} + +/** + * Handle a CSV buffer directly in memory + */ +async function handleCsvBuffer( + fileBuffer: Buffer, + filename: string, + fileType?: string, + originalPath?: string, + maxParsedOutputBytes?: number +): Promise { + try { + logger.info(`Parsing CSV in memory: ${filename}`) + + const { parseBuffer } = await import('@/lib/file-parsers') + const result = await parseBuffer(fileBuffer, 'csv') + + return { + success: true, + content: assertParsedContentWithinLimit(result.content, maxParsedOutputBytes), + filePath: originalPath || filename, + metadata: { + fileType: fileType || 'text/csv', + size: fileBuffer.length, + hash: createHash('md5').update(fileBuffer).digest('hex'), + processingTime: 0, + }, + } + } catch (error) { + if (isPayloadSizeLimitError(error)) throw error + + logger.error('Failed to parse CSV in memory:', error) + return { + success: false, + error: `Failed to parse CSV: ${(error as Error).message}`, + filePath: originalPath || filename, + metadata: { + fileType: 'text/csv', + size: 0, + hash: '', + processingTime: 0, + }, + } + } +} + +/** + * Handle a generic text file buffer in memory + */ +async function handleGenericTextBuffer( + fileBuffer: Buffer, + filename: string, + extension: string, + fileType?: string, + originalPath?: string, + maxParsedOutputBytes?: number +): Promise { + try { + logger.info(`Parsing text file in memory: ${filename}`) + + try { + const { parseBuffer, isSupportedFileType } = await import('@/lib/file-parsers') + + if (isSupportedFileType(extension)) { + const result = await parseBuffer(fileBuffer, extension) + + return { + success: true, + content: assertParsedContentWithinLimit(result.content, maxParsedOutputBytes), + filePath: originalPath || filename, + metadata: { + fileType: fileType || getMimeTypeFromExtension(extension), + size: fileBuffer.length, + hash: createHash('md5').update(fileBuffer).digest('hex'), + processingTime: 0, + }, + } + } + } catch (parserError) { + if (isPayloadSizeLimitError(parserError)) throw parserError + if (isFileParserError(parserError) && parserError.code === 'complexity_limit') { + throw parserError + } + + logger.warn('Specialized parser failed, falling back to generic parsing:', parserError) + } + + const content = fileBuffer.toString('utf-8') + const limitedContent = assertParsedContentWithinLimit(content, maxParsedOutputBytes) + + return { + success: true, + content: limitedContent, + filePath: originalPath || filename, + metadata: { + fileType: fileType || getMimeTypeFromExtension(extension), + size: fileBuffer.length, + hash: createHash('md5').update(fileBuffer).digest('hex'), + processingTime: 0, + }, + } + } catch (error) { + if (isPayloadSizeLimitError(error)) throw error + + logger.error('Failed to parse text file in memory:', error) + return { + success: false, + error: `Failed to parse file: ${(error as Error).message}`, + filePath: originalPath || filename, + metadata: { + fileType: 'text/plain', + size: 0, + hash: '', + processingTime: 0, + }, + } + } +} + +/** + * Handle a generic binary buffer + */ +function handleGenericBuffer( + fileBuffer: Buffer, + filename: string, + extension: string, + fileType?: string, + maxParsedOutputBytes?: number +): ParseResult { + const normalizedExtension = extension.toLowerCase() + const content = + !BINARY_EXTENSIONS.has(normalizedExtension) && isLikelyTextBuffer(fileBuffer) + ? assertParsedContentWithinLimit(fileBuffer.toString('utf-8'), maxParsedOutputBytes) + : `[Binary ${normalizedExtension.toUpperCase()} file - ${fileBuffer.length} bytes]` + + return { + success: true, + content, + filePath: filename, + metadata: { + fileType: fileType || getMimeTypeFromExtension(extension), + size: fileBuffer.length, + hash: createHash('md5').update(fileBuffer).digest('hex'), + processingTime: 0, + }, + } +} + +/** + * Parse a PDF buffer + */ +async function parseBufferAsPdf(buffer: Buffer) { + try { + const { PdfParser } = await import('@/lib/file-parsers/pdf-parser') + const parser = new PdfParser() + logger.info('Using main PDF parser for buffer') + + return await parser.parseBuffer(buffer) + } catch (error) { + throw new Error(`PDF parsing failed: ${(error as Error).message}`) + } +} + +/** + * Format bytes to human readable size + */ +function prettySize(bytes: number): string { + if (bytes === 0) return '0 Bytes' + + const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'] + const i = Math.floor(Math.log(bytes) / Math.log(1024)) + + return `${Number.parseFloat((bytes / 1024 ** i).toFixed(2))} ${sizes[i]}` +} + +/** + * Create a formatted message for PDF content + */ +function createPdfFallbackMessage(pageCount: number, size: number, path?: string): string { + const formattedPath = path || 'Unknown path' + + return `PDF document - ${pageCount} page(s), ${prettySize(size)} +Path: ${formattedPath} + +This file appears to be a PDF document that could not be fully processed as text. +Please use a PDF viewer for best results.` +} + +/** + * Create error message for PDF parsing failure and make it more readable + */ +function createPdfFailureMessage( + pageCount: number, + size: number, + path: string, + error: string +): string { + return `PDF document - Processing failed, ${prettySize(size)} +Path: ${path} +Error: ${error} + +This file appears to be a PDF document that could not be processed. +Please use a PDF viewer for best results.` +} diff --git a/apps/sim/lib/internal/firecrawl/execute-tool.test.ts b/apps/sim/lib/internal/firecrawl/execute-tool.test.ts new file mode 100644 index 00000000000..d9c9b5bcb68 --- /dev/null +++ b/apps/sim/lib/internal/firecrawl/execute-tool.test.ts @@ -0,0 +1,71 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockExecuteFirecrawlParse } = vi.hoisted(() => ({ + mockExecuteFirecrawlParse: vi.fn(), +})) + +vi.mock('@/lib/internal/firecrawl/operations', () => ({ + executeFirecrawlParse: mockExecuteFirecrawlParse, +})) + +import { executeFirecrawlTool } from '@/lib/internal/firecrawl/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const FILE = { + key: 'workspace/workspace-1/document.pdf', + name: 'document.pdf', + size: 42, + type: 'application/pdf', +} + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'firecrawl_parse', + input: { apiKey: 'firecrawl-key', file: FILE, options: { formats: ['markdown'] } }, + headers: new Headers(), + context: { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + userId: 'user-1', + metadata: {}, + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeFirecrawlTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mockExecuteFirecrawlParse.mockResolvedValue(Response.json({ success: true })) + }) + + it('dispatches typed input and provenance headers with trusted identity', async () => { + const request = createRequest() + const response = await executeFirecrawlTool(request) + + expect(response.status).toBe(200) + expect(mockExecuteFirecrawlParse).toHaveBeenCalledWith(request.input, { + headers: request.headers, + userId: 'user-1', + requestId: 'request-1', + signal: undefined, + }) + }) + + it('requires trusted execution identity', async () => { + const response = await executeFirecrawlTool( + createRequest({ + context: { workflowId: 'workflow-1', workspaceId: 'workspace-1', metadata: {} }, + }) + ) + + expect(response.status).toBe(401) + expect(mockExecuteFirecrawlParse).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/firecrawl/execute-tool.ts b/apps/sim/lib/internal/firecrawl/execute-tool.ts new file mode 100644 index 00000000000..4ba7b784839 --- /dev/null +++ b/apps/sim/lib/internal/firecrawl/execute-tool.ts @@ -0,0 +1,32 @@ +import { getValidationErrorMessage } from '@/lib/api/server' +import { executeFirecrawlParse } from '@/lib/internal/firecrawl/operations' +import { firecrawlParseInputSchema } from '@/lib/internal/firecrawl/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +export const executeFirecrawlTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'firecrawl_parse') { + return Response.json( + { error: `Unsupported Firecrawl tool: ${request.toolId}` }, + { status: 500 } + ) + } + if (!request.context.userId) { + return Response.json({ success: false, error: 'Unauthorized' }, { status: 401 }) + } + + const parsed = firecrawlParseInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { success: false, error: getValidationErrorMessage(parsed.error, 'Invalid request data') }, + { status: 400 } + ) + } + + return executeFirecrawlParse(parsed.data, { + headers: request.headers, + userId: request.context.userId, + requestId: request.requestId, + signal: request.signal, + }) +} diff --git a/apps/sim/lib/internal/firecrawl/operations.test.ts b/apps/sim/lib/internal/firecrawl/operations.test.ts new file mode 100644 index 00000000000..ac004bfa4bb --- /dev/null +++ b/apps/sim/lib/internal/firecrawl/operations.test.ts @@ -0,0 +1,123 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { PRIVATE_MODEL_INPUT_PROVENANCE_HEADER } from '@/lib/execution/model-input-provenance' +import { + RESOLVED_SECRET_PROVENANCE_FIELD, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, +} from '@/lib/execution/private-tool-metadata' + +const mocks = vi.hoisted(() => ({ + assertToolFileAccess: vi.fn(), + downloadServableFileFromStorage: vi.fn(), + isModelSafeWorkspaceFileKey: vi.fn(), +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertToolFileAccess, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mocks.downloadServableFileFromStorage, +})) + +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({ + isModelSafeWorkspaceFileKey: mocks.isModelSafeWorkspaceFileKey, + MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE: 'File is not model-safe', +})) + +import { executeFirecrawlParse } from '@/lib/internal/firecrawl/operations' + +const FILE = { + key: 'workspace/workspace-1/document.pdf', + name: 'document.pdf', + size: 42, + type: 'application/pdf', +} + +function createContext(headers = new Headers()) { + return { headers, userId: 'user-1', requestId: 'request-1' } +} + +describe('executeFirecrawlParse', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.assertToolFileAccess.mockResolvedValue(null) + mocks.isModelSafeWorkspaceFileKey.mockResolvedValue(true) + mocks.downloadServableFileFromStorage.mockResolvedValue({ + buffer: Buffer.from('document'), + contentType: 'application/pdf', + }) + vi.stubGlobal( + 'fetch', + vi + .fn() + .mockResolvedValue( + Response.json({ data: { markdown: '# Parsed' }, creditsUsed: 2 }, { status: 200 }) + ) + ) + }) + + it('authorizes and parses a model-safe file once while retaining credits', async () => { + const response = await executeFirecrawlParse( + { apiKey: 'firecrawl-key', file: FILE, options: { formats: ['markdown'] } }, + createContext() + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: true, + output: { markdown: '# Parsed', creditsUsed: 2 }, + }) + expect(mocks.assertToolFileAccess).toHaveBeenCalledOnce() + expect(mocks.isModelSafeWorkspaceFileKey).toHaveBeenCalledWith(FILE.key) + expect(fetch).toHaveBeenCalledOnce() + }) + + it('rejects incomplete private provenance before file or provider work', async () => { + const headers = new Headers({ + [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1, + }) + const response = await executeFirecrawlParse( + { + apiKey: 'firecrawl-key', + file: FILE, + options: { formats: ['markdown'] }, + [RESOLVED_SECRET_PROVENANCE_FIELD]: { version: 1, complete: false, entries: [] }, + }, + createContext(headers) + ) + + expect(response.status).toBe(400) + expect(mocks.assertToolFileAccess).not.toHaveBeenCalled() + expect(fetch).not.toHaveBeenCalled() + }) + + it('rejects model-unsafe files before reading bytes', async () => { + mocks.isModelSafeWorkspaceFileKey.mockResolvedValue(false) + const response = await executeFirecrawlParse( + { apiKey: 'firecrawl-key', file: FILE, options: { formats: ['markdown'] } }, + createContext() + ) + + expect(response.status).toBe(400) + expect(mocks.downloadServableFileFromStorage).not.toHaveBeenCalled() + expect(fetch).not.toHaveBeenCalled() + }) + + it('forwards cancellation to Firecrawl', async () => { + const controller = new AbortController() + vi.mocked(fetch).mockImplementationOnce(async (_url, init) => { + controller.abort(new DOMException('cancelled', 'AbortError')) + throw init?.signal?.reason + }) + + await expect( + executeFirecrawlParse( + { apiKey: 'firecrawl-key', file: FILE, options: {} }, + { ...createContext(), signal: controller.signal } + ) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) +}) diff --git a/apps/sim/lib/internal/firecrawl/operations.ts b/apps/sim/lib/internal/firecrawl/operations.ts new file mode 100644 index 00000000000..0a8c641fc93 --- /dev/null +++ b/apps/sim/lib/internal/firecrawl/operations.ts @@ -0,0 +1,132 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + isPayloadSizeLimitError, + readResponseJsonWithLimit, + readResponseTextWithLimit, +} from '@/lib/core/utils/stream-limits' +import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' +import type { FirecrawlParseInput } from '@/lib/internal/firecrawl/schema' +import { + isModelSafeWorkspaceFileKey, + MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, +} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' +import { assertToolFileAccess } from '@/app/api/files/authorization' +import { hasFirecrawlParseModelInput } from '@/tools/firecrawl/model-input' + +const logger = createLogger('FirecrawlParse') + +export interface FirecrawlOperationContext { + headers: Headers + userId: string + requestId: string + signal?: AbortSignal +} + +function failureResponse(error: string, status: number): Response { + return Response.json({ success: false, error }, { status }) +} + +export async function executeFirecrawlParse( + input: FirecrawlParseInput, + context: FirecrawlOperationContext +): Promise { + try { + context.signal?.throwIfAborted() + const hasModelInput = hasFirecrawlParseModelInput({ + file: input.file, + formats: Array.isArray(input.options?.formats) ? input.options.formats : undefined, + parsers: Array.isArray(input.options?.parsers) ? input.options.parsers : undefined, + }) + if (hasModelInput) { + const provenance = validateOpaqueModelInputProvenance({ + headers: context.headers, + payload: input, + isInternalRequest: true, + }) + if (!provenance.success) return failureResponse(provenance.error, provenance.status) + } + + const userFile = processFilesToUserFiles([input.file], context.requestId, logger)[0] + if (!userFile) return failureResponse('File input is required', 400) + + const denied = await assertToolFileAccess( + userFile.key, + context.userId, + context.requestId, + logger + ) + context.signal?.throwIfAborted() + if (denied) return denied + if (hasModelInput && !(await isModelSafeWorkspaceFileKey(userFile.key))) { + return failureResponse(MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, 400) + } + + const { buffer, contentType } = await downloadServableFileFromStorage( + userFile, + context.requestId, + logger, + { maxBytes: MAX_BUFFERED_TRANSFER_BYTES, signal: context.signal } + ) + const formData = new FormData() + formData.append( + 'file', + new Blob([new Uint8Array(buffer)], { + type: contentType || userFile.type || 'application/octet-stream', + }), + userFile.name + ) + if (input.options && Object.keys(input.options).length > 0) { + formData.append('options', JSON.stringify(input.options)) + } + + const response = await fetch('https://api.firecrawl.dev/v2/parse', { + method: 'POST', + headers: { Authorization: `Bearer ${input.apiKey}` }, + body: formData, + signal: context.signal, + }) + if (!response.ok) { + const errorText = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Firecrawl error response', + signal: context.signal, + }) + return failureResponse( + `Firecrawl API error: ${errorText || response.statusText}`, + response.status + ) + } + + const data = await readResponseJsonWithLimit>(response, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + label: 'Firecrawl parse response', + signal: context.signal, + }) + const nested = data.data + const document = nested && typeof nested === 'object' ? nested : data + return Response.json({ + success: true, + output: + data.creditsUsed != null + ? { ...(document as Record), creditsUsed: data.creditsUsed } + : document, + }) + } catch (error) { + context.signal?.throwIfAborted() + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + logger.error(`[${context.requestId}] Firecrawl parse failed`, { + error: getErrorMessage(error), + }) + return failureResponse( + getErrorMessage(error, 'Internal server error'), + isPayloadSizeLimitError(error) ? 413 : 500 + ) + } +} diff --git a/apps/sim/lib/internal/firecrawl/schema.ts b/apps/sim/lib/internal/firecrawl/schema.ts new file mode 100644 index 00000000000..ab9fba21ace --- /dev/null +++ b/apps/sim/lib/internal/firecrawl/schema.ts @@ -0,0 +1,13 @@ +import { z } from 'zod' +import { resolvedSecretTraceProvenanceSchema } from '@/lib/api/contracts/primitives' +import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' + +export const firecrawlParseInputSchema = z.object({ + apiKey: z.string().min(1, 'API key is required'), + file: RawFileInputSchema, + options: z.record(z.string(), z.unknown()).optional(), + [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), +}) + +export type FirecrawlParseInput = z.output diff --git a/apps/sim/lib/internal/fireflies/execute-tool.test.ts b/apps/sim/lib/internal/fireflies/execute-tool.test.ts new file mode 100644 index 00000000000..e2c11efd57f --- /dev/null +++ b/apps/sim/lib/internal/fireflies/execute-tool.test.ts @@ -0,0 +1,76 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockExecuteFirefliesUploadAudio } = vi.hoisted(() => ({ + mockExecuteFirefliesUploadAudio: vi.fn(), +})) + +vi.mock('@/lib/internal/fireflies/operations', () => ({ + executeFirefliesUploadAudio: mockExecuteFirefliesUploadAudio, +})) + +import { executeFirefliesTool } from '@/lib/internal/fireflies/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'fireflies_upload_audio', + input: { + apiKey: 'fireflies-key', + audioUrl: 'https://media.example.com/audio.mp3', + }, + headers: new Headers(), + context: { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + userId: 'user-1', + metadata: {}, + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeFirefliesTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mockExecuteFirefliesUploadAudio.mockResolvedValue(Response.json({ data: {} })) + }) + + it('validates and dispatches the typed operation input', async () => { + const request = createRequest() + const response = await executeFirefliesTool(request) + + expect(response.status).toBe(200) + expect(mockExecuteFirefliesUploadAudio).toHaveBeenCalledWith(request.input, { + headers: request.headers, + userId: 'user-1', + requestId: 'request-1', + signal: undefined, + }) + }) + + it('rejects missing audio before provider work', async () => { + const response = await executeFirefliesTool( + createRequest({ input: { apiKey: 'fireflies-key' } }) + ) + + expect(response.status).toBe(400) + expect(mockExecuteFirefliesUploadAudio).not.toHaveBeenCalled() + }) + + it('requires trusted execution identity', async () => { + const response = await executeFirefliesTool( + createRequest({ + context: { workflowId: 'workflow-1', workspaceId: 'workspace-1', metadata: {} }, + }) + ) + + expect(response.status).toBe(401) + expect(mockExecuteFirefliesUploadAudio).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/fireflies/execute-tool.ts b/apps/sim/lib/internal/fireflies/execute-tool.ts new file mode 100644 index 00000000000..f5ee3ec7e7e --- /dev/null +++ b/apps/sim/lib/internal/fireflies/execute-tool.ts @@ -0,0 +1,32 @@ +import { getValidationErrorMessage } from '@/lib/api/server' +import { executeFirefliesUploadAudio } from '@/lib/internal/fireflies/operations' +import { firefliesUploadAudioInputSchema } from '@/lib/internal/fireflies/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +export const executeFirefliesTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'fireflies_upload_audio') { + return Response.json( + { error: `Unsupported Fireflies tool: ${request.toolId}` }, + { status: 500 } + ) + } + if (!request.context.userId) { + return Response.json({ errors: [{ message: 'Unauthorized' }] }, { status: 401 }) + } + + const parsed = firefliesUploadAudioInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { errors: [{ message: getValidationErrorMessage(parsed.error, 'Invalid request data') }] }, + { status: 400 } + ) + } + + return executeFirefliesUploadAudio(parsed.data, { + headers: request.headers, + userId: request.context.userId, + requestId: request.requestId, + signal: request.signal, + }) +} diff --git a/apps/sim/lib/internal/fireflies/operations.test.ts b/apps/sim/lib/internal/fireflies/operations.test.ts new file mode 100644 index 00000000000..e480b8d5e98 --- /dev/null +++ b/apps/sim/lib/internal/fireflies/operations.test.ts @@ -0,0 +1,114 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { PRIVATE_MODEL_INPUT_PROVENANCE_HEADER } from '@/lib/execution/model-input-provenance' +import { + RESOLVED_SECRET_PROVENANCE_FIELD, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, +} from '@/lib/execution/private-tool-metadata' + +const { mockResolveFileInputToUrl } = vi.hoisted(() => ({ + mockResolveFileInputToUrl: vi.fn(), +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + resolveFileInputToUrl: mockResolveFileInputToUrl, +})) + +import { executeFirefliesUploadAudio } from '@/lib/internal/fireflies/operations' + +const upstreamSuccess = { + data: { uploadAudio: { success: true, title: 'Uploaded meeting', message: 'Queued' } }, +} + +function createContext(headers = new Headers()) { + return { headers, userId: 'user-1', requestId: 'request-1' } +} + +describe('executeFirefliesUploadAudio', () => { + beforeEach(() => { + vi.clearAllMocks() + mockResolveFileInputToUrl.mockResolvedValue({ + fileUrl: 'https://media.example.com/audio.mp3', + }) + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(Response.json(upstreamSuccess))) + }) + + it('preserves the GraphQL request and response', async () => { + const response = await executeFirefliesUploadAudio( + { + apiKey: 'fireflies-key', + audioUrl: 'https://media.example.com/audio.mp3', + title: 'Uploaded meeting', + attendees: [{ displayName: 'Ada', email: 'ada@example.com' }], + }, + createContext() + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual(upstreamSuccess) + const init = vi.mocked(fetch).mock.calls[0][1] + expect(JSON.parse(String(init?.body)).variables.input).toEqual({ + url: 'https://media.example.com/audio.mp3', + title: 'Uploaded meeting', + attendees: [{ displayName: 'Ada', email: 'ada@example.com' }], + }) + }) + + it('rejects incomplete private provenance before file resolution', async () => { + const headers = new Headers({ + [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1, + }) + const response = await executeFirefliesUploadAudio( + { + apiKey: 'fireflies-key', + audioUrl: 'https://media.example.com/audio.mp3', + [RESOLVED_SECRET_PROVENANCE_FIELD]: { version: 1, complete: false, entries: [] }, + }, + createContext(headers) + ) + + expect(response.status).toBe(400) + expect(mockResolveFileInputToUrl).not.toHaveBeenCalled() + expect(fetch).not.toHaveBeenCalled() + }) + + it('normalizes key-only stored files before model-safe URL resolution', async () => { + const headers = new Headers({ + [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1, + }) + const response = await executeFirefliesUploadAudio( + { + apiKey: 'fireflies-key', + audioFile: { key: 'workspace/workspace-1/audio.mp3' }, + [RESOLVED_SECRET_PROVENANCE_FIELD]: { version: 1, complete: true, entries: [] }, + }, + createContext(headers) + ) + + expect(response.status).toBe(200) + expect(mockResolveFileInputToUrl).toHaveBeenCalledWith( + expect.objectContaining({ + file: expect.objectContaining({ name: 'audio', size: 0 }), + modelEgress: true, + presignExpirySeconds: 3600, + }) + ) + }) + + it('forwards cancellation to the provider request', async () => { + const controller = new AbortController() + vi.mocked(fetch).mockImplementationOnce(async (_url, init) => { + controller.abort(new DOMException('cancelled', 'AbortError')) + throw init?.signal?.reason + }) + + await expect( + executeFirefliesUploadAudio( + { apiKey: 'fireflies-key', audioUrl: 'https://media.example.com/audio.mp3' }, + { ...createContext(), signal: controller.signal } + ) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) +}) diff --git a/apps/sim/lib/internal/fireflies/operations.ts b/apps/sim/lib/internal/fireflies/operations.ts new file mode 100644 index 00000000000..10bbdc1e0fd --- /dev/null +++ b/apps/sim/lib/internal/fireflies/operations.ts @@ -0,0 +1,125 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' +import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' +import type { FirefliesUploadAudioInput } from '@/lib/internal/fireflies/schema' +import type { RawFileInput } from '@/lib/uploads/utils/file-utils' +import { resolveFileInputToUrl } from '@/lib/uploads/utils/file-utils.server' + +const logger = createLogger('FirefliesUploadAudio') +const FIREFLIES_API_URL = 'https://api.fireflies.ai/graphql' +const FIREFLIES_AUDIO_PRESIGN_EXPIRY_SECONDS = 60 * 60 +const MAX_FIREFLIES_RESPONSE_BYTES = 10 * 1024 * 1024 +const UPLOAD_AUDIO_MUTATION = ` + mutation UploadAudio($input: AudioUploadInput) { + uploadAudio(input: $input) { + success + title + message + } + } +` + +export interface FirefliesOperationContext { + headers: Headers + userId: string + requestId: string + signal?: AbortSignal +} + +function errorResponse(message: string, status: number): Response { + return Response.json({ errors: [{ message }] }, { status }) +} + +function normalizeStoredAudioFile( + file: NonNullable +): RawFileInput { + return { + ...file, + name: file.name || 'audio', + size: file.size ?? 0, + } as RawFileInput +} + +async function resolveAudioUrl( + body: FirefliesUploadAudioInput, + context: FirefliesOperationContext +): Promise<{ fileUrl?: string; error?: { status: number; message: string } }> { + const file = body.audioFile + const shared = { + userId: context.userId, + requestId: context.requestId, + logger, + presignExpirySeconds: FIREFLIES_AUDIO_PRESIGN_EXPIRY_SECONDS, + modelEgress: true, + } as const + + if (file?.key) { + return resolveFileInputToUrl({ + ...shared, + file: normalizeStoredAudioFile(file), + }) + } + + return resolveFileInputToUrl({ + ...shared, + filePath: file?.url || file?.path || body.audioUrl, + }) +} + +export async function executeFirefliesUploadAudio( + body: FirefliesUploadAudioInput, + context: FirefliesOperationContext +): Promise { + try { + context.signal?.throwIfAborted() + const modelInputProvenance = validateOpaqueModelInputProvenance({ + headers: context.headers, + payload: body, + isInternalRequest: true, + }) + if (!modelInputProvenance.success) { + return errorResponse(modelInputProvenance.error, modelInputProvenance.status) + } + + const resolution = await resolveAudioUrl(body, context) + context.signal?.throwIfAborted() + if (resolution.error) return errorResponse(resolution.error.message, resolution.error.status) + if (!resolution.fileUrl?.startsWith('https://')) { + return errorResponse('Audio URL must be a valid HTTPS URL', 400) + } + + const input: Record = { url: resolution.fileUrl } + if (body.title) input.title = body.title + if (body.webhook) input.webhook = body.webhook + if (body.language) input.custom_language = body.language + if (body.clientReferenceId) input.client_reference_id = body.clientReferenceId + if (body.attendees !== undefined) input.attendees = body.attendees + + const response = await fetch(FIREFLIES_API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${body.apiKey}`, + }, + body: JSON.stringify({ query: UPLOAD_AUDIO_MUTATION, variables: { input } }), + signal: context.signal, + }) + const data = await readResponseJsonWithLimit(response, { + maxBytes: MAX_FIREFLIES_RESPONSE_BYTES, + label: 'Fireflies response', + signal: context.signal, + }) + + logger.info(`[${context.requestId}] Fireflies upload request completed`, { + status: response.status, + }) + return Response.json(data, { status: response.status }) + } catch (error) { + if (context.signal?.aborted) throw error + logger.error(`[${context.requestId}] Fireflies upload failed`, { + error: getErrorMessage(error, 'Unknown error'), + }) + return errorResponse('Failed to upload audio', 500) + } +} diff --git a/apps/sim/lib/internal/fireflies/schema.ts b/apps/sim/lib/internal/fireflies/schema.ts new file mode 100644 index 00000000000..b9c5cd7dc4e --- /dev/null +++ b/apps/sim/lib/internal/fireflies/schema.ts @@ -0,0 +1,41 @@ +import { z } from 'zod' +import { resolvedSecretTraceProvenanceSchema } from '@/lib/api/contracts/primitives' +import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' + +const firefliesAudioFileSchema = z + .object({ + id: z.string().optional(), + key: z.string().optional(), + path: z.string().optional(), + url: z.string().optional(), + name: z.string().optional(), + size: z.number().nonnegative().optional(), + type: z.string().optional(), + context: z.string().optional(), + }) + .passthrough() + +export const firefliesUploadAudioInputSchema = z + .object({ + apiKey: z.string().min(1, 'Missing API key for Fireflies API request'), + audioFile: firefliesAudioFileSchema.optional(), + audioUrl: z.string().optional(), + title: z.string().optional(), + webhook: z.string().optional(), + language: z.string().optional(), + attendees: z.unknown().optional(), + clientReferenceId: z.string().optional(), + [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), + }) + .superRefine((data, context) => { + const file = data.audioFile + if (!file?.key && !file?.url && !file?.path && !data.audioUrl) { + context.addIssue({ + code: 'custom', + message: 'Either an audio file or audio URL is required', + path: ['audioUrl'], + }) + } + }) + +export type FirefliesUploadAudioInput = z.output diff --git a/apps/sim/lib/internal/function/execute.test.ts b/apps/sim/lib/internal/function/execute.test.ts new file mode 100644 index 00000000000..a5ed37edef3 --- /dev/null +++ b/apps/sim/lib/internal/function/execute.test.ts @@ -0,0 +1,98 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + createPrincipal: vi.fn(), + execute: vi.fn(), +})) + +vi.mock('@/lib/internal/principals/executor', () => ({ + createExecutorPrincipalFromExecutionContext: mocks.createPrincipal, +})) + +vi.mock('@/lib/function-execution/application/execute-function', () => ({ + executeFunction: { execute: mocks.execute }, +})) + +import { FUNCTION_EXECUTION_DELEGATION_AUDIENCE } from '@/lib/function-execution/application/authorization' +import { executeFunctionTool } from '@/lib/internal/function/execute' + +describe('executeFunctionTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.execute.mockResolvedValue(Response.json({ success: true })) + }) + + it('binds executor calls from the canonical origin instead of the compatibility user ID', async () => { + const startedAt = Date.now() + const origin = { + workflowId: 'workflow-1', + executionId: 'execution-1', + principal: { + kind: 'system' as const, + serviceId: 'schedule' as const, + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + }, + currentWorkflow: { + workflowId: 'workflow-1', + mode: 'deployment' as const, + deploymentVersionId: 'deployment-1', + }, + } + const principal = { + kind: 'delegated' as const, + serviceId: 'executor' as const, + workspaceId: 'workspace-1', + delegationId: 'delegation-1', + audience: FUNCTION_EXECUTION_DELEGATION_AUDIENCE, + issuedAt: new Date(), + expiresAt: new Date(Date.now() + 60_000), + delegationContext: { kind: 'workflow_execution' as const, ...origin }, + } + mocks.createPrincipal.mockResolvedValue(principal) + const context = { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionId: 'execution-1', + userId: 'workspace-owner', + executorDelegationOrigin: origin, + } + const headers = new Headers() + + await executeFunctionTool({ + body: { + code: 'return 1', + timeout: 60_000, + userId: 'forged-user', + workspaceId: 'forged-workspace', + }, + headers, + context, + requestId: 'request-1', + }) + + expect(mocks.createPrincipal).toHaveBeenCalledWith({ + context, + audience: FUNCTION_EXECUTION_DELEGATION_AUDIENCE, + expiresAt: expect.any(Date), + resourceScope: { executionId: 'execution-1' }, + }) + const delegatedExpiry = mocks.createPrincipal.mock.calls[0]?.[0].expiresAt as Date + expect(delegatedExpiry.getTime()).toBeGreaterThanOrEqual(startedAt + 60_000) + expect(delegatedExpiry.getTime()).toBeLessThanOrEqual(Date.now() + 60_000) + expect(mocks.execute).toHaveBeenCalledWith({ + principal, + input: expect.objectContaining({ + workspaceId: 'workspace-1', + body: expect.objectContaining({ + workspaceId: 'workspace-1', + userId: undefined, + }), + headers, + }), + }) + }) +}) diff --git a/apps/sim/lib/internal/function/execute.ts b/apps/sim/lib/internal/function/execute.ts new file mode 100644 index 00000000000..4d68ba712a4 --- /dev/null +++ b/apps/sim/lib/internal/function/execute.ts @@ -0,0 +1,78 @@ +import type { DelegatedPrincipal } from '@sim/auth/principal' +import type { FunctionExecuteBody } from '@/lib/api/contracts' +import type { InternalSandboxProfile } from '@/lib/auth/internal' +import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' +import { serializeExecutionDeadlineHeader } from '@/lib/execution/execution-deadline-header' +import { FUNCTION_EXECUTION_DELEGATION_AUDIENCE } from '@/lib/function-execution/application/authorization' +import { executeFunction } from '@/lib/function-execution/application/execute-function' +import { createExecutorPrincipalFromExecutionContext } from '@/lib/internal/principals/executor' +import type { InternalToolOperationContext } from '@/lib/internal/tool-operations/types' + +export type TrustedFunctionToolExecutionContext = InternalToolOperationContext & { + workspaceId: string +} + +export interface ExecuteFunctionToolInput { + body: FunctionExecuteBody + headers: Headers + context: TrustedFunctionToolExecutionContext + requestId: string + signal?: AbortSignal + sandboxProfile?: InternalSandboxProfile +} + +/** Executes a Function tool with authority taken only from trusted server execution context. */ +export async function executeFunctionTool(input: ExecuteFunctionToolInput): Promise { + const { body, context, headers, requestId, signal, sandboxProfile } = input + const issuedAt = new Date() + const serializedDeadline = serializeExecutionDeadlineHeader(signal) + const requestedTimeout = + typeof body.timeout === 'number' ? body.timeout : DEFAULT_EXECUTION_TIMEOUT_MS + const expiresAt = serializedDeadline + ? new Date(Number(serializedDeadline)) + : new Date(issuedAt.getTime() + requestedTimeout) + const trustedBody: FunctionExecuteBody = { + ...body, + workflowId: context.workflowId, + executionId: context.executionId, + userId: undefined, + workspaceId: context.workspaceId, + largeValueExecutionIds: context.largeValueExecutionIds, + largeValueKeys: context.largeValueKeys, + fileKeys: context.fileKeys, + allowLargeValueWorkflowScope: context.allowLargeValueWorkflowScope, + } + let principal: DelegatedPrincipal + if (context.copilotToolExecution === true) { + if (!context.userId) throw new Error('Copilot Function execution requires a user') + principal = { + kind: 'delegated', + serviceId: 'copilot', + subjectUserId: context.userId, + workspaceId: context.workspaceId, + delegationId: `function-execute:${requestId}`, + audience: FUNCTION_EXECUTION_DELEGATION_AUDIENCE, + issuedAt, + expiresAt, + ...(context.executionId ? { resourceScope: { executionId: context.executionId } } : {}), + } + } else { + principal = await createExecutorPrincipalFromExecutionContext({ + context, + audience: FUNCTION_EXECUTION_DELEGATION_AUDIENCE, + expiresAt, + ...(context.executionId ? { resourceScope: { executionId: context.executionId } } : {}), + }) + } + + return executeFunction.execute({ + principal, + input: { + workspaceId: context.workspaceId, + body: trustedBody, + headers, + ...(signal ? { signal } : {}), + ...(sandboxProfile ? { sandboxProfile } : {}), + }, + }) +} diff --git a/apps/sim/lib/internal/github/errors.ts b/apps/sim/lib/internal/github/errors.ts new file mode 100644 index 00000000000..f7c98cf8f9a --- /dev/null +++ b/apps/sim/lib/internal/github/errors.ts @@ -0,0 +1,9 @@ +export class GitHubOperationError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'GitHubOperationError' + } +} diff --git a/apps/sim/lib/internal/github/execute-tool.test.ts b/apps/sim/lib/internal/github/execute-tool.test.ts new file mode 100644 index 00000000000..9d27a42ef9c --- /dev/null +++ b/apps/sim/lib/internal/github/execute-tool.test.ts @@ -0,0 +1,43 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ getGitHubLatestCommit: vi.fn() })) + +vi.mock('@/lib/internal/github/operations', () => ({ + getGitHubLatestCommit: mocks.getGitHubLatestCommit, +})) + +import { executeGitHubTool } from '@/lib/internal/github/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +describe('executeGitHubTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.getGitHubLatestCommit.mockResolvedValue({ success: true, output: {} }) + }) + + it.each(['github_latest_commit', 'github_latest_commit_v2'])( + 'dispatches %s to the same typed operation', + async (toolId) => { + const controller = new AbortController() + const input = { owner: 'simstudioai', repo: 'sim', branch: 'staging', apiKey: 'token' } + const request: InternalToolOperationCall = { + toolId, + input, + headers: new Headers(), + context: createExecutionContext(), + requestId: 'request-1', + signal: controller.signal, + } + + expect((await executeGitHubTool(request)).status).toBe(200) + expect(mocks.getGitHubLatestCommit).toHaveBeenCalledWith(input, { + requestId: 'request-1', + signal: controller.signal, + }) + } + ) +}) diff --git a/apps/sim/lib/internal/github/execute-tool.ts b/apps/sim/lib/internal/github/execute-tool.ts new file mode 100644 index 00000000000..0b185b8679b --- /dev/null +++ b/apps/sim/lib/internal/github/execute-tool.ts @@ -0,0 +1,48 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { z } from 'zod' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { GitHubOperationError } from '@/lib/internal/github/errors' +import { getGitHubLatestCommit } from '@/lib/internal/github/operations' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const TOOL_IDS = new Set(['github_latest_commit', 'github_latest_commit_v2']) + +const inputSchema = z.object({ + owner: z.string().min(1, 'Owner is required'), + repo: z.string().min(1, 'Repo is required'), + branch: z.string().optional(), + apiKey: z.string().min(1, 'API key is required'), +}) + +export const executeGitHubTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!TOOL_IDS.has(request.toolId)) { + return Response.json( + { success: false, error: `Unsupported GitHub tool: ${request.toolId}` }, + { status: 500 } + ) + } + const parsed = inputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + try { + return Response.json( + await getGitHubLatestCommit(parsed.data, { + requestId: request.requestId, + signal: request.signal, + }) + ) + } catch (error) { + request.signal?.throwIfAborted() + const status = isPayloadSizeLimitError(error) + ? 413 + : error instanceof GitHubOperationError + ? error.status + : 500 + return Response.json( + { success: false, error: getErrorMessage(error, 'Unknown error occurred') }, + { status } + ) + } +} diff --git a/apps/sim/lib/internal/github/operations.test.ts b/apps/sim/lib/internal/github/operations.test.ts new file mode 100644 index 00000000000..e4686f9af3b --- /dev/null +++ b/apps/sim/lib/internal/github/operations.test.ts @@ -0,0 +1,63 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + secureFetchWithPinnedIP: vi.fn(), + validateUrlWithDNS: vi.fn(), +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + secureFetchWithPinnedIP: mocks.secureFetchWithPinnedIP, + validateUrlWithDNS: mocks.validateUrlWithDNS, +})) + +import { getGitHubLatestCommit } from '@/lib/internal/github/operations' + +describe('getGitHubLatestCommit', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.validateUrlWithDNS.mockResolvedValue({ isValid: true, resolvedIP: '203.0.113.1' }) + mocks.secureFetchWithPinnedIP + .mockResolvedValueOnce( + Response.json({ + sha: 'abc123', + html_url: 'https://github.com/simstudioai/sim/commit/abc123', + commit: { + message: 'ship it', + author: { name: 'Author', email: 'author@example.com', date: '2026-08-27' }, + committer: { name: 'Committer', email: 'committer@example.com', date: '2026-08-27' }, + }, + files: [ + { + filename: 'README.md', + status: 'modified', + additions: 1, + deletions: 0, + changes: 1, + raw_url: 'https://raw.githubusercontent.com/simstudioai/sim/abc123/README.md', + }, + ], + }) + ) + .mockResolvedValueOnce(new Response('updated readme')) + }) + + it('pins provider requests, forwards cancellation, and includes changed file content', async () => { + const controller = new AbortController() + const result = await getGitHubLatestCommit( + { owner: 'simstudioai', repo: 'sim', branch: 'feature/test', apiKey: 'token' }, + { requestId: 'request-1', signal: controller.signal } + ) + + expect(mocks.secureFetchWithPinnedIP).toHaveBeenCalledTimes(2) + expect(mocks.secureFetchWithPinnedIP.mock.calls[0][0]).toContain('/commits/feature%2Ftest') + expect(mocks.secureFetchWithPinnedIP.mock.calls[0][2]).toEqual( + expect.objectContaining({ signal: controller.signal }) + ) + expect(result.output.metadata.files?.[0]).toEqual( + expect.objectContaining({ filename: 'README.md', content: 'updated readme' }) + ) + }) +}) diff --git a/apps/sim/lib/internal/github/operations.ts b/apps/sim/lib/internal/github/operations.ts new file mode 100644 index 00000000000..57c7501c10a --- /dev/null +++ b/apps/sim/lib/internal/github/operations.ts @@ -0,0 +1,168 @@ +import { createLogger } from '@sim/logger' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + readResponseJsonWithLimit, + readResponseTextWithLimit, +} from '@/lib/core/utils/stream-limits' +import { GitHubOperationError } from '@/lib/internal/github/errors' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import type { LatestCommitParams, LatestCommitResponse } from '@/tools/github/types' + +const logger = createLogger('GitHubLatestCommitOperation') +const MAX_COMMIT_RESPONSE_BYTES = 10 * 1024 * 1024 + +interface GitHubCommitFile { + filename: string + status: string + additions: number + deletions: number + changes: number + patch?: string + raw_url?: string + blob_url?: string +} + +interface GitHubCommitResponse { + sha: string + html_url: string + commit: { + message: string + author: { name: string; email: string; date: string } + committer: { name: string; email: string; date: string } + } + author?: { login: string; avatar_url: string; html_url: string } + committer?: { login: string; avatar_url: string; html_url: string } + stats?: { additions: number; deletions: number; total: number } + files?: GitHubCommitFile[] +} + +interface GitHubCommitFileOutput extends Omit { + raw_url: string + blob_url: string + content?: string +} + +export interface GitHubOperationContext { + requestId: string + signal?: AbortSignal +} + +async function fetchChangedFileContent( + file: GitHubCommitFile, + apiKey: string, + remainingBytes: number, + context: GitHubOperationContext +): Promise { + if (file.status === 'removed' || !file.raw_url || remainingBytes <= 0) return undefined + try { + const validation = await validateUrlWithDNS(file.raw_url, 'rawUrl') + context.signal?.throwIfAborted() + if (!validation.isValid || !validation.resolvedIP) return undefined + const response = await secureFetchWithPinnedIP(file.raw_url, validation.resolvedIP, { + headers: { + Authorization: `Bearer ${apiKey}`, + 'X-GitHub-Api-Version': '2022-11-28', + }, + maxResponseBytes: remainingBytes, + signal: context.signal, + }) + if (!response.ok) return undefined + return await readResponseTextWithLimit(response, { + maxBytes: remainingBytes, + label: `GitHub changed file ${file.filename}`, + signal: context.signal, + }) + } catch (error) { + context.signal?.throwIfAborted() + logger.warn('Failed to fetch changed file content', { + requestId: context.requestId, + filename: file.filename, + error, + }) + return undefined + } +} + +export async function getGitHubLatestCommit( + input: LatestCommitParams, + context: GitHubOperationContext +): Promise { + context.signal?.throwIfAborted() + const owner = encodeURIComponent(input.owner) + const repo = encodeURIComponent(input.repo) + const revision = encodeURIComponent(input.branch || 'HEAD') + const commitUrl = `https://api.github.com/repos/${owner}/${repo}/commits/${revision}` + const validation = await validateUrlWithDNS(commitUrl, 'commitUrl') + context.signal?.throwIfAborted() + if (!validation.isValid || !validation.resolvedIP) { + throw new GitHubOperationError(validation.error || 'Invalid GitHub commit URL', 400) + } + + const response = await secureFetchWithPinnedIP(commitUrl, validation.resolvedIP, { + method: 'GET', + headers: { + Accept: 'application/vnd.github.v3+json', + Authorization: `Bearer ${input.apiKey}`, + 'X-GitHub-Api-Version': '2022-11-28', + }, + maxResponseBytes: MAX_COMMIT_RESPONSE_BYTES, + signal: context.signal, + }) + if (!response.ok) { + const error = await readResponseJsonWithLimit<{ message?: string }>(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'GitHub error response', + signal: context.signal, + }).catch(() => ({ message: undefined })) + throw new GitHubOperationError(error.message || `GitHub API error: ${response.status}`, 400) + } + + const data = await readResponseJsonWithLimit(response, { + maxBytes: MAX_COMMIT_RESPONSE_BYTES, + label: 'GitHub latest commit response', + signal: context.signal, + }) + const files: GitHubCommitFileOutput[] = [] + let remainingBytes = MAX_BUFFERED_TRANSFER_BYTES + for (const file of data.files ?? []) { + context.signal?.throwIfAborted() + const content = await fetchChangedFileContent(file, input.apiKey, remainingBytes, context) + if (content) remainingBytes -= Buffer.byteLength(content) + files.push({ + ...file, + raw_url: file.raw_url || '', + blob_url: file.blob_url || '', + content, + }) + } + + return { + success: true, + output: { + content: `Latest commit: "${data.commit.message}" by ${data.commit.author.name} on ${data.commit.author.date}. SHA: ${data.sha}`, + metadata: { + sha: data.sha, + html_url: data.html_url, + commit_message: data.commit.message, + author: { + name: data.commit.author.name, + login: data.author?.login || 'Unknown', + avatar_url: data.author?.avatar_url || '', + html_url: data.author?.html_url || '', + }, + committer: { + name: data.commit.committer.name, + login: data.committer?.login || 'Unknown', + avatar_url: data.committer?.avatar_url || '', + html_url: data.committer?.html_url || '', + }, + stats: data.stats, + files: files.length > 0 ? files : undefined, + }, + }, + } +} diff --git a/apps/sim/lib/internal/gmail/client.ts b/apps/sim/lib/internal/gmail/client.ts new file mode 100644 index 00000000000..f3a9bbf426b --- /dev/null +++ b/apps/sim/lib/internal/gmail/client.ts @@ -0,0 +1,102 @@ +import { + type ReadResponseWithLimitOptions, + readResponseJsonWithLimit, +} from '@/lib/core/utils/stream-limits' +import { GmailOperationError } from '@/lib/internal/gmail/errors' + +const GMAIL_API_BASE = 'https://gmail.googleapis.com/gmail/v1/users/me' +const GMAIL_METADATA_RESPONSE_MAX_BYTES = 1024 * 1024 +const RESPONSE_LIMIT: ReadResponseWithLimitOptions = { + maxBytes: GMAIL_METADATA_RESPONSE_MAX_BYTES, + label: 'Gmail API response', +} + +export type JsonObject = Record + +export function asObject(value: unknown): JsonObject { + return value && typeof value === 'object' && !Array.isArray(value) ? (value as JsonObject) : {} +} + +export function asArray(value: unknown): unknown[] { + return Array.isArray(value) ? value : [] +} + +export function nested(value: unknown, ...keys: string[]): unknown { + let current = value + for (const key of keys) current = asObject(current)[key] + return current +} + +export class GmailClient { + constructor(private readonly accessToken: string) {} + + api(path: string): string { + return `${GMAIL_API_BASE}${path}` + } + + async fetch(path: string, init: RequestInit = {}, signal?: AbortSignal): Promise { + signal?.throwIfAborted() + return fetch(path, { + ...init, + headers: { + Authorization: `Bearer ${this.accessToken}`, + 'Content-Type': 'application/json', + ...init.headers, + }, + signal, + }) + } + + async json(path: string, init: RequestInit = {}, signal?: AbortSignal): Promise { + const response = await this.fetch(path, init, signal) + if (!response.ok) { + await response.body?.cancel().catch(() => {}) + throw new GmailOperationError(`Gmail API error: ${response.statusText}`, response.status, { + success: false, + error: `Gmail API error: ${response.statusText}`, + }) + } + return asObject(await readResponseJsonWithLimit(response, RESPONSE_LIMIT)) + } + + async threadingHeaders( + messageId: string, + signal?: AbortSignal + ): Promise<{ + messageId?: string + references?: string + subject?: string + }> { + try { + const query = new URLSearchParams({ format: 'metadata' }) + query.append('metadataHeaders', 'Message-ID') + query.append('metadataHeaders', 'References') + query.append('metadataHeaders', 'Subject') + const response = await this.fetch( + this.api(`/messages/${encodeURIComponent(messageId)}?${query}`), + {}, + signal + ) + if (!response.ok) { + await response.body?.cancel().catch(() => {}) + return {} + } + const data = asObject(await readResponseJsonWithLimit(response, RESPONSE_LIMIT)) + const headers = asArray(nested(data, 'payload', 'headers')).map(asObject) + const value = (name: string) => { + const header = headers.find( + (entry) => typeof entry.name === 'string' && entry.name.toLowerCase() === name + ) + return typeof header?.value === 'string' ? header.value : undefined + } + return { + messageId: value('message-id'), + references: value('references'), + subject: value('subject'), + } + } catch { + signal?.throwIfAborted() + return {} + } + } +} diff --git a/apps/sim/lib/internal/gmail/errors.ts b/apps/sim/lib/internal/gmail/errors.ts new file mode 100644 index 00000000000..82dd8b54d65 --- /dev/null +++ b/apps/sim/lib/internal/gmail/errors.ts @@ -0,0 +1,10 @@ +export class GmailOperationError extends Error { + constructor( + message: string, + readonly status: number, + readonly body?: Record + ) { + super(message) + this.name = 'GmailOperationError' + } +} diff --git a/apps/sim/lib/internal/gmail/execute-tool.test.ts b/apps/sim/lib/internal/gmail/execute-tool.test.ts new file mode 100644 index 00000000000..cdbe35cbc40 --- /dev/null +++ b/apps/sim/lib/internal/gmail/execute-tool.test.ts @@ -0,0 +1,183 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operationMocks = vi.hoisted(() => ({ + executeGmailAddLabel: vi.fn(), + executeGmailArchive: vi.fn(), + executeGmailDelete: vi.fn(), + executeGmailDraft: vi.fn(), + executeGmailEditDraft: vi.fn(), + executeGmailMarkRead: vi.fn(), + executeGmailMarkUnread: vi.fn(), + executeGmailMove: vi.fn(), + executeGmailRemoveLabel: vi.fn(), + executeGmailSend: vi.fn(), + executeGmailUnarchive: vi.fn(), +})) + +vi.mock('@/lib/internal/gmail/mail', () => ({ + executeGmailDraft: operationMocks.executeGmailDraft, + executeGmailEditDraft: operationMocks.executeGmailEditDraft, + executeGmailSend: operationMocks.executeGmailSend, +})) + +vi.mock('@/lib/internal/gmail/messages', () => ({ + executeGmailAddLabel: operationMocks.executeGmailAddLabel, + executeGmailArchive: operationMocks.executeGmailArchive, + executeGmailDelete: operationMocks.executeGmailDelete, + executeGmailMarkRead: operationMocks.executeGmailMarkRead, + executeGmailMarkUnread: operationMocks.executeGmailMarkUnread, + executeGmailMove: operationMocks.executeGmailMove, + executeGmailRemoveLabel: operationMocks.executeGmailRemoveLabel, + executeGmailUnarchive: operationMocks.executeGmailUnarchive, +})) + +import { GmailOperationError } from '@/lib/internal/gmail/errors' +import { executeGmailTool } from '@/lib/internal/gmail/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const MESSAGE_BODY = { accessToken: 'access-token', messageId: 'message-1' } +const LABEL_BODY = { ...MESSAGE_BODY, labelIds: 'INBOX,STARRED' } +const MOVE_BODY = { ...MESSAGE_BODY, addLabelIds: 'IMPORTANT', removeLabelIds: 'INBOX' } +const MAIL_BODY = { + accessToken: 'access-token', + to: 'recipient@example.com', + body: 'Hello', +} +const EDIT_DRAFT_BODY = { ...MAIL_BODY, draftId: 'draft-1' } + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'gmail_archive', + input: MESSAGE_BODY, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +const TOOL_CASES = [ + ['gmail_add_label', LABEL_BODY, operationMocks.executeGmailAddLabel, 'message'], + ['gmail_add_label_v2', LABEL_BODY, operationMocks.executeGmailAddLabel, 'message'], + ['gmail_archive', MESSAGE_BODY, operationMocks.executeGmailArchive, 'message'], + ['gmail_archive_v2', MESSAGE_BODY, operationMocks.executeGmailArchive, 'message'], + ['gmail_delete', MESSAGE_BODY, operationMocks.executeGmailDelete, 'message'], + ['gmail_delete_v2', MESSAGE_BODY, operationMocks.executeGmailDelete, 'message'], + ['gmail_draft', MAIL_BODY, operationMocks.executeGmailDraft, 'mail'], + ['gmail_draft_v2', MAIL_BODY, operationMocks.executeGmailDraft, 'mail'], + ['gmail_edit_draft_v2', EDIT_DRAFT_BODY, operationMocks.executeGmailEditDraft, 'mail'], + ['gmail_mark_read', MESSAGE_BODY, operationMocks.executeGmailMarkRead, 'message'], + ['gmail_mark_read_v2', MESSAGE_BODY, operationMocks.executeGmailMarkRead, 'message'], + ['gmail_mark_unread', MESSAGE_BODY, operationMocks.executeGmailMarkUnread, 'message'], + ['gmail_mark_unread_v2', MESSAGE_BODY, operationMocks.executeGmailMarkUnread, 'message'], + ['gmail_move', MOVE_BODY, operationMocks.executeGmailMove, 'message'], + ['gmail_move_v2', MOVE_BODY, operationMocks.executeGmailMove, 'message'], + ['gmail_remove_label', LABEL_BODY, operationMocks.executeGmailRemoveLabel, 'message'], + ['gmail_remove_label_v2', LABEL_BODY, operationMocks.executeGmailRemoveLabel, 'message'], + ['gmail_send', MAIL_BODY, operationMocks.executeGmailSend, 'mail'], + ['gmail_send_v2', MAIL_BODY, operationMocks.executeGmailSend, 'mail'], + ['gmail_unarchive', MESSAGE_BODY, operationMocks.executeGmailUnarchive, 'message'], + ['gmail_unarchive_v2', MESSAGE_BODY, operationMocks.executeGmailUnarchive, 'message'], +] as const + +describe('executeGmailTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it.each(TOOL_CASES)( + 'validates and dispatches %s', + async (toolId, input, operation, operationKind) => { + const controller = new AbortController() + operation.mockResolvedValue({ toolId }) + + const response = await executeGmailTool( + createRequest({ toolId, input, signal: controller.signal }) + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ toolId }) + if (operationKind === 'mail') { + expect(operation).toHaveBeenCalledWith(input, { + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + }) + } else { + expect(operation).toHaveBeenCalledWith(input, controller.signal) + } + } + ) + + it('returns the canonical validation envelope before provider work', async () => { + const response = await executeGmailTool( + createRequest({ input: { accessToken: '', messageId: '' } }) + ) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(operationMocks.executeGmailArchive).not.toHaveBeenCalled() + }) + + it('rejects non-object operation input', async () => { + const response = await executeGmailTool(createRequest({ input: '{' })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(operationMocks.executeGmailArchive).not.toHaveBeenCalled() + }) + + it('preserves typed provider status and error envelopes', async () => { + operationMocks.executeGmailArchive.mockRejectedValue( + new GmailOperationError('Gmail API error: Not Found', 404, { + success: false, + error: 'Gmail API error: Not Found', + }) + ) + + const response = await executeGmailTool(createRequest()) + + expect(response.status).toBe(404) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Gmail API error: Not Found', + }) + }) + + it('rejects unsupported Gmail IDs without provider work', async () => { + const response = await executeGmailTool(createRequest({ toolId: 'gmail_unknown' })) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Unsupported Gmail tool: gmail_unknown', + }) + expect(operationMocks.executeGmailArchive).not.toHaveBeenCalled() + }) + + it('propagates cancellation without starting provider work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeGmailTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(operationMocks.executeGmailArchive).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/gmail/execute-tool.ts b/apps/sim/lib/internal/gmail/execute-tool.ts new file mode 100644 index 00000000000..4ea9ee4836a --- /dev/null +++ b/apps/sim/lib/internal/gmail/execute-tool.ts @@ -0,0 +1,164 @@ +import { getErrorMessage } from '@sim/utils/errors' +import type { AnyApiRouteContract, ContractBody } from '@/lib/api/contracts' +import { + gmailAddLabelContract, + gmailArchiveContract, + gmailDeleteContract, + gmailDraftContract, + gmailEditDraftContract, + gmailMarkReadContract, + gmailMarkUnreadContract, + gmailMoveContract, + gmailRemoveLabelContract, + gmailSendContract, + gmailUnarchiveContract, +} from '@/lib/api/contracts/tools/google' +import { GmailOperationError } from '@/lib/internal/gmail/errors' +import { + executeGmailDraft, + executeGmailEditDraft, + executeGmailSend, + type GmailMailOperationContext, +} from '@/lib/internal/gmail/mail' +import { + executeGmailAddLabel, + executeGmailArchive, + executeGmailDelete, + executeGmailMarkRead, + executeGmailMarkUnread, + executeGmailMove, + executeGmailRemoveLabel, + executeGmailUnarchive, +} from '@/lib/internal/gmail/messages' +import { parseInternalToolInput } from '@/lib/internal/tool-operations/parse-input' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +async function executeOperation( + contract: C, + input: unknown, + execute: (input: ContractBody) => Promise, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const parsed = parseInternalToolInput(contract, input) + if (!parsed.success) return parsed.response + try { + const result = await execute(parsed.data) + signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + signal?.throwIfAborted() + if (error instanceof GmailOperationError) { + return Response.json(error.body ?? { success: false, error: error.message }, { + status: error.status, + }) + } + return Response.json( + { success: false, error: getErrorMessage(error, 'Internal server error') }, + { status: 500 } + ) + } +} + +export const executeGmailTool: InternalToolOperationHandler = async (request) => { + const { input, context, requestId, signal, toolId } = request + const mailContext: GmailMailOperationContext = { + requestId, + signal, + userId: context.userId, + } + switch (toolId) { + case 'gmail_add_label': + case 'gmail_add_label_v2': + return executeOperation( + gmailAddLabelContract, + input, + (input) => executeGmailAddLabel(input, signal), + signal + ) + case 'gmail_archive': + case 'gmail_archive_v2': + return executeOperation( + gmailArchiveContract, + input, + (input) => executeGmailArchive(input, signal), + signal + ) + case 'gmail_delete': + case 'gmail_delete_v2': + return executeOperation( + gmailDeleteContract, + input, + (input) => executeGmailDelete(input, signal), + signal + ) + case 'gmail_draft': + case 'gmail_draft_v2': + return executeOperation( + gmailDraftContract, + input, + (input) => executeGmailDraft(input, mailContext), + signal + ) + case 'gmail_edit_draft_v2': + return executeOperation( + gmailEditDraftContract, + input, + (input) => executeGmailEditDraft(input, mailContext), + signal + ) + case 'gmail_mark_read': + case 'gmail_mark_read_v2': + return executeOperation( + gmailMarkReadContract, + input, + (input) => executeGmailMarkRead(input, signal), + signal + ) + case 'gmail_mark_unread': + case 'gmail_mark_unread_v2': + return executeOperation( + gmailMarkUnreadContract, + input, + (input) => executeGmailMarkUnread(input, signal), + signal + ) + case 'gmail_move': + case 'gmail_move_v2': + return executeOperation( + gmailMoveContract, + input, + (input) => executeGmailMove(input, signal), + signal + ) + case 'gmail_remove_label': + case 'gmail_remove_label_v2': + return executeOperation( + gmailRemoveLabelContract, + input, + (input) => executeGmailRemoveLabel(input, signal), + signal + ) + case 'gmail_send': + case 'gmail_send_v2': + return executeOperation( + gmailSendContract, + input, + (input) => executeGmailSend(input, mailContext), + signal + ) + case 'gmail_unarchive': + case 'gmail_unarchive_v2': + return executeOperation( + gmailUnarchiveContract, + input, + (input) => executeGmailUnarchive(input, signal), + signal + ) + default: + return Response.json( + { success: false, error: `Unsupported Gmail tool: ${toolId}` }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/gmail/mail.ts b/apps/sim/lib/internal/gmail/mail.ts new file mode 100644 index 00000000000..859914d4932 --- /dev/null +++ b/apps/sim/lib/internal/gmail/mail.ts @@ -0,0 +1,222 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { + GmailDraftBody, + GmailEditDraftBody, + GmailSendBody, +} from '@/lib/api/contracts/tools/google' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { asObject, GmailClient } from '@/lib/internal/gmail/client' +import { GmailOperationError } from '@/lib/internal/gmail/errors' +import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/doc-not-ready' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' +import { base64UrlEncode, buildMimeMessage, buildSimpleEmailMessage } from '@/tools/gmail/utils' + +const logger = createLogger('GmailMailOperations') +const GMAIL_ATTACHMENT_MAX_BYTES = 25 * 1024 * 1024 + +export interface GmailMailOperationContext { + requestId: string + signal?: AbortSignal + userId?: string +} + +type MailInput = GmailDraftBody | GmailEditDraftBody | GmailSendBody + +function attachmentSizeError(observedBytes: number): GmailOperationError { + const sizeMB = (observedBytes / (1024 * 1024)).toFixed(2) + return new GmailOperationError( + `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`, + 400, + { + success: false, + error: `Total attachment size (${sizeMB}MB) exceeds Gmail's limit of 25MB`, + } + ) +} + +async function buildRawMessage( + input: MailInput, + context: GmailMailOperationContext, + client: GmailClient +): Promise { + const { requestId, signal, userId } = context + const threading = input.replyToMessageId + ? await client.threadingHeaders(input.replyToMessageId, signal) + : {} + signal?.throwIfAborted() + let rawMessage: string | undefined + + if (input.attachments?.length) { + const attachments = processFilesToUserFiles(input.attachments, requestId, logger) + if (attachments.length > 0) { + const declaredSize = attachments.reduce((total, file) => total + file.size, 0) + if (declaredSize > GMAIL_ATTACHMENT_MAX_BYTES) throw attachmentSizeError(declaredSize) + if (!userId) { + throw new GmailOperationError('Authentication required', 401, { + success: false, + error: 'Authentication required', + }) + } + for (const file of attachments) { + signal?.throwIfAborted() + const denied = await assertToolFileAccess(file.key, userId, requestId, logger) + if (denied) { + throw new GmailOperationError('File not found', denied.status, { + success: false, + error: 'File not found', + }) + } + } + + let resolved: Awaited> + try { + resolved = await downloadServableFilesWithinBudget(attachments, requestId, logger, { + totalMaxBytes: GMAIL_ATTACHMENT_MAX_BYTES, + label: 'Total attachment size', + signal, + }) + } catch (error) { + signal?.throwIfAborted() + if (isDocNotReadyError(error)) { + throw new GmailOperationError(docNotReadyMessage(), 409, { + success: false, + error: docNotReadyMessage(), + }) + } + if (isPayloadSizeLimitError(error)) { + throw attachmentSizeError(error.observedBytes ?? declaredSize) + } + const message = `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}` + throw new GmailOperationError(message, 500, { success: false, error: message }) + } + + const attachmentBuffers = attachments.map((file, index) => { + const resolvedFile = resolved[index] + if (!resolvedFile) { + throw new GmailOperationError('Failed to download attachment: Missing file data', 500, { + success: false, + error: 'Failed to download attachment: Missing file data', + }) + } + return { + filename: file.name, + mimeType: resolvedFile.contentType || file.type || 'application/octet-stream', + content: resolvedFile.buffer, + } + }) + const mimeMessage = buildMimeMessage({ + to: input.to, + cc: input.cc ?? undefined, + bcc: input.bcc ?? undefined, + subject: input.subject || threading.subject || '', + body: input.body, + contentType: input.contentType || 'text', + inReplyTo: threading.messageId, + references: threading.references, + attachments: attachmentBuffers, + }) + rawMessage = base64UrlEncode(mimeMessage) + } + } + + return ( + rawMessage || + buildSimpleEmailMessage({ + to: input.to, + cc: input.cc, + bcc: input.bcc, + subject: input.subject || threading.subject, + body: input.body, + contentType: input.contentType || 'text', + inReplyTo: threading.messageId, + references: threading.references, + }) + ) +} + +function requireUser(context: GmailMailOperationContext): void { + context.signal?.throwIfAborted() + if (!context.userId) { + throw new GmailOperationError('Authentication required', 401, { + success: false, + error: 'Authentication required', + }) + } +} + +export async function executeGmailSend(input: GmailSendBody, context: GmailMailOperationContext) { + requireUser(context) + const client = new GmailClient(input.accessToken) + const raw = await buildRawMessage(input, context, client) + const requestBody: { raw: string; threadId?: string } = { raw } + if (input.threadId) requestBody.threadId = input.threadId + const data = await client.json( + client.api('/messages/send'), + { method: 'POST', body: JSON.stringify(requestBody) }, + context.signal + ) + return { + success: true, + output: { + content: 'Email sent successfully', + metadata: { id: data.id, threadId: data.threadId, labelIds: data.labelIds }, + }, + } +} + +export async function executeGmailDraft(input: GmailDraftBody, context: GmailMailOperationContext) { + requireUser(context) + const client = new GmailClient(input.accessToken) + const raw = await buildRawMessage(input, context, client) + const message: { raw: string; threadId?: string } = { raw } + if (input.threadId) message.threadId = input.threadId + const data = await client.json( + client.api('/drafts'), + { method: 'POST', body: JSON.stringify({ message }) }, + context.signal + ) + const draftMessage = asObject(data.message) + return { + success: true, + output: { + content: 'Email drafted successfully', + metadata: { + id: data.id, + message: { + id: draftMessage.id, + threadId: draftMessage.threadId, + labelIds: draftMessage.labelIds, + }, + }, + }, + } +} + +export async function executeGmailEditDraft( + input: GmailEditDraftBody, + context: GmailMailOperationContext +) { + requireUser(context) + const client = new GmailClient(input.accessToken) + const raw = await buildRawMessage(input, context, client) + const message: { raw: string; threadId?: string } = { raw } + if (input.threadId) message.threadId = input.threadId + const data = await client.json( + client.api(`/drafts/${encodeURIComponent(input.draftId)}`), + { method: 'PUT', body: JSON.stringify({ id: input.draftId, message }) }, + context.signal + ) + const draftMessage = asObject(data.message) + return { + success: true, + output: { + draftId: data.id ?? null, + messageId: draftMessage.id ?? null, + threadId: draftMessage.threadId ?? null, + labelIds: draftMessage.labelIds ?? null, + }, + } +} diff --git a/apps/sim/lib/internal/gmail/messages.ts b/apps/sim/lib/internal/gmail/messages.ts new file mode 100644 index 00000000000..12515bdfec9 --- /dev/null +++ b/apps/sim/lib/internal/gmail/messages.ts @@ -0,0 +1,163 @@ +import type { + GmailAddLabelBody, + GmailArchiveBody, + GmailDeleteBody, + GmailMarkReadBody, + GmailMarkUnreadBody, + GmailMoveBody, + GmailRemoveLabelBody, + GmailUnarchiveBody, +} from '@/lib/api/contracts/tools/google' +import { validateAlphanumericId } from '@/lib/core/security/input-validation' +import { GmailClient, type JsonObject } from '@/lib/internal/gmail/client' +import { GmailOperationError } from '@/lib/internal/gmail/errors' + +interface GmailMessageResultOptions { + content: string + data: JsonObject +} + +function messageResult({ content, data }: GmailMessageResultOptions) { + return { + success: true, + output: { + content, + metadata: { id: data.id, threadId: data.threadId, labelIds: data.labelIds }, + }, + } +} + +function csv(value: string): string[] { + return value + .split(',') + .map((item) => item.trim()) + .filter(Boolean) +} + +function validateMessageAndLabels(messageId: string, labelIds: string[]): void { + if (labelIds.length === 0) { + const message = 'At least one label ID is required' + throw new GmailOperationError(message, 400, { success: false, error: message }) + } + for (const labelId of labelIds) { + const validation = validateAlphanumericId(labelId, 'labelId', 255) + if (!validation.isValid) { + const message = validation.error || 'Invalid labelId' + throw new GmailOperationError(message, 400, { + success: false, + error: message, + }) + } + } + const validation = validateAlphanumericId(messageId, 'messageId', 255) + if (!validation.isValid) { + const message = validation.error || 'Invalid messageId' + throw new GmailOperationError(message, 400, { + success: false, + error: message, + }) + } +} + +async function modifyMessage( + input: + | GmailAddLabelBody + | GmailArchiveBody + | GmailMarkReadBody + | GmailMarkUnreadBody + | GmailMoveBody + | GmailRemoveLabelBody + | GmailUnarchiveBody, + body: { addLabelIds?: string[]; removeLabelIds?: string[] }, + content: string, + signal?: AbortSignal +) { + const client = new GmailClient(input.accessToken) + const data = await client.json( + client.api(`/messages/${encodeURIComponent(input.messageId)}/modify`), + { method: 'POST', body: JSON.stringify(body) }, + signal + ) + return messageResult({ content, data }) +} + +export function executeGmailArchive(input: GmailArchiveBody, signal?: AbortSignal) { + return modifyMessage(input, { removeLabelIds: ['INBOX'] }, 'Email archived successfully', signal) +} + +export function executeGmailMarkRead(input: GmailMarkReadBody, signal?: AbortSignal) { + return modifyMessage( + input, + { removeLabelIds: ['UNREAD'] }, + 'Email marked as read successfully', + signal + ) +} + +export function executeGmailMarkUnread(input: GmailMarkUnreadBody, signal?: AbortSignal) { + return modifyMessage( + input, + { addLabelIds: ['UNREAD'] }, + 'Email marked as unread successfully', + signal + ) +} + +export function executeGmailUnarchive(input: GmailUnarchiveBody, signal?: AbortSignal) { + return modifyMessage( + input, + { addLabelIds: ['INBOX'] }, + 'Email moved back to inbox successfully', + signal + ) +} + +export async function executeGmailDelete(input: GmailDeleteBody, signal?: AbortSignal) { + const client = new GmailClient(input.accessToken) + const data = await client.json( + client.api(`/messages/${encodeURIComponent(input.messageId)}/trash`), + { method: 'POST' }, + signal + ) + return messageResult({ content: 'Email moved to trash successfully', data }) +} + +export function executeGmailAddLabel(input: GmailAddLabelBody, signal?: AbortSignal) { + const labelIds = csv(input.labelIds) + validateMessageAndLabels(input.messageId, labelIds) + return modifyMessage( + input, + { addLabelIds: labelIds }, + `Successfully added ${labelIds.length} label(s) to email`, + signal + ) +} + +export function executeGmailRemoveLabel(input: GmailRemoveLabelBody, signal?: AbortSignal) { + const labelIds = csv(input.labelIds) + validateMessageAndLabels(input.messageId, labelIds) + return modifyMessage( + input, + { removeLabelIds: labelIds }, + `Successfully removed ${labelIds.length} label(s) from email`, + signal + ) +} + +export function executeGmailMove(input: GmailMoveBody, signal?: AbortSignal) { + const addLabelIds = csv(input.addLabelIds) + const removeLabelIds = input.removeLabelIds ? csv(input.removeLabelIds) : [] + validateMessageAndLabels(input.messageId, addLabelIds) + if (removeLabelIds.length > 0) { + validateMessageAndLabels(input.messageId, removeLabelIds) + } + return modifyMessage( + input, + { + ...(addLabelIds.length ? { addLabelIds } : {}), + ...(removeLabelIds.length ? { removeLabelIds } : {}), + }, + 'Email moved successfully', + signal + ) +} diff --git a/apps/sim/lib/internal/gmail/operations.test.ts b/apps/sim/lib/internal/gmail/operations.test.ts new file mode 100644 index 00000000000..bf4ec26feb5 --- /dev/null +++ b/apps/sim/lib/internal/gmail/operations.test.ts @@ -0,0 +1,226 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const fileMocks = vi.hoisted(() => ({ + assertToolFileAccess: vi.fn(), + downloadServableFilesWithinBudget: vi.fn(), + processFilesToUserFiles: vi.fn(), +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: fileMocks.assertToolFileAccess, +})) + +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + processFilesToUserFiles: fileMocks.processFilesToUserFiles, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFilesWithinBudget: fileMocks.downloadServableFilesWithinBudget, +})) + +import { GmailClient } from '@/lib/internal/gmail/client' +import { GmailOperationError } from '@/lib/internal/gmail/errors' +import { executeGmailSend } from '@/lib/internal/gmail/mail' +import { executeGmailAddLabel, executeGmailMove } from '@/lib/internal/gmail/messages' + +const MESSAGE = { accessToken: 'access-token', messageId: 'message-1' } +const MAIL = { + accessToken: 'access-token', + to: 'recipient@example.com', + body: 'Hello', +} +const STORED_FILE = { + id: 'file-1', + key: 'workspace/file.txt', + name: 'file.txt', + size: 4, + type: 'text/plain', +} + +describe('Gmail operations', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.unstubAllGlobals() + fileMocks.assertToolFileAccess.mockResolvedValue(null) + fileMocks.processFilesToUserFiles.mockReturnValue([STORED_FILE]) + fileMocks.downloadServableFilesWithinBudget.mockResolvedValue([ + { buffer: Buffer.from('test'), contentType: 'text/plain' }, + ]) + }) + + it('forwards OAuth credentials and cancellation through message operations', async () => { + const response = Response.json({ id: 'message-1', threadId: 'thread-1', labelIds: ['INBOX'] }) + const fetchMock = vi.fn().mockResolvedValue(response) + vi.stubGlobal('fetch', fetchMock) + const controller = new AbortController() + + await expect( + executeGmailAddLabel({ ...MESSAGE, labelIds: 'INBOX, STARRED' }, controller.signal) + ).resolves.toEqual({ + success: true, + output: { + content: 'Successfully added 2 label(s) to email', + metadata: { id: 'message-1', threadId: 'thread-1', labelIds: ['INBOX'] }, + }, + }) + + expect(fetchMock).toHaveBeenCalledWith( + 'https://gmail.googleapis.com/gmail/v1/users/me/messages/message-1/modify', + expect.objectContaining({ + method: 'POST', + headers: expect.objectContaining({ Authorization: 'Bearer access-token' }), + body: JSON.stringify({ addLabelIds: ['INBOX', 'STARRED'] }), + signal: controller.signal, + }) + ) + expect(response.bodyUsed).toBe(true) + }) + + it('rejects an empty parsed label list before provider work', async () => { + const fetchMock = vi.fn() + vi.stubGlobal('fetch', fetchMock) + + expect(() => executeGmailAddLabel({ ...MESSAGE, labelIds: ' , ' })).toThrow( + 'At least one label ID is required' + ) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('preserves move label parsing and omits empty removals', async () => { + const response = Response.json({ id: 'message-1' }) + const fetchMock = vi.fn().mockResolvedValue(response) + vi.stubGlobal('fetch', fetchMock) + + await executeGmailMove({ ...MESSAGE, addLabelIds: 'IMPORTANT', removeLabelIds: ' , ' }) + + expect(fetchMock).toHaveBeenCalledWith( + expect.stringContaining('/messages/message-1/modify'), + expect.objectContaining({ body: JSON.stringify({ addLabelIds: ['IMPORTANT'] }) }) + ) + }) + + it('rejects empty and invalid move labels before provider work', () => { + const fetchMock = vi.fn() + vi.stubGlobal('fetch', fetchMock) + + expect(() => executeGmailMove({ ...MESSAGE, addLabelIds: ' , ' })).toThrow( + 'At least one label ID is required' + ) + expect(() => executeGmailMove({ ...MESSAGE, addLabelIds: 'INVALID/LABEL' })).toThrow( + 'labelId cannot contain directory separators' + ) + expect(() => + executeGmailMove({ + ...MESSAGE, + addLabelIds: 'IMPORTANT', + removeLabelIds: 'INVALID/LABEL', + }) + ).toThrow('labelId cannot contain directory separators') + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('preserves provider status and cancels unread error bodies', async () => { + const response = new Response('provider details', { + status: 429, + statusText: 'Too Many Requests', + }) + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(response)) + + await expect(executeGmailMove({ ...MESSAGE, addLabelIds: 'IMPORTANT' })).rejects.toEqual( + new GmailOperationError('Gmail API error: Too Many Requests', 429, { + success: false, + error: 'Gmail API error: Too Many Requests', + }) + ) + expect(response.bodyUsed).toBe(true) + }) + + it('authorizes and bounds stored attachments before sending', async () => { + const response = Response.json({ id: 'message-1', threadId: 'thread-1', labelIds: ['SENT'] }) + const fetchMock = vi.fn().mockResolvedValue(response) + vi.stubGlobal('fetch', fetchMock) + const controller = new AbortController() + + await executeGmailSend( + { ...MAIL, attachments: [STORED_FILE] }, + { + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + } + ) + + expect(fileMocks.assertToolFileAccess).toHaveBeenCalledWith( + 'workspace/file.txt', + 'user-1', + 'request-1', + expect.anything() + ) + expect(fileMocks.downloadServableFilesWithinBudget).toHaveBeenCalledWith( + [STORED_FILE], + 'request-1', + expect.anything(), + { + totalMaxBytes: 25 * 1024 * 1024, + label: 'Total attachment size', + signal: controller.signal, + } + ) + expect(fetchMock).toHaveBeenCalledWith( + 'https://gmail.googleapis.com/gmail/v1/users/me/messages/send', + expect.objectContaining({ signal: controller.signal }) + ) + }) + + it('fails closed before file download and provider work when access is denied', async () => { + fileMocks.assertToolFileAccess.mockResolvedValue( + Response.json({ success: false, error: 'File not found' }, { status: 404 }) + ) + const fetchMock = vi.fn() + vi.stubGlobal('fetch', fetchMock) + + await expect( + executeGmailSend( + { ...MAIL, attachments: [STORED_FILE] }, + { requestId: 'request-1', userId: 'user-1' } + ) + ).rejects.toEqual( + new GmailOperationError('File not found', 404, { + success: false, + error: 'File not found', + }) + ) + expect(fileMocks.downloadServableFilesWithinBudget).not.toHaveBeenCalled() + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('requires an acting user before constructing mail', async () => { + const fetchMock = vi.fn() + vi.stubGlobal('fetch', fetchMock) + + await expect(executeGmailSend(MAIL, { requestId: 'request-1' })).rejects.toEqual( + new GmailOperationError('Authentication required', 401, { + success: false, + error: 'Authentication required', + }) + ) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('uses best-effort threading metadata while preserving cancellation', async () => { + const controller = new AbortController() + const fetchMock = vi.fn().mockImplementation(async (_url: string, init: RequestInit) => { + expect(init.signal).toBe(controller.signal) + controller.abort(new DOMException('cancelled', 'AbortError')) + throw controller.signal.reason + }) + vi.stubGlobal('fetch', fetchMock) + + await expect( + new GmailClient('access-token').threadingHeaders('message-1', controller.signal) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) +}) diff --git a/apps/sim/lib/internal/google-drive/client.test.ts b/apps/sim/lib/internal/google-drive/client.test.ts new file mode 100644 index 00000000000..c2f1f965248 --- /dev/null +++ b/apps/sim/lib/internal/google-drive/client.test.ts @@ -0,0 +1,86 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + secureFetch: vi.fn(), + validateUrl: vi.fn(), +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + MAX_JSON_API_RESPONSE_BYTES: 10 * 1024 * 1024, + secureFetchWithPinnedIP: mocks.secureFetch, + validateUrlWithDNS: mocks.validateUrl, +})) + +import { requestGoogleDrive, responseErrorObject } from '@/lib/internal/google-drive/client' + +describe('requestGoogleDrive', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.validateUrl.mockResolvedValue({ isValid: true, resolvedIP: '93.184.216.34' }) + mocks.secureFetch.mockResolvedValue({ ok: true }) + }) + + it('pins the Google host and forwards credentials, caps, and cancellation', async () => { + const controller = new AbortController() + await requestGoogleDrive({ + accessToken: 'token', + headers: { 'Content-Type': 'application/json' }, + label: 'metadataUrl', + method: 'GET', + signal: controller.signal, + url: 'https://www.googleapis.com/drive/v3/files/file-1', + }) + + expect(mocks.validateUrl).toHaveBeenCalledWith( + 'https://www.googleapis.com/drive/v3/files/file-1', + 'metadataUrl' + ) + expect(mocks.secureFetch).toHaveBeenCalledWith( + 'https://www.googleapis.com/drive/v3/files/file-1', + '93.184.216.34', + expect.objectContaining({ + headers: { Authorization: 'Bearer token', 'Content-Type': 'application/json' }, + maxResponseBytes: 10 * 1024 * 1024, + redirectPolicy: { + mode: 'standard', + sendCredentialsOnCrossOriginRedirect: false, + }, + signal: controller.signal, + }) + ) + }) + + it('fails closed before fetching when URL validation fails', async () => { + mocks.validateUrl.mockResolvedValue({ isValid: false, error: 'URL blocked' }) + + await expect( + requestGoogleDrive({ + accessToken: 'token', + label: 'downloadUrl', + url: 'https://www.googleapis.com/drive/v3/files/file-1', + }) + ).rejects.toMatchObject({ + status: 400, + body: { success: false, error: 'URL blocked' }, + }) + expect(mocks.secureFetch).not.toHaveBeenCalled() + }) + + it('discards oversized provider error details within the request cap', async () => { + let cancelled = false + const response = new Response( + new ReadableStream({ + cancel: () => { + cancelled = true + }, + }), + { headers: { 'content-length': String(64 * 1024 + 1) } } + ) + + await expect(responseErrorObject(response)).resolves.toEqual({}) + expect(cancelled).toBe(true) + }) +}) diff --git a/apps/sim/lib/internal/google-drive/client.ts b/apps/sim/lib/internal/google-drive/client.ts new file mode 100644 index 00000000000..3397fcf1fb6 --- /dev/null +++ b/apps/sim/lib/internal/google-drive/client.ts @@ -0,0 +1,85 @@ +import { + MAX_JSON_API_RESPONSE_BYTES, + type SecureFetchResponse, + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + readResponseTextWithLimit, +} from '@/lib/core/utils/stream-limits' +import { GoogleDriveOperationError } from '@/lib/internal/google-drive/errors' + +export interface GoogleDriveRequestOptions { + accessToken: string + body?: Buffer | string | Uint8Array + headers?: Record + label: string + maxResponseBytes?: number + method?: string + signal?: AbortSignal + url: string +} + +export async function requestGoogleDrive( + options: GoogleDriveRequestOptions +): Promise { + options.signal?.throwIfAborted() + const validation = await validateUrlWithDNS(options.url, options.label) + options.signal?.throwIfAborted() + if (!validation.isValid) { + throw new GoogleDriveOperationError(400, { + success: false, + error: validation.error, + }) + } + + return secureFetchWithPinnedIP(options.url, validation.resolvedIP!, { + method: options.method, + headers: { + Authorization: `Bearer ${options.accessToken}`, + ...options.headers, + }, + body: options.body, + maxResponseBytes: options.maxResponseBytes ?? MAX_JSON_API_RESPONSE_BYTES, + redirectPolicy: { + mode: 'standard', + sendCredentialsOnCrossOriginRedirect: false, + }, + signal: options.signal, + }) +} + +export type JsonObject = Record + +export function asObject(value: unknown): JsonObject { + return value !== null && typeof value === 'object' && !Array.isArray(value) + ? (value as JsonObject) + : {} +} + +export async function responseObject(response: SecureFetchResponse): Promise { + return asObject(await response.json()) +} + +export async function responseErrorObject( + response: Pick, + signal?: AbortSignal +): Promise { + try { + const text = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Google Drive error response', + signal, + }) + return text ? asObject(JSON.parse(text)) : {} + } catch { + signal?.throwIfAborted() + return {} + } +} + +export function googleApiErrorMessage(data: JsonObject, fallback: string): string { + const error = asObject(data.error) + return typeof error.message === 'string' && error.message ? error.message : fallback +} diff --git a/apps/sim/lib/internal/google-drive/errors.ts b/apps/sim/lib/internal/google-drive/errors.ts new file mode 100644 index 00000000000..8f6c588cd08 --- /dev/null +++ b/apps/sim/lib/internal/google-drive/errors.ts @@ -0,0 +1,13 @@ +export class GoogleDriveOperationError extends Error { + constructor( + readonly status: number, + readonly body: unknown + ) { + super( + typeof body === 'object' && body !== null && 'error' in body + ? String(body.error) + : 'Google Drive operation failed' + ) + this.name = 'GoogleDriveOperationError' + } +} diff --git a/apps/sim/lib/internal/google-drive/execute-tool.test.ts b/apps/sim/lib/internal/google-drive/execute-tool.test.ts new file mode 100644 index 00000000000..b11eb9de86d --- /dev/null +++ b/apps/sim/lib/internal/google-drive/execute-tool.test.ts @@ -0,0 +1,139 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + download: vi.fn(), + exportFile: vi.fn(), + upload: vi.fn(), +})) + +vi.mock('@/lib/internal/google-drive/operations', () => ({ + executeGoogleDriveDownload: mocks.download, + executeGoogleDriveExport: mocks.exportFile, + executeGoogleDriveUpload: mocks.upload, +})) + +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { GoogleDriveOperationError } from '@/lib/internal/google-drive/errors' +import { executeGoogleDriveTool } from '@/lib/internal/google-drive/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const INPUTS = { + google_drive_download: { accessToken: 'token', fileId: 'file-1' }, + google_drive_export: { accessToken: 'token', fileId: 'file-1', mimeType: 'application/pdf' }, + google_drive_upload: { + accessToken: 'token', + fileName: 'notes.txt', + content: 'hello', + }, +} as const + +const OPERATIONS = { + google_drive_download: mocks.download, + google_drive_export: mocks.exportFile, + google_drive_upload: mocks.upload, +} as const + +function request( + toolId: keyof typeof INPUTS, + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId, + input: INPUTS[toolId], + headers: new Headers(), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + executionId: 'execution-1', + userId: 'user-1', + workspaceId: 'workspace-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeGoogleDriveTool', () => { + beforeEach(() => { + vi.clearAllMocks() + for (const operation of Object.values(OPERATIONS)) { + operation.mockResolvedValue({ success: true, output: { ok: true } }) + } + }) + + it.each(Object.keys(INPUTS) as Array)( + 'validates and dispatches %s with trusted context', + async (toolId) => { + const controller = new AbortController() + const response = await executeGoogleDriveTool(request(toolId, { signal: controller.signal })) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ success: true, output: { ok: true } }) + expect(OPERATIONS[toolId]).toHaveBeenCalledWith(expect.objectContaining(INPUTS[toolId]), { + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + }) + } + ) + + it('preserves validation and provider error envelopes', async () => { + const invalid = await executeGoogleDriveTool( + request('google_drive_export', { input: { accessToken: 'token' } }) + ) + expect(invalid.status).toBe(400) + await expect(invalid.json()).resolves.toEqual({ + success: false, + error: 'Invalid input: expected string, received undefined', + }) + + mocks.exportFile.mockRejectedValue( + new GoogleDriveOperationError(400, { success: false, error: 'Unsupported export' }) + ) + const provider = await executeGoogleDriveTool(request('google_drive_export')) + expect(provider.status).toBe(400) + await expect(provider.json()).resolves.toEqual({ + success: false, + error: 'Unsupported export', + }) + }) + + it('maps oversized export responses to 413', async () => { + mocks.exportFile.mockRejectedValueOnce( + new PayloadSizeLimitError({ + label: 'Google Drive export', + maxBytes: 10, + observedBytes: 11, + }) + ) + + const response = await executeGoogleDriveTool(request('google_drive_export')) + + expect(response.status).toBe(413) + await expect(response.json()).resolves.toMatchObject({ + success: false, + error: expect.stringContaining('Google Drive export'), + }) + }) + + it('propagates cancellation before and after operation work', async () => { + const before = new AbortController() + before.abort(new DOMException('cancelled', 'AbortError')) + await expect( + executeGoogleDriveTool(request('google_drive_download', { signal: before.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mocks.download).not.toHaveBeenCalled() + + const after = new AbortController() + mocks.download.mockImplementationOnce(async () => { + after.abort(new DOMException('cancelled', 'AbortError')) + return { success: true } + }) + await expect( + executeGoogleDriveTool(request('google_drive_download', { signal: after.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) +}) diff --git a/apps/sim/lib/internal/google-drive/execute-tool.ts b/apps/sim/lib/internal/google-drive/execute-tool.ts new file mode 100644 index 00000000000..ed267fe5770 --- /dev/null +++ b/apps/sim/lib/internal/google-drive/execute-tool.ts @@ -0,0 +1,122 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { z } from 'zod' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { + isPayloadSizeLimitError, + MAX_MULTIPART_OVERHEAD_BYTES, +} from '@/lib/core/utils/stream-limits' +import { GoogleDriveOperationError } from '@/lib/internal/google-drive/errors' +import { + googleDriveDownloadInputSchema, + googleDriveExportInputSchema, + googleDriveUploadInputSchema, +} from '@/lib/internal/google-drive/input' +import { + executeGoogleDriveDownload, + executeGoogleDriveExport, + executeGoogleDriveUpload, + type GoogleDriveOperationContext, +} from '@/lib/internal/google-drive/operations' +import type { + InternalToolOperationCall, + InternalToolOperationHandler, +} from '@/lib/internal/tool-operations/types' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' + +const logger = createLogger('GoogleDriveToolExecution') +const MAX_UPLOAD_INPUT_BYTES = MAX_BUFFERED_TRANSFER_BYTES + MAX_MULTIPART_OVERHEAD_BYTES + +function inputLimit(toolId: string): number { + return toolId === 'google_drive_upload' ? MAX_UPLOAD_INPUT_BYTES : DEFAULT_MAX_JSON_BODY_BYTES +} + +function validationResponse(error: z.ZodError): Response { + return Response.json( + { success: false, error: error.issues[0]?.message || 'Invalid request' }, + { status: 400 } + ) +} + +function parseInput(schema: S, input: unknown): z.output | Response { + const parsed = schema.safeParse(input) + return parsed.success ? parsed.data : validationResponse(parsed.error) +} + +async function dispatch( + request: InternalToolOperationCall, + context: GoogleDriveOperationContext +): Promise { + switch (request.toolId) { + case 'google_drive_download': { + const input = parseInput(googleDriveDownloadInputSchema, request.input) + return input instanceof Response ? input : executeGoogleDriveDownload(input, context) + } + case 'google_drive_export': { + const input = parseInput(googleDriveExportInputSchema, request.input) + return input instanceof Response ? input : executeGoogleDriveExport(input, context) + } + case 'google_drive_upload': { + const input = parseInput(googleDriveUploadInputSchema, request.input) + return input instanceof Response ? input : executeGoogleDriveUpload(input, context) + } + default: + return Response.json( + { success: false, error: `Unsupported Google Drive tool: ${request.toolId}` }, + { status: 500 } + ) + } +} + +function unexpectedResponse(request: InternalToolOperationCall, error: unknown): Response { + const fallback = + request.toolId === 'google_drive_upload' ? 'Internal server error' : 'Unknown error occurred' + const message = getErrorMessage(error, fallback) + logger.error('Google Drive operation failed', { + error: message, + requestId: request.requestId, + toolId: request.toolId, + }) + const status = + ['google_drive_download', 'google_drive_export'].includes(request.toolId) && + isPayloadSizeLimitError(error) + ? 413 + : 500 + return Response.json({ success: false, error: message }, { status }) +} + +export const executeGoogleDriveTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + let serialized: string + try { + serialized = JSON.stringify(request.input) ?? '' + } catch { + return Response.json({ success: false, error: 'Invalid request' }, { status: 400 }) + } + const maxBytes = inputLimit(request.toolId) + if (Buffer.byteLength(serialized, 'utf8') > maxBytes) { + return Response.json( + { + success: false, + error: `Request body exceeds the maximum allowed size of ${maxBytes} bytes`, + }, + { status: 413 } + ) + } + + try { + const result = await dispatch(request, { + requestId: request.requestId, + signal: request.signal, + userId: request.context.userId, + }) + request.signal?.throwIfAborted() + return result instanceof Response ? result : Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof GoogleDriveOperationError) { + return Response.json(error.body, { status: error.status }) + } + return unexpectedResponse(request, error) + } +} diff --git a/apps/sim/lib/internal/google-drive/file-input.test.ts b/apps/sim/lib/internal/google-drive/file-input.test.ts new file mode 100644 index 00000000000..7169621a744 --- /dev/null +++ b/apps/sim/lib/internal/google-drive/file-input.test.ts @@ -0,0 +1,97 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + assertAccess: vi.fn(), + download: vi.fn(), + process: vi.fn(), +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertAccess, +})) + +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + processSingleFileToUserFile: mocks.process, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mocks.download, +})) + +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { resolveGoogleDriveUploadFile } from '@/lib/internal/google-drive/file-input' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' + +const file = { key: 'workspace/file.txt', name: 'file.txt', size: 4 } + +describe('resolveGoogleDriveUploadFile', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.process.mockReturnValue({ ...file, type: 'text/plain' }) + mocks.assertAccess.mockResolvedValue(null) + mocks.download.mockResolvedValue({ buffer: Buffer.from('test'), contentType: 'text/plain' }) + }) + + it('requires a trusted user before protected file access', async () => { + await expect( + resolveGoogleDriveUploadFile(file, { requestId: 'request-1' }) + ).rejects.toMatchObject({ + status: 401, + body: { success: false, error: 'Authentication required' }, + }) + expect(mocks.assertAccess).not.toHaveBeenCalled() + }) + + it('fails closed on denied access', async () => { + mocks.assertAccess.mockResolvedValue( + Response.json({ success: false, error: 'File not found' }, { status: 404 }) + ) + + await expect( + resolveGoogleDriveUploadFile(file, { + requestId: 'request-1', + userId: 'user-1', + }) + ).rejects.toMatchObject({ + status: 404, + body: { success: false, error: 'File not found' }, + }) + expect(mocks.download).not.toHaveBeenCalled() + }) + + it('passes aggregate cap and cancellation through the servable-file resolver', async () => { + const controller = new AbortController() + await resolveGoogleDriveUploadFile(file, { + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + }) + + expect(mocks.download).toHaveBeenCalledWith( + expect.objectContaining({ key: file.key }), + 'request-1', + expect.anything(), + { maxBytes: MAX_BUFFERED_TRANSFER_BYTES, signal: controller.signal } + ) + }) + + it('preserves the clean 413 file-download envelope', async () => { + mocks.download.mockRejectedValue( + new PayloadSizeLimitError({ + label: 'file', + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + observedBytes: MAX_BUFFERED_TRANSFER_BYTES + 1, + }) + ) + + await expect( + resolveGoogleDriveUploadFile(file, { + requestId: 'request-1', + userId: 'user-1', + }) + ).rejects.toMatchObject({ status: 413 }) + }) +}) diff --git a/apps/sim/lib/internal/google-drive/file-input.ts b/apps/sim/lib/internal/google-drive/file-input.ts new file mode 100644 index 00000000000..1f383a943b3 --- /dev/null +++ b/apps/sim/lib/internal/google-drive/file-input.ts @@ -0,0 +1,68 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { GoogleDriveOperationError } from '@/lib/internal/google-drive/errors' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' +import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +const logger = createLogger('GoogleDriveFileInput') + +async function bodyFromResponse(response: Response): Promise { + try { + return await response.json() + } catch { + return { success: false, error: response.statusText || 'File operation failed' } + } +} + +export async function resolveGoogleDriveUploadFile( + file: RawFileInput, + context: { requestId: string; signal?: AbortSignal; userId?: string } +) { + context.signal?.throwIfAborted() + if (!context.userId) { + throw new GoogleDriveOperationError(401, { + success: false, + error: 'Authentication required', + }) + } + + let userFile + try { + userFile = processSingleFileToUserFile(file, context.requestId, logger) + } catch (error) { + throw new GoogleDriveOperationError(400, { + success: false, + error: getErrorMessage(error, 'Failed to process file'), + }) + } + + const denied = await assertToolFileAccess(userFile.key, context.userId, context.requestId, logger) + context.signal?.throwIfAborted() + if (denied) { + throw new GoogleDriveOperationError(denied.status, await bodyFromResponse(denied)) + } + + try { + const downloaded = await downloadServableFileFromStorage(userFile, context.requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + signal: context.signal, + }) + context.signal?.throwIfAborted() + return { userFile, ...downloaded } + } catch (error) { + context.signal?.throwIfAborted() + const notReady = docNotReadyResponse(error) + if (notReady) { + throw new GoogleDriveOperationError(notReady.status, await bodyFromResponse(notReady)) + } + throw new GoogleDriveOperationError(isPayloadSizeLimitError(error) ? 413 : 500, { + success: false, + error: `Failed to download file: ${getErrorMessage(error, 'Unknown error')}`, + }) + } +} diff --git a/apps/sim/lib/internal/google-drive/input.ts b/apps/sim/lib/internal/google-drive/input.ts new file mode 100644 index 00000000000..062f06907ba --- /dev/null +++ b/apps/sim/lib/internal/google-drive/input.ts @@ -0,0 +1,32 @@ +import { z } from 'zod' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' + +const googleAccessTokenSchema = z.string().min(1, 'Access token is required') + +export const googleDriveUploadInputSchema = z.object({ + accessToken: googleAccessTokenSchema, + fileName: z.string().min(1, 'File name is required'), + file: RawFileInputSchema.optional().nullable(), + content: z.string().optional(), + mimeType: z.string().optional().nullable(), + folderId: z.string().optional().nullable(), +}) + +export const googleDriveDownloadInputSchema = z.object({ + accessToken: googleAccessTokenSchema, + fileId: z.string().min(1, 'File ID is required'), + mimeType: z.string().optional().nullable(), + fileName: z.string().optional().nullable(), + includeRevisions: z.boolean().optional().default(true), +}) + +export const googleDriveExportInputSchema = z.object({ + accessToken: googleAccessTokenSchema, + fileId: z.string().min(1, 'File ID is required'), + mimeType: z.string().min(1, 'Target export MIME type is required'), + fileName: z.string().optional().nullable(), +}) + +export type GoogleDriveUploadInput = z.output +export type GoogleDriveDownloadInput = z.output +export type GoogleDriveExportInput = z.output diff --git a/apps/sim/lib/internal/google-drive/operations.test.ts b/apps/sim/lib/internal/google-drive/operations.test.ts new file mode 100644 index 00000000000..bd3046b17cb --- /dev/null +++ b/apps/sim/lib/internal/google-drive/operations.test.ts @@ -0,0 +1,202 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + request: vi.fn(), + resolveFile: vi.fn(), +})) + +vi.mock('@/lib/internal/google-drive/client', () => ({ + asObject: (value: unknown) => + value !== null && typeof value === 'object' && !Array.isArray(value) ? value : {}, + googleApiErrorMessage: (data: { error?: { message?: string } }, fallback: string) => + data.error?.message || fallback, + requestGoogleDrive: mocks.request, + responseObject: async (response: { json: () => Promise }) => response.json(), +})) + +vi.mock('@/lib/internal/google-drive/file-input', () => ({ + resolveGoogleDriveUploadFile: mocks.resolveFile, +})) + +import { + executeGoogleDriveDownload, + executeGoogleDriveExport, + executeGoogleDriveUpload, +} from '@/lib/internal/google-drive/operations' +import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' +import { MAX_EXPORT_BYTES } from '@/tools/google_drive/utils' + +function response(body: unknown, options: { ok?: boolean; status?: number; bytes?: number } = {}) { + const bytes = options.bytes ?? 0 + return { + ok: options.ok ?? true, + status: options.status ?? 200, + statusText: options.ok === false ? 'Bad Request' : 'OK', + headers: new Headers(), + body: null, + json: async () => body, + text: async () => JSON.stringify(body), + arrayBuffer: async () => new ArrayBuffer(bytes), + } +} + +const context = { + requestId: 'request-1', + signal: new AbortController().signal, + userId: 'user-1', +} + +describe('Google Drive operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.resolveFile.mockResolvedValue({ + buffer: Buffer.from('file'), + contentType: 'text/plain', + userFile: { key: 'workspace/file.txt', name: 'file.txt', size: 4, type: 'text/plain' }, + }) + }) + + it('downloads regular files with metadata and binary caps', async () => { + mocks.request + .mockResolvedValueOnce( + response({ + id: 'file-1', + name: 'report.pdf', + mimeType: 'application/pdf', + size: '4', + capabilities: { canReadRevisions: false }, + }) + ) + .mockResolvedValueOnce(response({}, { bytes: 4 })) + + const result = await executeGoogleDriveDownload( + { accessToken: 'token', fileId: 'file-1', includeRevisions: true }, + context + ) + + expect(mocks.request.mock.calls[1]?.[0]).toMatchObject({ + label: 'downloadUrl', + maxResponseBytes: MAX_FILE_SIZE, + signal: context.signal, + }) + expect(result.output.file).toEqual({ + name: 'report.pdf', + mimeType: 'application/pdf', + data: 'AAAAAA==', + size: 4, + }) + }) + + it('keeps revision lookup optional and bounded', async () => { + mocks.request + .mockResolvedValueOnce( + response({ + id: 'file-1', + name: 'report.pdf', + mimeType: 'application/pdf', + capabilities: { canReadRevisions: true }, + }) + ) + .mockResolvedValueOnce(response({}, { bytes: 1 })) + .mockResolvedValueOnce(response({ revisions: [{ id: 'rev-1' }] })) + + const result = await executeGoogleDriveDownload( + { accessToken: 'token', fileId: 'file-1', includeRevisions: true }, + context + ) + + expect(mocks.request.mock.calls[2]?.[0]).toMatchObject({ + label: 'revisionsUrl', + signal: context.signal, + }) + expect(result.output.metadata.revisions).toEqual([{ id: 'rev-1' }]) + }) + + it('preserves the export byte limit and exact error', async () => { + mocks.request + .mockResolvedValueOnce( + response({ + id: 'doc-1', + name: 'Doc', + mimeType: 'application/vnd.google-apps.document', + }) + ) + .mockResolvedValueOnce(response({}, { bytes: MAX_EXPORT_BYTES + 1 })) + + await expect( + executeGoogleDriveExport( + { accessToken: 'token', fileId: 'doc-1', mimeType: 'application/pdf' }, + context + ) + ).rejects.toMatchObject({ + status: 413, + body: { + success: false, + error: `Exported content (${MAX_EXPORT_BYTES + 1} bytes) exceeds the ${MAX_EXPORT_BYTES}-byte export limit.`, + }, + }) + expect(mocks.request.mock.calls[1]?.[0]).toMatchObject({ + maxResponseBytes: MAX_EXPORT_BYTES, + signal: context.signal, + }) + }) + + it('runs text uploads entirely inside the typed operation', async () => { + mocks.request + .mockResolvedValueOnce(response({ id: 'file-1' })) + .mockResolvedValueOnce(response({})) + .mockResolvedValueOnce(response({ id: 'file-1', name: 'notes.txt', mimeType: 'text/plain' })) + + const result = await executeGoogleDriveUpload( + { + accessToken: 'token', + fileName: 'notes.txt', + content: 'hello', + mimeType: 'text/plain', + }, + context + ) + + expect(mocks.request.mock.calls.map((call) => call[0].label)).toEqual([ + 'createFileUrl', + 'uploadContentUrl', + 'finalFileUrl', + ]) + expect(mocks.request.mock.calls[1]?.[0]).toMatchObject({ + body: 'hello', + method: 'PATCH', + signal: context.signal, + }) + expect(result.output.file).toMatchObject({ id: 'file-1', name: 'notes.txt' }) + }) + + it('uses the authorized stored-file resolver and multipart provider upload', async () => { + mocks.request + .mockResolvedValueOnce(response({ id: 'file-1' })) + .mockResolvedValueOnce(response({ id: 'file-1', name: 'file.txt', mimeType: 'text/plain' })) + + const result = await executeGoogleDriveUpload( + { + accessToken: 'token', + fileName: 'file.txt', + file: { key: 'workspace/file.txt', name: 'file.txt', size: 4 }, + }, + context + ) + + expect(mocks.resolveFile).toHaveBeenCalledWith( + { key: 'workspace/file.txt', name: 'file.txt', size: 4 }, + context + ) + expect(mocks.request.mock.calls[0]?.[0]).toMatchObject({ + label: 'uploadFileUrl', + method: 'POST', + signal: context.signal, + }) + expect(String(mocks.request.mock.calls[0]?.[0].body)).toContain('ZmlsZQ==') + expect(result.output.file).toMatchObject({ id: 'file-1', name: 'file.txt' }) + }) +}) diff --git a/apps/sim/lib/internal/google-drive/operations.ts b/apps/sim/lib/internal/google-drive/operations.ts new file mode 100644 index 00000000000..48554453620 --- /dev/null +++ b/apps/sim/lib/internal/google-drive/operations.ts @@ -0,0 +1,471 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { generateShortId } from '@sim/utils/id' +import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { assertKnownSizeWithinLimit, isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { + googleApiErrorMessage, + requestGoogleDrive, + responseErrorObject, + responseObject, +} from '@/lib/internal/google-drive/client' +import { GoogleDriveOperationError } from '@/lib/internal/google-drive/errors' +import { resolveGoogleDriveUploadFile } from '@/lib/internal/google-drive/file-input' +import type { + GoogleDriveDownloadInput, + GoogleDriveExportInput, + GoogleDriveUploadInput, +} from '@/lib/internal/google-drive/input' +import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' +import type { GoogleDriveFile, GoogleDriveRevision } from '@/tools/google_drive/types' +import { + ALL_FILE_FIELDS, + ALL_REVISION_FIELDS, + DEFAULT_EXPORT_FORMATS, + GOOGLE_WORKSPACE_MIME_TYPES, + handleSheetsFormat, + MAX_EXPORT_BYTES, + SOURCE_MIME_TYPES, + VALID_EXPORT_FORMATS, +} from '@/tools/google_drive/utils' + +const logger = createLogger('GoogleDriveInternalOperation') +const DRIVE_FILES_URL = 'https://www.googleapis.com/drive/v3/files' +const DRIVE_UPLOAD_URL = 'https://www.googleapis.com/upload/drive/v3/files' + +export interface GoogleDriveOperationContext { + requestId: string + signal?: AbortSignal + userId?: string +} + +interface GoogleDriveRevisionsResponse { + revisions?: GoogleDriveRevision[] +} + +function operationError(status: number, error: string): GoogleDriveOperationError { + return new GoogleDriveOperationError(status, { success: false, error }) +} + +async function providerJsonError( + response: Awaited>, + fallback: string, + status = 400, + signal?: AbortSignal +): Promise { + const data = await responseErrorObject(response, signal) + throw operationError(status, googleApiErrorMessage(data, fallback)) +} + +async function getFileMetadata( + fileId: string, + accessToken: string, + signal?: AbortSignal +): Promise { + const response = await requestGoogleDrive({ + accessToken, + label: 'metadataUrl', + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + signal, + url: `${DRIVE_FILES_URL}/${encodeURIComponent(fileId)}?fields=${ALL_FILE_FIELDS}&supportsAllDrives=true`, + }) + if (!response.ok) await providerJsonError(response, 'Failed to get file metadata', 400, signal) + const data = await responseObject(response) + return { + ...data, + id: typeof data.id === 'string' ? data.id : '', + name: typeof data.name === 'string' ? data.name : '', + mimeType: typeof data.mimeType === 'string' ? data.mimeType : '', + } as GoogleDriveFile +} + +async function updateWorkspaceFileName( + fileId: string, + fileName: string, + accessToken: string, + signal?: AbortSignal +): Promise { + const response = await requestGoogleDrive({ + accessToken, + body: JSON.stringify({ name: fileName }), + headers: { 'Content-Type': 'application/json' }, + label: 'updateNameUrl', + method: 'PATCH', + signal, + url: `${DRIVE_FILES_URL}/${encodeURIComponent(fileId)}?supportsAllDrives=true`, + }) + await response.text() + if (!response.ok) { + logger.warn('Failed to update filename after conversion, but content was uploaded', { + fileId, + status: response.status, + }) + } +} + +async function fetchFinalFile( + fileId: string, + accessToken: string, + fields: string, + signal?: AbortSignal +): Promise> { + const response = await requestGoogleDrive({ + accessToken, + label: 'finalFileUrl', + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + signal, + url: `${DRIVE_FILES_URL}/${encodeURIComponent(fileId)}?supportsAllDrives=true&fields=${fields}`, + }) + return responseObject(response) +} + +export async function executeGoogleDriveDownload( + input: GoogleDriveDownloadInput, + context: GoogleDriveOperationContext +) { + context.signal?.throwIfAborted() + const metadata = await getFileMetadata(input.fileId, input.accessToken, context.signal) + const fileMimeType = metadata.mimeType + const requestedExportMimeType = + input.mimeType && input.mimeType !== 'auto' ? input.mimeType : null + let fileBuffer: Buffer + let finalMimeType = fileMimeType + + if (GOOGLE_WORKSPACE_MIME_TYPES.includes(fileMimeType)) { + const exportFormat = + requestedExportMimeType || DEFAULT_EXPORT_FORMATS[fileMimeType] || 'text/plain' + const validFormats = VALID_EXPORT_FORMATS[fileMimeType] + if (validFormats && !validFormats.includes(exportFormat)) { + throw operationError( + 400, + `Export format "${exportFormat}" is not supported for this file type. Supported formats: ${validFormats.join(', ')}` + ) + } + finalMimeType = exportFormat + const response = await requestGoogleDrive({ + accessToken: input.accessToken, + label: 'exportUrl', + maxResponseBytes: MAX_FILE_SIZE, + signal: context.signal, + url: `${DRIVE_FILES_URL}/${encodeURIComponent(input.fileId)}/export?mimeType=${encodeURIComponent(exportFormat)}&supportsAllDrives=true`, + }) + if (!response.ok) { + await providerJsonError( + response, + 'Failed to export Google Workspace file', + 400, + context.signal + ) + } + fileBuffer = Buffer.from(await response.arrayBuffer()) + } else { + if (metadata.size) { + const parsedSize = Number.parseInt(metadata.size, 10) + if (Number.isFinite(parsedSize)) { + assertKnownSizeWithinLimit(parsedSize, MAX_FILE_SIZE, `Google Drive file ${input.fileId}`) + } + } + const response = await requestGoogleDrive({ + accessToken: input.accessToken, + label: 'downloadUrl', + maxResponseBytes: MAX_FILE_SIZE, + signal: context.signal, + url: `${DRIVE_FILES_URL}/${encodeURIComponent(input.fileId)}?alt=media&supportsAllDrives=true`, + }) + if (!response.ok) { + await providerJsonError(response, 'Failed to download file', 400, context.signal) + } + fileBuffer = Buffer.from(await response.arrayBuffer()) + } + + if (input.includeRevisions && metadata.capabilities?.canReadRevisions === true) { + try { + const response = await requestGoogleDrive({ + accessToken: input.accessToken, + label: 'revisionsUrl', + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + signal: context.signal, + url: `${DRIVE_FILES_URL}/${encodeURIComponent(input.fileId)}/revisions?fields=revisions(${ALL_REVISION_FIELDS})&pageSize=100`, + }) + if (response.ok) { + const revisions = (await response.json()) as GoogleDriveRevisionsResponse + metadata.revisions = revisions.revisions + } else { + await response.text() + } + } catch (error) { + context.signal?.throwIfAborted() + logger.warn('Error fetching Google Drive revisions, continuing without them', { + error: getErrorMessage(error), + fileId: input.fileId, + }) + } + } + + context.signal?.throwIfAborted() + return { + success: true, + output: { + file: { + name: input.fileName || metadata.name || 'download', + mimeType: finalMimeType, + data: fileBuffer.toString('base64'), + size: fileBuffer.length, + }, + metadata, + }, + } +} + +export async function executeGoogleDriveExport( + input: GoogleDriveExportInput, + context: GoogleDriveOperationContext +) { + context.signal?.throwIfAborted() + const metadata = await getFileMetadata(input.fileId, input.accessToken, context.signal) + if (!GOOGLE_WORKSPACE_MIME_TYPES.includes(metadata.mimeType)) { + throw operationError( + 400, + `Export only supports Google Workspace files (Docs, Sheets, Slides, Drawings). This file is "${metadata.mimeType}" — use the Download operation instead.` + ) + } + const validFormats = VALID_EXPORT_FORMATS[metadata.mimeType] + if (validFormats && !validFormats.includes(input.mimeType)) { + throw operationError( + 400, + `Export format "${input.mimeType}" is not supported for this file type. Supported formats: ${validFormats.join(', ')}` + ) + } + + let response: Awaited> + try { + response = await requestGoogleDrive({ + accessToken: input.accessToken, + label: 'exportUrl', + maxResponseBytes: MAX_EXPORT_BYTES, + signal: context.signal, + url: `${DRIVE_FILES_URL}/${encodeURIComponent(input.fileId)}/export?mimeType=${encodeURIComponent(input.mimeType)}`, + }) + } catch (error) { + context.signal?.throwIfAborted() + if (isPayloadSizeLimitError(error)) { + const observed = error.observedBytes + throw operationError( + 413, + observed === undefined + ? `Exported content exceeds the ${MAX_EXPORT_BYTES}-byte export limit.` + : `Exported content (${observed} bytes) exceeds the ${MAX_EXPORT_BYTES}-byte export limit.` + ) + } + throw error + } + if (!response.ok) { + await providerJsonError(response, 'Failed to export Google Workspace file', 400, context.signal) + } + + const declaredSize = Number(response.headers.get('content-length')) + if (Number.isFinite(declaredSize) && declaredSize > MAX_EXPORT_BYTES) { + throw operationError( + 413, + `Exported content (${declaredSize} bytes) exceeds the ${MAX_EXPORT_BYTES}-byte export limit.` + ) + } + const arrayBuffer = await response.arrayBuffer() + if (arrayBuffer.byteLength > MAX_EXPORT_BYTES) { + throw operationError( + 413, + `Exported content (${arrayBuffer.byteLength} bytes) exceeds the ${MAX_EXPORT_BYTES}-byte export limit.` + ) + } + const fileBuffer = Buffer.from(arrayBuffer) + return { + success: true, + output: { + file: { + name: input.fileName || metadata.name || 'export', + mimeType: input.mimeType, + data: fileBuffer.toString('base64'), + size: fileBuffer.length, + }, + exportedMimeType: input.mimeType, + }, + } +} + +function uploadMetadata(input: GoogleDriveUploadInput, requestedMimeType: string) { + return { + name: input.fileName, + mimeType: requestedMimeType, + ...(input.folderId?.trim() ? { parents: [input.folderId.trim()] } : {}), + } +} + +function prepareTextContent(input: GoogleDriveUploadInput, requestedMimeType: string): string { + if (requestedMimeType !== 'application/vnd.google-apps.spreadsheet' || !input.content) { + return input.content || '' + } + const { csv } = handleSheetsFormat(input.content as unknown) + return csv ?? input.content +} + +async function uploadTextContent( + input: GoogleDriveUploadInput, + context: GoogleDriveOperationContext +) { + const requestedMimeType = input.mimeType || 'text/plain' + const createResponse = await requestGoogleDrive({ + accessToken: input.accessToken, + body: JSON.stringify(uploadMetadata(input, requestedMimeType)), + headers: { 'Content-Type': 'application/json' }, + label: 'createFileUrl', + method: 'POST', + signal: context.signal, + url: `${DRIVE_FILES_URL}?supportsAllDrives=true`, + }) + const created = await responseObject(createResponse) + if (!createResponse.ok) { + throw operationError( + createResponse.status, + googleApiErrorMessage(created, 'Failed to create file in Google Drive') + ) + } + const fileId = typeof created.id === 'string' ? created.id : '' + const uploadMimeType = GOOGLE_WORKSPACE_MIME_TYPES.includes(requestedMimeType) + ? SOURCE_MIME_TYPES[requestedMimeType] || 'text/plain' + : requestedMimeType + const contentResponse = await requestGoogleDrive({ + accessToken: input.accessToken, + body: prepareTextContent(input, requestedMimeType), + headers: { 'Content-Type': uploadMimeType }, + label: 'uploadContentUrl', + method: 'PATCH', + signal: context.signal, + url: `https://www.googleapis.com/upload/drive/v3/files/${encodeURIComponent(fileId)}?uploadType=media&supportsAllDrives=true`, + }) + if (!contentResponse.ok) { + await providerJsonError( + contentResponse, + 'Failed to upload content to file', + contentResponse.status, + context.signal + ) + } + await contentResponse.text() + + if (GOOGLE_WORKSPACE_MIME_TYPES.includes(requestedMimeType)) { + await updateWorkspaceFileName(fileId, input.fileName, input.accessToken, context.signal) + } + const finalFile = await fetchFinalFile(fileId, input.accessToken, ALL_FILE_FIELDS, context.signal) + return { success: true, output: { file: finalFile } } +} + +function buildMultipartBody( + metadata: Record, + fileBuffer: Buffer, + mimeType: string, + boundary: string +): string { + return [ + `--${boundary}`, + 'Content-Type: application/json; charset=UTF-8', + '', + JSON.stringify(metadata), + `--${boundary}`, + `Content-Type: ${mimeType}`, + 'Content-Transfer-Encoding: base64', + '', + fileBuffer.toString('base64'), + `--${boundary}--`, + ].join('\r\n') +} + +async function uploadStoredFile( + input: GoogleDriveUploadInput & { file: NonNullable }, + context: GoogleDriveOperationContext +) { + const resolved = await resolveGoogleDriveUploadFile(input.file, context) + let fileBuffer = resolved.buffer + let uploadMimeType = + input.mimeType || resolved.contentType || resolved.userFile.type || 'application/octet-stream' + const requestedMimeType = uploadMimeType + if (GOOGLE_WORKSPACE_MIME_TYPES.includes(requestedMimeType)) { + uploadMimeType = SOURCE_MIME_TYPES[requestedMimeType] || 'text/plain' + } + if (requestedMimeType === 'application/vnd.google-apps.spreadsheet') { + try { + const { csv } = handleSheetsFormat(fileBuffer.toString('utf-8')) + if (csv !== undefined) { + fileBuffer = Buffer.from(csv, 'utf-8') + uploadMimeType = 'text/csv' + } + } catch (error) { + logger.warn('Could not convert Google Sheets upload to CSV, uploading as-is', { + error: getErrorMessage(error), + }) + } + } + + const boundary = `boundary_${Date.now()}_${generateShortId(7)}` + const body = buildMultipartBody( + uploadMetadata(input, requestedMimeType), + fileBuffer, + uploadMimeType, + boundary + ) + const uploadResponse = await requestGoogleDrive({ + accessToken: input.accessToken, + body, + headers: { + 'Content-Type': `multipart/related; boundary=${boundary}`, + 'Content-Length': Buffer.byteLength(body, 'utf-8').toString(), + }, + label: 'uploadFileUrl', + method: 'POST', + signal: context.signal, + url: `${DRIVE_UPLOAD_URL}?uploadType=multipart&supportsAllDrives=true`, + }) + if (!uploadResponse.ok) { + await responseErrorObject(uploadResponse, context.signal) + throw operationError( + uploadResponse.status, + `Google Drive API error: ${uploadResponse.statusText}` + ) + } + const uploaded = await responseObject(uploadResponse) + const fileId = typeof uploaded.id === 'string' ? uploaded.id : '' + if (GOOGLE_WORKSPACE_MIME_TYPES.includes(requestedMimeType)) { + await updateWorkspaceFileName(fileId, input.fileName, input.accessToken, context.signal) + } + const finalFile = await fetchFinalFile( + fileId, + input.accessToken, + 'id,name,mimeType,webViewLink,webContentLink,size,createdTime,modifiedTime,parents', + context.signal + ) + return { + success: true, + output: { + file: { + id: finalFile.id, + name: finalFile.name, + mimeType: finalFile.mimeType, + webViewLink: finalFile.webViewLink, + webContentLink: finalFile.webContentLink, + size: finalFile.size, + createdTime: finalFile.createdTime, + modifiedTime: finalFile.modifiedTime, + parents: finalFile.parents, + }, + }, + } +} + +export async function executeGoogleDriveUpload( + input: GoogleDriveUploadInput, + context: GoogleDriveOperationContext +) { + context.signal?.throwIfAborted() + return input.file + ? uploadStoredFile({ ...input, file: input.file }, context) + : uploadTextContent(input, context) +} diff --git a/apps/sim/lib/internal/google-slides/errors.ts b/apps/sim/lib/internal/google-slides/errors.ts new file mode 100644 index 00000000000..d8fb80e6024 --- /dev/null +++ b/apps/sim/lib/internal/google-slides/errors.ts @@ -0,0 +1,9 @@ +export class GoogleSlidesOperationError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'GoogleSlidesOperationError' + } +} diff --git a/apps/sim/lib/internal/google-slides/execute-tool.test.ts b/apps/sim/lib/internal/google-slides/execute-tool.test.ts new file mode 100644 index 00000000000..a7ad27614cf --- /dev/null +++ b/apps/sim/lib/internal/google-slides/execute-tool.test.ts @@ -0,0 +1,70 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ exportGoogleSlidesPresentation: vi.fn() })) + +vi.mock('@/lib/internal/google-slides/operations', () => ({ + exportGoogleSlidesPresentation: mocks.exportGoogleSlidesPresentation, +})) + +import { executeGoogleSlidesTool } from '@/lib/internal/google-slides/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +function request(overrides: Partial = {}): InternalToolOperationCall { + return { + toolId: 'google_slides_export_presentation', + input: { accessToken: 'token', presentationId: 'presentation-1' }, + headers: new Headers(), + context: { + ...createExecutionContext({ + workflowId: 'workflow-1', + executionId: 'execution-1', + }), + userId: 'user-1', + workspaceId: 'workspace-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeGoogleSlidesTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.exportGoogleSlidesPresentation.mockResolvedValue({ success: true, output: {} }) + }) + + it('uses trusted execution context and normalized input', async () => { + const controller = new AbortController() + const response = await executeGoogleSlidesTool( + request({ + signal: controller.signal, + input: { accessToken: 'token', presentationId: 'presentation-1', exportFormat: ' pdf ' }, + }) + ) + + expect(response.status).toBe(200) + expect(mocks.exportGoogleSlidesPresentation).toHaveBeenCalledWith( + { accessToken: 'token', presentationId: 'presentation-1', exportFormat: 'PDF' }, + { + userId: 'user-1', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + signal: controller.signal, + } + ) + }) + + it('rejects invalid presentation IDs before provider work', async () => { + const response = await executeGoogleSlidesTool( + request({ input: { accessToken: 'token', presentationId: '../presentation' } }) + ) + + expect(response.status).toBe(400) + expect(mocks.exportGoogleSlidesPresentation).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/google-slides/execute-tool.ts b/apps/sim/lib/internal/google-slides/execute-tool.ts new file mode 100644 index 00000000000..c8e9bf51d0f --- /dev/null +++ b/apps/sim/lib/internal/google-slides/execute-tool.ts @@ -0,0 +1,46 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { GoogleSlidesOperationError } from '@/lib/internal/google-slides/errors' +import { googleSlidesExportInputSchema } from '@/lib/internal/google-slides/input' +import { exportGoogleSlidesPresentation } from '@/lib/internal/google-slides/operations' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +export const executeGoogleSlidesTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'google_slides_export_presentation') { + return Response.json( + { success: false, error: `Unsupported Google Slides tool: ${request.toolId}` }, + { status: 500 } + ) + } + const userId = request.context.userId + if (!userId) { + return Response.json({ success: false, error: 'Unauthorized' }, { status: 401 }) + } + const parsed = googleSlidesExportInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + try { + return Response.json( + await exportGoogleSlidesPresentation(parsed.data, { + userId, + workspaceId: request.context.workspaceId, + workflowId: request.context.workflowId, + executionId: request.context.executionId, + signal: request.signal, + }) + ) + } catch (error) { + request.signal?.throwIfAborted() + const status = isPayloadSizeLimitError(error) + ? 413 + : error instanceof GoogleSlidesOperationError + ? error.status + : 500 + return Response.json( + { success: false, error: getErrorMessage(error, 'Failed to export presentation') }, + { status } + ) + } +} diff --git a/apps/sim/lib/internal/google-slides/input.ts b/apps/sim/lib/internal/google-slides/input.ts new file mode 100644 index 00000000000..bd4d2f13181 --- /dev/null +++ b/apps/sim/lib/internal/google-slides/input.ts @@ -0,0 +1,17 @@ +import { z } from 'zod' + +export const googleSlidesExportInputSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + presentationId: z + .string() + .trim() + .min(1, 'Presentation ID is required') + .regex(/^[a-zA-Z0-9_-]+$/, 'Presentation ID contains invalid characters'), + exportFormat: z.preprocess((value) => { + if (typeof value !== 'string') return value + const normalized = value.trim().toUpperCase() + return normalized || undefined + }, z.enum(['PDF', 'PPTX', 'ODP', 'TXT', 'PNG', 'JPEG', 'SVG']).optional()), +}) + +export type GoogleSlidesExportInput = z.output diff --git a/apps/sim/lib/internal/google-slides/operations.test.ts b/apps/sim/lib/internal/google-slides/operations.test.ts new file mode 100644 index 00000000000..8e237fca9f3 --- /dev/null +++ b/apps/sim/lib/internal/google-slides/operations.test.ts @@ -0,0 +1,76 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + secureFetchWithPinnedIP: vi.fn(), + validateUrlWithDNS: vi.fn(), + uploadCopilotFile: vi.fn(), + uploadExecutionFile: vi.fn(), +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + secureFetchWithPinnedIP: mocks.secureFetchWithPinnedIP, + validateUrlWithDNS: mocks.validateUrlWithDNS, +})) + +vi.mock('@/lib/uploads/contexts/copilot', () => ({ + uploadCopilotFile: mocks.uploadCopilotFile, +})) + +vi.mock('@/lib/uploads/contexts/execution', () => ({ + uploadExecutionFile: mocks.uploadExecutionFile, +})) + +import { exportGoogleSlidesPresentation } from '@/lib/internal/google-slides/operations' + +describe('exportGoogleSlidesPresentation', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.validateUrlWithDNS.mockResolvedValue({ isValid: true, resolvedIP: '203.0.113.1' }) + mocks.secureFetchWithPinnedIP.mockResolvedValue( + new Response(new Uint8Array([1, 2, 3]), { status: 200 }) + ) + mocks.uploadExecutionFile.mockResolvedValue({ + id: 'file-1', + name: 'presentation-1.pdf', + url: '/api/files/serve/file-1', + }) + }) + + it('pins the provider request and stores output in execution scope', async () => { + const controller = new AbortController() + const result = await exportGoogleSlidesPresentation( + { accessToken: 'token', presentationId: 'presentation-1', exportFormat: 'PDF' }, + { + userId: 'user-1', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + signal: controller.signal, + } + ) + + expect(mocks.secureFetchWithPinnedIP).toHaveBeenCalledWith( + expect.stringContaining('/drive/v3/files/presentation-1/export?'), + '203.0.113.1', + expect.objectContaining({ signal: controller.signal }) + ) + expect(mocks.uploadExecutionFile).toHaveBeenCalledWith( + { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }, + expect.any(Buffer), + 'presentation-1.pdf', + 'application/pdf', + 'user-1' + ) + expect(result.output.file).toEqual( + expect.objectContaining({ id: 'file-1', mimeType: 'application/pdf' }) + ) + expect(result.output.contentBase64).toBe('AQID') + }) +}) diff --git a/apps/sim/lib/internal/google-slides/operations.ts b/apps/sim/lib/internal/google-slides/operations.ts new file mode 100644 index 00000000000..9aa5158533e --- /dev/null +++ b/apps/sim/lib/internal/google-slides/operations.ts @@ -0,0 +1,137 @@ +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + readResponseTextWithLimit, + readResponseToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' +import { GoogleSlidesOperationError } from '@/lib/internal/google-slides/errors' +import type { GoogleSlidesExportInput } from '@/lib/internal/google-slides/input' +import { uploadCopilotFile } from '@/lib/uploads/contexts/copilot' +import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' +import { presentationUrl } from '@/tools/google_slides/utils' + +const MAX_GOOGLE_SLIDES_EXPORT_BYTES = 10 * 1024 * 1024 +const MAX_LEGACY_INLINE_EXPORT_BYTES = 7 * 1024 * 1024 + +const FORMAT_TO_MIME = { + PDF: 'application/pdf', + PPTX: 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + ODP: 'application/vnd.oasis.opendocument.presentation', + TXT: 'text/plain', + PNG: 'image/png', + JPEG: 'image/jpeg', + SVG: 'image/svg+xml', +} as const + +export interface GoogleSlidesOperationContext { + userId: string + workspaceId?: string + workflowId?: string + executionId?: string + signal?: AbortSignal +} + +export async function exportGoogleSlidesPresentation( + input: GoogleSlidesExportInput, + context: GoogleSlidesOperationContext +) { + context.signal?.throwIfAborted() + const exportFormat = input.exportFormat ?? 'PDF' + const mimeType = FORMAT_TO_MIME[exportFormat] + const exportUrl = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(input.presentationId)}/export?mimeType=${encodeURIComponent(mimeType)}` + const validation = await validateUrlWithDNS(exportUrl, 'googleSlidesExportUrl') + context.signal?.throwIfAborted() + if (!validation.isValid || !validation.resolvedIP) { + throw new GoogleSlidesOperationError( + validation.error || 'Invalid Google Slides export URL', + 400 + ) + } + + const response = await secureFetchWithPinnedIP(exportUrl, validation.resolvedIP, { + headers: { Authorization: `Bearer ${input.accessToken}` }, + maxResponseBytes: MAX_GOOGLE_SLIDES_EXPORT_BYTES, + signal: context.signal, + }) + if (!response.ok) { + const errorText = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Google Slides export error response', + signal: context.signal, + }).catch(() => '') + throw new GoogleSlidesOperationError( + `Failed to export presentation: ${response.status} ${errorText}`, + response.status + ) + } + + const buffer = await readResponseToBufferWithLimit(response, { + maxBytes: MAX_GOOGLE_SLIDES_EXPORT_BYTES, + label: 'Google Slides export response', + signal: context.signal, + }) + context.signal?.throwIfAborted() + const filename = `${input.presentationId}.${exportFormat.toLowerCase()}` + const legacyInlineContent = + buffer.length <= MAX_LEGACY_INLINE_EXPORT_BYTES + ? { contentBase64: buffer.toString('base64') } + : {} + + if (context.workspaceId && context.workflowId && context.executionId) { + const file = await uploadExecutionFile( + { + workspaceId: context.workspaceId, + workflowId: context.workflowId, + executionId: context.executionId, + }, + buffer, + filename, + mimeType, + context.userId + ) + context.signal?.throwIfAborted() + return { + success: true, + output: { + file: { ...file, mimeType }, + exportFormat, + mimeType, + sizeBytes: buffer.length, + exportUrl: file.url, + ...legacyInlineContent, + metadata: { + presentationId: input.presentationId, + url: presentationUrl(input.presentationId), + exportFormat, + }, + }, + } + } + + const file = await uploadCopilotFile({ + buffer, + fileName: filename, + contentType: mimeType, + userId: context.userId, + }) + context.signal?.throwIfAborted() + return { + success: true, + output: { + file, + exportUrl: file.url, + exportFormat, + mimeType, + sizeBytes: buffer.length, + ...legacyInlineContent, + metadata: { + presentationId: input.presentationId, + url: presentationUrl(input.presentationId), + exportFormat, + }, + }, + } +} diff --git a/apps/sim/lib/internal/google-vault/errors.ts b/apps/sim/lib/internal/google-vault/errors.ts new file mode 100644 index 00000000000..df53b5670f7 --- /dev/null +++ b/apps/sim/lib/internal/google-vault/errors.ts @@ -0,0 +1,9 @@ +export class GoogleVaultOperationError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'GoogleVaultOperationError' + } +} diff --git a/apps/sim/lib/internal/google-vault/execute-tool.test.ts b/apps/sim/lib/internal/google-vault/execute-tool.test.ts new file mode 100644 index 00000000000..cebe945359a --- /dev/null +++ b/apps/sim/lib/internal/google-vault/execute-tool.test.ts @@ -0,0 +1,45 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ downloadGoogleVaultExportFile: vi.fn() })) + +vi.mock('@/lib/internal/google-vault/operations', () => ({ + downloadGoogleVaultExportFile: mocks.downloadGoogleVaultExportFile, +})) + +import { executeGoogleVaultTool } from '@/lib/internal/google-vault/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +describe('executeGoogleVaultTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.downloadGoogleVaultExportFile.mockResolvedValue({ success: true, output: {} }) + }) + + it('dispatches typed input and cancellation without HTTP metadata', async () => { + const controller = new AbortController() + const request: InternalToolOperationCall = { + toolId: 'google_vault_download_export_file', + input: { + accessToken: 'token', + matterId: 'matter-1', + bucketName: 'bucket-1', + objectName: 'exports/result.zip', + }, + headers: new Headers(), + context: createExecutionContext(), + requestId: 'request-1', + signal: controller.signal, + } + + const response = await executeGoogleVaultTool(request) + + expect(response.status).toBe(200) + expect(mocks.downloadGoogleVaultExportFile).toHaveBeenCalledWith(request.input, { + signal: controller.signal, + }) + }) +}) diff --git a/apps/sim/lib/internal/google-vault/execute-tool.ts b/apps/sim/lib/internal/google-vault/execute-tool.ts new file mode 100644 index 00000000000..c18a7c55327 --- /dev/null +++ b/apps/sim/lib/internal/google-vault/execute-tool.ts @@ -0,0 +1,44 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { z } from 'zod' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { GoogleVaultOperationError } from '@/lib/internal/google-vault/errors' +import { downloadGoogleVaultExportFile } from '@/lib/internal/google-vault/operations' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const inputSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + matterId: z.string().min(1, 'Matter ID is required'), + bucketName: z.string().min(1, 'Bucket name is required'), + objectName: z.string().min(1, 'Object name is required'), + fileName: z.string().optional(), +}) + +export const executeGoogleVaultTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'google_vault_download_export_file') { + return Response.json( + { success: false, error: `Unsupported Google Vault tool: ${request.toolId}` }, + { status: 500 } + ) + } + const parsed = inputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + try { + return Response.json( + await downloadGoogleVaultExportFile(parsed.data, { signal: request.signal }) + ) + } catch (error) { + request.signal?.throwIfAborted() + const status = isPayloadSizeLimitError(error) + ? 413 + : error instanceof GoogleVaultOperationError + ? error.status + : 500 + return Response.json( + { success: false, error: getErrorMessage(error, 'Unknown error occurred') }, + { status } + ) + } +} diff --git a/apps/sim/lib/internal/google-vault/operations.test.ts b/apps/sim/lib/internal/google-vault/operations.test.ts new file mode 100644 index 00000000000..69f16be89ef --- /dev/null +++ b/apps/sim/lib/internal/google-vault/operations.test.ts @@ -0,0 +1,62 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + secureFetchWithPinnedIP: vi.fn(), + validateUrlWithDNS: vi.fn(), +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + secureFetchWithPinnedIP: mocks.secureFetchWithPinnedIP, + validateUrlWithDNS: mocks.validateUrlWithDNS, +})) + +import { downloadGoogleVaultExportFile } from '@/lib/internal/google-vault/operations' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' + +describe('downloadGoogleVaultExportFile', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.validateUrlWithDNS.mockResolvedValue({ isValid: true, resolvedIP: '203.0.113.1' }) + mocks.secureFetchWithPinnedIP.mockResolvedValue( + new Response(new Uint8Array([1, 2, 3]), { + headers: { + 'content-type': 'application/zip', + 'content-disposition': "attachment; filename*=UTF-8''vault%20export.zip", + }, + }) + ) + }) + + it('pins and bounds the GCS download while preserving file output', async () => { + const controller = new AbortController() + const result = await downloadGoogleVaultExportFile( + { + accessToken: 'token', + matterId: 'matter-1', + bucketName: 'bucket-1', + objectName: 'exports/result.zip', + }, + { signal: controller.signal } + ) + + expect(mocks.secureFetchWithPinnedIP).toHaveBeenCalledWith( + expect.stringContaining('/storage/v1/b/bucket-1/o/exports%2Fresult.zip?alt=media'), + '203.0.113.1', + { + method: 'GET', + headers: { Authorization: 'Bearer token' }, + maxResponseBytes: MAX_BUFFERED_TRANSFER_BYTES, + signal: controller.signal, + } + ) + expect(result.output.file).toEqual({ + name: 'vault export.zip', + mimeType: 'application/zip', + data: 'AQID', + size: 3, + }) + }) +}) diff --git a/apps/sim/lib/internal/google-vault/operations.ts b/apps/sim/lib/internal/google-vault/operations.ts new file mode 100644 index 00000000000..14762da9e66 --- /dev/null +++ b/apps/sim/lib/internal/google-vault/operations.ts @@ -0,0 +1,92 @@ +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + readResponseTextWithLimit, + readResponseToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' +import { GoogleVaultOperationError } from '@/lib/internal/google-vault/errors' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import type { GoogleVaultDownloadExportFileParams } from '@/tools/google_vault/types' +import { enhanceGoogleVaultError } from '@/tools/google_vault/utils' + +export interface GoogleVaultOperationContext { + signal?: AbortSignal +} + +function resolveFilename( + disposition: string, + requestedName: string | undefined, + objectName: string +): string { + if (requestedName) return requestedName + const match = disposition.match(/filename\*=UTF-8''([^;]+)|filename="([^"]+)"/) + if (match?.[1]) { + try { + return decodeURIComponent(match[1]) + } catch { + return match[1] + } + } + if (match?.[2]) return match[2] + return objectName.split('/').at(-1) || 'vault-export.bin' +} + +export async function downloadGoogleVaultExportFile( + input: GoogleVaultDownloadExportFileParams, + context: GoogleVaultOperationContext +) { + context.signal?.throwIfAborted() + const bucket = encodeURIComponent(input.bucketName) + const object = encodeURIComponent(input.objectName) + const downloadUrl = `https://storage.googleapis.com/storage/v1/b/${bucket}/o/${object}?alt=media` + const validation = await validateUrlWithDNS(downloadUrl, 'downloadUrl') + context.signal?.throwIfAborted() + if (!validation.isValid || !validation.resolvedIP) { + throw new GoogleVaultOperationError( + enhanceGoogleVaultError(validation.error || 'Invalid URL'), + 400 + ) + } + + const response = await secureFetchWithPinnedIP(downloadUrl, validation.resolvedIP, { + method: 'GET', + headers: { Authorization: `Bearer ${input.accessToken}` }, + maxResponseBytes: MAX_BUFFERED_TRANSFER_BYTES, + signal: context.signal, + }) + if (!response.ok) { + const errorText = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Google Vault export error response', + signal: context.signal, + }).catch(() => '') + throw new GoogleVaultOperationError( + enhanceGoogleVaultError( + `Failed to download file: ${errorText || response.statusText || response.status}` + ), + 400 + ) + } + + const buffer = await readResponseToBufferWithLimit(response, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + label: 'Google Vault export file', + signal: context.signal, + }) + context.signal?.throwIfAborted() + const mimeType = response.headers.get('content-type') || 'application/octet-stream' + const name = resolveFilename( + response.headers.get('content-disposition') || '', + input.fileName, + input.objectName + ) + return { + success: true, + output: { + file: { name, mimeType, data: buffer.toString('base64'), size: buffer.length }, + }, + } +} diff --git a/apps/sim/lib/internal/grafana/client.test.ts b/apps/sim/lib/internal/grafana/client.test.ts new file mode 100644 index 00000000000..e041a9ecb21 --- /dev/null +++ b/apps/sim/lib/internal/grafana/client.test.ts @@ -0,0 +1,90 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + secureFetchWithPinnedIP: vi.fn(), + validateUrlWithDNS: vi.fn(), +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + MAX_JSON_API_RESPONSE_BYTES: 10 * 1024 * 1024, + secureFetchWithPinnedIP: mocks.secureFetchWithPinnedIP, + validateUrlWithDNS: mocks.validateUrlWithDNS, +})) + +import { GrafanaClient } from '@/lib/internal/grafana/client' + +describe('GrafanaClient', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.validateUrlWithDNS.mockResolvedValue({ isValid: true, resolvedIP: '203.0.113.10' }) + mocks.secureFetchWithPinnedIP.mockResolvedValue({ ok: true, status: 200 }) + }) + + it('pins the validated host and bounds every request', async () => { + const controller = new AbortController() + const client = new GrafanaClient( + 'https://grafana.example.com/', + 'glsa_token', + '2', + controller.signal + ) + + await client.request('/api/folders/folder-1', { + method: 'PUT', + body: { title: 'New title' }, + headers: { 'X-Disable-Provenance': 'true' }, + }) + + expect(mocks.validateUrlWithDNS).toHaveBeenCalledWith( + 'https://grafana.example.com/api/folders/folder-1', + 'baseUrl' + ) + expect(mocks.secureFetchWithPinnedIP).toHaveBeenCalledWith( + 'https://grafana.example.com/api/folders/folder-1', + '203.0.113.10', + expect.objectContaining({ + method: 'PUT', + body: JSON.stringify({ title: 'New title' }), + maxResponseBytes: 10 * 1024 * 1024, + timeout: 30_000, + stripAuthOnRedirect: true, + signal: controller.signal, + headers: expect.objectContaining({ + Authorization: 'Bearer glsa_token', + 'X-Grafana-Org-Id': '2', + 'X-Disable-Provenance': 'true', + }), + }) + ) + }) + + it('rejects invalid destinations before sending credentials', async () => { + mocks.validateUrlWithDNS.mockResolvedValue({ isValid: false, error: 'private address' }) + const client = new GrafanaClient('http://127.0.0.1:3000', 'secret') + + await expect(client.request('/api/health', { method: 'GET' })).resolves.toEqual({ + success: false, + error: 'Invalid Grafana baseUrl: private address', + }) + expect(mocks.secureFetchWithPinnedIP).not.toHaveBeenCalled() + }) + + it('stops before network work when cancelled', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + const client = new GrafanaClient( + 'https://grafana.example.com', + 'secret', + undefined, + controller.signal + ) + + await expect(client.request('/api/health', { method: 'GET' })).rejects.toMatchObject({ + name: 'AbortError', + }) + expect(mocks.validateUrlWithDNS).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/grafana/client.ts b/apps/sim/lib/internal/grafana/client.ts new file mode 100644 index 00000000000..bccf83c8f95 --- /dev/null +++ b/apps/sim/lib/internal/grafana/client.ts @@ -0,0 +1,57 @@ +import { + MAX_JSON_API_RESPONSE_BYTES, + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' + +const OUTBOUND_FETCH_TIMEOUT_MS = 30_000 + +export type GrafanaClientResult = + | { success: true; response: Awaited> } + | { success: false; error: string } + +export class GrafanaClient { + private readonly baseUrl: string + + constructor( + baseUrl: string, + private readonly apiKey: string, + private readonly organizationId?: string, + private readonly signal?: AbortSignal + ) { + this.baseUrl = baseUrl.replace(/\/$/, '') + } + + async request( + path: string, + options: { method: 'GET' | 'POST' | 'PUT'; body?: unknown; headers?: Record } + ): Promise { + this.signal?.throwIfAborted() + const url = `${this.baseUrl}${path}` + const validation = await validateUrlWithDNS(url, 'baseUrl') + this.signal?.throwIfAborted() + if (!validation.isValid || !validation.resolvedIP) { + return { success: false, error: `Invalid Grafana baseUrl: ${validation.error}` } + } + + const headers: Record = { + Accept: 'application/json', + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.apiKey}`, + ...options.headers, + } + if (this.organizationId) headers['X-Grafana-Org-Id'] = this.organizationId + + const response = await secureFetchWithPinnedIP(url, validation.resolvedIP, { + method: options.method, + headers, + ...(options.body === undefined ? {} : { body: JSON.stringify(options.body) }), + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + timeout: OUTBOUND_FETCH_TIMEOUT_MS, + stripAuthOnRedirect: true, + signal: this.signal, + }) + this.signal?.throwIfAborted() + return { success: true, response } + } +} diff --git a/apps/sim/lib/internal/grafana/execute-tool.test.ts b/apps/sim/lib/internal/grafana/execute-tool.test.ts new file mode 100644 index 00000000000..67337684388 --- /dev/null +++ b/apps/sim/lib/internal/grafana/execute-tool.test.ts @@ -0,0 +1,112 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operations = vi.hoisted(() => ({ + checkGrafanaDataSourceHealth: vi.fn(), + updateGrafanaAlertRule: vi.fn(), + updateGrafanaDashboard: vi.fn(), + updateGrafanaFolder: vi.fn(), +})) + +vi.mock('@/lib/internal/grafana/operations', () => operations) + +import { executeGrafanaTool } from '@/lib/internal/grafana/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +function toolRequest(overrides: Partial = {}) { + return { + toolId: 'grafana_check_data_source_health', + input: { + apiKey: 'key', + baseUrl: 'https://grafana.example.com', + dataSourceUid: 'source-1', + }, + headers: new Headers(), + context: { ...createExecutionContext({ workflowId: 'workflow-1' }), userId: 'user-1' }, + requestId: 'request-1', + ...overrides, + } as InternalToolOperationCall +} + +const CASES = [ + ['grafana_check_data_source_health', 'dataSourceUid', operations.checkGrafanaDataSourceHealth], + ['grafana_update_alert_rule', 'alertRuleUid', operations.updateGrafanaAlertRule], + ['grafana_update_dashboard', 'dashboardUid', operations.updateGrafanaDashboard], + ['grafana_update_folder', 'folderUid', operations.updateGrafanaFolder], +] as const + +describe('executeGrafanaTool', () => { + beforeEach(() => { + vi.clearAllMocks() + for (const [, , operation] of CASES) operation.mockResolvedValue({ success: true, output: {} }) + }) + + it.each(CASES)( + 'dispatches %s to its authoritative operation', + async (toolId, idField, operation) => { + const input = { + apiKey: 'key', + baseUrl: 'https://grafana.example.com', + [idField]: 'resource-1', + ...(toolId === 'grafana_update_folder' ? { title: 'New' } : {}), + } + const response = await executeGrafanaTool(toolRequest({ toolId, input })) + + expect(response.status).toBe(200) + expect(operation).toHaveBeenCalledWith(input, { requestId: 'request-1', signal: undefined }) + } + ) + + it('authenticates before validating input', async () => { + const response = await executeGrafanaTool( + toolRequest({ input: null, context: createExecutionContext({ workflowId: 'workflow-1' }) }) + ) + + expect(response.status).toBe(401) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Authentication required', + }) + expect(operations.checkGrafanaDataSourceHealth).not.toHaveBeenCalled() + }) + + it('preserves exact contract validation details', async () => { + const response = await executeGrafanaTool(toolRequest({ input: {} })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + success: false, + error: 'Invalid input: expected string, received undefined', + details: expect.any(Array), + }) + }) + + it('returns a non-2xx response when an operation reports failure', async () => { + operations.updateGrafanaDashboard.mockResolvedValueOnce({ + success: false, + output: {}, + error: 'Grafana rejected the update', + }) + + const response = await executeGrafanaTool( + toolRequest({ + toolId: 'grafana_update_dashboard', + input: { + apiKey: 'key', + baseUrl: 'https://grafana.example.com', + dashboardUid: 'dashboard-1', + }, + }) + ) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + success: false, + output: {}, + error: 'Grafana rejected the update', + }) + }) +}) diff --git a/apps/sim/lib/internal/grafana/execute-tool.ts b/apps/sim/lib/internal/grafana/execute-tool.ts new file mode 100644 index 00000000000..df56ea591c5 --- /dev/null +++ b/apps/sim/lib/internal/grafana/execute-tool.ts @@ -0,0 +1,84 @@ +import type { z } from 'zod' +import { getValidationErrorMessage } from '@/lib/api/server' +import { + checkGrafanaDataSourceHealth, + type GrafanaOperationContext, + updateGrafanaAlertRule, + updateGrafanaDashboard, + updateGrafanaFolder, +} from '@/lib/internal/grafana/operations' +import { + grafanaCheckDataSourceHealthInputSchema, + grafanaUpdateAlertRuleInputSchema, + grafanaUpdateDashboardInputSchema, + grafanaUpdateFolderInputSchema, +} from '@/lib/internal/grafana/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +async function executeOperation( + schema: z.ZodType, + input: unknown, + context: GrafanaOperationContext, + execute: (input: Input, context: GrafanaOperationContext) => Promise +): Promise { + context.signal?.throwIfAborted() + const parsed = schema.safeParse(input) + if (!parsed.success) { + return Response.json( + { + success: false, + error: getValidationErrorMessage(parsed.error, 'Invalid request data'), + details: parsed.error.issues, + }, + { status: 400 } + ) + } + const result = await execute(parsed.data, context) + context.signal?.throwIfAborted() + const failed = + typeof result === 'object' && result !== null && 'success' in result && result.success === false + return Response.json(result, { status: failed ? 500 : 200 }) +} + +export const executeGrafanaTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!request.context.userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + const context = { requestId: request.requestId, signal: request.signal } + switch (request.toolId) { + case 'grafana_check_data_source_health': + return executeOperation( + grafanaCheckDataSourceHealthInputSchema, + request.input, + context, + checkGrafanaDataSourceHealth + ) + case 'grafana_update_alert_rule': + return executeOperation( + grafanaUpdateAlertRuleInputSchema, + request.input, + context, + updateGrafanaAlertRule + ) + case 'grafana_update_dashboard': + return executeOperation( + grafanaUpdateDashboardInputSchema, + request.input, + context, + updateGrafanaDashboard + ) + case 'grafana_update_folder': + return executeOperation( + grafanaUpdateFolderInputSchema, + request.input, + context, + updateGrafanaFolder + ) + default: + return Response.json( + { success: false, error: `Unsupported Grafana tool: ${request.toolId}` }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/grafana/operations.test.ts b/apps/sim/lib/internal/grafana/operations.test.ts new file mode 100644 index 00000000000..d223aeedf73 --- /dev/null +++ b/apps/sim/lib/internal/grafana/operations.test.ts @@ -0,0 +1,133 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const request = vi.hoisted(() => vi.fn()) + +vi.mock('@/lib/internal/grafana/client', () => ({ + GrafanaClient: class { + request = request + }, +})) + +import { + checkGrafanaDataSourceHealth, + updateGrafanaAlertRule, + updateGrafanaDashboard, + updateGrafanaFolder, +} from '@/lib/internal/grafana/operations' + +function response(body: unknown, status = 200) { + return { + ok: status >= 200 && status < 300, + status, + text: async () => (typeof body === 'string' ? body : JSON.stringify(body)), + json: async () => body, + } +} + +const auth = { apiKey: 'key', baseUrl: 'https://grafana.example.com' } +const context = { requestId: 'request-1' } + +describe('Grafana operations', () => { + beforeEach(() => vi.clearAllMocks()) + + it('returns Grafana unhealthy verdicts as successful health checks', async () => { + request.mockResolvedValue({ + success: true, + response: response( + { status: 'ERROR', message: 'dial tcp refused', details: { code: 1 } }, + 400 + ), + }) + + await expect( + checkGrafanaDataSourceHealth({ ...auth, dataSourceUid: 'a/../../admin' }, context) + ).resolves.toEqual({ + success: true, + output: { status: 'ERROR', message: 'dial tcp refused', details: { code: 1 } }, + }) + expect(request).toHaveBeenCalledWith('/api/datasources/uid/a%2F..%2F..%2Fadmin/health', { + method: 'GET', + }) + }) + + it('fetches and merges a dashboard before updating once', async () => { + request + .mockResolvedValueOnce({ + success: true, + response: response({ + dashboard: { uid: 'dash-1', title: 'Old', version: 7, untouched: true }, + meta: { folderUid: 'folder-1' }, + }), + }) + .mockResolvedValueOnce({ + success: true, + response: response({ id: 1, uid: 'dash-1', status: 'success', version: 8 }), + }) + + const result = await updateGrafanaDashboard( + { + ...auth, + dashboardUid: 'dash-1', + title: 'New', + tags: 'one, two', + panels: '[{"id":1}]', + }, + context + ) + + expect(result).toMatchObject({ success: true, output: { uid: 'dash-1', version: 8 } }) + expect(request).toHaveBeenNthCalledWith(2, '/api/dashboards/db', { + method: 'POST', + body: { + dashboard: { + uid: 'dash-1', + title: 'New', + version: 7, + untouched: true, + tags: ['one', 'two'], + panels: [{ id: 1 }], + }, + overwrite: false, + folderUid: 'folder-1', + }, + }) + }) + + it('fails invalid alert JSON before the update request', async () => { + request.mockResolvedValueOnce({ + success: true, + response: response({ uid: 'rule-1', annotations: {} }), + }) + + await expect( + updateGrafanaAlertRule({ ...auth, alertRuleUid: 'rule-1', annotations: '{not json' }, context) + ).resolves.toEqual({ + success: false, + output: {}, + error: 'Invalid JSON for annotations parameter', + }) + expect(request).toHaveBeenCalledTimes(1) + }) + + it('preserves folder version concurrency on update', async () => { + request + .mockResolvedValueOnce({ + success: true, + response: response({ uid: 'folder-1', version: 4 }), + }) + .mockResolvedValueOnce({ + success: true, + response: response({ uid: 'folder-1', title: 'New', version: 5 }), + }) + + await updateGrafanaFolder({ ...auth, folderUid: 'folder-1', title: 'New' }, context) + + expect(request).toHaveBeenNthCalledWith(2, '/api/folders/folder-1', { + method: 'PUT', + body: { title: 'New', version: 4 }, + }) + }) +}) diff --git a/apps/sim/lib/internal/grafana/operations.ts b/apps/sim/lib/internal/grafana/operations.ts new file mode 100644 index 00000000000..c2d63e95ca5 --- /dev/null +++ b/apps/sim/lib/internal/grafana/operations.ts @@ -0,0 +1,293 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { truncate } from '@sim/utils/string' +import { GrafanaClient } from '@/lib/internal/grafana/client' +import type { + GrafanaCheckDataSourceHealthInput, + GrafanaUpdateAlertRuleInput, + GrafanaUpdateDashboardInput, + GrafanaUpdateFolderInput, +} from '@/lib/internal/grafana/schema' +import { mapAlertRule } from '@/tools/grafana/utils' + +const logger = createLogger('GrafanaOperations') +const MAX_ERROR_MESSAGE_LENGTH = 2000 + +export interface GrafanaOperationContext { + requestId: string + signal?: AbortSignal +} + +function failure(error: string, withOutput = true) { + return withOutput + ? { success: false as const, output: {}, error } + : { success: false as const, error } +} + +async function errorText(response: { text(): Promise }): Promise { + return truncate(await response.text(), MAX_ERROR_MESSAGE_LENGTH) +} + +function parseJsonField(value: string | undefined, field: string): unknown | undefined { + if (!value) return undefined + try { + return JSON.parse(value) + } catch { + throw new Error(`Invalid JSON for ${field} parameter`) + } +} + +export async function checkGrafanaDataSourceHealth( + input: GrafanaCheckDataSourceHealthInput, + context: GrafanaOperationContext +) { + try { + const client = new GrafanaClient( + input.baseUrl, + input.apiKey, + input.organizationId, + context.signal + ) + const result = await client.request( + `/api/datasources/uid/${encodeURIComponent(input.dataSourceUid.trim())}/health`, + { method: 'GET' } + ) + if (!result.success) return failure(result.error, false) + + const raw = await result.response.text() + let body: unknown = null + if (raw.length > 0) { + try { + body = JSON.parse(raw) + } catch { + body = null + } + } + const payload = + body && typeof body === 'object' + ? (body as { status?: unknown; message?: unknown; details?: unknown }) + : null + if (payload && typeof payload.status === 'string') { + return { + success: true as const, + output: { + status: payload.status, + message: typeof payload.message === 'string' ? payload.message : null, + ...(payload.details === undefined ? {} : { details: payload.details }), + }, + } + } + return failure( + `Failed to check data source health: HTTP ${result.response.status} ${truncate(raw, MAX_ERROR_MESSAGE_LENGTH)}`, + false + ) + } catch (error) { + context.signal?.throwIfAborted() + logger.error('Error checking Grafana data source health', { + requestId: context.requestId, + error: getErrorMessage(error), + }) + return failure(getErrorMessage(error), false) + } +} + +export async function updateGrafanaDashboard( + input: GrafanaUpdateDashboardInput, + context: GrafanaOperationContext +) { + try { + const client = new GrafanaClient( + input.baseUrl, + input.apiKey, + input.organizationId, + context.signal + ) + const existingResult = await client.request( + `/api/dashboards/uid/${encodeURIComponent(input.dashboardUid.trim())}`, + { method: 'GET' } + ) + if (!existingResult.success) return failure(existingResult.error) + if (!existingResult.response.ok) { + return failure( + `Failed to fetch existing dashboard: ${await errorText(existingResult.response)}` + ) + } + const existing = (await existingResult.response.json()) as { + dashboard?: Record + meta?: { folderUid?: string } + } + const dashboard = existing.dashboard + if (!dashboard?.uid) return failure('Failed to fetch existing dashboard') + + const updated: Record = { ...dashboard } + if (input.title) updated.title = input.title + if (input.timezone) updated.timezone = input.timezone + if (input.refresh) updated.refresh = input.refresh + if (input.tags) { + updated.tags = input.tags + .split(',') + .map((tag) => tag.trim()) + .filter(Boolean) + } + const panels = parseJsonField(input.panels, 'panels') + if (panels !== undefined) updated.panels = panels + if (dashboard.version) updated.version = dashboard.version + + const body: Record = { + dashboard: updated, + overwrite: input.overwrite === true, + } + if (input.folderUid) body.folderUid = input.folderUid + else if (existing.meta?.folderUid) body.folderUid = existing.meta.folderUid + if (input.message) body.message = input.message + + const updateResult = await client.request('/api/dashboards/db', { method: 'POST', body }) + if (!updateResult.success) return failure(updateResult.error) + if (!updateResult.response.ok) { + return failure(`Failed to update dashboard: ${await errorText(updateResult.response)}`) + } + const data = (await updateResult.response.json()) as Record + return { + success: true as const, + output: { + id: data.id, + uid: data.uid, + url: data.url, + status: data.status, + version: data.version, + slug: data.slug, + }, + } + } catch (error) { + context.signal?.throwIfAborted() + return failure(getErrorMessage(error)) + } +} + +export async function updateGrafanaAlertRule( + input: GrafanaUpdateAlertRuleInput, + context: GrafanaOperationContext +) { + try { + const client = new GrafanaClient( + input.baseUrl, + input.apiKey, + input.organizationId, + context.signal + ) + const path = `/api/v1/provisioning/alert-rules/${encodeURIComponent(input.alertRuleUid.trim())}` + const existingResult = await client.request(path, { method: 'GET' }) + if (!existingResult.success) return failure(existingResult.error) + if (!existingResult.response.ok) { + return failure( + `Failed to fetch existing alert rule: ${await errorText(existingResult.response)}` + ) + } + const existing = (await existingResult.response.json()) as Record + if (!existing.uid) return failure('Failed to fetch existing alert rule') + + const updated: Record = { ...existing } + if (input.title) updated.title = input.title + if (input.folderUid) updated.folderUID = input.folderUid + if (input.ruleGroup) updated.ruleGroup = input.ruleGroup + if (input.condition) updated.condition = input.condition + if (input.forDuration) updated.for = input.forDuration + if (input.noDataState) updated.noDataState = input.noDataState + if (input.execErrState) updated.execErrState = input.execErrState + if (input.isPaused !== undefined) updated.isPaused = input.isPaused + if (input.keepFiringFor) updated.keep_firing_for = input.keepFiringFor + if (input.missingSeriesEvalsToResolve !== undefined) { + updated.missingSeriesEvalsToResolve = input.missingSeriesEvalsToResolve + } + const replacements = [ + ['notificationSettings', 'notification_settings'], + ['record', 'record'], + ['data', 'data'], + ] as const + for (const [inputKey, outputKey] of replacements) { + const value = parseJsonField(input[inputKey], inputKey) + if (value !== undefined) updated[outputKey] = value + } + for (const key of ['annotations', 'labels'] as const) { + const value = parseJsonField(input[key], key) + if (value !== undefined) { + updated[key] = { + ...(typeof existing[key] === 'object' ? existing[key] : {}), + ...(value as object), + } + } + } + + const updateResult = await client.request(path, { + method: 'PUT', + body: updated, + headers: input.disableProvenance ? { 'X-Disable-Provenance': 'true' } : undefined, + }) + if (!updateResult.success) return failure(updateResult.error) + if (!updateResult.response.ok) { + return failure(`Failed to update alert rule: ${await errorText(updateResult.response)}`) + } + return { + success: true as const, + output: mapAlertRule((await updateResult.response.json()) as Record), + } + } catch (error) { + context.signal?.throwIfAborted() + return failure(getErrorMessage(error)) + } +} + +export async function updateGrafanaFolder( + input: GrafanaUpdateFolderInput, + context: GrafanaOperationContext +) { + try { + const client = new GrafanaClient( + input.baseUrl, + input.apiKey, + input.organizationId, + context.signal + ) + const path = `/api/folders/${encodeURIComponent(input.folderUid.trim())}` + const existingResult = await client.request(path, { method: 'GET' }) + if (!existingResult.success) return failure(existingResult.error) + if (!existingResult.response.ok) { + return failure(`Failed to fetch existing folder: ${await errorText(existingResult.response)}`) + } + const existing = (await existingResult.response.json()) as Record + if (!existing.uid) return failure('Failed to fetch existing folder') + + const updateResult = await client.request(path, { + method: 'PUT', + body: { title: input.title, version: existing.version }, + }) + if (!updateResult.success) return failure(updateResult.error) + if (!updateResult.response.ok) { + return failure(`Failed to update folder: ${await errorText(updateResult.response)}`) + } + const data = (await updateResult.response.json()) as Record + return { + success: true as const, + output: { + id: (data.id as number) ?? null, + uid: (data.uid as string) ?? null, + title: (data.title as string) ?? null, + url: (data.url as string) ?? null, + parentUid: (data.parentUid as string) ?? null, + parents: (data.parents as { uid: string; title: string; url: string }[]) ?? [], + hasAcl: (data.hasAcl as boolean) ?? null, + canSave: (data.canSave as boolean) ?? null, + canEdit: (data.canEdit as boolean) ?? null, + canAdmin: (data.canAdmin as boolean) ?? null, + createdBy: (data.createdBy as string) ?? null, + created: (data.created as string) ?? null, + updatedBy: (data.updatedBy as string) ?? null, + updated: (data.updated as string) ?? null, + version: (data.version as number) ?? null, + }, + } + } catch (error) { + context.signal?.throwIfAborted() + return failure(getErrorMessage(error)) + } +} diff --git a/apps/sim/lib/internal/grafana/schema.ts b/apps/sim/lib/internal/grafana/schema.ts new file mode 100644 index 00000000000..1e2f8b26fe3 --- /dev/null +++ b/apps/sim/lib/internal/grafana/schema.ts @@ -0,0 +1,61 @@ +import { z } from 'zod' + +export const grafanaUpdateDashboardInputSchema = z.object({ + apiKey: z.string().min(1, 'Grafana Service Account Token is required'), + baseUrl: z.string().min(1, 'Grafana instance URL is required'), + organizationId: z.string().optional(), + dashboardUid: z.string().min(1, 'Dashboard UID is required'), + title: z.string().optional(), + folderUid: z.string().optional(), + tags: z.string().optional(), + timezone: z.string().optional(), + refresh: z.string().optional(), + panels: z.string().optional(), + overwrite: z.boolean().optional(), + message: z.string().optional(), +}) + +export const grafanaUpdateAlertRuleInputSchema = z.object({ + apiKey: z.string().min(1, 'Grafana Service Account Token is required'), + baseUrl: z.string().min(1, 'Grafana instance URL is required'), + organizationId: z.string().optional(), + alertRuleUid: z.string().min(1, 'Alert rule UID is required'), + title: z.string().optional(), + folderUid: z.string().optional(), + ruleGroup: z.string().optional(), + condition: z.string().optional(), + data: z.string().optional(), + forDuration: z.string().optional(), + noDataState: z.string().optional(), + execErrState: z.string().optional(), + annotations: z.string().optional(), + labels: z.string().optional(), + isPaused: z.boolean().optional(), + keepFiringFor: z.string().optional(), + missingSeriesEvalsToResolve: z.number().optional(), + notificationSettings: z.string().optional(), + record: z.string().optional(), + disableProvenance: z.boolean().optional(), +}) + +export const grafanaUpdateFolderInputSchema = z.object({ + apiKey: z.string().min(1, 'Grafana Service Account Token is required'), + baseUrl: z.string().min(1, 'Grafana instance URL is required'), + organizationId: z.string().optional(), + folderUid: z.string().min(1, 'Folder UID is required'), + title: z.string().min(1, 'Folder title is required'), +}) + +export const grafanaCheckDataSourceHealthInputSchema = z.object({ + apiKey: z.string().min(1, 'Grafana Service Account Token is required'), + baseUrl: z.string().min(1, 'Grafana instance URL is required'), + organizationId: z.string().optional(), + dataSourceUid: z.string().min(1, 'Data source UID is required').max(40, 'UID is too long'), +}) + +export type GrafanaUpdateDashboardInput = z.input +export type GrafanaUpdateAlertRuleInput = z.input +export type GrafanaUpdateFolderInput = z.input +export type GrafanaCheckDataSourceHealthInput = z.input< + typeof grafanaCheckDataSourceHealthInputSchema +> diff --git a/apps/sim/lib/internal/guardrails/errors.ts b/apps/sim/lib/internal/guardrails/errors.ts new file mode 100644 index 00000000000..67f7028884c --- /dev/null +++ b/apps/sim/lib/internal/guardrails/errors.ts @@ -0,0 +1,9 @@ +export class GuardrailsOperationError extends Error { + constructor( + readonly status: number, + readonly body: { error: string } + ) { + super(body.error) + this.name = 'GuardrailsOperationError' + } +} diff --git a/apps/sim/lib/internal/guardrails/execute-tool.test.ts b/apps/sim/lib/internal/guardrails/execute-tool.test.ts new file mode 100644 index 00000000000..a54f56123d3 --- /dev/null +++ b/apps/sim/lib/internal/guardrails/execute-tool.test.ts @@ -0,0 +1,95 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const executeOperation = vi.hoisted(() => vi.fn()) + +vi.mock('@/lib/internal/guardrails/operations', () => ({ + executeGuardrailsValidation: executeOperation, +})) + +import { GuardrailsOperationError } from '@/lib/internal/guardrails/errors' +import { executeGuardrailsTool } from '@/lib/internal/guardrails/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +function request(overrides: Partial = {}): InternalToolOperationCall { + return { + toolId: 'guardrails_validate', + input: { + input: 'claim', + validationType: 'hallucination', + knowledgeBaseId: 'knowledge-1', + model: 'gpt-4o', + workflowId: 'untrusted-workflow', + }, + headers: new Headers({ 'x-sim-billing-attribution': 'attribution' }), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + userId: 'user-1', + workspaceId: 'workspace-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeGuardrailsTool', () => { + beforeEach(() => { + vi.clearAllMocks() + executeOperation.mockResolvedValue({ + success: true, + output: { passed: true, validationType: 'hallucination', input: 'claim' }, + }) + }) + + it('binds hallucination validation to trusted workflow scope', async () => { + const controller = new AbortController() + const executionRequest = request({ signal: controller.signal }) + const response = await executeGuardrailsTool(executionRequest) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + output: { passed: true }, + }) + expect(executeOperation).toHaveBeenCalledWith( + expect.objectContaining({ workflowId: 'workflow-1' }), + expect.objectContaining({ + actorUserId: 'user-1', + headers: executionRequest.headers, + signal: controller.signal, + }) + ) + }) + + it('preserves classified admission errors', async () => { + executeOperation.mockRejectedValueOnce( + new GuardrailsOperationError(402, { error: 'Usage limit exceeded' }) + ) + + const response = await executeGuardrailsTool(request()) + + expect(response.status).toBe(402) + await expect(response.json()).resolves.toEqual({ error: 'Usage limit exceeded' }) + }) + + it('propagates cancellation before and after guardrail work', async () => { + const before = new AbortController() + before.abort(new DOMException('cancelled', 'AbortError')) + await expect(executeGuardrailsTool(request({ signal: before.signal }))).rejects.toMatchObject({ + name: 'AbortError', + }) + expect(executeOperation).not.toHaveBeenCalled() + + const after = new AbortController() + executeOperation.mockImplementationOnce(async () => { + after.abort(new DOMException('cancelled', 'AbortError')) + return { success: true, output: { passed: true } } + }) + await expect(executeGuardrailsTool(request({ signal: after.signal }))).rejects.toMatchObject({ + name: 'AbortError', + }) + }) +}) diff --git a/apps/sim/lib/internal/guardrails/execute-tool.ts b/apps/sim/lib/internal/guardrails/execute-tool.ts new file mode 100644 index 00000000000..17fd2dfabe2 --- /dev/null +++ b/apps/sim/lib/internal/guardrails/execute-tool.ts @@ -0,0 +1,85 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { isPlainRecord } from '@sim/utils/object' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { GuardrailsOperationError } from '@/lib/internal/guardrails/errors' +import { guardrailsValidationInputSchema } from '@/lib/internal/guardrails/input' +import { executeGuardrailsValidation } from '@/lib/internal/guardrails/operations' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { isAbortError } from '@/providers/streaming-tool-loop-shared' + +const logger = createLogger('GuardrailsToolExecution') + +function unexpectedFailure(error: unknown, requestId: string): Response { + const message = getErrorMessage(error, 'Validation failed due to unexpected error') + logger.error(`[${requestId}] Guardrails validation failed`, { error: message }) + return Response.json({ + success: true, + output: { + passed: false, + validationType: 'unknown', + input: '', + error: message, + }, + }) +} + +export const executeGuardrailsTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'guardrails_validate') { + return Response.json( + { success: false, error: `Unsupported Guardrails tool: ${request.toolId}` }, + { status: 500 } + ) + } + if (!request.context.userId) { + return Response.json({ error: 'Unauthorized' }, { status: 401 }) + } + if (!isPlainRecord(request.input)) { + return Response.json({ error: 'Invalid request data' }, { status: 400 }) + } + + let serializedInput: string + try { + serializedInput = JSON.stringify(request.input) + } catch { + return Response.json({ error: 'Invalid request data' }, { status: 400 }) + } + if (Buffer.byteLength(serializedInput, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + + const parsed = guardrailsValidationInputSchema.safeParse({ + ...request.input, + workflowId: request.context.workflowId, + }) + if (!parsed.success) { + return Response.json( + { error: 'Invalid request data', details: parsed.error.issues }, + { status: 400 } + ) + } + + try { + const result = await executeGuardrailsValidation(parsed.data, { + actorUserId: request.context.userId, + executionContext: request.context, + headers: request.headers, + requestId: request.requestId, + signal: request.signal, + }) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + if (isAbortError(error) || request.signal?.aborted) throw error + if (error instanceof GuardrailsOperationError) { + return Response.json(error.body, { status: error.status }) + } + return unexpectedFailure(error, request.requestId) + } +} diff --git a/apps/sim/lib/internal/guardrails/input.ts b/apps/sim/lib/internal/guardrails/input.ts new file mode 100644 index 00000000000..8bdaf7af57d --- /dev/null +++ b/apps/sim/lib/internal/guardrails/input.ts @@ -0,0 +1,33 @@ +import { z } from 'zod' +import { + customPatternSchema, + resolvedSecretTraceProvenanceSchema, +} from '@/lib/api/contracts/primitives' +import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' + +export const guardrailsValidationInputSchema = z.object({ + validationType: z.string().optional(), + input: z.unknown().optional(), + regex: z.string().optional(), + knowledgeBaseId: z.string().optional(), + threshold: z.string().optional(), + topK: z.string().optional(), + model: z.string().optional(), + apiKey: z.string().optional(), + azureEndpoint: z.string().optional(), + azureApiVersion: z.string().optional(), + vertexProject: z.string().optional(), + vertexLocation: z.string().optional(), + vertexCredential: z.string().optional(), + bedrockAccessKeyId: z.string().optional(), + bedrockSecretKey: z.string().optional(), + bedrockRegion: z.string().optional(), + workflowId: z.string().optional(), + piiEntityTypes: z.array(z.string()).optional(), + piiMode: z.string().optional(), + piiLanguage: z.string().optional(), + piiCustomPatterns: z.array(customPatternSchema).max(20).optional(), + [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), +}) + +export type GuardrailsValidationInput = z.input diff --git a/apps/sim/lib/internal/guardrails/operations.test.ts b/apps/sim/lib/internal/guardrails/operations.test.ts new file mode 100644 index 00000000000..b64a6e41e24 --- /dev/null +++ b/apps/sim/lib/internal/guardrails/operations.test.ts @@ -0,0 +1,206 @@ +/** + * @vitest-environment node + */ +import { workflowAuthzMockFns } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { PRIVATE_MODEL_INPUT_PROVENANCE_HEADER } from '@/lib/execution/model-input-provenance' +import { + RESOLVED_SECRET_PROVENANCE_FIELD, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, +} from '@/lib/execution/private-tool-metadata' + +const mocks = vi.hoisted(() => ({ + assertPermissionsAllowed: vi.fn(), + authorizeCredential: vi.fn(), + checkAttributedUsageLimits: vi.fn(), + importProvenance: vi.fn(), + isComplete: vi.fn(), + prepareEnvironment: vi.fn(), + requireBillingAttribution: vi.fn(), + validateHallucination: vi.fn(), + validateJson: vi.fn(), + validatePII: vi.fn(), + validateRegex: vi.fn(), +})) + +vi.mock('@/lib/auth/credential-access', () => ({ + authorizeCredentialUseForAuth: mocks.authorizeCredential, +})) +vi.mock('@/lib/billing/core/billing-attribution', () => ({ + checkAttributedUsageLimits: mocks.checkAttributedUsageLimits, + requireBillingAttributionHeader: mocks.requireBillingAttribution, + toBillingContext: vi.fn(() => ({})), +})) +vi.mock('@/lib/billing/threshold-billing', () => ({ + checkAndBillPayerOverageThreshold: vi.fn(), +})) +vi.mock('@/lib/copilot/environment-context', () => ({ + prepareCopilotEnvironmentContext: mocks.prepareEnvironment, +})) +vi.mock('@/lib/guardrails/validate_hallucination', () => ({ + validateHallucination: mocks.validateHallucination, +})) +vi.mock('@/lib/guardrails/validate_json', () => ({ validateJson: mocks.validateJson })) +vi.mock('@/lib/guardrails/validate_pii', () => ({ validatePII: mocks.validatePII })) +vi.mock('@/lib/guardrails/validate_regex', () => ({ validateRegex: mocks.validateRegex })) +vi.mock('@/ee/access-control/utils/permission-check', () => ({ + assertPermissionsAllowed: mocks.assertPermissionsAllowed, + ModelNotAllowedError: class ModelNotAllowedError extends Error {}, + ProviderNotAllowedError: class ProviderNotAllowedError extends Error {}, +})) + +import { executeGuardrailsValidation } from '@/lib/internal/guardrails/operations' + +const BILLING_ATTRIBUTION = { + actorUserId: 'user-1', + workspaceId: 'workspace-1', + billingEntity: { type: 'user' as const, id: 'user-1' }, +} + +describe('executeGuardrailsValidation', () => { + beforeEach(() => { + vi.clearAllMocks() + workflowAuthzMockFns.mockAuthorizeWorkflowByWorkspacePermission.mockResolvedValue({ + allowed: true, + workflow: { id: 'workflow-1', workspaceId: 'workspace-1' }, + }) + mocks.requireBillingAttribution.mockReturnValue(BILLING_ATTRIBUTION) + mocks.checkAttributedUsageLimits.mockResolvedValue({ isExceeded: false }) + mocks.importProvenance.mockResolvedValue({ success: true, matched: true }) + mocks.isComplete.mockReturnValue(true) + mocks.prepareEnvironment.mockResolvedValue({ + resolvedSecretTraceRegistry: { + importProvenanceForValueAtInputPath: mocks.importProvenance, + isComplete: mocks.isComplete, + }, + }) + mocks.authorizeCredential.mockResolvedValue({ ok: true }) + mocks.validateHallucination.mockResolvedValue({ passed: true, score: 8 }) + mocks.validateJson.mockReturnValue({ passed: true }) + mocks.validateRegex.mockReturnValue({ passed: true }) + mocks.validatePII.mockResolvedValue({ passed: true, detectedEntities: [] }) + }) + + it('runs hallucination work once with authorized scope, billing, provenance, and signal', async () => { + const controller = new AbortController() + const provenance = { version: 1, complete: true, entries: [] } + const headers = new Headers({ + 'x-sim-billing-attribution': 'attribution', + [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1, + }) + const result = await executeGuardrailsValidation( + { + validationType: 'hallucination', + input: 'claim', + knowledgeBaseId: 'knowledge-1', + model: 'gpt-4o', + workflowId: 'workflow-1', + [RESOLVED_SECRET_PROVENANCE_FIELD]: provenance, + }, + { + actorUserId: 'user-1', + headers, + requestId: 'request-1', + signal: controller.signal, + } + ) + + expect(result.output).toMatchObject({ passed: true, score: 8 }) + expect(workflowAuthzMockFns.mockAuthorizeWorkflowByWorkspacePermission).toHaveBeenCalledWith({ + workflowId: 'workflow-1', + userId: 'user-1', + action: 'read', + }) + expect(mocks.requireBillingAttribution).toHaveBeenCalledWith(headers, { + actorUserId: 'user-1', + workspaceId: 'workspace-1', + }) + expect(mocks.importProvenance).toHaveBeenCalledWith(provenance, 'claim', ['input'], { + trusted: true, + origin: 'guardrailsTool.inputProvenance', + }) + expect(mocks.validateHallucination).toHaveBeenCalledTimes(1) + expect(mocks.validateHallucination).toHaveBeenCalledWith( + expect.objectContaining({ + abortSignal: controller.signal, + actorUserId: 'user-1', + billingAttribution: BILLING_ATTRIBUTION, + workspaceId: 'workspace-1', + }) + ) + }) + + it('keeps local validators outside protected hallucination admission', async () => { + const result = await executeGuardrailsValidation( + { validationType: 'regex', input: 'claim', regex: '^claim$' }, + { + actorUserId: 'user-1', + headers: new Headers(), + requestId: 'request-1', + } + ) + + expect(result.output.passed).toBe(true) + expect(mocks.validateRegex).toHaveBeenCalledOnce() + expect(workflowAuthzMockFns.mockAuthorizeWorkflowByWorkspacePermission).not.toHaveBeenCalled() + expect(mocks.requireBillingAttribution).not.toHaveBeenCalled() + }) + + it('conceals inaccessible workflow validation as a failed verdict without provider work', async () => { + workflowAuthzMockFns.mockAuthorizeWorkflowByWorkspacePermission.mockResolvedValueOnce({ + allowed: false, + message: 'Workflow not found or access denied.', + }) + + const result = await executeGuardrailsValidation( + { + validationType: 'hallucination', + input: 'claim', + knowledgeBaseId: 'knowledge-1', + model: 'gpt-4o', + workflowId: 'workflow-1', + }, + { + actorUserId: 'user-1', + headers: new Headers(), + requestId: 'request-1', + } + ) + + expect(result.output).toMatchObject({ + passed: false, + error: 'Workflow not found or access denied.', + }) + expect(mocks.validateHallucination).not.toHaveBeenCalled() + }) + + it('does not conceal cancellation during hallucination authorization', async () => { + const controller = new AbortController() + const abortError = new DOMException('Operation aborted', 'AbortError') + workflowAuthzMockFns.mockAuthorizeWorkflowByWorkspacePermission.mockImplementationOnce( + async () => { + controller.abort() + throw abortError + } + ) + + await expect( + executeGuardrailsValidation( + { + validationType: 'hallucination', + input: 'claim', + knowledgeBaseId: 'knowledge-1', + model: 'gpt-4o', + workflowId: 'workflow-1', + }, + { + actorUserId: 'user-1', + headers: new Headers(), + requestId: 'request-1', + signal: controller.signal, + } + ) + ).rejects.toBe(abortError) + expect(mocks.validateHallucination).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/guardrails/operations.ts b/apps/sim/lib/internal/guardrails/operations.ts new file mode 100644 index 00000000000..18eda1d2461 --- /dev/null +++ b/apps/sim/lib/internal/guardrails/operations.ts @@ -0,0 +1,381 @@ +import { createLogger } from '@sim/logger' +import { authorizeWorkflowByWorkspacePermission } from '@sim/platform-authz/workflow' +import { getErrorMessage } from '@sim/utils/errors' +import { authorizeCredentialUseForAuth } from '@/lib/auth/credential-access' +import { AuthType } from '@/lib/auth/hybrid' +import { + type BillingAttributionSnapshot, + checkAttributedUsageLimits, + requireBillingAttributionHeader, + toBillingContext, +} from '@/lib/billing/core/billing-attribution' +import { checkAndBillPayerOverageThreshold } from '@/lib/billing/threshold-billing' +import { prepareCopilotEnvironmentContext } from '@/lib/copilot/environment-context' +import { inspectModelInputProvenanceRequest } from '@/lib/execution/model-input-provenance' +import type { CustomPiiPattern } from '@/lib/guardrails/pii-entities' +import { validateHallucination } from '@/lib/guardrails/validate_hallucination' +import { validateJson } from '@/lib/guardrails/validate_json' +import { validatePII } from '@/lib/guardrails/validate_pii' +import { validateRegex } from '@/lib/guardrails/validate_regex' +import { GuardrailsOperationError } from '@/lib/internal/guardrails/errors' +import type { GuardrailsValidationInput } from '@/lib/internal/guardrails/input' +import type { InternalToolOperationContext } from '@/lib/internal/tool-operations/types' +import { + assertPermissionsAllowed, + ModelNotAllowedError, + ProviderNotAllowedError, +} from '@/ee/access-control/utils/permission-check' +import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' +import { isAbortError } from '@/providers/streaming-tool-loop-shared' +import { getProviderFromModel } from '@/providers/utils' + +const logger = createLogger('GuardrailsOperation') + +interface GuardrailsValidationResult { + passed: boolean + error?: string + score?: number + reasoning?: string + detectedEntities?: unknown[] + maskedText?: string + cost?: number +} + +export interface GuardrailsOperationOutput { + success: true + output: { + passed: boolean + validationType: string + input?: unknown + error?: string + score?: number + reasoning?: string + detectedEntities?: unknown[] + maskedText?: string + } +} + +export interface GuardrailsOperationContext { + actorUserId: string + executionContext: InternalToolOperationContext + headers: Headers + requestId: string + signal?: AbortSignal +} + +function fail(status: number, error: string): never { + throw new GuardrailsOperationError(status, { error }) +} + +function failedVerdict( + validationType: string, + input: unknown, + error: string +): GuardrailsOperationOutput { + return { + success: true, + output: { + passed: false, + validationType, + input, + error, + }, + } +} + +function authenticatedCaller(context: GuardrailsOperationContext) { + return { + success: true, + userId: context.actorUserId, + authType: AuthType.INTERNAL_JWT, + } as const +} + +async function authorizeVertexCredential( + input: GuardrailsValidationInput, + context: GuardrailsOperationContext +): Promise { + if (!input.vertexCredential || !input.model || getProviderFromModel(input.model) !== 'vertex') { + return + } + const access = await authorizeCredentialUseForAuth(authenticatedCaller(context), { + credentialId: input.vertexCredential, + workflowId: input.workflowId, + callerUserId: context.actorUserId, + }) + if (!access.ok) { + logger.warn(`[${context.requestId}] Vertex credential access denied`, { + error: access.error, + credentialId: input.vertexCredential, + }) + fail(401, access.error || 'Unauthorized') + } +} + +async function prepareHallucinationContext( + input: GuardrailsValidationInput, + inputString: string, + context: GuardrailsOperationContext +): Promise<{ + workspaceId: string + billingAttribution: BillingAttributionSnapshot + resolvedSecretTraceRegistry: ResolvedSecretTraceRegistry +}> { + if (!input.workflowId || typeof input.workflowId !== 'string') { + throw new Error('Workflow context missing') + } + + const authorization = await authorizeWorkflowByWorkspacePermission({ + workflowId: input.workflowId, + userId: context.actorUserId, + action: 'read', + }) + if (!authorization.allowed || !authorization.workflow?.workspaceId) { + throw new Error(authorization.message || 'Workflow not found or access denied.') + } + + const workspaceId = authorization.workflow.workspaceId + const resolvedSecretTraceRegistry = ( + await prepareCopilotEnvironmentContext(context.actorUserId, workspaceId) + ).resolvedSecretTraceRegistry + + let billingAttribution: BillingAttributionSnapshot + try { + billingAttribution = requireBillingAttributionHeader(context.headers, { + actorUserId: context.actorUserId, + workspaceId, + }) + } catch (error) { + logger.error(`[${context.requestId}] Failed to establish billing attribution`, { error }) + fail(400, 'Invalid billing attribution') + } + + if (!input.model) throw new Error('Model missing') + try { + await assertPermissionsAllowed({ + userId: context.actorUserId, + workspaceId, + model: input.model, + }) + } catch (error) { + if (error instanceof ProviderNotAllowedError || error instanceof ModelNotAllowedError) { + throw error + } + throw error + } + + const usage = await checkAttributedUsageLimits(billingAttribution) + if (usage.isExceeded) { + fail(402, usage.message || 'Usage limit exceeded. Please upgrade your plan to continue.') + } + + await authorizeVertexCredential(input, context) + + const provenanceInspection = inspectModelInputProvenanceRequest(context.headers, input) + if (provenanceInspection.status === 'invalid') { + fail(400, 'Invalid model input provenance') + } + const provenanceReady = + provenanceInspection.status === 'verified' + ? ( + await resolvedSecretTraceRegistry.importProvenanceForValueAtInputPath( + provenanceInspection.value, + inputString, + ['input'], + { trusted: true, origin: 'guardrailsTool.inputProvenance' } + ) + ).success + : true + if (!provenanceReady || !resolvedSecretTraceRegistry.isComplete()) { + fail(400, 'Model input provenance is unavailable') + } + + return { workspaceId, billingAttribution, resolvedSecretTraceRegistry } +} + +function convertInputToString(input: unknown): string { + if (typeof input === 'string') return input + if (input === null || input === undefined) return '' + if (typeof input === 'object') return JSON.stringify(input) + return String(input) +} + +async function executeValidation( + input: GuardrailsValidationInput, + inputString: string, + context: GuardrailsOperationContext, + hallucinationContext?: Awaited> +): Promise { + if (input.validationType === 'json') return validateJson(inputString) + if (input.validationType === 'regex') { + return input.regex + ? validateRegex(inputString, input.regex) + : { passed: false, error: 'Regex pattern is required' } + } + if (input.validationType === 'hallucination') { + if (!input.knowledgeBaseId) { + return { passed: false, error: 'Knowledge base ID is required for hallucination check' } + } + if (!input.model) { + return { passed: false, error: 'Model is required for hallucination validation' } + } + if (!hallucinationContext) { + throw new Error('Hallucination authorization context is unavailable') + } + return validateHallucination({ + userInput: inputString, + knowledgeBaseId: input.knowledgeBaseId, + threshold: input.threshold != null ? Number.parseFloat(input.threshold) : 3, + topK: input.topK ? Number.parseInt(input.topK) : 10, + model: input.model, + apiKey: input.apiKey, + providerCredentials: { + azureEndpoint: input.azureEndpoint, + azureApiVersion: input.azureApiVersion, + vertexProject: input.vertexProject, + vertexLocation: input.vertexLocation, + vertexCredential: input.vertexCredential, + bedrockAccessKeyId: input.bedrockAccessKeyId, + bedrockSecretKey: input.bedrockSecretKey, + bedrockRegion: input.bedrockRegion, + }, + workflowId: input.workflowId, + workspaceId: hallucinationContext.workspaceId, + actorUserId: context.actorUserId, + executionContext: context.executionContext, + billingAttribution: hallucinationContext.billingAttribution, + requestId: context.requestId, + resolvedSecretTraceRegistry: hallucinationContext.resolvedSecretTraceRegistry, + abortSignal: context.signal, + }) + } + if (input.validationType === 'pii') { + return validatePII({ + text: inputString, + entityTypes: input.piiEntityTypes || [], + mode: input.piiMode === 'mask' ? 'mask' : 'block', + language: input.piiLanguage || 'en', + customPatterns: input.piiCustomPatterns as CustomPiiPattern[] | undefined, + requestId: context.requestId, + abortSignal: context.signal, + }) + } + return { passed: false, error: 'Unknown validation type' } +} + +async function recordHallucinationUsage( + input: GuardrailsValidationInput, + result: GuardrailsValidationResult, + context: GuardrailsOperationContext, + hallucinationContext: Awaited> | undefined +): Promise { + if (!hallucinationContext || typeof result.cost !== 'number' || result.cost <= 0) return + + const { recordUsage } = await import('@/lib/billing/core/usage-log') + try { + await recordUsage({ + userId: context.actorUserId, + workspaceId: hallucinationContext.workspaceId, + ...toBillingContext(hallucinationContext.billingAttribution), + entries: [ + { + category: 'model', + source: 'workflow', + description: `guardrail-hallucination:${input.model ?? 'unknown'}`, + cost: result.cost, + sourceReference: `guardrail:${input.workflowId ?? 'unknown'}:${context.requestId}`, + }, + ], + }) + await checkAndBillPayerOverageThreshold(hallucinationContext.billingAttribution.billingEntity) + } catch (error) { + logger.error(`[${context.requestId}] Failed to record guardrail usage`, { error }) + } +} + +/** Executes one guardrail verdict without retrying provider or guardrail work. */ +export async function executeGuardrailsValidation( + input: GuardrailsValidationInput, + context: GuardrailsOperationContext +): Promise { + context.signal?.throwIfAborted() + const validationType = input.validationType + const originalInput = input.input + + if (!validationType) { + return failedVerdict('unknown', originalInput || '', 'Missing required field: validationType') + } + if (originalInput === undefined || originalInput === null) { + return failedVerdict(validationType, '', 'Input is missing or undefined') + } + if (!['json', 'regex', 'hallucination', 'pii'].includes(validationType)) { + return failedVerdict( + validationType, + originalInput || '', + 'Invalid validationType. Must be "json", "regex", "hallucination", or "pii"' + ) + } + if (validationType === 'regex' && !input.regex) { + return failedVerdict( + validationType, + originalInput || '', + 'Regex pattern is required for regex validation' + ) + } + if (validationType === 'hallucination' && !input.model) { + return failedVerdict( + validationType, + originalInput || '', + 'Model is required for hallucination validation' + ) + } + if (validationType === 'hallucination' && !input.workflowId) { + return failedVerdict( + validationType, + originalInput || '', + 'Workflow context is required for hallucination validation. Call this endpoint via a workflow execution, not directly.' + ) + } + + const inputString = convertInputToString(originalInput) + let hallucinationContext: Awaited> | undefined + if (validationType === 'hallucination') { + try { + hallucinationContext = await prepareHallucinationContext(input, inputString, context) + } catch (error) { + if (isAbortError(error) || context.signal?.aborted) throw error + if ( + error instanceof GuardrailsOperationError || + error instanceof ProviderNotAllowedError || + error instanceof ModelNotAllowedError + ) { + if (error instanceof GuardrailsOperationError) throw error + return failedVerdict(validationType, originalInput || '', error.message) + } + return failedVerdict( + validationType, + originalInput || '', + getErrorMessage(error, 'Workflow not found or access denied.') + ) + } + } + + context.signal?.throwIfAborted() + const result = await executeValidation(input, inputString, context, hallucinationContext) + context.signal?.throwIfAborted() + await recordHallucinationUsage(input, result, context, hallucinationContext) + + return { + success: true, + output: { + passed: result.passed, + validationType, + input: originalInput, + error: result.error, + score: result.score, + reasoning: result.reasoning, + detectedEntities: result.detectedEntities, + maskedText: result.maskedText, + }, + } +} diff --git a/apps/sim/lib/internal/iam/client.ts b/apps/sim/lib/internal/iam/client.ts new file mode 100644 index 00000000000..bb654ca1784 --- /dev/null +++ b/apps/sim/lib/internal/iam/client.ts @@ -0,0 +1,501 @@ +import type { + AttachedPolicy, + Group, + Policy, + PolicyScopeType, + Role, + User, +} from '@aws-sdk/client-iam' +import { + AddUserToGroupCommand, + AttachRolePolicyCommand, + AttachUserPolicyCommand, + CreateAccessKeyCommand, + CreateRoleCommand, + CreateUserCommand, + DeleteAccessKeyCommand, + DeleteRoleCommand, + DeleteUserCommand, + DetachRolePolicyCommand, + DetachUserPolicyCommand, + GetRoleCommand, + GetUserCommand, + IAMClient, + ListAttachedRolePoliciesCommand, + ListAttachedUserPoliciesCommand, + ListGroupsCommand, + ListPoliciesCommand, + ListRolesCommand, + ListUsersCommand, + RemoveUserFromGroupCommand, + SimulatePrincipalPolicyCommand, +} from '@aws-sdk/client-iam' +import type { IAMConnectionConfig } from '@/tools/iam/types' + +export function createIAMClient(config: IAMConnectionConfig): IAMClient { + return new IAMClient({ + region: config.region, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + }) +} + +export async function listUsers( + client: IAMClient, + pathPrefix?: string | null, + maxItems?: number | null, + marker?: string | null, + signal?: AbortSignal +) { + const command = new ListUsersCommand({ + ...(pathPrefix ? { PathPrefix: pathPrefix } : {}), + ...(maxItems ? { MaxItems: maxItems } : {}), + ...(marker ? { Marker: marker } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + const users = (response.Users ?? []).map((user: User) => ({ + userName: user.UserName ?? '', + userId: user.UserId ?? '', + arn: user.Arn ?? '', + path: user.Path ?? '', + createDate: user.CreateDate?.toISOString() ?? null, + passwordLastUsed: user.PasswordLastUsed?.toISOString() ?? null, + })) + + return { + users, + isTruncated: response.IsTruncated ?? false, + marker: response.Marker ?? null, + count: users.length, + } +} + +export async function getUser(client: IAMClient, userName?: string | null, signal?: AbortSignal) { + const command = new GetUserCommand(userName ? { UserName: userName } : {}) + const response = await client.send(command, { abortSignal: signal }) + const user = response.User + + return { + userName: user?.UserName ?? '', + userId: user?.UserId ?? '', + arn: user?.Arn ?? '', + path: user?.Path ?? '', + createDate: user?.CreateDate?.toISOString() ?? null, + passwordLastUsed: user?.PasswordLastUsed?.toISOString() ?? null, + permissionsBoundaryArn: user?.PermissionsBoundary?.PermissionsBoundaryArn ?? null, + tags: user?.Tags?.map((t) => ({ key: t.Key ?? '', value: t.Value ?? '' })) ?? [], + } +} + +export async function createUser( + client: IAMClient, + userName: string, + path?: string | null, + signal?: AbortSignal +) { + const command = new CreateUserCommand({ + UserName: userName, + ...(path ? { Path: path } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + const user = response.User + + return { + userName: user?.UserName ?? '', + userId: user?.UserId ?? '', + arn: user?.Arn ?? '', + path: user?.Path ?? '', + createDate: user?.CreateDate?.toISOString() ?? null, + } +} + +export async function deleteUser(client: IAMClient, userName: string, signal?: AbortSignal) { + const command = new DeleteUserCommand({ UserName: userName }) + await client.send(command, { abortSignal: signal }) +} + +export async function listRoles( + client: IAMClient, + pathPrefix?: string | null, + maxItems?: number | null, + marker?: string | null, + signal?: AbortSignal +) { + const command = new ListRolesCommand({ + ...(pathPrefix ? { PathPrefix: pathPrefix } : {}), + ...(maxItems ? { MaxItems: maxItems } : {}), + ...(marker ? { Marker: marker } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + const roles = (response.Roles ?? []).map((role: Role) => ({ + roleName: role.RoleName ?? '', + roleId: role.RoleId ?? '', + arn: role.Arn ?? '', + path: role.Path ?? '', + createDate: role.CreateDate?.toISOString() ?? null, + description: role.Description ?? null, + maxSessionDuration: role.MaxSessionDuration ?? null, + })) + + return { + roles, + isTruncated: response.IsTruncated ?? false, + marker: response.Marker ?? null, + count: roles.length, + } +} + +export async function getRole(client: IAMClient, roleName: string, signal?: AbortSignal) { + const command = new GetRoleCommand({ RoleName: roleName }) + const response = await client.send(command, { abortSignal: signal }) + const role = response.Role + + let policyDocument: string | null = null + if (role?.AssumeRolePolicyDocument) { + try { + policyDocument = decodeURIComponent(role.AssumeRolePolicyDocument) + } catch { + policyDocument = role.AssumeRolePolicyDocument + } + } + + return { + roleName: role?.RoleName ?? '', + roleId: role?.RoleId ?? '', + arn: role?.Arn ?? '', + path: role?.Path ?? '', + createDate: role?.CreateDate?.toISOString() ?? null, + description: role?.Description ?? null, + maxSessionDuration: role?.MaxSessionDuration ?? null, + assumeRolePolicyDocument: policyDocument, + roleLastUsedDate: role?.RoleLastUsed?.LastUsedDate?.toISOString() ?? null, + roleLastUsedRegion: role?.RoleLastUsed?.Region ?? null, + } +} + +export async function createRole( + client: IAMClient, + roleName: string, + assumeRolePolicyDocument: string, + description?: string | null, + path?: string | null, + maxSessionDuration?: number | null, + signal?: AbortSignal +) { + const command = new CreateRoleCommand({ + RoleName: roleName, + AssumeRolePolicyDocument: assumeRolePolicyDocument, + ...(description ? { Description: description } : {}), + ...(path ? { Path: path } : {}), + ...(maxSessionDuration ? { MaxSessionDuration: maxSessionDuration } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + const role = response.Role + + return { + roleName: role?.RoleName ?? '', + roleId: role?.RoleId ?? '', + arn: role?.Arn ?? '', + path: role?.Path ?? '', + createDate: role?.CreateDate?.toISOString() ?? null, + } +} + +export async function deleteRole(client: IAMClient, roleName: string, signal?: AbortSignal) { + const command = new DeleteRoleCommand({ RoleName: roleName }) + await client.send(command, { abortSignal: signal }) +} + +export async function attachUserPolicy( + client: IAMClient, + userName: string, + policyArn: string, + signal?: AbortSignal +) { + const command = new AttachUserPolicyCommand({ + UserName: userName, + PolicyArn: policyArn, + }) + await client.send(command, { abortSignal: signal }) +} + +export async function detachUserPolicy( + client: IAMClient, + userName: string, + policyArn: string, + signal?: AbortSignal +) { + const command = new DetachUserPolicyCommand({ + UserName: userName, + PolicyArn: policyArn, + }) + await client.send(command, { abortSignal: signal }) +} + +export async function attachRolePolicy( + client: IAMClient, + roleName: string, + policyArn: string, + signal?: AbortSignal +) { + const command = new AttachRolePolicyCommand({ + RoleName: roleName, + PolicyArn: policyArn, + }) + await client.send(command, { abortSignal: signal }) +} + +export async function detachRolePolicy( + client: IAMClient, + roleName: string, + policyArn: string, + signal?: AbortSignal +) { + const command = new DetachRolePolicyCommand({ + RoleName: roleName, + PolicyArn: policyArn, + }) + await client.send(command, { abortSignal: signal }) +} + +export async function listPolicies( + client: IAMClient, + scope?: string | null, + onlyAttached?: boolean | null, + pathPrefix?: string | null, + maxItems?: number | null, + marker?: string | null, + signal?: AbortSignal +) { + const command = new ListPoliciesCommand({ + ...(scope ? { Scope: scope as PolicyScopeType } : {}), + ...(onlyAttached != null ? { OnlyAttached: onlyAttached } : {}), + ...(pathPrefix ? { PathPrefix: pathPrefix } : {}), + ...(maxItems ? { MaxItems: maxItems } : {}), + ...(marker ? { Marker: marker } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + const policies = (response.Policies ?? []).map((policy: Policy) => ({ + policyName: policy.PolicyName ?? '', + policyId: policy.PolicyId ?? '', + arn: policy.Arn ?? '', + path: policy.Path ?? '', + attachmentCount: policy.AttachmentCount ?? 0, + isAttachable: policy.IsAttachable ?? false, + createDate: policy.CreateDate?.toISOString() ?? null, + updateDate: policy.UpdateDate?.toISOString() ?? null, + description: policy.Description ?? null, + defaultVersionId: policy.DefaultVersionId ?? null, + permissionsBoundaryUsageCount: policy.PermissionsBoundaryUsageCount ?? 0, + })) + + return { + policies, + isTruncated: response.IsTruncated ?? false, + marker: response.Marker ?? null, + count: policies.length, + } +} + +export async function createAccessKey( + client: IAMClient, + userName?: string | null, + signal?: AbortSignal +) { + const command = new CreateAccessKeyCommand({ + ...(userName ? { UserName: userName } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + const key = response.AccessKey + + return { + accessKeyId: key?.AccessKeyId ?? '', + secretAccessKey: key?.SecretAccessKey ?? '', + userName: key?.UserName ?? '', + status: key?.Status ?? '', + createDate: key?.CreateDate?.toISOString() ?? null, + } +} + +export async function deleteAccessKey( + client: IAMClient, + accessKeyIdToDelete: string, + userName?: string | null, + signal?: AbortSignal +) { + const command = new DeleteAccessKeyCommand({ + AccessKeyId: accessKeyIdToDelete, + ...(userName ? { UserName: userName } : {}), + }) + await client.send(command, { abortSignal: signal }) +} + +export async function listGroups( + client: IAMClient, + pathPrefix?: string | null, + maxItems?: number | null, + marker?: string | null, + signal?: AbortSignal +) { + const command = new ListGroupsCommand({ + ...(pathPrefix ? { PathPrefix: pathPrefix } : {}), + ...(maxItems ? { MaxItems: maxItems } : {}), + ...(marker ? { Marker: marker } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + const groups = (response.Groups ?? []).map((group: Group) => ({ + groupName: group.GroupName ?? '', + groupId: group.GroupId ?? '', + arn: group.Arn ?? '', + path: group.Path ?? '', + createDate: group.CreateDate?.toISOString() ?? null, + })) + + return { + groups, + isTruncated: response.IsTruncated ?? false, + marker: response.Marker ?? null, + count: groups.length, + } +} + +export async function addUserToGroup( + client: IAMClient, + userName: string, + groupName: string, + signal?: AbortSignal +) { + const command = new AddUserToGroupCommand({ + UserName: userName, + GroupName: groupName, + }) + await client.send(command, { abortSignal: signal }) +} + +export async function removeUserFromGroup( + client: IAMClient, + userName: string, + groupName: string, + signal?: AbortSignal +) { + const command = new RemoveUserFromGroupCommand({ + UserName: userName, + GroupName: groupName, + }) + await client.send(command, { abortSignal: signal }) +} + +export async function listAttachedRolePolicies( + client: IAMClient, + roleName: string, + pathPrefix?: string | null, + maxItems?: number | null, + marker?: string | null, + signal?: AbortSignal +) { + const command = new ListAttachedRolePoliciesCommand({ + RoleName: roleName, + ...(pathPrefix ? { PathPrefix: pathPrefix } : {}), + ...(maxItems ? { MaxItems: maxItems } : {}), + ...(marker ? { Marker: marker } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + const attachedPolicies = (response.AttachedPolicies ?? []).map((p: AttachedPolicy) => ({ + policyName: p.PolicyName ?? '', + policyArn: p.PolicyArn ?? '', + })) + + return { + attachedPolicies, + isTruncated: response.IsTruncated ?? false, + marker: response.Marker ?? null, + count: attachedPolicies.length, + } +} + +export async function listAttachedUserPolicies( + client: IAMClient, + userName: string, + pathPrefix?: string | null, + maxItems?: number | null, + marker?: string | null, + signal?: AbortSignal +) { + const command = new ListAttachedUserPoliciesCommand({ + UserName: userName, + ...(pathPrefix ? { PathPrefix: pathPrefix } : {}), + ...(maxItems ? { MaxItems: maxItems } : {}), + ...(marker ? { Marker: marker } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + const attachedPolicies = (response.AttachedPolicies ?? []).map((p: AttachedPolicy) => ({ + policyName: p.PolicyName ?? '', + policyArn: p.PolicyArn ?? '', + })) + + return { + attachedPolicies, + isTruncated: response.IsTruncated ?? false, + marker: response.Marker ?? null, + count: attachedPolicies.length, + } +} + +export async function simulatePrincipalPolicy( + client: IAMClient, + policySourceArn: string, + actionNames: string, + resourceArns?: string | null, + maxResults?: number | null, + marker?: string | null, + signal?: AbortSignal +) { + const actions = actionNames + .split(',') + .map((a) => a.trim()) + .filter(Boolean) + const resources = resourceArns + ? resourceArns + .split(',') + .map((r) => r.trim()) + .filter(Boolean) + : ['*'] + + const command = new SimulatePrincipalPolicyCommand({ + PolicySourceArn: policySourceArn, + ActionNames: actions, + ResourceArns: resources, + ...(maxResults ? { MaxItems: maxResults } : {}), + ...(marker ? { Marker: marker } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + const evaluationResults = (response.EvaluationResults ?? []).map((r) => ({ + evalActionName: r.EvalActionName ?? '', + evalResourceName: r.EvalResourceName ?? '', + evalDecision: r.EvalDecision ?? '', + matchedStatements: (r.MatchedStatements ?? []).map((s) => ({ + sourcePolicyId: s.SourcePolicyId ?? '', + sourcePolicyType: s.SourcePolicyType ?? '', + })), + missingContextValues: (r.MissingContextValues ?? []).map((v) => String(v)), + })) + + return { + evaluationResults, + isTruncated: response.IsTruncated ?? false, + marker: response.Marker ?? null, + count: evaluationResults.length, + } +} diff --git a/apps/sim/lib/internal/iam/execute-tool.test.ts b/apps/sim/lib/internal/iam/execute-tool.test.ts new file mode 100644 index 00000000000..e8ddad046db --- /dev/null +++ b/apps/sim/lib/internal/iam/execute-tool.test.ts @@ -0,0 +1,224 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mockOperations = vi.hoisted(() => ({ + executeIamAddUserToGroup: vi.fn(), + executeIamAttachRolePolicy: vi.fn(), + executeIamAttachUserPolicy: vi.fn(), + executeIamCreateAccessKey: vi.fn(), + executeIamCreateRole: vi.fn(), + executeIamCreateUser: vi.fn(), + executeIamDeleteAccessKey: vi.fn(), + executeIamDeleteRole: vi.fn(), + executeIamDeleteUser: vi.fn(), + executeIamDetachRolePolicy: vi.fn(), + executeIamDetachUserPolicy: vi.fn(), + executeIamGetRole: vi.fn(), + executeIamGetUser: vi.fn(), + executeIamListAttachedRolePolicies: vi.fn(), + executeIamListAttachedUserPolicies: vi.fn(), + executeIamListGroups: vi.fn(), + executeIamListPolicies: vi.fn(), + executeIamListRoles: vi.fn(), + executeIamListUsers: vi.fn(), + executeIamRemoveUserFromGroup: vi.fn(), + executeIamSimulatePrincipalPolicy: vi.fn(), +})) + +vi.mock('@/lib/internal/iam/operations', () => mockOperations) + +import { executeIamTool } from '@/lib/internal/iam/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const CONNECTION = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', +} + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'iam_list_users', + input: CONNECTION, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + userId: 'user-1', + metadata: {}, + }, + requestId: 'request-1', + ...overrides, + } +} + +const TOOL_CASES = [ + { + toolId: 'iam_add_user_to_group', + input: { ...CONNECTION, userName: 'test-user', groupName: 'test-group' }, + operation: mockOperations.executeIamAddUserToGroup, + }, + { + toolId: 'iam_attach_role_policy', + input: { ...CONNECTION, roleName: 'test-role', policyArn: 'arn:aws:iam::aws:policy/Test' }, + operation: mockOperations.executeIamAttachRolePolicy, + }, + { + toolId: 'iam_attach_user_policy', + input: { ...CONNECTION, userName: 'test-user', policyArn: 'arn:aws:iam::aws:policy/Test' }, + operation: mockOperations.executeIamAttachUserPolicy, + }, + { + toolId: 'iam_create_access_key', + input: { ...CONNECTION, userName: 'test-user' }, + operation: mockOperations.executeIamCreateAccessKey, + }, + { + toolId: 'iam_create_role', + input: { + ...CONNECTION, + roleName: 'test-role', + assumeRolePolicyDocument: '{"Version":"2012-10-17"}', + }, + operation: mockOperations.executeIamCreateRole, + }, + { + toolId: 'iam_create_user', + input: { ...CONNECTION, userName: 'test-user' }, + operation: mockOperations.executeIamCreateUser, + }, + { + toolId: 'iam_delete_access_key', + input: { ...CONNECTION, accessKeyIdToDelete: 'AKIADELETE' }, + operation: mockOperations.executeIamDeleteAccessKey, + }, + { + toolId: 'iam_delete_role', + input: { ...CONNECTION, roleName: 'test-role' }, + operation: mockOperations.executeIamDeleteRole, + }, + { + toolId: 'iam_delete_user', + input: { ...CONNECTION, userName: 'test-user' }, + operation: mockOperations.executeIamDeleteUser, + }, + { + toolId: 'iam_detach_role_policy', + input: { ...CONNECTION, roleName: 'test-role', policyArn: 'arn:aws:iam::aws:policy/Test' }, + operation: mockOperations.executeIamDetachRolePolicy, + }, + { + toolId: 'iam_detach_user_policy', + input: { ...CONNECTION, userName: 'test-user', policyArn: 'arn:aws:iam::aws:policy/Test' }, + operation: mockOperations.executeIamDetachUserPolicy, + }, + { + toolId: 'iam_get_role', + input: { ...CONNECTION, roleName: 'test-role' }, + operation: mockOperations.executeIamGetRole, + }, + { + toolId: 'iam_get_user', + input: { ...CONNECTION, userName: 'test-user' }, + operation: mockOperations.executeIamGetUser, + }, + { + toolId: 'iam_list_attached_role_policies', + input: { ...CONNECTION, roleName: 'test-role' }, + operation: mockOperations.executeIamListAttachedRolePolicies, + }, + { + toolId: 'iam_list_attached_user_policies', + input: { ...CONNECTION, userName: 'test-user' }, + operation: mockOperations.executeIamListAttachedUserPolicies, + }, + { + toolId: 'iam_list_groups', + input: CONNECTION, + operation: mockOperations.executeIamListGroups, + }, + { + toolId: 'iam_list_policies', + input: CONNECTION, + operation: mockOperations.executeIamListPolicies, + }, + { + toolId: 'iam_list_roles', + input: CONNECTION, + operation: mockOperations.executeIamListRoles, + }, + { + toolId: 'iam_list_users', + input: CONNECTION, + operation: mockOperations.executeIamListUsers, + }, + { + toolId: 'iam_remove_user_from_group', + input: { ...CONNECTION, userName: 'test-user', groupName: 'test-group' }, + operation: mockOperations.executeIamRemoveUserFromGroup, + }, + { + toolId: 'iam_simulate_principal_policy', + input: { + ...CONNECTION, + policySourceArn: 'arn:aws:iam::123456789012:user/test-user', + actionNames: 's3:GetObject', + }, + operation: mockOperations.executeIamSimulatePrincipalPolicy, + }, +] as const + +describe('executeIamTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it.each(TOOL_CASES)('validates and dispatches $toolId', async ({ toolId, input, operation }) => { + const controller = new AbortController() + operation.mockResolvedValue({ toolId }) + + const response = await executeIamTool( + createRequest({ toolId, input, signal: controller.signal }) + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ toolId }) + expect(operation).toHaveBeenCalledWith(input, controller.signal) + }) + + it('returns the route-compatible validation envelope before provider work', async () => { + const response = await executeIamTool(createRequest({ input: { region: 'invalid' } })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(mockOperations.executeIamListUsers).not.toHaveBeenCalled() + }) + + it('preserves the provider error envelope', async () => { + mockOperations.executeIamListUsers.mockRejectedValue(new Error('AWS rejected credentials')) + + const response = await executeIamTool(createRequest()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'Failed to list IAM users: AWS rejected credentials', + }) + }) + + it('propagates cancellation without starting provider work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeIamTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mockOperations.executeIamListUsers).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/iam/execute-tool.ts b/apps/sim/lib/internal/iam/execute-tool.ts new file mode 100644 index 00000000000..e9074744ac2 --- /dev/null +++ b/apps/sim/lib/internal/iam/execute-tool.ts @@ -0,0 +1,223 @@ +import { awsIamAddUserToGroupContract } from '@/lib/api/contracts/tools/aws/iam-add-user-to-group' +import { awsIamAttachRolePolicyContract } from '@/lib/api/contracts/tools/aws/iam-attach-role-policy' +import { awsIamAttachUserPolicyContract } from '@/lib/api/contracts/tools/aws/iam-attach-user-policy' +import { awsIamCreateAccessKeyContract } from '@/lib/api/contracts/tools/aws/iam-create-access-key' +import { awsIamCreateRoleContract } from '@/lib/api/contracts/tools/aws/iam-create-role' +import { awsIamCreateUserContract } from '@/lib/api/contracts/tools/aws/iam-create-user' +import { awsIamDeleteAccessKeyContract } from '@/lib/api/contracts/tools/aws/iam-delete-access-key' +import { awsIamDeleteRoleContract } from '@/lib/api/contracts/tools/aws/iam-delete-role' +import { awsIamDeleteUserContract } from '@/lib/api/contracts/tools/aws/iam-delete-user' +import { awsIamDetachRolePolicyContract } from '@/lib/api/contracts/tools/aws/iam-detach-role-policy' +import { awsIamDetachUserPolicyContract } from '@/lib/api/contracts/tools/aws/iam-detach-user-policy' +import { awsIamGetRoleContract } from '@/lib/api/contracts/tools/aws/iam-get-role' +import { awsIamGetUserContract } from '@/lib/api/contracts/tools/aws/iam-get-user' +import { awsIamListAttachedRolePoliciesContract } from '@/lib/api/contracts/tools/aws/iam-list-attached-role-policies' +import { awsIamListAttachedUserPoliciesContract } from '@/lib/api/contracts/tools/aws/iam-list-attached-user-policies' +import { awsIamListGroupsContract } from '@/lib/api/contracts/tools/aws/iam-list-groups' +import { awsIamListPoliciesContract } from '@/lib/api/contracts/tools/aws/iam-list-policies' +import { awsIamListRolesContract } from '@/lib/api/contracts/tools/aws/iam-list-roles' +import { awsIamListUsersContract } from '@/lib/api/contracts/tools/aws/iam-list-users' +import { awsIamRemoveUserFromGroupContract } from '@/lib/api/contracts/tools/aws/iam-remove-user-from-group' +import { awsIamSimulatePrincipalPolicyContract } from '@/lib/api/contracts/tools/aws/iam-simulate-principal-policy' +import { + executeIamAddUserToGroup, + executeIamAttachRolePolicy, + executeIamAttachUserPolicy, + executeIamCreateAccessKey, + executeIamCreateRole, + executeIamCreateUser, + executeIamDeleteAccessKey, + executeIamDeleteRole, + executeIamDeleteUser, + executeIamDetachRolePolicy, + executeIamDetachUserPolicy, + executeIamGetRole, + executeIamGetUser, + executeIamListAttachedRolePolicies, + executeIamListAttachedUserPolicies, + executeIamListGroups, + executeIamListPolicies, + executeIamListRoles, + executeIamListUsers, + executeIamRemoveUserFromGroup, + executeIamSimulatePrincipalPolicy, +} from '@/lib/internal/iam/operations' +import { executeInternalJsonToolOperation } from '@/lib/internal/tool-operations/execute-json-operation' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +export const executeIamTool: InternalToolOperationHandler = async ({ toolId, input, signal }) => { + signal?.throwIfAborted() + + switch (toolId) { + case 'iam_add_user_to_group': + return executeInternalJsonToolOperation( + awsIamAddUserToGroupContract, + input, + executeIamAddUserToGroup, + 'Failed to add user to group', + signal + ) + case 'iam_attach_role_policy': + return executeInternalJsonToolOperation( + awsIamAttachRolePolicyContract, + input, + executeIamAttachRolePolicy, + 'Failed to attach role policy', + signal + ) + case 'iam_attach_user_policy': + return executeInternalJsonToolOperation( + awsIamAttachUserPolicyContract, + input, + executeIamAttachUserPolicy, + 'Failed to attach user policy', + signal + ) + case 'iam_create_access_key': + return executeInternalJsonToolOperation( + awsIamCreateAccessKeyContract, + input, + executeIamCreateAccessKey, + 'Failed to create access key', + signal + ) + case 'iam_create_role': + return executeInternalJsonToolOperation( + awsIamCreateRoleContract, + input, + executeIamCreateRole, + 'Failed to create IAM role', + signal + ) + case 'iam_create_user': + return executeInternalJsonToolOperation( + awsIamCreateUserContract, + input, + executeIamCreateUser, + 'Failed to create IAM user', + signal + ) + case 'iam_delete_access_key': + return executeInternalJsonToolOperation( + awsIamDeleteAccessKeyContract, + input, + executeIamDeleteAccessKey, + 'Failed to delete access key', + signal + ) + case 'iam_delete_role': + return executeInternalJsonToolOperation( + awsIamDeleteRoleContract, + input, + executeIamDeleteRole, + 'Failed to delete IAM role', + signal + ) + case 'iam_delete_user': + return executeInternalJsonToolOperation( + awsIamDeleteUserContract, + input, + executeIamDeleteUser, + 'Failed to delete IAM user', + signal + ) + case 'iam_detach_role_policy': + return executeInternalJsonToolOperation( + awsIamDetachRolePolicyContract, + input, + executeIamDetachRolePolicy, + 'Failed to detach role policy', + signal + ) + case 'iam_detach_user_policy': + return executeInternalJsonToolOperation( + awsIamDetachUserPolicyContract, + input, + executeIamDetachUserPolicy, + 'Failed to detach user policy', + signal + ) + case 'iam_get_role': + return executeInternalJsonToolOperation( + awsIamGetRoleContract, + input, + executeIamGetRole, + 'Failed to get IAM role', + signal + ) + case 'iam_get_user': + return executeInternalJsonToolOperation( + awsIamGetUserContract, + input, + executeIamGetUser, + 'Failed to get IAM user', + signal + ) + case 'iam_list_attached_role_policies': + return executeInternalJsonToolOperation( + awsIamListAttachedRolePoliciesContract, + input, + executeIamListAttachedRolePolicies, + 'Failed to list attached role policies', + signal + ) + case 'iam_list_attached_user_policies': + return executeInternalJsonToolOperation( + awsIamListAttachedUserPoliciesContract, + input, + executeIamListAttachedUserPolicies, + 'Failed to list attached user policies', + signal + ) + case 'iam_list_groups': + return executeInternalJsonToolOperation( + awsIamListGroupsContract, + input, + executeIamListGroups, + 'Failed to list IAM groups', + signal + ) + case 'iam_list_policies': + return executeInternalJsonToolOperation( + awsIamListPoliciesContract, + input, + executeIamListPolicies, + 'Failed to list IAM policies', + signal + ) + case 'iam_list_roles': + return executeInternalJsonToolOperation( + awsIamListRolesContract, + input, + executeIamListRoles, + 'Failed to list IAM roles', + signal + ) + case 'iam_list_users': + return executeInternalJsonToolOperation( + awsIamListUsersContract, + input, + executeIamListUsers, + 'Failed to list IAM users', + signal + ) + case 'iam_remove_user_from_group': + return executeInternalJsonToolOperation( + awsIamRemoveUserFromGroupContract, + input, + executeIamRemoveUserFromGroup, + 'Failed to remove user from group', + signal + ) + case 'iam_simulate_principal_policy': + return executeInternalJsonToolOperation( + awsIamSimulatePrincipalPolicyContract, + input, + executeIamSimulatePrincipalPolicy, + 'Failed to simulate principal policy', + signal + ) + default: + return Response.json({ error: `Unsupported IAM tool: ${toolId}` }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/iam/operations.test.ts b/apps/sim/lib/internal/iam/operations.test.ts new file mode 100644 index 00000000000..c7b46afcbd0 --- /dev/null +++ b/apps/sim/lib/internal/iam/operations.test.ts @@ -0,0 +1,76 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockCreateIAMClient, mockDestroy, mockListUsers } = vi.hoisted(() => ({ + mockCreateIAMClient: vi.fn(), + mockDestroy: vi.fn(), + mockListUsers: vi.fn(), +})) + +vi.mock('@/lib/internal/iam/client', () => ({ + addUserToGroup: vi.fn(), + attachRolePolicy: vi.fn(), + attachUserPolicy: vi.fn(), + createAccessKey: vi.fn(), + createIAMClient: mockCreateIAMClient, + createRole: vi.fn(), + createUser: vi.fn(), + deleteAccessKey: vi.fn(), + deleteRole: vi.fn(), + deleteUser: vi.fn(), + detachRolePolicy: vi.fn(), + detachUserPolicy: vi.fn(), + getRole: vi.fn(), + getUser: vi.fn(), + listAttachedRolePolicies: vi.fn(), + listAttachedUserPolicies: vi.fn(), + listGroups: vi.fn(), + listPolicies: vi.fn(), + listRoles: vi.fn(), + listUsers: mockListUsers, + removeUserFromGroup: vi.fn(), + simulatePrincipalPolicy: vi.fn(), +})) + +import { executeIamListUsers } from '@/lib/internal/iam/operations' + +const INPUT = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', + pathPrefix: '/engineering/', + maxItems: 25, + marker: 'next-page', +} + +describe('IAM operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCreateIAMClient.mockReturnValue({ destroy: mockDestroy }) + }) + + it('forwards cancellation and destroys the AWS client after success', async () => { + const controller = new AbortController() + const result = { users: [], isTruncated: false, marker: null, count: 0 } + mockListUsers.mockResolvedValue(result) + + await expect(executeIamListUsers(INPUT, controller.signal)).resolves.toBe(result) + expect(mockListUsers).toHaveBeenCalledWith( + { destroy: mockDestroy }, + '/engineering/', + 25, + 'next-page', + controller.signal + ) + expect(mockDestroy).toHaveBeenCalledOnce() + }) + + it('destroys the AWS client when provider execution fails', async () => { + mockListUsers.mockRejectedValue(new Error('provider failure')) + + await expect(executeIamListUsers(INPUT)).rejects.toThrow('provider failure') + expect(mockDestroy).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/internal/iam/operations.ts b/apps/sim/lib/internal/iam/operations.ts new file mode 100644 index 00000000000..1780c0d4c56 --- /dev/null +++ b/apps/sim/lib/internal/iam/operations.ts @@ -0,0 +1,334 @@ +import type { IAMClient } from '@aws-sdk/client-iam' +import type { AwsIamAddUserToGroupBody } from '@/lib/api/contracts/tools/aws/iam-add-user-to-group' +import type { AwsIamAttachRolePolicyBody } from '@/lib/api/contracts/tools/aws/iam-attach-role-policy' +import type { AwsIamAttachUserPolicyBody } from '@/lib/api/contracts/tools/aws/iam-attach-user-policy' +import type { AwsIamCreateAccessKeyBody } from '@/lib/api/contracts/tools/aws/iam-create-access-key' +import type { AwsIamCreateRoleBody } from '@/lib/api/contracts/tools/aws/iam-create-role' +import type { AwsIamCreateUserBody } from '@/lib/api/contracts/tools/aws/iam-create-user' +import type { AwsIamDeleteAccessKeyBody } from '@/lib/api/contracts/tools/aws/iam-delete-access-key' +import type { AwsIamDeleteRoleBody } from '@/lib/api/contracts/tools/aws/iam-delete-role' +import type { AwsIamDeleteUserBody } from '@/lib/api/contracts/tools/aws/iam-delete-user' +import type { AwsIamDetachRolePolicyBody } from '@/lib/api/contracts/tools/aws/iam-detach-role-policy' +import type { AwsIamDetachUserPolicyBody } from '@/lib/api/contracts/tools/aws/iam-detach-user-policy' +import type { AwsIamGetRoleBody } from '@/lib/api/contracts/tools/aws/iam-get-role' +import type { AwsIamGetUserBody } from '@/lib/api/contracts/tools/aws/iam-get-user' +import type { AwsIamListAttachedRolePoliciesBody } from '@/lib/api/contracts/tools/aws/iam-list-attached-role-policies' +import type { AwsIamListAttachedUserPoliciesBody } from '@/lib/api/contracts/tools/aws/iam-list-attached-user-policies' +import type { AwsIamListGroupsBody } from '@/lib/api/contracts/tools/aws/iam-list-groups' +import type { AwsIamListPoliciesBody } from '@/lib/api/contracts/tools/aws/iam-list-policies' +import type { AwsIamListRolesBody } from '@/lib/api/contracts/tools/aws/iam-list-roles' +import type { AwsIamListUsersBody } from '@/lib/api/contracts/tools/aws/iam-list-users' +import type { AwsIamRemoveUserFromGroupBody } from '@/lib/api/contracts/tools/aws/iam-remove-user-from-group' +import type { AwsIamSimulatePrincipalPolicyBody } from '@/lib/api/contracts/tools/aws/iam-simulate-principal-policy' +import { + addUserToGroup, + attachRolePolicy, + attachUserPolicy, + createAccessKey, + createIAMClient, + createRole, + createUser, + deleteAccessKey, + deleteRole, + deleteUser, + detachRolePolicy, + detachUserPolicy, + getRole, + getUser, + listAttachedRolePolicies, + listAttachedUserPolicies, + listGroups, + listPolicies, + listRoles, + listUsers, + removeUserFromGroup, + simulatePrincipalPolicy, +} from '@/lib/internal/iam/client' +import type { IAMConnectionConfig } from '@/tools/iam/types' + +async function withIamClient( + config: IAMConnectionConfig, + operation: (client: IAMClient) => Promise, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const client = createIAMClient(config) + try { + const result = await operation(client) + signal?.throwIfAborted() + return result + } finally { + client.destroy() + } +} + +export async function executeIamListUsers(input: AwsIamListUsersBody, signal?: AbortSignal) { + return withIamClient( + input, + (client) => listUsers(client, input.pathPrefix, input.maxItems, input.marker, signal), + signal + ) +} + +export async function executeIamGetUser(input: AwsIamGetUserBody, signal?: AbortSignal) { + return withIamClient(input, (client) => getUser(client, input.userName, signal), signal) +} + +export async function executeIamCreateUser(input: AwsIamCreateUserBody, signal?: AbortSignal) { + return withIamClient( + input, + async (client) => { + const result = await createUser(client, input.userName, input.path, signal) + return { message: `User "${result.userName}" created successfully`, ...result } + }, + signal + ) +} + +export async function executeIamDeleteUser(input: AwsIamDeleteUserBody, signal?: AbortSignal) { + return withIamClient( + input, + async (client) => { + await deleteUser(client, input.userName, signal) + return { message: `User "${input.userName}" deleted successfully` } + }, + signal + ) +} + +export async function executeIamListRoles(input: AwsIamListRolesBody, signal?: AbortSignal) { + return withIamClient( + input, + (client) => listRoles(client, input.pathPrefix, input.maxItems, input.marker, signal), + signal + ) +} + +export async function executeIamGetRole(input: AwsIamGetRoleBody, signal?: AbortSignal) { + return withIamClient(input, (client) => getRole(client, input.roleName, signal), signal) +} + +export async function executeIamCreateRole(input: AwsIamCreateRoleBody, signal?: AbortSignal) { + return withIamClient( + input, + async (client) => { + const result = await createRole( + client, + input.roleName, + input.assumeRolePolicyDocument, + input.description, + input.path, + input.maxSessionDuration, + signal + ) + return { message: `Role "${result.roleName}" created successfully`, ...result } + }, + signal + ) +} + +export async function executeIamDeleteRole(input: AwsIamDeleteRoleBody, signal?: AbortSignal) { + return withIamClient( + input, + async (client) => { + await deleteRole(client, input.roleName, signal) + return { message: `Role "${input.roleName}" deleted successfully` } + }, + signal + ) +} + +export async function executeIamAttachUserPolicy( + input: AwsIamAttachUserPolicyBody, + signal?: AbortSignal +) { + return withIamClient( + input, + async (client) => { + await attachUserPolicy(client, input.userName, input.policyArn, signal) + return { message: `Policy "${input.policyArn}" attached to user "${input.userName}"` } + }, + signal + ) +} + +export async function executeIamDetachUserPolicy( + input: AwsIamDetachUserPolicyBody, + signal?: AbortSignal +) { + return withIamClient( + input, + async (client) => { + await detachUserPolicy(client, input.userName, input.policyArn, signal) + return { message: `Policy "${input.policyArn}" detached from user "${input.userName}"` } + }, + signal + ) +} + +export async function executeIamAttachRolePolicy( + input: AwsIamAttachRolePolicyBody, + signal?: AbortSignal +) { + return withIamClient( + input, + async (client) => { + await attachRolePolicy(client, input.roleName, input.policyArn, signal) + return { message: `Policy "${input.policyArn}" attached to role "${input.roleName}"` } + }, + signal + ) +} + +export async function executeIamDetachRolePolicy( + input: AwsIamDetachRolePolicyBody, + signal?: AbortSignal +) { + return withIamClient( + input, + async (client) => { + await detachRolePolicy(client, input.roleName, input.policyArn, signal) + return { message: `Policy "${input.policyArn}" detached from role "${input.roleName}"` } + }, + signal + ) +} + +export async function executeIamListPolicies(input: AwsIamListPoliciesBody, signal?: AbortSignal) { + return withIamClient( + input, + (client) => + listPolicies( + client, + input.scope, + input.onlyAttached, + input.pathPrefix, + input.maxItems, + input.marker, + signal + ), + signal + ) +} + +export async function executeIamCreateAccessKey( + input: AwsIamCreateAccessKeyBody, + signal?: AbortSignal +) { + return withIamClient( + input, + async (client) => { + const result = await createAccessKey(client, input.userName, signal) + return { message: `Access key created for user "${result.userName}"`, ...result } + }, + signal + ) +} + +export async function executeIamDeleteAccessKey( + input: AwsIamDeleteAccessKeyBody, + signal?: AbortSignal +) { + return withIamClient( + input, + async (client) => { + await deleteAccessKey(client, input.accessKeyIdToDelete, input.userName, signal) + return { message: `Access key "${input.accessKeyIdToDelete}" deleted` } + }, + signal + ) +} + +export async function executeIamListGroups(input: AwsIamListGroupsBody, signal?: AbortSignal) { + return withIamClient( + input, + (client) => listGroups(client, input.pathPrefix, input.maxItems, input.marker, signal), + signal + ) +} + +export async function executeIamAddUserToGroup( + input: AwsIamAddUserToGroupBody, + signal?: AbortSignal +) { + return withIamClient( + input, + async (client) => { + await addUserToGroup(client, input.userName, input.groupName, signal) + return { message: `User "${input.userName}" added to group "${input.groupName}"` } + }, + signal + ) +} + +export async function executeIamRemoveUserFromGroup( + input: AwsIamRemoveUserFromGroupBody, + signal?: AbortSignal +) { + return withIamClient( + input, + async (client) => { + await removeUserFromGroup(client, input.userName, input.groupName, signal) + return { message: `User "${input.userName}" removed from group "${input.groupName}"` } + }, + signal + ) +} + +export async function executeIamListAttachedRolePolicies( + input: AwsIamListAttachedRolePoliciesBody, + signal?: AbortSignal +) { + return withIamClient( + input, + (client) => + listAttachedRolePolicies( + client, + input.roleName, + input.pathPrefix, + input.maxItems, + input.marker, + signal + ), + signal + ) +} + +export async function executeIamListAttachedUserPolicies( + input: AwsIamListAttachedUserPoliciesBody, + signal?: AbortSignal +) { + return withIamClient( + input, + (client) => + listAttachedUserPolicies( + client, + input.userName, + input.pathPrefix, + input.maxItems, + input.marker, + signal + ), + signal + ) +} + +export async function executeIamSimulatePrincipalPolicy( + input: AwsIamSimulatePrincipalPolicyBody, + signal?: AbortSignal +) { + return withIamClient( + input, + (client) => + simulatePrincipalPolicy( + client, + input.policySourceArn, + input.actionNames, + input.resourceArns, + input.maxResults, + input.marker, + signal + ), + signal + ) +} diff --git a/apps/sim/lib/internal/identity-center/client.ts b/apps/sim/lib/internal/identity-center/client.ts new file mode 100644 index 00000000000..539a2381d94 --- /dev/null +++ b/apps/sim/lib/internal/identity-center/client.ts @@ -0,0 +1,373 @@ +import { + DescribeGroupCommand, + DescribeUserCommand, + GetGroupIdCommand, + GetUserIdCommand, + IdentitystoreClient, + ListGroupsCommand, +} from '@aws-sdk/client-identitystore' +import { + DescribeAccountCommand, + ListAccountsCommand, + OrganizationsClient, +} from '@aws-sdk/client-organizations' +import { + type AccountAssignmentOperationStatus, + CreateAccountAssignmentCommand, + DeleteAccountAssignmentCommand, + DescribeAccountAssignmentCreationStatusCommand, + DescribeAccountAssignmentDeletionStatusCommand, + DescribePermissionSetCommand, + ListAccountAssignmentsForPrincipalCommand, + ListInstancesCommand, + ListPermissionSetsCommand, + type PrincipalType, + SSOAdminClient, + type TargetType, +} from '@aws-sdk/client-sso-admin' + +interface IdentityCenterConnectionConfig { + region: string + accessKeyId: string + secretAccessKey: string +} + +const AWS_ORGANIZATIONS_REGION = 'us-east-1' + +export function createSSOAdminClient(config: IdentityCenterConnectionConfig): SSOAdminClient { + return new SSOAdminClient({ + region: config.region, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + }) +} + +export function createIdentityStoreClient( + config: IdentityCenterConnectionConfig +): IdentitystoreClient { + return new IdentitystoreClient({ + region: config.region, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + }) +} + +export function createOrganizationsClient(config: IdentityCenterConnectionConfig) { + return new OrganizationsClient({ + region: AWS_ORGANIZATIONS_REGION, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + }) +} + +export async function listInstances( + client: SSOAdminClient, + maxResults?: number | null, + nextToken?: string | null, + signal?: AbortSignal +) { + const command = new ListInstancesCommand({ + ...(maxResults ? { MaxResults: maxResults } : {}), + ...(nextToken ? { NextToken: nextToken } : {}), + }) + const response = await client.send(command, { abortSignal: signal }) + const instances = (response.Instances ?? []).map((instance) => ({ + instanceArn: instance.InstanceArn ?? '', + identityStoreId: instance.IdentityStoreId ?? '', + name: instance.Name ?? null, + status: instance.Status ?? '', + statusReason: instance.StatusReason ?? null, + ownerAccountId: instance.OwnerAccountId ?? null, + createdDate: instance.CreatedDate?.toISOString() ?? null, + })) + return { instances, nextToken: response.NextToken ?? null, count: instances.length } +} + +export async function listAccounts( + client: OrganizationsClient, + maxResults?: number | null, + nextToken?: string | null, + signal?: AbortSignal +) { + const command = new ListAccountsCommand({ + ...(maxResults ? { MaxResults: maxResults } : {}), + ...(nextToken ? { NextToken: nextToken } : {}), + }) + const response = await client.send(command, { abortSignal: signal }) + const accounts = (response.Accounts ?? []).map((account) => ({ + id: account.Id ?? '', + arn: account.Arn ?? '', + name: account.Name ?? '', + email: account.Email ?? '', + status: account.State ?? '', + joinedTimestamp: account.JoinedTimestamp?.toISOString() ?? null, + })) + return { accounts, nextToken: response.NextToken ?? null, count: accounts.length } +} + +export async function listPermissionSets( + client: SSOAdminClient, + instanceArn: string, + maxResults?: number | null, + nextToken?: string | null, + signal?: AbortSignal +) { + const listCommand = new ListPermissionSetsCommand({ + InstanceArn: instanceArn, + ...(maxResults ? { MaxResults: maxResults } : {}), + ...(nextToken ? { NextToken: nextToken } : {}), + }) + const listResponse = await client.send(listCommand, { abortSignal: signal }) + const permissionSetArns = listResponse.PermissionSets ?? [] + + const permissionSets = await Promise.all( + permissionSetArns.map(async (arn) => { + const describeCommand = new DescribePermissionSetCommand({ + InstanceArn: instanceArn, + PermissionSetArn: arn, + }) + const describeResponse = await client.send(describeCommand, { abortSignal: signal }) + const permissionSet = describeResponse.PermissionSet + return { + permissionSetArn: permissionSet?.PermissionSetArn ?? arn, + name: permissionSet?.Name ?? '', + description: permissionSet?.Description ?? null, + sessionDuration: permissionSet?.SessionDuration ?? null, + createdDate: permissionSet?.CreatedDate?.toISOString() ?? null, + } + }) + ) + + return { + permissionSets, + nextToken: listResponse.NextToken ?? null, + count: permissionSets.length, + } +} + +export async function getUserByEmail( + client: IdentitystoreClient, + identityStoreId: string, + email: string, + signal?: AbortSignal +) { + const getUserIdCommand = new GetUserIdCommand({ + IdentityStoreId: identityStoreId, + AlternateIdentifier: { + UniqueAttribute: { + AttributePath: 'emails.value', + AttributeValue: email, + }, + }, + }) + const getUserIdResponse = await client.send(getUserIdCommand, { abortSignal: signal }) + const userId = getUserIdResponse.UserId ?? '' + + const describeCommand = new DescribeUserCommand({ + IdentityStoreId: identityStoreId, + UserId: userId, + }) + const describeResponse = await client.send(describeCommand, { abortSignal: signal }) + + const primaryEmail = + describeResponse.Emails?.find((entry) => entry.Primary)?.Value ?? + describeResponse.Emails?.[0]?.Value ?? + null + + return { + userId, + userName: describeResponse.UserName ?? '', + displayName: describeResponse.DisplayName ?? null, + email: primaryEmail, + } +} + +function mapAssignmentStatus(status: AccountAssignmentOperationStatus) { + return { + status: status.Status ?? '', + requestId: status.RequestId ?? '', + accountId: status.TargetId ?? null, + permissionSetArn: status.PermissionSetArn ?? null, + principalType: status.PrincipalType ?? null, + principalId: status.PrincipalId ?? null, + failureReason: status.FailureReason ?? null, + createdDate: status.CreatedDate?.toISOString() ?? null, + } +} + +interface AccountAssignmentInput { + instanceArn: string + accountId: string + permissionSetArn: string + principalType: 'USER' | 'GROUP' + principalId: string +} + +export async function createAccountAssignment( + client: SSOAdminClient, + input: AccountAssignmentInput, + signal?: AbortSignal +) { + const command = new CreateAccountAssignmentCommand({ + InstanceArn: input.instanceArn, + TargetId: input.accountId, + TargetType: 'AWS_ACCOUNT' as TargetType, + PermissionSetArn: input.permissionSetArn, + PrincipalType: input.principalType as PrincipalType, + PrincipalId: input.principalId, + }) + const response = await client.send(command, { abortSignal: signal }) + return mapAssignmentStatus(response.AccountAssignmentCreationStatus ?? {}) +} + +export async function deleteAccountAssignment( + client: SSOAdminClient, + input: AccountAssignmentInput, + signal?: AbortSignal +) { + const command = new DeleteAccountAssignmentCommand({ + InstanceArn: input.instanceArn, + TargetId: input.accountId, + TargetType: 'AWS_ACCOUNT' as TargetType, + PermissionSetArn: input.permissionSetArn, + PrincipalType: input.principalType as PrincipalType, + PrincipalId: input.principalId, + }) + const response = await client.send(command, { abortSignal: signal }) + return mapAssignmentStatus(response.AccountAssignmentDeletionStatus ?? {}) +} + +export async function checkAssignmentCreationStatus( + client: SSOAdminClient, + instanceArn: string, + requestId: string, + signal?: AbortSignal +) { + const command = new DescribeAccountAssignmentCreationStatusCommand({ + InstanceArn: instanceArn, + AccountAssignmentCreationRequestId: requestId, + }) + const response = await client.send(command, { abortSignal: signal }) + return mapAssignmentStatus(response.AccountAssignmentCreationStatus ?? {}) +} + +export async function checkAssignmentDeletionStatus( + client: SSOAdminClient, + instanceArn: string, + requestId: string, + signal?: AbortSignal +) { + const command = new DescribeAccountAssignmentDeletionStatusCommand({ + InstanceArn: instanceArn, + AccountAssignmentDeletionRequestId: requestId, + }) + const response = await client.send(command, { abortSignal: signal }) + return mapAssignmentStatus(response.AccountAssignmentDeletionStatus ?? {}) +} + +export async function listGroups( + client: IdentitystoreClient, + identityStoreId: string, + maxResults?: number | null, + nextToken?: string | null, + signal?: AbortSignal +) { + const command = new ListGroupsCommand({ + IdentityStoreId: identityStoreId, + ...(maxResults ? { MaxResults: maxResults } : {}), + ...(nextToken ? { NextToken: nextToken } : {}), + }) + const response = await client.send(command, { abortSignal: signal }) + const groups = (response.Groups ?? []).map((group) => ({ + groupId: group.GroupId ?? '', + displayName: group.DisplayName ?? null, + description: group.Description ?? null, + externalIds: + group.ExternalIds?.map((externalId) => ({ + issuer: externalId.Issuer ?? '', + id: externalId.Id ?? '', + })) ?? [], + })) + return { groups, nextToken: response.NextToken ?? null, count: groups.length } +} + +export async function getGroupByDisplayName( + client: IdentitystoreClient, + identityStoreId: string, + displayName: string, + signal?: AbortSignal +) { + const getGroupIdCommand = new GetGroupIdCommand({ + IdentityStoreId: identityStoreId, + AlternateIdentifier: { + UniqueAttribute: { + AttributePath: 'displayName', + AttributeValue: displayName, + }, + }, + }) + const getGroupIdResponse = await client.send(getGroupIdCommand, { abortSignal: signal }) + const groupId = getGroupIdResponse.GroupId ?? '' + + const describeCommand = new DescribeGroupCommand({ + IdentityStoreId: identityStoreId, + GroupId: groupId, + }) + const describeResponse = await client.send(describeCommand, { abortSignal: signal }) + + return { + groupId, + displayName: describeResponse.DisplayName ?? null, + description: describeResponse.Description ?? null, + } +} + +export async function describeAccount( + client: OrganizationsClient, + accountId: string, + signal?: AbortSignal +) { + const command = new DescribeAccountCommand({ AccountId: accountId }) + const response = await client.send(command, { abortSignal: signal }) + const account = response.Account + return { + id: account?.Id ?? '', + arn: account?.Arn ?? '', + name: account?.Name ?? '', + email: account?.Email ?? '', + status: account?.State ?? '', + joinedTimestamp: account?.JoinedTimestamp?.toISOString() ?? null, + } +} + +export async function listAccountAssignmentsForPrincipal( + client: SSOAdminClient, + instanceArn: string, + principalId: string, + principalType: 'USER' | 'GROUP', + maxResults?: number | null, + nextToken?: string | null, + signal?: AbortSignal +) { + const command = new ListAccountAssignmentsForPrincipalCommand({ + InstanceArn: instanceArn, + PrincipalId: principalId, + PrincipalType: principalType as PrincipalType, + ...(maxResults ? { MaxResults: maxResults } : {}), + ...(nextToken ? { NextToken: nextToken } : {}), + }) + const response = await client.send(command, { abortSignal: signal }) + const assignments = (response.AccountAssignments ?? []).map((assignment) => ({ + accountId: assignment.AccountId ?? '', + permissionSetArn: assignment.PermissionSetArn ?? '', + principalType: assignment.PrincipalType ?? '', + principalId: assignment.PrincipalId ?? '', + })) + return { assignments, nextToken: response.NextToken ?? null, count: assignments.length } +} diff --git a/apps/sim/lib/internal/identity-center/execute-tool.test.ts b/apps/sim/lib/internal/identity-center/execute-tool.test.ts new file mode 100644 index 00000000000..8bff280c367 --- /dev/null +++ b/apps/sim/lib/internal/identity-center/execute-tool.test.ts @@ -0,0 +1,193 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mockOperations = vi.hoisted(() => ({ + executeIdentityCenterListInstances: vi.fn(), + executeIdentityCenterListAccounts: vi.fn(), + executeIdentityCenterDescribeAccount: vi.fn(), + executeIdentityCenterListPermissionSets: vi.fn(), + executeIdentityCenterGetUser: vi.fn(), + executeIdentityCenterGetGroup: vi.fn(), + executeIdentityCenterListGroups: vi.fn(), + executeIdentityCenterCreateAccountAssignment: vi.fn(), + executeIdentityCenterDeleteAccountAssignment: vi.fn(), + executeIdentityCenterCheckAssignmentStatus: vi.fn(), + executeIdentityCenterCheckAssignmentDeletionStatus: vi.fn(), + executeIdentityCenterListAccountAssignments: vi.fn(), +})) + +vi.mock('@/lib/internal/identity-center/operations', () => mockOperations) + +import { executeIdentityCenterTool } from '@/lib/internal/identity-center/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const CONNECTION = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', +} + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'identity_center_list_instances', + input: CONNECTION, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + userId: 'user-1', + metadata: {}, + }, + requestId: 'request-1', + ...overrides, + } +} + +const TOOL_CASES = [ + { + toolId: 'identity_center_list_instances', + input: CONNECTION, + operation: mockOperations.executeIdentityCenterListInstances, + }, + { + toolId: 'identity_center_list_accounts', + input: CONNECTION, + operation: mockOperations.executeIdentityCenterListAccounts, + }, + { + toolId: 'identity_center_describe_account', + input: { ...CONNECTION, accountId: '123456789012' }, + operation: mockOperations.executeIdentityCenterDescribeAccount, + }, + { + toolId: 'identity_center_list_permission_sets', + input: { ...CONNECTION, instanceArn: 'arn:aws:sso:::instance/ssoins-test' }, + operation: mockOperations.executeIdentityCenterListPermissionSets, + }, + { + toolId: 'identity_center_get_user', + input: { ...CONNECTION, identityStoreId: 'd-test', email: 'user@example.com' }, + operation: mockOperations.executeIdentityCenterGetUser, + }, + { + toolId: 'identity_center_get_group', + input: { ...CONNECTION, identityStoreId: 'd-test', displayName: 'Engineering' }, + operation: mockOperations.executeIdentityCenterGetGroup, + }, + { + toolId: 'identity_center_list_groups', + input: { ...CONNECTION, identityStoreId: 'd-test' }, + operation: mockOperations.executeIdentityCenterListGroups, + }, + { + toolId: 'identity_center_create_account_assignment', + input: { + ...CONNECTION, + instanceArn: 'arn:aws:sso:::instance/ssoins-test', + accountId: '123456789012', + permissionSetArn: 'arn:aws:sso:::permissionSet/ssoins-test/ps-test', + principalType: 'USER', + principalId: 'user-1', + }, + operation: mockOperations.executeIdentityCenterCreateAccountAssignment, + }, + { + toolId: 'identity_center_delete_account_assignment', + input: { + ...CONNECTION, + instanceArn: 'arn:aws:sso:::instance/ssoins-test', + accountId: '123456789012', + permissionSetArn: 'arn:aws:sso:::permissionSet/ssoins-test/ps-test', + principalType: 'GROUP', + principalId: 'group-1', + }, + operation: mockOperations.executeIdentityCenterDeleteAccountAssignment, + }, + { + toolId: 'identity_center_check_assignment_status', + input: { + ...CONNECTION, + instanceArn: 'arn:aws:sso:::instance/ssoins-test', + requestId: 'request-1', + }, + operation: mockOperations.executeIdentityCenterCheckAssignmentStatus, + }, + { + toolId: 'identity_center_check_assignment_deletion_status', + input: { + ...CONNECTION, + instanceArn: 'arn:aws:sso:::instance/ssoins-test', + requestId: 'request-1', + }, + operation: mockOperations.executeIdentityCenterCheckAssignmentDeletionStatus, + }, + { + toolId: 'identity_center_list_account_assignments', + input: { + ...CONNECTION, + instanceArn: 'arn:aws:sso:::instance/ssoins-test', + principalType: 'USER', + principalId: 'user-1', + }, + operation: mockOperations.executeIdentityCenterListAccountAssignments, + }, +] as const + +describe('executeIdentityCenterTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it.each(TOOL_CASES)('validates and dispatches $toolId', async ({ toolId, input, operation }) => { + const controller = new AbortController() + operation.mockResolvedValue({ toolId }) + + const response = await executeIdentityCenterTool( + createRequest({ toolId, input, signal: controller.signal }) + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ toolId }) + expect(operation).toHaveBeenCalledWith(input, controller.signal) + }) + + it('returns the route-compatible validation envelope before provider work', async () => { + const response = await executeIdentityCenterTool( + createRequest({ input: { region: 'invalid' } }) + ) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(mockOperations.executeIdentityCenterListInstances).not.toHaveBeenCalled() + }) + + it('preserves the provider error envelope', async () => { + mockOperations.executeIdentityCenterListInstances.mockRejectedValue( + new Error('AWS rejected credentials') + ) + + const response = await executeIdentityCenterTool(createRequest()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'Failed to list Identity Center instances: AWS rejected credentials', + }) + }) + + it('propagates cancellation without starting provider work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeIdentityCenterTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mockOperations.executeIdentityCenterListInstances).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/identity-center/execute-tool.ts b/apps/sim/lib/internal/identity-center/execute-tool.ts new file mode 100644 index 00000000000..47e21a478da --- /dev/null +++ b/apps/sim/lib/internal/identity-center/execute-tool.ts @@ -0,0 +1,140 @@ +import { awsIdentityCenterCheckAssignmentDeletionStatusContract } from '@/lib/api/contracts/tools/aws/identity-center-check-assignment-deletion-status' +import { awsIdentityCenterCheckAssignmentStatusContract } from '@/lib/api/contracts/tools/aws/identity-center-check-assignment-status' +import { awsIdentityCenterCreateAccountAssignmentContract } from '@/lib/api/contracts/tools/aws/identity-center-create-account-assignment' +import { awsIdentityCenterDeleteAccountAssignmentContract } from '@/lib/api/contracts/tools/aws/identity-center-delete-account-assignment' +import { awsIdentityCenterDescribeAccountContract } from '@/lib/api/contracts/tools/aws/identity-center-describe-account' +import { awsIdentityCenterGetGroupContract } from '@/lib/api/contracts/tools/aws/identity-center-get-group' +import { awsIdentityCenterGetUserContract } from '@/lib/api/contracts/tools/aws/identity-center-get-user' +import { awsIdentityCenterListAccountAssignmentsContract } from '@/lib/api/contracts/tools/aws/identity-center-list-account-assignments' +import { awsIdentityCenterListAccountsContract } from '@/lib/api/contracts/tools/aws/identity-center-list-accounts' +import { awsIdentityCenterListGroupsContract } from '@/lib/api/contracts/tools/aws/identity-center-list-groups' +import { awsIdentityCenterListInstancesContract } from '@/lib/api/contracts/tools/aws/identity-center-list-instances' +import { awsIdentityCenterListPermissionSetsContract } from '@/lib/api/contracts/tools/aws/identity-center-list-permission-sets' +import { + executeIdentityCenterCheckAssignmentDeletionStatus, + executeIdentityCenterCheckAssignmentStatus, + executeIdentityCenterCreateAccountAssignment, + executeIdentityCenterDeleteAccountAssignment, + executeIdentityCenterDescribeAccount, + executeIdentityCenterGetGroup, + executeIdentityCenterGetUser, + executeIdentityCenterListAccountAssignments, + executeIdentityCenterListAccounts, + executeIdentityCenterListGroups, + executeIdentityCenterListInstances, + executeIdentityCenterListPermissionSets, +} from '@/lib/internal/identity-center/operations' +import { executeInternalJsonToolOperation } from '@/lib/internal/tool-operations/execute-json-operation' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +export const executeIdentityCenterTool: InternalToolOperationHandler = async ({ + toolId, + input, + signal, +}) => { + signal?.throwIfAborted() + + switch (toolId) { + case 'identity_center_list_instances': + return executeInternalJsonToolOperation( + awsIdentityCenterListInstancesContract, + input, + executeIdentityCenterListInstances, + 'Failed to list Identity Center instances', + signal + ) + case 'identity_center_list_accounts': + return executeInternalJsonToolOperation( + awsIdentityCenterListAccountsContract, + input, + executeIdentityCenterListAccounts, + 'Failed to list AWS accounts', + signal + ) + case 'identity_center_describe_account': + return executeInternalJsonToolOperation( + awsIdentityCenterDescribeAccountContract, + input, + executeIdentityCenterDescribeAccount, + 'Failed to describe account', + signal + ) + case 'identity_center_list_permission_sets': + return executeInternalJsonToolOperation( + awsIdentityCenterListPermissionSetsContract, + input, + executeIdentityCenterListPermissionSets, + 'Failed to list permission sets', + signal + ) + case 'identity_center_get_user': + return executeInternalJsonToolOperation( + awsIdentityCenterGetUserContract, + input, + executeIdentityCenterGetUser, + 'Failed to get user', + signal + ) + case 'identity_center_get_group': + return executeInternalJsonToolOperation( + awsIdentityCenterGetGroupContract, + input, + executeIdentityCenterGetGroup, + 'Failed to get group', + signal + ) + case 'identity_center_list_groups': + return executeInternalJsonToolOperation( + awsIdentityCenterListGroupsContract, + input, + executeIdentityCenterListGroups, + 'Failed to list groups', + signal + ) + case 'identity_center_create_account_assignment': + return executeInternalJsonToolOperation( + awsIdentityCenterCreateAccountAssignmentContract, + input, + executeIdentityCenterCreateAccountAssignment, + 'Failed to create account assignment', + signal + ) + case 'identity_center_delete_account_assignment': + return executeInternalJsonToolOperation( + awsIdentityCenterDeleteAccountAssignmentContract, + input, + executeIdentityCenterDeleteAccountAssignment, + 'Failed to delete account assignment', + signal + ) + case 'identity_center_check_assignment_status': + return executeInternalJsonToolOperation( + awsIdentityCenterCheckAssignmentStatusContract, + input, + executeIdentityCenterCheckAssignmentStatus, + 'Failed to check assignment status', + signal + ) + case 'identity_center_check_assignment_deletion_status': + return executeInternalJsonToolOperation( + awsIdentityCenterCheckAssignmentDeletionStatusContract, + input, + executeIdentityCenterCheckAssignmentDeletionStatus, + 'Failed to check assignment deletion status', + signal + ) + case 'identity_center_list_account_assignments': + return executeInternalJsonToolOperation( + awsIdentityCenterListAccountAssignmentsContract, + input, + executeIdentityCenterListAccountAssignments, + 'Failed to list account assignments', + signal + ) + default: + return Response.json( + { error: `Unsupported Identity Center tool: ${toolId}` }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/identity-center/operations.test.ts b/apps/sim/lib/internal/identity-center/operations.test.ts new file mode 100644 index 00000000000..f78050438e0 --- /dev/null +++ b/apps/sim/lib/internal/identity-center/operations.test.ts @@ -0,0 +1,67 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockCreateSSOAdminClient, mockDestroy, mockListInstances } = vi.hoisted(() => ({ + mockCreateSSOAdminClient: vi.fn(), + mockDestroy: vi.fn(), + mockListInstances: vi.fn(), +})) + +vi.mock('@/lib/internal/identity-center/client', () => ({ + checkAssignmentCreationStatus: vi.fn(), + checkAssignmentDeletionStatus: vi.fn(), + createAccountAssignment: vi.fn(), + createIdentityStoreClient: vi.fn(), + createOrganizationsClient: vi.fn(), + createSSOAdminClient: mockCreateSSOAdminClient, + deleteAccountAssignment: vi.fn(), + describeAccount: vi.fn(), + getGroupByDisplayName: vi.fn(), + getUserByEmail: vi.fn(), + listAccountAssignmentsForPrincipal: vi.fn(), + listAccounts: vi.fn(), + listGroups: vi.fn(), + listInstances: mockListInstances, + listPermissionSets: vi.fn(), +})) + +import { executeIdentityCenterListInstances } from '@/lib/internal/identity-center/operations' + +const INPUT = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', + maxResults: 10, + nextToken: 'next-token', +} + +describe('Identity Center operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCreateSSOAdminClient.mockReturnValue({ destroy: mockDestroy }) + }) + + it('forwards cancellation and destroys the AWS client after success', async () => { + const controller = new AbortController() + const result = { instances: [], nextToken: null, count: 0 } + mockListInstances.mockResolvedValue(result) + + await expect(executeIdentityCenterListInstances(INPUT, controller.signal)).resolves.toBe(result) + expect(mockListInstances).toHaveBeenCalledWith( + { destroy: mockDestroy }, + 10, + 'next-token', + controller.signal + ) + expect(mockDestroy).toHaveBeenCalledOnce() + }) + + it('destroys the AWS client when provider execution fails', async () => { + mockListInstances.mockRejectedValue(new Error('provider failure')) + + await expect(executeIdentityCenterListInstances(INPUT)).rejects.toThrow('provider failure') + expect(mockDestroy).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/internal/identity-center/operations.ts b/apps/sim/lib/internal/identity-center/operations.ts new file mode 100644 index 00000000000..0f88ce9d9f3 --- /dev/null +++ b/apps/sim/lib/internal/identity-center/operations.ts @@ -0,0 +1,213 @@ +import type { AwsIdentityCenterCheckAssignmentDeletionStatusBody } from '@/lib/api/contracts/tools/aws/identity-center-check-assignment-deletion-status' +import type { AwsIdentityCenterCheckAssignmentStatusBody } from '@/lib/api/contracts/tools/aws/identity-center-check-assignment-status' +import type { AwsIdentityCenterCreateAccountAssignmentBody } from '@/lib/api/contracts/tools/aws/identity-center-create-account-assignment' +import type { AwsIdentityCenterDeleteAccountAssignmentBody } from '@/lib/api/contracts/tools/aws/identity-center-delete-account-assignment' +import type { AwsIdentityCenterDescribeAccountBody } from '@/lib/api/contracts/tools/aws/identity-center-describe-account' +import type { AwsIdentityCenterGetGroupBody } from '@/lib/api/contracts/tools/aws/identity-center-get-group' +import type { AwsIdentityCenterGetUserBody } from '@/lib/api/contracts/tools/aws/identity-center-get-user' +import type { AwsIdentityCenterListAccountAssignmentsBody } from '@/lib/api/contracts/tools/aws/identity-center-list-account-assignments' +import type { AwsIdentityCenterListAccountsBody } from '@/lib/api/contracts/tools/aws/identity-center-list-accounts' +import type { AwsIdentityCenterListGroupsBody } from '@/lib/api/contracts/tools/aws/identity-center-list-groups' +import type { AwsIdentityCenterListInstancesBody } from '@/lib/api/contracts/tools/aws/identity-center-list-instances' +import type { AwsIdentityCenterListPermissionSetsBody } from '@/lib/api/contracts/tools/aws/identity-center-list-permission-sets' +import { + checkAssignmentCreationStatus, + checkAssignmentDeletionStatus, + createAccountAssignment, + createIdentityStoreClient, + createOrganizationsClient, + createSSOAdminClient, + deleteAccountAssignment, + describeAccount, + getGroupByDisplayName, + getUserByEmail, + listAccountAssignmentsForPrincipal, + listAccounts, + listGroups, + listInstances, + listPermissionSets, +} from '@/lib/internal/identity-center/client' + +export async function executeIdentityCenterListInstances( + input: AwsIdentityCenterListInstancesBody, + signal?: AbortSignal +) { + const client = createSSOAdminClient(input) + try { + return await listInstances(client, input.maxResults, input.nextToken, signal) + } finally { + client.destroy() + } +} + +export async function executeIdentityCenterListAccounts( + input: AwsIdentityCenterListAccountsBody, + signal?: AbortSignal +) { + const client = createOrganizationsClient(input) + try { + return await listAccounts(client, input.maxResults, input.nextToken, signal) + } finally { + client.destroy() + } +} + +export async function executeIdentityCenterDescribeAccount( + input: AwsIdentityCenterDescribeAccountBody, + signal?: AbortSignal +) { + const client = createOrganizationsClient(input) + try { + return await describeAccount(client, input.accountId, signal) + } finally { + client.destroy() + } +} + +export async function executeIdentityCenterListPermissionSets( + input: AwsIdentityCenterListPermissionSetsBody, + signal?: AbortSignal +) { + const client = createSSOAdminClient(input) + try { + return await listPermissionSets( + client, + input.instanceArn, + input.maxResults, + input.nextToken, + signal + ) + } finally { + client.destroy() + } +} + +export async function executeIdentityCenterGetUser( + input: AwsIdentityCenterGetUserBody, + signal?: AbortSignal +) { + const client = createIdentityStoreClient(input) + try { + return await getUserByEmail(client, input.identityStoreId, input.email, signal) + } finally { + client.destroy() + } +} + +export async function executeIdentityCenterGetGroup( + input: AwsIdentityCenterGetGroupBody, + signal?: AbortSignal +) { + const client = createIdentityStoreClient(input) + try { + return await getGroupByDisplayName(client, input.identityStoreId, input.displayName, signal) + } finally { + client.destroy() + } +} + +export async function executeIdentityCenterListGroups( + input: AwsIdentityCenterListGroupsBody, + signal?: AbortSignal +) { + const client = createIdentityStoreClient(input) + try { + return await listGroups( + client, + input.identityStoreId, + input.maxResults, + input.nextToken, + signal + ) + } finally { + client.destroy() + } +} + +export async function executeIdentityCenterCreateAccountAssignment( + input: AwsIdentityCenterCreateAccountAssignmentBody, + signal?: AbortSignal +) { + const client = createSSOAdminClient(input) + try { + const result = await createAccountAssignment(client, input, signal) + return { + message: `Account assignment creation ${result.status === 'SUCCEEDED' ? 'succeeded' : 'initiated'}`, + ...result, + } + } finally { + client.destroy() + } +} + +export async function executeIdentityCenterDeleteAccountAssignment( + input: AwsIdentityCenterDeleteAccountAssignmentBody, + signal?: AbortSignal +) { + const client = createSSOAdminClient(input) + try { + const result = await deleteAccountAssignment(client, input, signal) + return { + message: `Account assignment deletion ${result.status === 'SUCCEEDED' ? 'succeeded' : 'initiated'}`, + ...result, + } + } finally { + client.destroy() + } +} + +export async function executeIdentityCenterCheckAssignmentStatus( + input: AwsIdentityCenterCheckAssignmentStatusBody, + signal?: AbortSignal +) { + const client = createSSOAdminClient(input) + try { + const result = await checkAssignmentCreationStatus( + client, + input.instanceArn, + input.requestId, + signal + ) + return { message: `Assignment status: ${result.status}`, ...result } + } finally { + client.destroy() + } +} + +export async function executeIdentityCenterCheckAssignmentDeletionStatus( + input: AwsIdentityCenterCheckAssignmentDeletionStatusBody, + signal?: AbortSignal +) { + const client = createSSOAdminClient(input) + try { + const result = await checkAssignmentDeletionStatus( + client, + input.instanceArn, + input.requestId, + signal + ) + return { message: `Assignment deletion status: ${result.status}`, ...result } + } finally { + client.destroy() + } +} + +export async function executeIdentityCenterListAccountAssignments( + input: AwsIdentityCenterListAccountAssignmentsBody, + signal?: AbortSignal +) { + const client = createSSOAdminClient(input) + try { + return await listAccountAssignmentsForPrincipal( + client, + input.instanceArn, + input.principalId, + input.principalType, + input.maxResults, + input.nextToken, + signal + ) + } finally { + client.destroy() + } +} diff --git a/apps/sim/lib/internal/image/execute-tool.ts b/apps/sim/lib/internal/image/execute-tool.ts new file mode 100644 index 00000000000..2f932991e50 --- /dev/null +++ b/apps/sim/lib/internal/image/execute-tool.ts @@ -0,0 +1,47 @@ +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { executeImageGeneration } from '@/lib/internal/image/operations' +import { imageGenerationInputSchema } from '@/lib/internal/image/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +export const executeImageTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!request.context.userId) { + return Response.json({ error: 'Unauthorized' }, { status: 401 }) + } + if (request.toolId !== 'image_generate') { + return Response.json({ error: `Unsupported image tool: ${request.toolId}` }, { status: 500 }) + } + + let serializedInput: string + try { + serializedInput = JSON.stringify(request.input) ?? '' + } catch { + return Response.json({ error: 'Invalid request data' }, { status: 400 }) + } + if (Buffer.byteLength(serializedInput, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + + const parsed = imageGenerationInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { error: getValidationErrorMessage(parsed.error, 'Invalid request data') }, + { status: 400 } + ) + } + + return executeImageGeneration(parsed.data, { + userId: request.context.userId, + workspaceId: request.context.workspaceId, + workflowId: request.context.workflowId, + executionId: request.context.executionId, + requestId: request.requestId, + signal: request.signal, + }) +} diff --git a/apps/sim/lib/internal/image/fetch.test.ts b/apps/sim/lib/internal/image/fetch.test.ts new file mode 100644 index 00000000000..14421b4985a --- /dev/null +++ b/apps/sim/lib/internal/image/fetch.test.ts @@ -0,0 +1,87 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockSecureFetchWithPinnedIP, mockValidateUrlWithDNS } = vi.hoisted(() => ({ + mockSecureFetchWithPinnedIP: vi.fn(), + mockValidateUrlWithDNS: vi.fn(), +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + secureFetchWithPinnedIP: mockSecureFetchWithPinnedIP, + validateUrlWithDNS: mockValidateUrlWithDNS, +})) + +import { + fetchRemoteImage, + MAX_REMOTE_IMAGE_BYTES, + type RemoteImageFetchError, +} from '@/lib/internal/image/fetch' + +describe('fetchRemoteImage', () => { + beforeEach(() => { + vi.clearAllMocks() + mockValidateUrlWithDNS.mockResolvedValue({ isValid: true, resolvedIP: '203.0.113.1' }) + }) + + it('validates, pins, bounds, and returns the image bytes', async () => { + mockSecureFetchWithPinnedIP.mockResolvedValue( + new Response(Buffer.from('image-bytes'), { + headers: { 'Content-Type': 'image/png' }, + }) + ) + const controller = new AbortController() + + const result = await fetchRemoteImage( + 'https://images.example.test/generated.png', + controller.signal + ) + + expect(result).toEqual({ buffer: Buffer.from('image-bytes'), contentType: 'image/png' }) + expect(mockSecureFetchWithPinnedIP).toHaveBeenCalledWith( + 'https://images.example.test/generated.png', + '203.0.113.1', + expect.objectContaining({ + method: 'GET', + maxResponseBytes: MAX_REMOTE_IMAGE_BYTES, + signal: controller.signal, + headers: expect.not.objectContaining({ 'Accept-Encoding': expect.anything() }), + }) + ) + }) + + it('rejects an unsafe URL before issuing a request', async () => { + mockValidateUrlWithDNS.mockResolvedValue({ + isValid: false, + error: 'Private addresses are not allowed', + }) + + await expect(fetchRemoteImage('http://127.0.0.1/private.png')).rejects.toMatchObject< + Partial + >({ status: 403, message: 'Private addresses are not allowed' }) + expect(mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() + }) + + it('preserves the upstream response status', async () => { + mockSecureFetchWithPinnedIP.mockResolvedValue( + new Response('missing', { status: 404, statusText: 'Not Found' }) + ) + + await expect(fetchRemoteImage('https://images.example.test/missing.png')).rejects.toMatchObject< + Partial + >({ status: 404, message: 'Failed to fetch image: Not Found' }) + }) + + it('maps an oversized response to 413', async () => { + mockSecureFetchWithPinnedIP.mockResolvedValue( + new Response('x', { + headers: { 'Content-Length': String(MAX_REMOTE_IMAGE_BYTES + 1) }, + }) + ) + + await expect(fetchRemoteImage('https://images.example.test/huge.png')).rejects.toMatchObject< + Partial + >({ status: 413 }) + }) +}) diff --git a/apps/sim/lib/internal/image/fetch.ts b/apps/sim/lib/internal/image/fetch.ts new file mode 100644 index 00000000000..23d8760d647 --- /dev/null +++ b/apps/sim/lib/internal/image/fetch.ts @@ -0,0 +1,84 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { + consumeOrCancelBody, + isPayloadSizeLimitError, + readResponseToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' + +export const MAX_REMOTE_IMAGE_BYTES = 25 * 1024 * 1024 + +export class RemoteImageFetchError extends Error { + constructor( + message: string, + readonly status: number, + options?: ErrorOptions + ) { + super(message, options) + this.name = 'RemoteImageFetchError' + } +} + +export interface FetchRemoteImageResult { + buffer: Buffer + contentType: string +} + +export async function fetchRemoteImage( + imageUrl: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const validation = await validateUrlWithDNS(imageUrl, 'imageUrl') + if (!validation.isValid || !validation.resolvedIP) { + throw new RemoteImageFetchError(validation.error || 'Invalid image URL', 403) + } + + try { + const response = await secureFetchWithPinnedIP(imageUrl, validation.resolvedIP, { + method: 'GET', + maxResponseBytes: MAX_REMOTE_IMAGE_BYTES, + signal, + headers: { + 'User-Agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', + Accept: 'image/webp,image/avif,image/apng,image/svg+xml,image/*,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.9', + Referer: 'https://sim.ai/', + 'Sec-Fetch-Dest': 'image', + 'Sec-Fetch-Mode': 'no-cors', + 'Sec-Fetch-Site': 'cross-site', + }, + }) + + if (!response.ok) { + await consumeOrCancelBody(response) + throw new RemoteImageFetchError( + `Failed to fetch image: ${response.statusText}`, + response.status + ) + } + + const buffer = await readResponseToBufferWithLimit(response, { + maxBytes: MAX_REMOTE_IMAGE_BYTES, + label: 'image proxy response', + signal, + }) + if (buffer.length === 0) throw new RemoteImageFetchError('Empty image received', 404) + + return { + buffer, + contentType: response.headers.get('content-type') || 'image/jpeg', + } + } catch (error) { + if (error instanceof RemoteImageFetchError) throw error + throw new RemoteImageFetchError( + `Failed to proxy image: ${getErrorMessage(error)}`, + isPayloadSizeLimitError(error) ? 413 : 500, + { cause: error } + ) + } +} diff --git a/apps/sim/lib/internal/image/operations.test.ts b/apps/sim/lib/internal/image/operations.test.ts new file mode 100644 index 00000000000..1ee4c7bef53 --- /dev/null +++ b/apps/sim/lib/internal/image/operations.test.ts @@ -0,0 +1,108 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + fetch: vi.fn(), + interruptibleSleep: vi.fn(), + uploadCopilotFile: vi.fn(), + uploadExecutionFile: vi.fn(), + getFalAICostMetadata: vi.fn(), +})) + +vi.stubGlobal('fetch', mocks.fetch) + +vi.mock('@sim/utils/helpers', () => ({ + interruptibleSleep: mocks.interruptibleSleep, +})) + +vi.mock('@/lib/core/execution-limits', () => ({ + getMaxExecutionTimeout: () => 9000, +})) + +vi.mock('@/lib/tools/falai-pricing', () => ({ + getFalAICostMetadata: mocks.getFalAICostMetadata, +})) + +vi.mock('@/lib/uploads/contexts/copilot', () => ({ + uploadCopilotFile: mocks.uploadCopilotFile, +})) + +vi.mock('@/lib/uploads/contexts/execution', () => ({ + uploadExecutionFile: mocks.uploadExecutionFile, +})) + +import { executeImageGeneration } from '@/lib/internal/image/operations' + +const falInput = { + provider: 'falai' as const, + apiKey: 'fal-key', + model: 'nano-banana-2', + prompt: 'draw a safe bounded image', +} + +describe('image operations', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mocks.fetch) + mocks.interruptibleSleep.mockResolvedValue(undefined) + mocks.uploadCopilotFile.mockResolvedValue({ url: 'https://sim.test/generated.png' }) + }) + + it('submits a Fal.ai job once and only polls the created job', async () => { + const inlineImage = `data:image/png;base64,${Buffer.from('png').toString('base64')}` + mocks.fetch + .mockResolvedValueOnce( + Response.json({ + request_id: 'job-1', + status_url: 'https://queue.fal.run/status/job-1', + response_url: 'https://queue.fal.run/result/job-1', + }) + ) + .mockResolvedValueOnce(Response.json({ status: 'IN_QUEUE' })) + .mockResolvedValueOnce(Response.json({ status: 'COMPLETED' })) + .mockResolvedValueOnce(Response.json({ images: [{ url: inlineImage }] })) + + const response = await executeImageGeneration(falInput, { + userId: 'user-1', + requestId: 'request-1', + }) + + expect(response.status).toBe(200) + expect((await response.json()).imageUrl).toBe('https://sim.test/generated.png') + const urls = mocks.fetch.mock.calls.map(([url]) => String(url)) + expect(urls.filter((url) => url === 'https://queue.fal.run/fal-ai/nano-banana-2')).toHaveLength( + 1 + ) + expect(urls).toEqual([ + 'https://queue.fal.run/fal-ai/nano-banana-2', + 'https://queue.fal.run/status/job-1', + 'https://queue.fal.run/status/job-1', + 'https://queue.fal.run/result/job-1', + ]) + }) + + it('cancels polling without resubmitting or storing an image', async () => { + const controller = new AbortController() + mocks.fetch.mockResolvedValueOnce( + Response.json({ + request_id: 'job-2', + status_url: 'https://queue.fal.run/status/job-2', + response_url: 'https://queue.fal.run/result/job-2', + }) + ) + mocks.interruptibleSleep.mockImplementationOnce(async () => controller.abort()) + + await expect( + executeImageGeneration(falInput, { + userId: 'user-1', + requestId: 'request-2', + signal: controller.signal, + }) + ).rejects.toMatchObject({ name: 'AbortError' }) + + expect(mocks.fetch).toHaveBeenCalledTimes(1) + expect(mocks.uploadCopilotFile).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/image/operations.ts b/apps/sim/lib/internal/image/operations.ts new file mode 100644 index 00000000000..591b4979168 --- /dev/null +++ b/apps/sim/lib/internal/image/operations.ts @@ -0,0 +1,949 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { interruptibleSleep } from '@sim/utils/helpers' +import { isRecordLike } from '@sim/utils/object' +import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { + assertKnownSizeWithinLimit, + DEFAULT_MAX_ERROR_BODY_BYTES, + isPayloadSizeLimitError, + readResponseJsonWithLimit, + readResponseTextWithLimit, + readResponseToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' +import { MAX_REMOTE_IMAGE_BYTES } from '@/lib/internal/image/fetch' +import type { ImageGenerationInput, ImageProvider } from '@/lib/internal/image/schema' +import { type FalAICostMetadata, getFalAICostMetadata } from '@/lib/tools/falai-pricing' +import { uploadCopilotFile } from '@/lib/uploads/contexts/copilot' +import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' + +const logger = createLogger('ImageOperations') +const MAX_IMAGE_BYTES = MAX_REMOTE_IMAGE_BYTES +const MAX_IMAGE_JSON_BYTES = Math.ceil((MAX_IMAGE_BYTES * 4) / 3) + 256 * 1024 + +interface GeneratedImageResult { + buffer: Buffer + contentType: string + fileName: string + provider: ImageProvider + model: string + sourceUrl?: string + description?: string + revisedPrompt?: string + seed?: number + jobId?: string + falaiCost?: FalAICostMetadata +} + +interface StoredImageResponse { + content: string + imageUrl: string + imageFile?: unknown + fileName: string + contentType: string + provider: ImageProvider + model: string + metadata: { + provider: ImageProvider + model: string + description?: string + revisedPrompt?: string + seed?: number + jobId?: string + contentType: string + } + __falaiCostDollars?: number + __falaiBilling?: FalAICostMetadata +} + +export interface ImageOperationContext { + userId: string + workspaceId?: string + workflowId?: string + executionId?: string + requestId: string + signal?: AbortSignal +} + +export async function executeImageGeneration( + body: ImageGenerationInput, + context: ImageOperationContext +): Promise { + const { requestId } = context + logger.info(`[${requestId}] Image generation request started`) + + try { + context.signal?.throwIfAborted() + const provider = body.provider as ImageProvider + const { apiKey, model, prompt } = body + + if (prompt.length < 3 || prompt.length > 4000) { + return Response.json( + { error: 'Prompt must be between 3 and 4000 characters' }, + { status: 400 } + ) + } + + logger.info(`[${requestId}] Generating image with ${provider}, model: ${model || 'default'}`) + + let imageResult: GeneratedImageResult + try { + if (provider === 'openai') { + imageResult = await generateWithOpenAI(apiKey, body, requestId, logger, context.signal) + } else if (provider === 'gemini') { + imageResult = await generateWithGemini(apiKey, body, requestId, logger, context.signal) + } else if (provider === 'falai') { + imageResult = await generateWithFalAI(apiKey, body, requestId, logger, context.signal) + } else { + return Response.json({ error: `Unknown provider: ${provider}` }, { status: 400 }) + } + } catch (error) { + context.signal?.throwIfAborted() + logger.error(`[${requestId}] Image generation failed:`, error) + const errorMessage = getErrorMessage(error, 'Image generation failed') + return Response.json( + { error: errorMessage }, + { status: isPayloadSizeLimitError(error) ? 413 : 500 } + ) + } + + const storedImage = await storeGeneratedImage(imageResult, context) + + logger.info(`[${requestId}] Image generation completed successfully`, { + provider, + model: storedImage.model, + contentType: storedImage.contentType, + }) + + return Response.json(storedImage) + } catch (error) { + context.signal?.throwIfAborted() + logger.error(`[${requestId}] Image generation operation error:`, error) + const errorMessage = getErrorMessage(error, 'Unknown error') + return Response.json( + { error: errorMessage }, + { status: isPayloadSizeLimitError(error) ? 413 : 500 } + ) + } +} + +const OPENAI_IMAGE_MODELS = [ + 'gpt-image-2', + 'gpt-image-1.5', + 'gpt-image-1', + 'gpt-image-1-mini', +] as const +const OPENAI_IMAGE_SIZES = ['auto', '1024x1024', '1536x1024', '1024x1536'] as const +const OPENAI_IMAGE_2_SIZES = [...OPENAI_IMAGE_SIZES, '2560x1440', '3840x2160'] as const +const OPENAI_IMAGE_QUALITIES = ['auto', 'low', 'medium', 'high'] as const +const OPENAI_IMAGE_BACKGROUNDS = ['auto', 'transparent', 'opaque'] as const +const IMAGE_OUTPUT_FORMATS = ['png', 'jpeg', 'webp'] as const +const OPENAI_MODERATION_LEVELS = ['auto', 'low'] as const + +const GEMINI_IMAGE_MODELS = [ + 'gemini-3.1-flash-image-preview', + 'gemini-3-pro-image-preview', + 'gemini-2.5-flash-image', +] as const +const GEMINI_BASE_ASPECT_RATIOS = [ + '1:1', + '2:3', + '3:2', + '3:4', + '4:3', + '4:5', + '5:4', + '9:16', + '16:9', + '21:9', +] as const +const GEMINI_EXTREME_ASPECT_RATIOS = ['1:4', '1:8', '4:1', '8:1'] as const +const GEMINI_IMAGE_SIZES = ['512', '1K', '2K', '4K'] as const +const GEMINI_PRO_IMAGE_SIZES = ['1K', '2K', '4K'] as const + +interface FalAIImageModelConfig { + endpoint: string + defaultSize?: string + sizeOptions?: readonly string[] + defaultAspectRatio?: string + aspectRatios?: readonly string[] + defaultResolution?: string + resolutionOptions?: readonly string[] + defaultOutputFormat?: string + outputFormats?: readonly string[] + defaultQuality?: string + qualityOptions?: readonly string[] + defaultBackground?: string + backgroundOptions?: readonly string[] + defaultSafetyTolerance?: string + safetyToleranceOptions?: readonly string[] + maxNumImages?: number + supportsSeed?: boolean + supportsEnableSafetyChecker?: boolean + supportsEnableWebSearch?: boolean + supportsThinkingLevel?: boolean +} + +const FALAI_NANO_BANANA_ASPECT_RATIOS = [ + 'auto', + '21:9', + '16:9', + '3:2', + '4:3', + '5:4', + '1:1', + '4:5', + '3:4', + '2:3', + '9:16', +] as const +const FALAI_EXTREME_ASPECT_RATIOS = ['4:1', '1:4', '8:1', '1:8'] as const +const FALAI_STANDARD_IMAGE_SIZES = [ + 'square_hd', + 'square', + 'portrait_4_3', + 'portrait_16_9', + 'landscape_4_3', + 'landscape_16_9', +] as const +const FALAI_SEEDREAM_IMAGE_SIZES = [...FALAI_STANDARD_IMAGE_SIZES, 'auto_2K', 'auto_4K'] as const + +const FALAI_IMAGE_MODEL_CONFIGS: Record = { + 'nano-banana-2': { + endpoint: 'fal-ai/nano-banana-2', + defaultAspectRatio: 'auto', + aspectRatios: [...FALAI_NANO_BANANA_ASPECT_RATIOS, ...FALAI_EXTREME_ASPECT_RATIOS], + defaultResolution: '1K', + resolutionOptions: ['0.5K', '1K', '2K', '4K'], + defaultOutputFormat: 'png', + outputFormats: IMAGE_OUTPUT_FORMATS, + defaultSafetyTolerance: '4', + safetyToleranceOptions: ['1', '2', '3', '4', '5', '6'], + maxNumImages: 4, + supportsSeed: true, + supportsEnableWebSearch: true, + supportsThinkingLevel: true, + }, + 'nano-banana-pro': { + endpoint: 'fal-ai/nano-banana-pro', + defaultAspectRatio: '1:1', + aspectRatios: FALAI_NANO_BANANA_ASPECT_RATIOS, + defaultResolution: '1K', + resolutionOptions: ['1K', '2K', '4K'], + defaultOutputFormat: 'png', + outputFormats: IMAGE_OUTPUT_FORMATS, + defaultSafetyTolerance: '4', + safetyToleranceOptions: ['1', '2', '3', '4', '5', '6'], + maxNumImages: 4, + supportsSeed: true, + supportsEnableWebSearch: true, + }, + 'nano-banana': { + endpoint: 'fal-ai/nano-banana', + defaultAspectRatio: '1:1', + aspectRatios: FALAI_NANO_BANANA_ASPECT_RATIOS.filter((ratio) => ratio !== 'auto'), + defaultOutputFormat: 'png', + outputFormats: IMAGE_OUTPUT_FORMATS, + defaultSafetyTolerance: '4', + safetyToleranceOptions: ['1', '2', '3', '4', '5', '6'], + maxNumImages: 4, + supportsSeed: true, + }, + 'gpt-image-1.5': { + endpoint: 'fal-ai/gpt-image-1.5', + defaultSize: '1024x1024', + sizeOptions: ['1024x1024', '1536x1024', '1024x1536'], + defaultQuality: 'high', + qualityOptions: ['low', 'medium', 'high'], + defaultBackground: 'auto', + backgroundOptions: OPENAI_IMAGE_BACKGROUNDS, + defaultOutputFormat: 'png', + outputFormats: IMAGE_OUTPUT_FORMATS, + maxNumImages: 4, + }, + 'seedream-v4.5': { + endpoint: 'fal-ai/bytedance/seedream/v4.5/text-to-image', + defaultSize: 'auto_2K', + sizeOptions: FALAI_SEEDREAM_IMAGE_SIZES, + maxNumImages: 6, + supportsSeed: true, + supportsEnableSafetyChecker: true, + }, + 'flux-2-pro': { + endpoint: 'fal-ai/flux-2-pro', + defaultSize: 'landscape_4_3', + sizeOptions: FALAI_STANDARD_IMAGE_SIZES, + defaultOutputFormat: 'jpeg', + outputFormats: ['jpeg', 'png'], + defaultSafetyTolerance: '2', + safetyToleranceOptions: ['1', '2', '3', '4', '5'], + supportsSeed: true, + supportsEnableSafetyChecker: true, + }, + 'grok-imagine-image': { + endpoint: 'xai/grok-imagine-image', + defaultAspectRatio: '1:1', + aspectRatios: [ + '2:1', + '20:9', + '19.5:9', + '16:9', + '4:3', + '3:2', + '1:1', + '2:3', + '3:4', + '9:16', + '9:19.5', + '9:20', + '1:2', + ], + defaultResolution: '1k', + resolutionOptions: ['1k', '2k'], + defaultOutputFormat: 'jpeg', + outputFormats: IMAGE_OUTPUT_FORMATS, + maxNumImages: 4, + }, +} + +function getStringProperty( + record: Record | undefined, + key: string +): string | undefined { + const value = record?.[key] + return typeof value === 'string' ? value : undefined +} + +function getNumberProperty( + record: Record | undefined, + key: string +): number | undefined { + const value = record?.[key] + return typeof value === 'number' ? value : undefined +} + +function firstRecord(value: unknown): Record | undefined { + return Array.isArray(value) ? value.find(isRecordLike) : undefined +} + +function pickAllowed( + value: string | undefined, + allowed: readonly string[], + fallback: string +): string { + return value && allowed.includes(value) ? value : fallback +} + +function clampInteger( + value: number | undefined, + min: number, + max: number, + fallback: number +): number { + if (typeof value !== 'number' || !Number.isInteger(value)) return fallback + return Math.min(Math.max(value, min), max) +} + +function getContentTypeForFormat(format: string | undefined): string { + if (format === 'jpeg') return 'image/jpeg' + if (format === 'webp') return 'image/webp' + return 'image/png' +} + +function extensionFromContentType(contentType: string): string { + if (contentType.includes('jpeg') || contentType.includes('jpg')) return 'jpg' + if (contentType.includes('webp')) return 'webp' + return 'png' +} + +async function bufferFromImageUrl( + url: string, + signal?: AbortSignal +): Promise<{ buffer: Buffer; contentType: string }> { + signal?.throwIfAborted() + if (url.startsWith('data:')) { + const match = /^data:([^;]+);base64,(.+)$/u.exec(url) + if (!match) throw new Error('Invalid data URI image response') + const buffer = Buffer.from(match[2], 'base64') + assertKnownSizeWithinLimit(buffer.length, MAX_IMAGE_BYTES, 'inline image response') + return { + contentType: match[1], + buffer, + } + } + + const urlValidation = await validateUrlWithDNS(url, 'imageUrl') + if (!urlValidation.isValid || !urlValidation.resolvedIP) { + throw new Error(urlValidation.error || 'Generated image URL failed validation') + } + + const imageResponse = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP, { + method: 'GET', + maxResponseBytes: MAX_IMAGE_BYTES, + signal, + }) + if (!imageResponse.ok) { + await readResponseTextWithLimit(imageResponse, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'generated image error response', + signal, + }).catch(() => '') + throw new Error(`Failed to download generated image: ${imageResponse.status}`) + } + + const contentType = imageResponse.headers.get('content-type') || 'image/png' + const buffer = await readResponseToBufferWithLimit(imageResponse, { + maxBytes: MAX_IMAGE_BYTES, + label: 'generated image download', + signal, + }) + return { buffer, contentType } +} + +async function generateWithOpenAI( + apiKey: string, + body: ImageGenerationInput, + requestId: string, + logger: ReturnType, + signal?: AbortSignal +): Promise { + const model = pickAllowed(body.model, OPENAI_IMAGE_MODELS, 'gpt-image-1.5') + const size = + model === 'gpt-image-2' + ? pickAllowed(body.size, OPENAI_IMAGE_2_SIZES, 'auto') + : pickAllowed(body.size, OPENAI_IMAGE_SIZES, 'auto') + const outputFormat = pickAllowed(body.outputFormat, IMAGE_OUTPUT_FORMATS, 'png') + const requestBody: Record = { + model, + prompt: body.prompt, + size, + n: 1, + } + + if (body.quality) { + requestBody.quality = pickAllowed(body.quality, OPENAI_IMAGE_QUALITIES, 'auto') + } + if (body.background) { + requestBody.background = pickAllowed(body.background, OPENAI_IMAGE_BACKGROUNDS, 'auto') + } + if (body.outputFormat) { + requestBody.output_format = outputFormat + } + if (body.moderation) { + requestBody.moderation = pickAllowed(body.moderation, OPENAI_MODERATION_LEVELS, 'auto') + } + + const openaiResponse = await fetch('https://api.openai.com/v1/images/generations', { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(requestBody), + signal, + }) + + if (!openaiResponse.ok) { + const error = await readResponseTextWithLimit(openaiResponse, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'OpenAI image error response', + signal, + }) + throw new Error(`OpenAI API error: ${openaiResponse.status} - ${error}`) + } + + const data = await readResponseJsonWithLimit(openaiResponse, { + maxBytes: MAX_IMAGE_JSON_BYTES, + label: 'OpenAI image response', + signal, + }) + if (!isRecordLike(data)) { + throw new Error('Invalid OpenAI image response') + } + + const firstImage = firstRecord(data.data) + const base64Image = getStringProperty(firstImage, 'b64_json') + const imageUrl = getStringProperty(firstImage, 'url') + const revisedPrompt = getStringProperty(firstImage, 'revised_prompt') + let buffer: Buffer + let contentType = getContentTypeForFormat(outputFormat) + + if (base64Image) { + buffer = Buffer.from(base64Image, 'base64') + assertKnownSizeWithinLimit(buffer.length, MAX_IMAGE_BYTES, 'OpenAI image response') + } else if (imageUrl) { + const downloaded = await bufferFromImageUrl(imageUrl, signal) + buffer = downloaded.buffer + contentType = downloaded.contentType + } else { + logger.error(`[${requestId}] OpenAI response missing image payload`) + throw new Error('No image data found in OpenAI response') + } + + return { + buffer, + contentType, + fileName: `openai-${model}.${extensionFromContentType(contentType)}`, + provider: 'openai', + model, + sourceUrl: imageUrl, + revisedPrompt, + } +} + +async function generateWithGemini( + apiKey: string, + body: ImageGenerationInput, + requestId: string, + logger: ReturnType, + signal?: AbortSignal +): Promise { + const model = pickAllowed(body.model, GEMINI_IMAGE_MODELS, 'gemini-3.1-flash-image-preview') + const aspectRatios = + model === 'gemini-3.1-flash-image-preview' + ? [...GEMINI_BASE_ASPECT_RATIOS, ...GEMINI_EXTREME_ASPECT_RATIOS] + : GEMINI_BASE_ASPECT_RATIOS + const imageConfig: Record = {} + + if (body.aspectRatio) { + imageConfig.aspectRatio = pickAllowed(body.aspectRatio, aspectRatios, '1:1') + } + + if (model === 'gemini-3.1-flash-image-preview' && body.resolution) { + imageConfig.imageSize = pickAllowed(body.resolution, GEMINI_IMAGE_SIZES, '1K') + } else if (model === 'gemini-3-pro-image-preview' && body.resolution) { + imageConfig.imageSize = pickAllowed(body.resolution, GEMINI_PRO_IMAGE_SIZES, '1K') + } + + const requestBody: Record = { + contents: [ + { + parts: [{ text: body.prompt }], + }, + ], + } + + requestBody.generationConfig = { + responseModalities: ['TEXT', 'IMAGE'], + ...(Object.keys(imageConfig).length > 0 && { imageConfig }), + } + + const geminiResponse = await fetch( + `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent`, + { + method: 'POST', + headers: { + 'x-goog-api-key': apiKey, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(requestBody), + signal, + } + ) + + if (!geminiResponse.ok) { + const error = await readResponseTextWithLimit(geminiResponse, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Gemini image error response', + signal, + }) + throw new Error(`Gemini API error: ${geminiResponse.status} - ${error}`) + } + + const data = await readResponseJsonWithLimit(geminiResponse, { + maxBytes: MAX_IMAGE_JSON_BYTES, + label: 'Gemini image response', + signal, + }) + if (!isRecordLike(data)) { + throw new Error('Invalid Gemini image response') + } + + const candidate = firstRecord(data.candidates) + const content = isRecordLike(candidate?.content) ? candidate.content : undefined + const parts = Array.isArray(content?.parts) ? content.parts : [] + const textPart = parts.find((part) => isRecordLike(part) && typeof part.text === 'string') + const imagePart = parts.find((part) => { + if (!isRecordLike(part)) return false + return isRecordLike(part.inlineData) || isRecordLike(part.inline_data) + }) + + if (!isRecordLike(imagePart)) { + logger.error(`[${requestId}] Gemini response missing image part`) + throw new Error('No image data found in Gemini response') + } + + const inlineData = isRecordLike(imagePart.inlineData) + ? imagePart.inlineData + : isRecordLike(imagePart.inline_data) + ? imagePart.inline_data + : undefined + const base64Image = getStringProperty(inlineData, 'data') + const contentType = + getStringProperty(inlineData, 'mimeType') || + getStringProperty(inlineData, 'mime_type') || + 'image/png' + + if (!base64Image) { + throw new Error('Gemini image response missing inline image data') + } + + return { + buffer: (() => { + const buffer = Buffer.from(base64Image, 'base64') + assertKnownSizeWithinLimit(buffer.length, MAX_IMAGE_BYTES, 'Gemini image response') + return buffer + })(), + contentType, + fileName: `gemini-${model}.${extensionFromContentType(contentType)}`, + provider: 'gemini', + model, + description: isRecordLike(textPart) ? getStringProperty(textPart, 'text') : undefined, + } +} + +function buildFalAIQueueUrl(endpoint: string, requestId: string, path: 'status' | 'response') { + return `https://queue.fal.run/${endpoint}/requests/${requestId}/${path}` +} + +function getFalAIErrorMessage(error: unknown): string { + if (typeof error === 'string') return error + if (isRecordLike(error)) { + return ( + getStringProperty(error, 'message') || + getStringProperty(error, 'detail') || + JSON.stringify(error) + ) + } + return 'Unknown Fal.ai error' +} + +async function generateWithFalAI( + apiKey: string, + body: ImageGenerationInput, + requestId: string, + logger: ReturnType, + signal?: AbortSignal +): Promise { + const model = body.model || 'nano-banana-2' + const modelConfig = FALAI_IMAGE_MODEL_CONFIGS[model] + if (!modelConfig) { + throw new Error(`Unknown Fal.ai image model: ${model}`) + } + + const requestBody: Record = { + prompt: body.prompt, + sync_mode: false, + } + + if (modelConfig.maxNumImages) { + requestBody.num_images = clampInteger(body.numImages, 1, modelConfig.maxNumImages, 1) + } + if (modelConfig.supportsSeed && body.seed !== undefined) { + requestBody.seed = body.seed + } + if (modelConfig.sizeOptions && modelConfig.defaultSize) { + requestBody.image_size = pickAllowed( + body.size, + modelConfig.sizeOptions, + modelConfig.defaultSize + ) + } + if (modelConfig.aspectRatios && modelConfig.defaultAspectRatio) { + requestBody.aspect_ratio = pickAllowed( + body.aspectRatio, + modelConfig.aspectRatios, + modelConfig.defaultAspectRatio + ) + } + if (modelConfig.resolutionOptions && modelConfig.defaultResolution) { + requestBody.resolution = pickAllowed( + body.resolution, + modelConfig.resolutionOptions, + modelConfig.defaultResolution + ) + } + if (modelConfig.outputFormats && modelConfig.defaultOutputFormat) { + requestBody.output_format = pickAllowed( + body.outputFormat, + modelConfig.outputFormats, + modelConfig.defaultOutputFormat + ) + } + if (modelConfig.qualityOptions && modelConfig.defaultQuality) { + requestBody.quality = pickAllowed( + body.quality, + modelConfig.qualityOptions, + modelConfig.defaultQuality + ) + } + if (modelConfig.backgroundOptions && modelConfig.defaultBackground) { + requestBody.background = pickAllowed( + body.background, + modelConfig.backgroundOptions, + modelConfig.defaultBackground + ) + } + if (modelConfig.safetyToleranceOptions && modelConfig.defaultSafetyTolerance) { + requestBody.safety_tolerance = pickAllowed( + body.safetyTolerance, + modelConfig.safetyToleranceOptions, + modelConfig.defaultSafetyTolerance + ) + } + if (modelConfig.supportsEnableSafetyChecker && body.enableSafetyChecker !== undefined) { + requestBody.enable_safety_checker = body.enableSafetyChecker + } + if (modelConfig.supportsEnableWebSearch && body.enableWebSearch !== undefined) { + requestBody.enable_web_search = body.enableWebSearch + } + if (modelConfig.supportsThinkingLevel && body.thinkingLevel) { + requestBody.thinking_level = pickAllowed(body.thinkingLevel, ['minimal', 'high'], 'minimal') + } + + const createResponse = await fetch(`https://queue.fal.run/${modelConfig.endpoint}`, { + method: 'POST', + headers: { + Authorization: `Key ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(requestBody), + signal, + }) + + if (!createResponse.ok) { + const error = await readResponseTextWithLimit(createResponse, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Fal.ai create error response', + signal, + }) + throw new Error(`Fal.ai API error: ${createResponse.status} - ${error}`) + } + + const createData = await readResponseJsonWithLimit(createResponse, { + maxBytes: MAX_IMAGE_JSON_BYTES, + label: 'Fal.ai create response', + signal, + }) + if (!isRecordLike(createData)) { + throw new Error('Invalid Fal.ai queue response') + } + + const falRequestId = getStringProperty(createData, 'request_id') + if (!falRequestId) { + throw new Error('Fal.ai queue response missing request_id') + } + + const statusUrl = + getStringProperty(createData, 'status_url') || + buildFalAIQueueUrl(modelConfig.endpoint, falRequestId, 'status') + const responseUrl = + getStringProperty(createData, 'response_url') || + buildFalAIQueueUrl(modelConfig.endpoint, falRequestId, 'response') + + logger.info(`[${requestId}] Fal.ai image request created: ${falRequestId}`) + + const pollIntervalMs = 3000 + const maxAttempts = Math.ceil(getMaxExecutionTimeout() / pollIntervalMs) + let attempts = 0 + + while (attempts < maxAttempts) { + await interruptibleSleep(pollIntervalMs, signal) + signal?.throwIfAborted() + + const statusResponse = await fetch(statusUrl, { + headers: { + Authorization: `Key ${apiKey}`, + }, + signal, + }) + + if (!statusResponse.ok) { + await readResponseTextWithLimit(statusResponse, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Fal.ai status error response', + signal, + }).catch(() => '') + throw new Error(`Fal.ai status check failed: ${statusResponse.status}`) + } + + const statusData = await readResponseJsonWithLimit(statusResponse, { + maxBytes: MAX_IMAGE_JSON_BYTES, + label: 'Fal.ai status response', + signal, + }) + if (!isRecordLike(statusData)) { + throw new Error('Invalid Fal.ai status response') + } + + const status = getStringProperty(statusData, 'status') + if (status === 'COMPLETED') { + const statusError = statusData.error + if (statusError) { + throw new Error(`Fal.ai generation failed: ${getFalAIErrorMessage(statusError)}`) + } + + const resultResponse = await fetch( + getStringProperty(statusData, 'response_url') || responseUrl, + { + headers: { + Authorization: `Key ${apiKey}`, + }, + signal, + } + ) + + if (!resultResponse.ok) { + await readResponseTextWithLimit(resultResponse, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Fal.ai result error response', + signal, + }).catch(() => '') + throw new Error(`Failed to fetch Fal.ai result: ${resultResponse.status}`) + } + + const resultData = await readResponseJsonWithLimit(resultResponse, { + maxBytes: MAX_IMAGE_JSON_BYTES, + label: 'Fal.ai result response', + signal, + }) + if (!isRecordLike(resultData)) { + throw new Error('Invalid Fal.ai result response') + } + + const firstImage = firstRecord(resultData.images) + const imageUrl = + getStringProperty(firstImage, 'url') || + getStringProperty(firstImage, 'data') || + getStringProperty(firstImage, 'content') + if (!imageUrl) { + throw new Error('No image URL in Fal.ai response') + } + + const downloaded = await bufferFromImageUrl(imageUrl, signal) + const contentType = + getStringProperty(firstImage, 'content_type') || + getStringProperty(firstImage, 'contentType') || + downloaded.contentType + const fileName = + getStringProperty(firstImage, 'file_name') || + getStringProperty(firstImage, 'fileName') || + `falai-${model}.${extensionFromContentType(contentType)}` + + return { + buffer: downloaded.buffer, + contentType, + fileName, + provider: 'falai', + model, + sourceUrl: imageUrl.startsWith('data:') ? undefined : imageUrl, + description: getStringProperty(resultData, 'description'), + revisedPrompt: getStringProperty(resultData, 'revised_prompt'), + seed: getNumberProperty(resultData, 'seed'), + jobId: falRequestId, + falaiCost: body.useHostedCostTracking + ? await getFalAICostMetadata({ + apiKey, + endpointId: modelConfig.endpoint, + requestId: falRequestId, + signal, + }) + : undefined, + } + } + + if (['ERROR', 'FAILED', 'CANCELLED'].includes(status || '')) { + throw new Error(`Fal.ai generation failed: ${getFalAIErrorMessage(statusData.error)}`) + } + + attempts += 1 + } + + throw new Error('Fal.ai image generation timed out') +} + +async function storeGeneratedImage( + imageResult: GeneratedImageResult, + context: ImageOperationContext +): Promise { + context.signal?.throwIfAborted() + const timestamp = Date.now() + const safeFileName = imageResult.fileName || `image-${imageResult.provider}-${timestamp}.png` + const executionContext = + context.workspaceId && context.workflowId && context.executionId + ? { + workspaceId: context.workspaceId, + workflowId: context.workflowId, + executionId: context.executionId, + } + : null + + if (executionContext) { + const imageFile = await uploadExecutionFile( + executionContext, + imageResult.buffer, + safeFileName, + imageResult.contentType, + context.userId + ) + context.signal?.throwIfAborted() + + return { + content: imageFile.url, + imageUrl: imageFile.url, + imageFile, + fileName: safeFileName, + contentType: imageResult.contentType, + provider: imageResult.provider, + model: imageResult.model, + metadata: { + provider: imageResult.provider, + model: imageResult.model, + description: imageResult.description, + revisedPrompt: imageResult.revisedPrompt, + seed: imageResult.seed, + jobId: imageResult.jobId, + contentType: imageResult.contentType, + }, + __falaiCostDollars: imageResult.falaiCost?.costDollars, + __falaiBilling: imageResult.falaiCost, + } + } + + const fileInfo = await uploadCopilotFile({ + buffer: imageResult.buffer, + fileName: safeFileName, + contentType: imageResult.contentType, + userId: context.userId, + }) + context.signal?.throwIfAborted() + const imageUrl = fileInfo.url + logger.info(`[${context.requestId}] Stored generated image fallback`, { + fileName: safeFileName, + size: imageResult.buffer.length, + }) + + return { + content: imageUrl, + imageUrl, + fileName: safeFileName, + contentType: imageResult.contentType, + provider: imageResult.provider, + model: imageResult.model, + metadata: { + provider: imageResult.provider, + model: imageResult.model, + description: imageResult.description, + revisedPrompt: imageResult.revisedPrompt, + seed: imageResult.seed, + jobId: imageResult.jobId, + contentType: imageResult.contentType, + }, + __falaiCostDollars: imageResult.falaiCost?.costDollars, + __falaiBilling: imageResult.falaiCost, + } +} diff --git a/apps/sim/lib/internal/image/schema.test.ts b/apps/sim/lib/internal/image/schema.test.ts new file mode 100644 index 00000000000..27413c07d7e --- /dev/null +++ b/apps/sim/lib/internal/image/schema.test.ts @@ -0,0 +1,50 @@ +/** + * @vitest-environment node + */ + +import { describe, expect, it } from 'vitest' +import { imageGenerationInputSchema } from '@/lib/internal/image/schema' + +const BASE_INPUT = { + provider: 'openai', + apiKey: 'key', + prompt: 'Draw a landscape', +} + +describe('image generation input', () => { + it.each([null, '', ' '])('treats optional numeric sentinel %j as omitted', (sentinel) => { + const parsed = imageGenerationInputSchema.parse({ + ...BASE_INPUT, + numImages: sentinel, + seed: sentinel, + }) + + expect(parsed.numImages).toBeUndefined() + expect(parsed.seed).toBeUndefined() + }) + + it('preserves an explicit zero seed while rejecting zero images', () => { + expect(imageGenerationInputSchema.parse({ ...BASE_INPUT, seed: 0 }).seed).toBe(0) + expect(imageGenerationInputSchema.safeParse({ ...BASE_INPUT, numImages: 0 }).success).toBe( + false + ) + }) + + it('rejects multiple OpenAI images because the tool contract returns one image', () => { + expect( + imageGenerationInputSchema.safeParse({ ...BASE_INPUT, numImages: 3 }).error?.issues + ).toContainEqual( + expect.objectContaining({ + path: ['numImages'], + message: 'OpenAI image generation returns one image per tool execution', + }) + ) + expect( + imageGenerationInputSchema.safeParse({ + ...BASE_INPUT, + provider: 'falai', + numImages: 3, + }).success + ).toBe(true) + }) +}) diff --git a/apps/sim/lib/internal/image/schema.ts b/apps/sim/lib/internal/image/schema.ts new file mode 100644 index 00000000000..86e3d9d9f1a --- /dev/null +++ b/apps/sim/lib/internal/image/schema.ts @@ -0,0 +1,57 @@ +import { z } from 'zod' + +export const imageProviders = ['openai', 'gemini', 'falai'] as const +const MISSING_IMAGE_FIELDS_ERROR = 'Missing required fields: provider, apiKey, and prompt' +const optionalNumberInput = (value: unknown) => + value === null || (typeof value === 'string' && value.trim() === '') ? undefined : value +const booleanInputSchema = z.preprocess( + (value) => { + if (typeof value === 'boolean') return value + if (typeof value !== 'string') return value + const normalized = value.trim().toLowerCase() + if (normalized === 'true' || normalized === '1') return true + if (normalized === 'false' || normalized === '0' || normalized === '') return false + return value + }, + z.boolean({ error: 'must be a boolean (true/false)' }) +) + +export const imageGenerationInputSchema = z + .object({ + provider: z + .string({ error: MISSING_IMAGE_FIELDS_ERROR }) + .min(1, MISSING_IMAGE_FIELDS_ERROR) + .refine((provider) => imageProviders.includes(provider as ImageProvider), { + message: `Invalid provider. Must be one of: ${imageProviders.join(', ')}`, + }), + apiKey: z.string({ error: MISSING_IMAGE_FIELDS_ERROR }).min(1, MISSING_IMAGE_FIELDS_ERROR), + model: z.string().optional(), + prompt: z.string({ error: MISSING_IMAGE_FIELDS_ERROR }).min(1, MISSING_IMAGE_FIELDS_ERROR), + size: z.string().optional(), + aspectRatio: z.string().optional(), + resolution: z.string().optional(), + quality: z.string().optional(), + background: z.string().optional(), + outputFormat: z.string().optional(), + moderation: z.string().optional(), + safetyTolerance: z.string().optional(), + numImages: z.preprocess(optionalNumberInput, z.coerce.number().int().min(1).max(6).optional()), + seed: z.preprocess(optionalNumberInput, z.coerce.number().int().optional()), + enableSafetyChecker: booleanInputSchema.optional(), + enableWebSearch: booleanInputSchema.optional(), + thinkingLevel: z.string().optional(), + useHostedCostTracking: z.boolean().optional(), + }) + .passthrough() + .superRefine((input, context) => { + if (input.provider === 'openai' && input.numImages !== undefined && input.numImages !== 1) { + context.addIssue({ + code: 'custom', + path: ['numImages'], + message: 'OpenAI image generation returns one image per tool execution', + }) + } + }) + +export type ImageGenerationInput = z.output +export type ImageProvider = (typeof imageProviders)[number] diff --git a/apps/sim/lib/internal/instagram/execute-tool.test.ts b/apps/sim/lib/internal/instagram/execute-tool.test.ts new file mode 100644 index 00000000000..e568ce9c39b --- /dev/null +++ b/apps/sim/lib/internal/instagram/execute-tool.test.ts @@ -0,0 +1,387 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockDeleteFileMetadata, + mockDeleteFiles, + mockDownloadFileFromUrl, + mockUploadExecutionFile, + mockUploadCopilotFile, +} = vi.hoisted(() => ({ + mockDeleteFileMetadata: vi.fn(), + mockDeleteFiles: vi.fn(), + mockDownloadFileFromUrl: vi.fn(), + mockUploadExecutionFile: vi.fn(), + mockUploadCopilotFile: vi.fn(), +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadFileFromUrl: mockDownloadFileFromUrl, +})) +vi.mock('@/lib/uploads/contexts/execution', () => ({ + uploadExecutionFile: mockUploadExecutionFile, +})) +vi.mock('@/lib/uploads/contexts/copilot', () => ({ + uploadCopilotFile: mockUploadCopilotFile, +})) +vi.mock('@/lib/uploads/core/storage-service', () => ({ + deleteFiles: mockDeleteFiles, +})) +vi.mock('@/lib/uploads/server/metadata', () => ({ + deleteFileMetadata: mockDeleteFileMetadata, +})) + +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { executeInstagramTool } from '@/lib/internal/instagram/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' +import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' +import { instagramDownloadMediaTool } from '@/tools/instagram/download_media' + +const mockFetch = vi.fn() +const JPEG_BYTES = Buffer.from([0xff, 0xd8, 0xff, 0x01]) + +function executionFile(name: string, type: string, size: number) { + return { + id: `file-${name}`, + name, + url: `/api/files/serve/execution/${name}`, + size, + type, + key: `execution/workflow-1/execution-1/${name}`, + context: 'execution', + } +} + +beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + mockDownloadFileFromUrl.mockResolvedValue(Buffer.from('instagram-media')) + mockDeleteFiles.mockResolvedValue({ deleted: 0, failed: [] }) + mockDeleteFileMetadata.mockResolvedValue(true) + mockUploadExecutionFile.mockImplementation( + async ( + _context: { workspaceId: string; workflowId: string; executionId: string }, + buffer: Buffer, + name: string, + type: string + ) => executionFile(name, type, buffer.length) + ) +}) + +afterEach(() => { + vi.unstubAllGlobals() +}) + +function request( + input: Record, + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'instagram_download_media', + input, + headers: new Headers(), + context: { + userId: 'user-1', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + metadata: {}, + }, + requestId: 'request-1', + signal: new AbortController().signal, + ...overrides, + } +} + +describe('executeInstagramTool download media', () => { + it('stores a single download as an execution-scoped UserFile', async () => { + mockFetch.mockResolvedValueOnce( + Response.json({ + id: 'media-1', + media_type: 'IMAGE', + media_url: 'https://scontent.example.com/media-1.jpg', + }) + ) + mockDownloadFileFromUrl.mockResolvedValueOnce(JPEG_BYTES) + + const response = await executeInstagramTool( + request({ + accessToken: 'instagram-token', + mediaId: 'media-1', + filename: 'campaign-cover.png', + }) + ) + + expect(response.status).toBe(200) + const data = await response.json() + expect(data).toEqual({ + success: true, + output: { + files: [executionFile('campaign-cover.jpg', 'image/jpeg', JPEG_BYTES.length)], + mediaId: 'media-1', + mediaType: 'IMAGE', + downloadedCount: 1, + }, + }) + expect(mockDownloadFileFromUrl).toHaveBeenCalledWith( + 'https://scontent.example.com/media-1.jpg', + expect.objectContaining({ + maxBytes: MAX_FILE_SIZE, + signal: expect.any(AbortSignal), + userId: 'user-1', + }) + ) + expect(mockUploadExecutionFile).toHaveBeenCalledWith( + { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }, + JPEG_BYTES, + 'campaign-cover.jpg', + 'image/jpeg', + 'user-1' + ) + expect(mockUploadCopilotFile).not.toHaveBeenCalled() + }) + + it('downloads carousel children sequentially and preserves their order', async () => { + mockFetch + .mockResolvedValueOnce( + Response.json({ + id: 'carousel-1', + media_type: 'CAROUSEL_ALBUM', + children: { data: [{ id: 'child-image' }, { id: 'child-video' }] }, + }) + ) + .mockResolvedValueOnce( + Response.json({ + id: 'child-image', + media_type: 'IMAGE', + media_url: 'https://scontent.example.com/child-image.jpg', + }) + ) + .mockResolvedValueOnce( + Response.json({ + id: 'child-video', + media_type: 'VIDEO', + media_url: 'https://scontent.example.com/child-video.mp4', + }) + ) + mockDownloadFileFromUrl + .mockResolvedValueOnce(JPEG_BYTES) + .mockResolvedValueOnce(Buffer.from('video')) + + const response = await executeInstagramTool( + request({ + accessToken: 'instagram-token', + mediaId: 'carousel-1', + filename: 'launch', + }) + ) + + expect(response.status).toBe(200) + const data = await response.json() + expect(data.output).toEqual({ + files: [ + executionFile('launch-1.jpg', 'image/jpeg', JPEG_BYTES.length), + executionFile('launch-2.mp4', 'video/mp4', 5), + ], + mediaId: 'carousel-1', + mediaType: 'CAROUSEL_ALBUM', + downloadedCount: 2, + }) + expect(mockDownloadFileFromUrl.mock.calls.map(([url]) => url)).toEqual([ + 'https://scontent.example.com/child-image.jpg', + 'https://scontent.example.com/child-video.mp4', + ]) + expect(mockUploadExecutionFile.mock.calls.map(([, , name]) => name)).toEqual([ + 'launch-1.jpg', + 'launch-2.mp4', + ]) + expect(mockUploadExecutionFile.mock.invocationCallOrder[0]).toBeLessThan( + mockFetch.mock.invocationCallOrder[2] + ) + }) + + it('rolls back earlier carousel files when a later child cannot be downloaded', async () => { + mockFetch + .mockResolvedValueOnce( + Response.json({ + id: 'carousel-1', + media_type: 'CAROUSEL_ALBUM', + children: { data: [{ id: 'child-image' }, { id: 'child-missing' }] }, + }) + ) + .mockResolvedValueOnce( + Response.json({ + id: 'child-image', + media_type: 'IMAGE', + media_url: 'https://scontent.example.com/child-image.jpg', + }) + ) + .mockResolvedValueOnce( + Response.json( + { error: { message: 'The second carousel item is unavailable' } }, + { status: 404 } + ) + ) + mockDownloadFileFromUrl.mockResolvedValueOnce(JPEG_BYTES) + mockDeleteFiles.mockResolvedValueOnce({ deleted: 1, failed: [] }) + + const response = await executeInstagramTool( + request({ + accessToken: 'instagram-token', + mediaId: 'carousel-1', + filename: 'launch', + }) + ) + + const storedFile = executionFile('launch-1.jpg', 'image/jpeg', JPEG_BYTES.length) + expect(response.status).toBe(404) + expect(mockDeleteFiles).toHaveBeenCalledWith([storedFile.key], 'execution') + expect(mockDeleteFileMetadata).toHaveBeenCalledWith(storedFile.key) + }) + + it('does not preserve an image MIME type when the downloaded bytes are not a raster image', async () => { + mockFetch.mockResolvedValueOnce( + Response.json({ + id: 'media-invalid-image', + media_type: 'IMAGE', + media_url: 'https://scontent.example.com/media-invalid-image.jpg', + }) + ) + const invalidImage = Buffer.from('not an image') + mockDownloadFileFromUrl.mockResolvedValueOnce(invalidImage) + + const response = await executeInstagramTool( + request({ + accessToken: 'instagram-token', + mediaId: 'media-invalid-image', + }) + ) + + expect(response.status).toBe(200) + expect(mockUploadExecutionFile).toHaveBeenCalledWith( + expect.any(Object), + invalidImage, + 'instagram-media-invalid-image.bin', + 'application/octet-stream', + 'user-1' + ) + }) + + it('returns 413 when a media download exceeds the size cap', async () => { + mockFetch.mockResolvedValueOnce( + Response.json({ + id: 'media-large', + media_type: 'VIDEO', + media_url: 'https://scontent.example.com/media-large.mp4', + }) + ) + mockDownloadFileFromUrl.mockRejectedValueOnce( + new PayloadSizeLimitError({ + label: 'Instagram media download', + maxBytes: MAX_FILE_SIZE, + observedBytes: MAX_FILE_SIZE + 1, + }) + ) + + const response = await executeInstagramTool( + request({ + accessToken: 'instagram-token', + mediaId: 'media-large', + }) + ) + + expect(response.status).toBe(413) + expect(await response.json()).toEqual({ + success: false, + error: 'Instagram media exceeds the 100 MB canonical User File limit', + }) + expect(mockUploadExecutionFile).not.toHaveBeenCalled() + expect(mockUploadCopilotFile).not.toHaveBeenCalled() + }) + + it('rolls back stored carousel files before propagating cancellation', async () => { + const controller = new AbortController() + mockFetch + .mockResolvedValueOnce( + Response.json({ + id: 'carousel-1', + media_type: 'CAROUSEL_ALBUM', + children: { data: [{ id: 'child-image' }, { id: 'child-cancelled' }] }, + }) + ) + .mockResolvedValueOnce( + Response.json({ + id: 'child-image', + media_type: 'IMAGE', + media_url: 'https://scontent.example.com/child-image.jpg', + }) + ) + .mockImplementationOnce(async () => { + controller.abort() + throw controller.signal.reason + }) + mockDownloadFileFromUrl.mockResolvedValueOnce(JPEG_BYTES) + mockDeleteFiles.mockResolvedValueOnce({ deleted: 1, failed: [] }) + + await expect( + executeInstagramTool( + request( + { accessToken: 'instagram-token', mediaId: 'carousel-1', filename: 'launch' }, + { signal: controller.signal } + ) + ) + ).rejects.toMatchObject({ name: 'AbortError' }) + + const storedFile = executionFile('launch-1.jpg', 'image/jpeg', JPEG_BYTES.length) + expect(mockDeleteFiles).toHaveBeenCalledWith([storedFile.key], 'execution') + expect(mockDeleteFileMetadata).toHaveBeenCalledWith(storedFile.key) + }) +}) + +describe('instagramDownloadMediaTool', () => { + it('forwards execution context and returns canonical file-array output', async () => { + const body = instagramDownloadMediaTool.operation.input({ + accessToken: 'instagram-token', + mediaId: 'media-1', + filename: 'campaign-cover', + }) + expect(body).toEqual({ + accessToken: 'instagram-token', + mediaId: 'media-1', + filename: 'campaign-cover', + }) + + const file = executionFile('campaign-cover.jpg', 'image/jpeg', 11) + const result = await instagramDownloadMediaTool.transformResponse?.( + Response.json({ + success: true, + output: { + files: [file], + mediaId: 'media-1', + mediaType: 'IMAGE', + downloadedCount: 1, + }, + }), + { + accessToken: 'instagram-token', + mediaId: 'media-1', + } + ) + + expect(result).toEqual({ + success: true, + output: { + files: [file], + mediaId: 'media-1', + mediaType: 'IMAGE', + downloadedCount: 1, + }, + }) + }) +}) diff --git a/apps/sim/lib/internal/instagram/execute-tool.ts b/apps/sim/lib/internal/instagram/execute-tool.ts new file mode 100644 index 00000000000..be6d47bc6b0 --- /dev/null +++ b/apps/sim/lib/internal/instagram/execute-tool.ts @@ -0,0 +1,113 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { z } from 'zod' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { + executeInstagramDownloadMedia, + executeInstagramPublishCarousel, + executeInstagramPublishImage, + executeInstagramPublishReel, + executeInstagramPublishStory, + executeInstagramPublishVideo, + type InstagramOperationContext, +} from '@/lib/internal/instagram/operations' +import { + instagramDownloadMediaBodySchema, + instagramPublishCarouselBodySchema, + instagramPublishImageBodySchema, + instagramPublishReelBodySchema, + instagramPublishStoryBodySchema, + instagramPublishVideoBodySchema, +} from '@/lib/internal/instagram/schema' +import type { + InternalToolOperationCall, + InternalToolOperationHandler, +} from '@/lib/internal/tool-operations/types' + +const logger = createLogger('InstagramToolExecution') + +async function executeParsed( + request: InternalToolOperationCall, + schema: S, + execute: (input: z.output, context: InstagramOperationContext) => Promise +): Promise { + const parsed = schema.safeParse(request.input) + if (!parsed.success) { + const error = getValidationErrorMessage(parsed.error, 'Invalid request data') + return request.toolId === 'instagram_download_media' + ? Response.json({ success: false, error }, { status: 400 }) + : Response.json({ error, details: parsed.error.issues }, { status: 400 }) + } + + const userId = request.context.userId + if (!userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + + return execute(parsed.data, { + userId, + workspaceId: request.context.workspaceId, + workflowId: request.context.workflowId, + executionId: request.context.executionId, + requestId: request.requestId, + signal: request.signal, + }) +} + +export const executeInstagramTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + let serializedInput: string + try { + serializedInput = JSON.stringify(request.input) ?? '' + } catch { + return Response.json({ error: 'Invalid request data' }, { status: 400 }) + } + if (Buffer.byteLength(serializedInput, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + + try { + switch (request.toolId) { + case 'instagram_download_media': + return executeParsed( + request, + instagramDownloadMediaBodySchema, + executeInstagramDownloadMedia + ) + case 'instagram_publish_carousel': + return executeParsed( + request, + instagramPublishCarouselBodySchema, + executeInstagramPublishCarousel + ) + case 'instagram_publish_image': + return executeParsed(request, instagramPublishImageBodySchema, executeInstagramPublishImage) + case 'instagram_publish_reel': + return executeParsed(request, instagramPublishReelBodySchema, executeInstagramPublishReel) + case 'instagram_publish_story': + return executeParsed(request, instagramPublishStoryBodySchema, executeInstagramPublishStory) + case 'instagram_publish_video': + return executeParsed(request, instagramPublishVideoBodySchema, executeInstagramPublishVideo) + default: + return Response.json( + { success: false, error: `Unsupported Instagram tool: ${request.toolId}` }, + { status: 500 } + ) + } + } catch (error) { + request.signal?.throwIfAborted() + const message = getErrorMessage(error, 'Unknown error') + logger.error('Instagram operation dispatch failed', { + error: message, + requestId: request.requestId, + toolId: request.toolId, + }) + return Response.json({ success: false, error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/instagram/operations.test.ts b/apps/sim/lib/internal/instagram/operations.test.ts new file mode 100644 index 00000000000..6df32266e00 --- /dev/null +++ b/apps/sim/lib/internal/instagram/operations.test.ts @@ -0,0 +1,263 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + createMediaContainer: vi.fn(), + publishMediaContainer: vi.fn(), + resolveIgUserId: vi.fn(), + resolveInstagramCarouselMedia: vi.fn(), + resolveInstagramMedia: vi.fn(), + waitForContainerReady: vi.fn(), +})) + +vi.mock('@/lib/internal/instagram/publishing', () => mocks) + +import { executeInstagramTool } from '@/lib/internal/instagram/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' +import { instagramDownloadMediaTool } from '@/tools/instagram/download_media' +import { instagramPublishCarouselTool } from '@/tools/instagram/publish_carousel' +import { instagramPublishImageTool } from '@/tools/instagram/publish_image' +import { instagramPublishReelTool } from '@/tools/instagram/publish_reel' +import { instagramPublishStoryTool } from '@/tools/instagram/publish_story' +import { instagramPublishVideoTool } from '@/tools/instagram/publish_video' + +const image = { + id: 'image-1', + name: 'image.jpg', + size: 1024, + type: 'image/jpeg', + key: 'execution/workflow-1/execution-1/image.jpg', +} +const video = { + id: 'video-1', + name: 'video.mp4', + size: 2048, + type: 'video/mp4', + key: 'execution/workflow-1/execution-1/video.mp4', +} + +function request( + toolId: string, + input: Record, + signal = new AbortController().signal +): InternalToolOperationCall { + return { + toolId, + input, + headers: new Headers(), + context: { + userId: 'user-1', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + metadata: {}, + }, + requestId: 'request-1', + signal, + } +} + +beforeEach(() => { + vi.clearAllMocks() + mocks.resolveIgUserId.mockImplementation(async (_token: string, override?: string) => { + return override || 'ig-user-1' + }) + mocks.createMediaContainer.mockResolvedValue('container-1') + mocks.waitForContainerReady.mockResolvedValue({ statusCode: 'FINISHED', status: null }) + mocks.publishMediaContainer.mockResolvedValue('media-1') +}) + +describe('Instagram operation declarations', () => { + it('contains no HTTP-shaped request metadata for all six internal tools', () => { + const tools = [ + instagramDownloadMediaTool, + instagramPublishCarouselTool, + instagramPublishImageTool, + instagramPublishReelTool, + instagramPublishStoryTool, + instagramPublishVideoTool, + ] + + for (const tool of tools) { + expect(tool.operation.input).toBeTypeOf('function') + expect(tool).not.toHaveProperty('request') + expect(tool.operation).not.toHaveProperty('transport') + expect(tool.operation).not.toHaveProperty('url') + expect(tool.operation).not.toHaveProperty('method') + expect(tool.operation).not.toHaveProperty('headers') + expect(tool.operation).not.toHaveProperty('body') + } + }) + + it('does not serialize trusted execution scope into download input', () => { + const input = instagramDownloadMediaTool.operation.input({ + accessToken: 'token', + mediaId: 'media-1', + _context: { + workspaceId: 'untrusted-workspace', + workflowId: 'untrusted-workflow', + executionId: 'untrusted-execution', + }, + }) + + expect(input).toEqual({ accessToken: 'token', mediaId: 'media-1' }) + }) +}) + +describe('Instagram publish operations', () => { + it('publishes an image with the exact optional Meta fields', async () => { + mocks.resolveInstagramMedia.mockResolvedValue({ + media: { url: 'https://signed.example/image.jpg', kind: 'image' }, + }) + + const response = await executeInstagramTool( + request('instagram_publish_image', { + accessToken: 'token', + image, + caption: 'Caption', + altText: 'Alt text', + isAiGenerated: true, + }) + ) + + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ + success: true, + output: { containerId: 'container-1', mediaId: 'media-1', statusCode: 'FINISHED' }, + }) + expect(mocks.resolveInstagramMedia).toHaveBeenCalledWith( + expect.objectContaining({ userId: 'user-1', role: 'image', input: image }) + ) + expect(mocks.createMediaContainer).toHaveBeenCalledWith( + 'token', + 'ig-user-1', + { + image_url: 'https://signed.example/image.jpg', + caption: 'Caption', + alt_text: 'Alt text', + is_ai_generated: true, + }, + expect.any(AbortSignal) + ) + }) + + it('publishes video and reel variants without conflating share-to-feed semantics', async () => { + mocks.resolveInstagramMedia.mockImplementation(async ({ role }: { role: string }) => ({ + media: + role === 'cover' + ? { url: 'https://signed.example/cover.jpg', kind: 'image' } + : { url: 'https://signed.example/video.mp4', kind: 'video' }, + })) + + await executeInstagramTool( + request('instagram_publish_video', { + accessToken: 'token', + video, + cover: image, + caption: 'Video caption', + }) + ) + expect(mocks.createMediaContainer).toHaveBeenLastCalledWith( + 'token', + 'ig-user-1', + { + media_type: 'REELS', + video_url: 'https://signed.example/video.mp4', + share_to_feed: true, + caption: 'Video caption', + cover_url: 'https://signed.example/cover.jpg', + }, + expect.any(AbortSignal) + ) + + vi.clearAllMocks() + mocks.resolveIgUserId.mockResolvedValue('ig-user-1') + mocks.createMediaContainer.mockResolvedValue('container-1') + mocks.waitForContainerReady.mockResolvedValue({ statusCode: 'FINISHED', status: null }) + mocks.publishMediaContainer.mockResolvedValue('media-1') + mocks.resolveInstagramMedia.mockResolvedValue({ + media: { url: 'https://signed.example/video.mp4', kind: 'video' }, + }) + await executeInstagramTool( + request('instagram_publish_reel', { + accessToken: 'token', + video, + shareToFeed: false, + thumbOffset: 0, + }) + ) + expect(mocks.createMediaContainer).toHaveBeenLastCalledWith( + 'token', + 'ig-user-1', + { + media_type: 'REELS', + video_url: 'https://signed.example/video.mp4', + share_to_feed: false, + thumb_offset: 0, + }, + expect.any(AbortSignal) + ) + }) + + it('selects the correct story URL field from resolved media kind', async () => { + mocks.resolveInstagramMedia.mockResolvedValue({ + media: { url: 'https://signed.example/story.mp4', kind: 'video' }, + }) + + await executeInstagramTool( + request('instagram_publish_story', { accessToken: 'token', media: video }) + ) + + expect(mocks.createMediaContainer).toHaveBeenCalledWith( + 'token', + 'ig-user-1', + { media_type: 'STORIES', video_url: 'https://signed.example/story.mp4' }, + expect.any(AbortSignal) + ) + }) + + it('creates ordered carousel children before the parent container', async () => { + mocks.resolveInstagramCarouselMedia.mockResolvedValue({ + items: [ + { url: 'https://signed.example/one.jpg', kind: 'image' }, + { url: 'https://signed.example/two.mp4', kind: 'video' }, + ], + }) + mocks.createMediaContainer + .mockResolvedValueOnce('child-1') + .mockResolvedValueOnce('child-2') + .mockResolvedValueOnce('parent-1') + + const response = await executeInstagramTool( + request('instagram_publish_carousel', { + accessToken: 'token', + media: [image, video], + caption: 'Carousel caption', + }) + ) + + expect(response.status).toBe(200) + expect(mocks.createMediaContainer.mock.calls.map((call) => call[2])).toEqual([ + { is_carousel_item: true, image_url: 'https://signed.example/one.jpg' }, + { + is_carousel_item: true, + media_type: 'VIDEO', + video_url: 'https://signed.example/two.mp4', + }, + { media_type: 'CAROUSEL', children: 'child-1,child-2', caption: 'Carousel caption' }, + ]) + }) + + it('propagates cancellation without returning a provider retry error', async () => { + const controller = new AbortController() + controller.abort() + + await expect( + executeInstagramTool( + request('instagram_publish_image', { accessToken: 'token', image }, controller.signal) + ) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) +}) diff --git a/apps/sim/lib/internal/instagram/operations.ts b/apps/sim/lib/internal/instagram/operations.ts new file mode 100644 index 00000000000..805c60e8453 --- /dev/null +++ b/apps/sim/lib/internal/instagram/operations.ts @@ -0,0 +1,622 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { isRecordLike } from '@sim/utils/object' +import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' +import { + createMediaContainer, + publishMediaContainer, + resolveIgUserId, + resolveInstagramCarouselMedia, + resolveInstagramMedia, + waitForContainerReady, +} from '@/lib/internal/instagram/publishing' +import { + type InstagramDownloadMediaBody, + type InstagramDownloadMediaRouteResponse, + type InstagramPublishCarouselBody, + type InstagramPublishImageBody, + type InstagramPublishReelBody, + type InstagramPublishStoryBody, + type InstagramPublishVideoBody, + instagramDownloadMediaOutputSchema, +} from '@/lib/internal/instagram/schema' +import { uploadCopilotFile } from '@/lib/uploads/contexts/copilot' +import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' +import { deleteFiles } from '@/lib/uploads/core/storage-service' +import { deleteFileMetadata } from '@/lib/uploads/server/metadata' +import type { StorageContext } from '@/lib/uploads/shared/types' +import { + getExtensionFromMimeType, + getFileExtension, + getMimeTypeFromExtension, +} from '@/lib/uploads/utils/file-utils' +import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server' +import { MAX_FILE_SIZE, sniffImageContentType } from '@/lib/uploads/utils/validation' +import { sanitizeFileName } from '@/executor/constants' +import type { UserFile } from '@/executor/types' +import { bearerHeaders, graphUrl, idString, readGraphError } from '@/tools/instagram/utils' + +const logger = createLogger('InstagramOperations') +const MAX_GRAPH_METADATA_BYTES = 256 * 1024 +const MAX_CAROUSEL_ITEMS = 10 +const ROOT_MEDIA_FIELDS = 'id,media_type,media_url,children{id}' +const CHILD_MEDIA_FIELDS = 'id,media_type,media_url' + +interface InstagramMediaMetadata { + id: string + mediaType: string | null + mediaUrl: string | null + childIds: string[] +} + +type InstagramMediaMetadataResult = + | { success: true; data: InstagramMediaMetadata } + | { success: false; error: string; status: number } + +function failureResponse(error: string, status: number) { + const body = { success: false, error } satisfies InstagramDownloadMediaRouteResponse + return Response.json(body, { status }) +} + +function normalizedId(value: unknown): string | null { + return typeof value === 'string' || typeof value === 'number' ? idString(value) : null +} + +function parseMediaMetadata(data: unknown): InstagramMediaMetadataResult { + if (!isRecordLike(data)) { + return { success: false, error: 'Instagram returned invalid media metadata', status: 502 } + } + + const id = normalizedId(data.id) + if (!id) { + return { success: false, error: 'Instagram media metadata did not include an ID', status: 502 } + } + + const mediaType = typeof data.media_type === 'string' ? data.media_type : null + const mediaUrl = + typeof data.media_url === 'string' && data.media_url.length > 0 ? data.media_url : null + const children = data.children + + if (children === undefined) { + return { success: true, data: { id, mediaType, mediaUrl, childIds: [] } } + } + + if (!isRecordLike(children) || !Array.isArray(children.data)) { + return { success: false, error: 'Instagram returned invalid carousel metadata', status: 502 } + } + + if (children.data.length > MAX_CAROUSEL_ITEMS) { + return { + success: false, + error: `Instagram carousel exceeds the ${MAX_CAROUSEL_ITEMS}-item download limit`, + status: 502, + } + } + + const childIds: string[] = [] + for (const child of children.data) { + if (!isRecordLike(child)) { + return { success: false, error: 'Instagram returned an invalid carousel item', status: 502 } + } + const childId = normalizedId(child.id) + if (!childId) { + return { + success: false, + error: 'Instagram carousel item did not include an ID', + status: 502, + } + } + childIds.push(childId) + } + + return { success: true, data: { id, mediaType, mediaUrl, childIds } } +} + +async function fetchMediaMetadata({ + accessToken, + mediaId, + fields, + signal, +}: { + accessToken: string + mediaId: string + fields: string + signal?: AbortSignal +}): Promise { + const response = await fetch(graphUrl(`/${encodeURIComponent(mediaId)}`, { fields }), { + headers: bearerHeaders(accessToken), + signal, + }) + + if (!response.ok) { + return { + success: false, + error: await readGraphError(response), + status: response.status >= 400 && response.status < 500 ? response.status : 502, + } + } + + const data = await readResponseJsonWithLimit(response, { + maxBytes: MAX_GRAPH_METADATA_BYTES, + label: `Instagram media ${mediaId} metadata`, + signal, + }) + return parseMediaMetadata(data) +} + +function inferContentType(mediaUrl: string, mediaType: string | null): string { + if (mediaType === 'VIDEO') return 'video/mp4' + if (mediaType === 'IMAGE') return 'image/jpeg' + + let extension = '' + try { + extension = getFileExtension(new URL(mediaUrl).pathname) + } catch { + extension = '' + } + + const mimeType = getMimeTypeFromExtension(extension) + if (mimeType !== 'application/octet-stream') return mimeType + return 'application/octet-stream' +} + +function resolveDownloadedContentType( + buffer: Buffer, + mediaUrl: string, + mediaType: string | null +): string { + const inferred = inferContentType(mediaUrl, mediaType) + if (mediaType === 'IMAGE' || inferred.startsWith('image/')) { + return sniffImageContentType(buffer) ?? 'application/octet-stream' + } + return inferred +} + +function buildFilename({ + filename, + mediaId, + contentType, + itemIndex, + itemCount, +}: { + filename?: string + mediaId: string + contentType: string + itemIndex: number + itemCount: number +}): string { + const extension = getExtensionFromMimeType(contentType) ?? 'bin' + if (!filename) return sanitizeFileName(`instagram-${mediaId}.${extension}`) + + const sanitized = sanitizeFileName(filename).replace(/^\.+/, '') + const lastDot = sanitized.lastIndexOf('.') + const base = (lastDot > 0 ? sanitized.slice(0, lastDot) : sanitized) || `instagram-${mediaId}` + const suffix = itemCount > 1 ? `-${itemIndex + 1}` : '' + return `${base}${suffix}.${extension}` +} + +async function downloadAndStoreMedia({ + metadata, + filename, + itemIndex, + itemCount, + userId, + executionContext, + signal, +}: { + metadata: InstagramMediaMetadata + filename?: string + itemIndex: number + itemCount: number + userId: string + executionContext?: { workspaceId: string; workflowId: string; executionId: string } + signal?: AbortSignal +}): Promise { + if (!metadata.mediaUrl) { + throw new Error(`Instagram media ${metadata.id} did not include a downloadable URL`) + } + + const buffer = await downloadFileFromUrl(metadata.mediaUrl, { + maxBytes: MAX_FILE_SIZE, + signal, + userId, + }) + const contentType = resolveDownloadedContentType(buffer, metadata.mediaUrl, metadata.mediaType) + const storedFilename = buildFilename({ + filename, + mediaId: metadata.id, + contentType, + itemIndex, + itemCount, + }) + if (executionContext) { + return uploadExecutionFile(executionContext, buffer, storedFilename, contentType, userId) + } + + return uploadCopilotFile({ + buffer, + fileName: storedFilename, + contentType, + userId, + }) +} + +/** Removes successfully stored files when a multi-item download cannot return a complete result. */ +async function rollbackStoredFiles(files: UserFile[], context: StorageContext): Promise { + if (files.length === 0) return + + const keys = files.map((file) => file.key) + let failedKeys: Set + try { + const deletion = await deleteFiles(keys, context) + failedKeys = new Set(deletion.failed.map((failure) => failure.key)) + if (deletion.failed.length > 0) { + logger.warn('Instagram media rollback could not delete every stored object', { + context, + failedKeys: [...failedKeys], + }) + } + } catch (error) { + logger.warn('Instagram media rollback failed before metadata cleanup', { + context, + error: getErrorMessage(error), + keys, + }) + return + } + + for (const key of keys) { + if (failedKeys.has(key)) continue + try { + await deleteFileMetadata(key) + } catch (error) { + logger.warn('Instagram media rollback could not delete file metadata', { + error: getErrorMessage(error), + key, + }) + } + } +} + +export interface InstagramOperationContext { + userId: string + workspaceId?: string + workflowId?: string + executionId?: string + requestId: string + signal?: AbortSignal +} + +export async function executeInstagramDownloadMedia( + body: InstagramDownloadMediaBody, + context: InstagramOperationContext +): Promise { + const { executionId, requestId, signal, userId, workflowId, workspaceId } = context + signal?.throwIfAborted() + const files: UserFile[] = [] + let storageContext: StorageContext = 'copilot' + try { + const rootResult = await fetchMediaMetadata({ + accessToken: body.accessToken, + mediaId: body.mediaId, + fields: ROOT_MEDIA_FIELDS, + signal, + }) + if (!rootResult.success) return failureResponse(rootResult.error, rootResult.status) + + const rootMedia = rootResult.data + const itemCount = rootMedia.childIds.length || 1 + const executionContext = + workspaceId && workflowId && executionId + ? { + workspaceId, + workflowId, + executionId, + } + : undefined + storageContext = executionContext ? 'execution' : 'copilot' + + if (rootMedia.childIds.length === 0) { + files.push( + await downloadAndStoreMedia({ + metadata: rootMedia, + filename: body.filename, + itemIndex: 0, + itemCount, + userId, + executionContext, + signal, + }) + ) + } else { + for (const [itemIndex, childId] of rootMedia.childIds.entries()) { + const childResult = await fetchMediaMetadata({ + accessToken: body.accessToken, + mediaId: childId, + fields: CHILD_MEDIA_FIELDS, + signal, + }) + if (!childResult.success) { + await rollbackStoredFiles(files, storageContext) + return failureResponse(childResult.error, childResult.status) + } + + files.push( + await downloadAndStoreMedia({ + metadata: childResult.data, + filename: body.filename, + itemIndex, + itemCount, + userId, + executionContext, + signal, + }) + ) + } + } + + const output = instagramDownloadMediaOutputSchema.parse({ + files, + mediaId: rootMedia.id, + mediaType: rootMedia.mediaType, + downloadedCount: files.length, + }) + const responseBody = { + success: true, + output, + } satisfies InstagramDownloadMediaRouteResponse + + return Response.json(responseBody) + } catch (error) { + await rollbackStoredFiles(files, storageContext) + signal?.throwIfAborted() + logger.error('Instagram media download failed', { error }) + + if (isPayloadSizeLimitError(error) && error.maxBytes === MAX_FILE_SIZE) { + return failureResponse('Instagram media exceeds the 100 MB canonical User File limit', 413) + } + + return failureResponse( + getErrorMessage(error, 'Failed to download Instagram media'), + isPayloadSizeLimitError(error) ? 413 : 500 + ) + } +} + +const FAILED_PUBLISH_OUTPUT = { + containerId: null, + mediaId: null, + statusCode: null, +} as const + +function publishFailure(error: string, status: number): Response { + return Response.json({ success: false, error, output: FAILED_PUBLISH_OUTPUT }, { status }) +} + +async function publishContainer( + accessToken: string, + igUserIdOverride: string | null | undefined, + containerBody: Record, + context: InstagramOperationContext +): Promise { + context.signal?.throwIfAborted() + const igUserId = await resolveIgUserId(accessToken, igUserIdOverride ?? undefined, context.signal) + const containerId = await createMediaContainer( + accessToken, + igUserId, + containerBody, + context.signal + ) + const { statusCode } = await waitForContainerReady(accessToken, containerId, context.signal) + const mediaId = await publishMediaContainer(accessToken, igUserId, containerId, context.signal) + return Response.json({ + success: true, + output: { containerId, mediaId, statusCode }, + }) +} + +export async function executeInstagramPublishImage( + body: InstagramPublishImageBody, + context: InstagramOperationContext +): Promise { + try { + const resolved = await resolveInstagramMedia({ + input: body.image, + userId: context.userId, + requestId: context.requestId, + logger, + role: 'image', + label: 'Image', + }) + if (resolved.error || !resolved.media) { + return publishFailure( + resolved.error?.message || 'Failed to resolve image', + resolved.error?.status || 400 + ) + } + + const containerBody: Record = { image_url: resolved.media.url } + if (body.caption) containerBody.caption = body.caption + if (body.altText) containerBody.alt_text = body.altText + if (body.isAiGenerated === true) containerBody.is_ai_generated = true + return await publishContainer(body.accessToken, body.igUserId, containerBody, context) + } catch (error) { + context.signal?.throwIfAborted() + logger.error('Instagram publish image failed', { error }) + return publishFailure(getErrorMessage(error, 'Failed to publish image'), 500) + } +} + +async function resolveOptionalCover( + cover: InstagramPublishVideoBody['cover'], + context: InstagramOperationContext +): Promise<{ url?: string; response?: Response }> { + if (cover == null) return {} + const resolved = await resolveInstagramMedia({ + input: cover, + userId: context.userId, + requestId: context.requestId, + logger, + role: 'cover', + required: false, + label: 'Cover image', + }) + if (resolved.error) { + return { response: publishFailure(resolved.error.message, resolved.error.status) } + } + return { url: resolved.media?.url } +} + +async function executeInstagramPublishVideoLike( + body: InstagramPublishVideoBody | InstagramPublishReelBody, + context: InstagramOperationContext, + mode: 'video' | 'reel' +): Promise { + try { + const resolvedVideo = await resolveInstagramMedia({ + input: body.video, + userId: context.userId, + requestId: context.requestId, + logger, + role: 'video', + label: 'Video', + }) + if (resolvedVideo.error || !resolvedVideo.media) { + return publishFailure( + resolvedVideo.error?.message || 'Failed to resolve video', + resolvedVideo.error?.status || 400 + ) + } + + const cover = await resolveOptionalCover(body.cover, context) + if (cover.response) return cover.response + + const containerBody: Record = { + media_type: 'REELS', + video_url: resolvedVideo.media.url, + } + if (mode === 'video') containerBody.share_to_feed = true + if (body.caption) containerBody.caption = body.caption + if (cover.url) containerBody.cover_url = cover.url + if (mode === 'reel') { + const reel = body as InstagramPublishReelBody + if (reel.shareToFeed !== undefined && reel.shareToFeed !== null) { + containerBody.share_to_feed = reel.shareToFeed + } + if (reel.thumbOffset != null) containerBody.thumb_offset = reel.thumbOffset + } + return await publishContainer(body.accessToken, body.igUserId, containerBody, context) + } catch (error) { + context.signal?.throwIfAborted() + const action = mode === 'video' ? 'video' : 'reel' + logger.error(`Instagram publish ${action} failed`, { error }) + return publishFailure(getErrorMessage(error, `Failed to publish ${action}`), 500) + } +} + +export function executeInstagramPublishVideo( + body: InstagramPublishVideoBody, + context: InstagramOperationContext +): Promise { + return executeInstagramPublishVideoLike(body, context, 'video') +} + +export function executeInstagramPublishReel( + body: InstagramPublishReelBody, + context: InstagramOperationContext +): Promise { + return executeInstagramPublishVideoLike(body, context, 'reel') +} + +export async function executeInstagramPublishStory( + body: InstagramPublishStoryBody, + context: InstagramOperationContext +): Promise { + try { + const resolved = await resolveInstagramMedia({ + input: body.media, + userId: context.userId, + requestId: context.requestId, + logger, + role: 'story', + label: 'Story media', + }) + if (resolved.error || !resolved.media) { + return publishFailure( + resolved.error?.message || 'Failed to resolve story media', + resolved.error?.status || 400 + ) + } + + const containerBody: Record = { media_type: 'STORIES' } + if (resolved.media.kind === 'video') containerBody.video_url = resolved.media.url + else containerBody.image_url = resolved.media.url + return await publishContainer(body.accessToken, body.igUserId, containerBody, context) + } catch (error) { + context.signal?.throwIfAborted() + logger.error('Instagram publish story failed', { error }) + return publishFailure(getErrorMessage(error, 'Failed to publish story'), 500) + } +} + +export async function executeInstagramPublishCarousel( + body: InstagramPublishCarouselBody, + context: InstagramOperationContext +): Promise { + try { + const resolved = await resolveInstagramCarouselMedia( + body.media, + context.userId, + context.requestId, + logger + ) + if (resolved.error || !resolved.items) { + return publishFailure( + resolved.error?.message || 'Failed to resolve carousel media', + resolved.error?.status || 400 + ) + } + + const igUserId = await resolveIgUserId( + body.accessToken, + body.igUserId ?? undefined, + context.signal + ) + const childIds: string[] = [] + for (const item of resolved.items) { + context.signal?.throwIfAborted() + const childBody: Record = { is_carousel_item: true } + if (item.kind === 'video') { + childBody.media_type = 'VIDEO' + childBody.video_url = item.url + } else { + childBody.image_url = item.url + } + childIds.push( + await createMediaContainer(body.accessToken, igUserId, childBody, context.signal) + ) + } + + const childResults = await Promise.allSettled( + childIds.map((childId) => waitForContainerReady(body.accessToken, childId, context.signal)) + ) + const failedChild = childResults.find( + (result): result is PromiseRejectedResult => result.status === 'rejected' + ) + if (failedChild) throw failedChild.reason + + const parentBody: Record = { + media_type: 'CAROUSEL', + children: childIds.join(','), + } + if (body.caption) parentBody.caption = body.caption + return await publishContainer(body.accessToken, igUserId, parentBody, { + ...context, + signal: context.signal, + }) + } catch (error) { + context.signal?.throwIfAborted() + logger.error('Instagram publish carousel failed', { error }) + return publishFailure(getErrorMessage(error, 'Failed to publish carousel'), 500) + } +} diff --git a/apps/sim/lib/internal/instagram/publishing.test.ts b/apps/sim/lib/internal/instagram/publishing.test.ts new file mode 100644 index 00000000000..e8d090dc97c --- /dev/null +++ b/apps/sim/lib/internal/instagram/publishing.test.ts @@ -0,0 +1,245 @@ +/** + * @vitest-environment node + */ +import type { Logger } from '@sim/logger' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockHasCloudStorage, mockResolveFileInputToUrl } = vi.hoisted(() => ({ + mockHasCloudStorage: vi.fn(), + mockResolveFileInputToUrl: vi.fn(), +})) + +vi.mock('@/lib/uploads/core/storage-service', () => ({ + hasCloudStorage: mockHasCloudStorage, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + resolveFileInputToUrl: mockResolveFileInputToUrl, +})) + +import { + createMediaContainer, + INSTAGRAM_MEDIA_URL_TTL_SECONDS, + publishMediaContainer, + resolveIgUserId, + resolveInstagramCarouselMedia, + resolveInstagramMedia, +} from '@/lib/internal/instagram/publishing' + +const logger = {} as Logger +const context = { + userId: 'user-1', + requestId: 'request-1', + logger, +} + +function uploadedFile(overrides: Record = {}) { + return { + id: 'file-1', + key: 'execution/workflow-1/execution-1/photo.jpg', + name: 'photo.jpg', + size: 1024, + type: 'image/jpeg', + ...overrides, + } +} + +beforeEach(() => { + vi.clearAllMocks() + mockHasCloudStorage.mockReturnValue(true) + mockResolveFileInputToUrl.mockImplementation(async ({ file }: { file?: { name?: string } }) => ({ + fileUrl: `https://signed.example.com/${file?.name || 'media'}`, + })) +}) + +afterEach(() => { + vi.unstubAllGlobals() +}) + +describe('resolveInstagramMedia', () => { + it('rejects non-file inputs before resolving them', async () => { + const result = await resolveInstagramMedia({ + ...context, + input: 'https://cdn.example.com/photo.jpg', + role: 'image', + }) + + expect(result.error).toEqual({ + status: 400, + message: 'Media must be a Sim file', + }) + expect(mockResolveFileInputToUrl).not.toHaveBeenCalled() + }) + + it('resolves an uploaded file with the Instagram publishing URL lifetime', async () => { + const file = uploadedFile() + const result = await resolveInstagramMedia({ ...context, input: file, role: 'image' }) + + expect(result.media).toEqual({ + url: 'https://signed.example.com/photo.jpg', + kind: 'image', + mimeType: 'image/jpeg', + size: 1024, + name: 'photo.jpg', + }) + expect(mockResolveFileInputToUrl).toHaveBeenCalledWith({ + file, + ...context, + presignExpirySeconds: INSTAGRAM_MEDIA_URL_TTL_SECONDS, + }) + }) + + it('requires cloud storage for publishing files', async () => { + mockHasCloudStorage.mockReturnValue(false) + + const result = await resolveInstagramMedia({ ...context, input: uploadedFile(), role: 'image' }) + expect(result.error).toEqual({ + status: 400, + message: expect.stringContaining('Cloud storage is required'), + }) + expect(mockResolveFileInputToUrl).not.toHaveBeenCalled() + }) + + it('validates JPEG MIME type and size without loading file bytes', async () => { + const invalidType = await resolveInstagramMedia({ + ...context, + input: uploadedFile({ name: 'photo.png', type: 'image/png' }), + role: 'image', + label: 'Image', + }) + const oversized = await resolveInstagramMedia({ + ...context, + input: uploadedFile({ size: 8 * 1024 * 1024 + 1 }), + role: 'image', + label: 'Image', + }) + + expect(invalidType.error?.message).toBe('Image must be a JPEG image (got image/png)') + expect(oversized.error?.message).toContain("Instagram's 8MB JPEG limit") + }) + + it.each([ + { role: 'video' as const, maxBytes: 300 * 1024 * 1024, label: 'Video' }, + { role: 'story' as const, maxBytes: 100 * 1024 * 1024, label: 'Story' }, + ])('enforces the $role video size limit', async ({ role, maxBytes, label }) => { + const result = await resolveInstagramMedia({ + ...context, + input: uploadedFile({ + key: 'execution/workflow-1/execution-1/video.mp4', + name: 'video.mp4', + size: maxBytes + 1, + type: 'video/mp4', + }), + role, + label, + }) + + expect(result.error?.message).toContain(`video limit for ${role}`) + }) + + it('rejects unsupported video formats', async () => { + const result = await resolveInstagramMedia({ + ...context, + input: uploadedFile({ name: 'video.webm', type: 'video/webm' }), + role: 'video', + label: 'Video', + }) + + expect(result.error?.message).toBe('Video must be an MP4 or MOV video (got video/webm)') + }) +}) + +describe('resolveInstagramCarouselMedia', () => { + it('resolves canonical files in order and infers image and video types sequentially', async () => { + let activeResolutions = 0 + let maxActiveResolutions = 0 + mockResolveFileInputToUrl.mockImplementation(async ({ file }: { file?: { name?: string } }) => { + activeResolutions += 1 + maxActiveResolutions = Math.max(maxActiveResolutions, activeResolutions) + await Promise.resolve() + activeResolutions -= 1 + return { fileUrl: `https://signed.example.com/${file?.name}` } + }) + + const result = await resolveInstagramCarouselMedia( + [ + uploadedFile({ name: 'carousel-1.jpg' }), + uploadedFile({ name: 'carousel-2.mp4', type: 'video/mp4' }), + ], + context.userId, + context.requestId, + logger + ) + + expect(result.items?.map(({ url, kind }) => ({ url, kind }))).toEqual([ + { url: 'https://signed.example.com/carousel-1.jpg', kind: 'image' }, + { url: 'https://signed.example.com/carousel-2.mp4', kind: 'video' }, + ]) + expect(maxActiveResolutions).toBe(1) + }) + + it.each([ + { count: 1, label: 'too few' }, + { count: 11, label: 'too many' }, + ])('rejects $label carousel items before resolving them', async ({ count }) => { + const input = Array.from({ length: count }, (_, index) => + uploadedFile({ id: `file-${index + 1}`, name: `carousel-${index + 1}.jpg` }) + ) + + const result = await resolveInstagramCarouselMedia( + input, + context.userId, + context.requestId, + logger + ) + + expect(result.error).toEqual({ + status: 400, + message: 'Carousels require between 2 and 10 items', + }) + expect(mockResolveFileInputToUrl).not.toHaveBeenCalled() + }) + + it('rejects non-file string inputs', async () => { + const result = await resolveInstagramCarouselMedia( + 'https://example.com/one.jpg,https://example.com/two.jpg', + context.userId, + context.requestId, + logger + ) + + expect(result.error).toEqual({ status: 400, message: 'Carousel media is required' }) + }) +}) + +describe('Instagram publishing requests', () => { + it('resolves the connected account when no override is supplied', async () => { + const fetchMock = vi.fn().mockResolvedValue(Response.json({ user_id: 123 }, { status: 200 })) + vi.stubGlobal('fetch', fetchMock) + + await expect(resolveIgUserId('token')).resolves.toBe('123') + expect(fetchMock).toHaveBeenCalledOnce() + }) + + it('creates and publishes form-encoded containers', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(Response.json({ id: 'container-1' }, { status: 200 })) + .mockResolvedValueOnce(Response.json({ id: 'media-1' }, { status: 200 })) + vi.stubGlobal('fetch', fetchMock) + + await expect( + createMediaContainer('token', 'user-1', { image_url: 'https://signed.example/image.jpg' }) + ).resolves.toBe('container-1') + await expect(publishMediaContainer('token', 'user-1', 'container-1')).resolves.toBe('media-1') + + expect(fetchMock.mock.calls[0]?.[1]).toMatchObject({ + method: 'POST', + body: 'image_url=https%3A%2F%2Fsigned.example%2Fimage.jpg', + }) + expect(fetchMock.mock.calls[1]?.[1]).toMatchObject({ + method: 'POST', + body: 'creation_id=container-1', + }) + }) +}) diff --git a/apps/sim/app/api/tools/instagram/server-utils.ts b/apps/sim/lib/internal/instagram/publishing.ts similarity index 100% rename from apps/sim/app/api/tools/instagram/server-utils.ts rename to apps/sim/lib/internal/instagram/publishing.ts diff --git a/apps/sim/lib/internal/instagram/schema.ts b/apps/sim/lib/internal/instagram/schema.ts new file mode 100644 index 00000000000..c1e75b58dbd --- /dev/null +++ b/apps/sim/lib/internal/instagram/schema.ts @@ -0,0 +1,157 @@ +import { z } from 'zod' +import { userFileSchema } from '@/lib/api/contracts/primitives' +import { RawFileInputArraySchema, RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' + +const MAX_ACCESS_TOKEN_LENGTH = 8192 +const MAX_GRAPH_ID_LENGTH = 256 +const MAX_CAPTION_LENGTH = 2200 +const MAX_ALT_TEXT_LENGTH = 1000 + +const instagramOptionalUserIdSchema = z + .string() + .trim() + .max(MAX_GRAPH_ID_LENGTH, 'Instagram user ID is too long') + .optional() + .nullable() + +const instagramOptionalCaptionSchema = z + .string() + .max(MAX_CAPTION_LENGTH, `Caption cannot exceed ${MAX_CAPTION_LENGTH} characters`) + .optional() + .nullable() + +export const instagramAccessTokenSchema = z + .string() + .min(1, 'Access token is required') + .max(MAX_ACCESS_TOKEN_LENGTH, 'Access token is too long') + +export const instagramDownloadMediaBodySchema = z.object({ + accessToken: instagramAccessTokenSchema, + mediaId: z.string().trim().min(1, 'Media ID is required').max(256, 'Media ID is too long'), + filename: z + .string() + .trim() + .min(1, 'Filename cannot be empty') + .max(180, 'Filename is too long') + .optional(), +}) + +export const instagramDownloadMediaOutputSchema = z + .object({ + files: z.array(userFileSchema).min(1, 'At least one downloaded file is required').max(10), + mediaId: z.string().min(1).max(MAX_GRAPH_ID_LENGTH), + mediaType: z.string().max(64).nullable(), + downloadedCount: z.number().int().min(1).max(10), + }) + .superRefine((output, context) => { + if (output.downloadedCount !== output.files.length) { + context.addIssue({ + code: 'custom', + path: ['downloadedCount'], + message: 'Downloaded count must match the number of files', + }) + } + }) + +export const instagramDownloadMediaResponseSchema = z.discriminatedUnion('success', [ + z.object({ + success: z.literal(true), + output: instagramDownloadMediaOutputSchema, + }), + z.object({ + success: z.literal(false), + error: z.string().min(1), + }), +]) + +/** Canonical Sim file uploaded in basic mode or referenced from a prior block. */ +export const instagramMediaInputSchema = RawFileInputSchema + +/** Canonical Sim files uploaded in basic mode or referenced from prior blocks. */ +export const instagramCarouselMediaSchema = RawFileInputArraySchema.min( + 2, + 'Carousels require at least 2 items' +).max(10, 'Carousels support at most 10 items') + +export const instagramPublishOutputSchema = z.object({ + containerId: z.string().min(1, 'Container ID is required'), + mediaId: z.string().min(1, 'Media ID is required'), + statusCode: z.string().min(1, 'Status code is required'), +}) + +const instagramFailedPublishOutputSchema = z.object({ + containerId: z.null(), + mediaId: z.null(), + statusCode: z.null(), +}) + +export const instagramPublishResponseSchema = z.discriminatedUnion('success', [ + z.object({ + success: z.literal(true), + output: instagramPublishOutputSchema, + }), + z.object({ + success: z.literal(false), + error: z.string().min(1), + output: instagramFailedPublishOutputSchema.optional(), + }), +]) + +export const instagramPublishImageBodySchema = z.object({ + accessToken: instagramAccessTokenSchema, + igUserId: instagramOptionalUserIdSchema, + image: instagramMediaInputSchema, + caption: instagramOptionalCaptionSchema, + altText: z + .string() + .max(MAX_ALT_TEXT_LENGTH, `Alt text cannot exceed ${MAX_ALT_TEXT_LENGTH} characters`) + .optional() + .nullable(), + isAiGenerated: z.boolean().optional().nullable(), +}) + +export const instagramPublishVideoBodySchema = z.object({ + accessToken: instagramAccessTokenSchema, + igUserId: instagramOptionalUserIdSchema, + video: instagramMediaInputSchema, + cover: instagramMediaInputSchema.optional().nullable(), + caption: instagramOptionalCaptionSchema, +}) + +export const instagramPublishReelBodySchema = z.object({ + accessToken: instagramAccessTokenSchema, + igUserId: instagramOptionalUserIdSchema, + video: instagramMediaInputSchema, + cover: instagramMediaInputSchema.optional().nullable(), + caption: instagramOptionalCaptionSchema, + shareToFeed: z.boolean().optional().nullable(), + thumbOffset: z.number().optional().nullable(), +}) + +export const instagramPublishStoryBodySchema = z.object({ + accessToken: instagramAccessTokenSchema, + igUserId: instagramOptionalUserIdSchema, + media: instagramMediaInputSchema, +}) + +export const instagramPublishCarouselBodySchema = z.object({ + accessToken: instagramAccessTokenSchema, + igUserId: instagramOptionalUserIdSchema, + media: instagramCarouselMediaSchema, + caption: instagramOptionalCaptionSchema, +}) + +export type InstagramDownloadMediaBody = z.output +export type InstagramDownloadMediaRouteResponse = z.output< + typeof instagramDownloadMediaResponseSchema +> +export type InstagramPublishImageBody = z.output +export type InstagramPublishVideoBody = z.output +export type InstagramPublishReelBody = z.output +export type InstagramPublishStoryBody = z.output +export type InstagramPublishCarouselBody = z.output +export type InstagramPublishImageResponse = z.output +export type InstagramPublishVideoResponse = z.output +export type InstagramPublishReelResponse = z.output +export type InstagramPublishStoryResponse = z.output +export type InstagramPublishCarouselResponse = z.output diff --git a/apps/sim/lib/internal/jira/client.test.ts b/apps/sim/lib/internal/jira/client.test.ts new file mode 100644 index 00000000000..c92412ca7b4 --- /dev/null +++ b/apps/sim/lib/internal/jira/client.test.ts @@ -0,0 +1,114 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + getCloudId: vi.fn(), +})) + +vi.mock('@/tools/jira/utils', () => ({ + getJiraCloudId: mocks.getCloudId, +})) + +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { createJiraClient } from '@/lib/internal/jira/client' + +describe('JiraClient', () => { + const fetchMock = vi.fn() + + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', fetchMock) + mocks.getCloudId.mockResolvedValue('cloud-1') + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('passes credentials and cancellation to Jira requests', async () => { + fetchMock.mockResolvedValue(new Response('{"id":"1"}', { status: 200 })) + const controller = new AbortController() + const client = await createJiraClient( + { accessToken: 'token', domain: 'example.atlassian.net', cloudId: 'cloud-1' }, + { signal: controller.signal, validateCloudId: true } + ) + + await expect( + client.request( + client.issuePath(), + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: '{}', + }, + controller.signal + ) + ).resolves.toMatchObject({ ok: true, text: '{"id":"1"}' }) + + expect(fetchMock).toHaveBeenCalledWith( + 'https://api.atlassian.com/ex/jira/cloud-1/rest/api/3/issue', + expect.objectContaining({ + method: 'POST', + signal: controller.signal, + headers: { + Authorization: 'Bearer token', + 'Content-Type': 'application/json', + }, + }) + ) + }) + + it('caps provider response bodies before materializing them', async () => { + let cancelled = false + const stream = new ReadableStream({ + cancel: () => { + cancelled = true + }, + }) + fetchMock.mockResolvedValue( + new Response(stream, { headers: { 'Content-Length': String(10 * 1024 * 1024 + 1) } }) + ) + const client = await createJiraClient( + { accessToken: 'token', domain: 'example.atlassian.net', cloudId: 'cloud-1' }, + { validateCloudId: true } + ) + + await expect(client.request(client.issuePath(), { method: 'GET' })).rejects.toEqual( + new PayloadSizeLimitError({ + label: 'Jira response', + maxBytes: 10 * 1024 * 1024, + observedBytes: 10 * 1024 * 1024 + 1, + }) + ) + expect(cancelled).toBe(true) + }) + + it('cancels the caller wait while shared Atlassian discovery is in flight', async () => { + mocks.getCloudId.mockReturnValue(new Promise(() => {})) + const controller = new AbortController() + const pending = createJiraClient( + { accessToken: 'token', domain: 'example.atlassian.net' }, + { signal: controller.signal, validateCloudId: true } + ) + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect(pending).rejects.toMatchObject({ name: 'AbortError' }) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('rejects malformed attachment cloud IDs before constructing request URLs', async () => { + await expect( + createJiraClient( + { + accessToken: 'token', + domain: 'example.atlassian.net', + cloudId: '../rest/api/3', + }, + { validateCloudId: true } + ) + ).rejects.toMatchObject({ status: 400 }) + expect(fetchMock).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/jira/client.ts b/apps/sim/lib/internal/jira/client.ts new file mode 100644 index 00000000000..62243e95e08 --- /dev/null +++ b/apps/sim/lib/internal/jira/client.ts @@ -0,0 +1,102 @@ +import { validateJiraCloudId } from '@/lib/core/security/input-validation' +import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' +import { JiraOperationError } from '@/lib/internal/jira/errors' +import { getJiraCloudId } from '@/tools/jira/utils' + +interface JiraConnectionConfig { + domain: string + accessToken: string + cloudId?: string | null +} + +export interface JiraProviderResponse { + ok: boolean + status: number + statusText: string + text: string +} + +function waitForDiscovery(promise: Promise, signal?: AbortSignal): Promise { + if (!signal) return promise + signal.throwIfAborted() + return new Promise((resolve, reject) => { + const cleanup = () => signal.removeEventListener('abort', onAbort) + const onAbort = () => { + cleanup() + reject(signal.reason) + } + signal.addEventListener('abort', onAbort, { once: true }) + promise.then( + (result) => { + cleanup() + resolve(result) + }, + (error: unknown) => { + cleanup() + reject(error) + } + ) + }) +} + +export class JiraClient { + constructor( + readonly cloudId: string, + private readonly accessToken: string + ) {} + + issuePath(path = ''): string { + return `https://api.atlassian.com/ex/jira/${this.cloudId}/rest/api/3/issue${path}` + } + + async request( + url: string, + init: RequestInit, + signal?: AbortSignal + ): Promise { + signal?.throwIfAborted() + const response = await fetch(url, { + ...init, + headers: { + Authorization: `Bearer ${this.accessToken}`, + ...init.headers, + }, + signal, + }) + const text = await readResponseTextWithLimit(response, { + maxBytes: MAX_JSON_API_RESPONSE_BYTES, + label: 'Jira response', + signal, + requestMethod: init.method, + }) + signal?.throwIfAborted() + return { + ok: response.ok, + status: response.status, + statusText: response.statusText, + text, + } + } +} + +export async function createJiraClient( + config: JiraConnectionConfig, + options: { signal?: AbortSignal; validateCloudId: boolean } +): Promise { + options.signal?.throwIfAborted() + const cloudId = + config.cloudId || + (await waitForDiscovery(getJiraCloudId(config.domain, config.accessToken), options.signal)) + options.signal?.throwIfAborted() + + if (options.validateCloudId) { + const validation = validateJiraCloudId(cloudId, 'cloudId') + if (!validation.isValid) { + throw new JiraOperationError(400, { error: validation.error }) + } + return new JiraClient(validation.sanitized ?? cloudId, config.accessToken) + } + + return new JiraClient(cloudId, config.accessToken) +} diff --git a/apps/sim/lib/internal/jira/errors.ts b/apps/sim/lib/internal/jira/errors.ts new file mode 100644 index 00000000000..a41713d6772 --- /dev/null +++ b/apps/sim/lib/internal/jira/errors.ts @@ -0,0 +1,13 @@ +export class JiraOperationError extends Error { + constructor( + readonly status: number, + readonly body: unknown + ) { + super( + typeof body === 'object' && body !== null && 'error' in body + ? String(body.error) + : 'Jira operation failed' + ) + this.name = 'JiraOperationError' + } +} diff --git a/apps/sim/lib/internal/jira/execute-tool.test.ts b/apps/sim/lib/internal/jira/execute-tool.test.ts new file mode 100644 index 00000000000..47343c7cf63 --- /dev/null +++ b/apps/sim/lib/internal/jira/execute-tool.test.ts @@ -0,0 +1,162 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + addAttachment: vi.fn(), + update: vi.fn(), + write: vi.fn(), +})) + +vi.mock('@/lib/internal/jira/operations', () => ({ + executeJiraAddAttachment: mocks.addAttachment, + executeJiraUpdate: mocks.update, + executeJiraWrite: mocks.write, +})) + +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { JiraOperationError } from '@/lib/internal/jira/errors' +import { executeJiraTool } from '@/lib/internal/jira/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const INPUTS = { + jira_write: { + accessToken: 'token', + domain: 'example.atlassian.net', + projectId: 'PROJ', + summary: 'New issue', + issueType: 'Task', + }, + jira_update: { + accessToken: 'token', + domain: 'example.atlassian.net', + issueKey: 'PROJ-1', + summary: 'Updated issue', + }, + jira_add_attachment: { + accessToken: 'token', + domain: 'example.atlassian.net', + issueKey: 'PROJ-1', + files: [{ key: 'workspace/file.txt', name: 'file.txt', size: 4 }], + }, +} as const + +const OPERATIONS = { + jira_write: mocks.write, + jira_update: mocks.update, + jira_add_attachment: mocks.addAttachment, +} as const + +function request( + toolId: keyof typeof INPUTS, + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId, + input: INPUTS[toolId], + headers: new Headers(), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + userId: 'user-1', + workspaceId: 'workspace-1', + executionId: 'execution-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeJiraTool', () => { + beforeEach(() => { + vi.clearAllMocks() + for (const operation of Object.values(OPERATIONS)) { + operation.mockResolvedValue({ success: true, output: { ok: true } }) + } + }) + + it.each(Object.keys(INPUTS) as Array)( + 'validates and dispatches %s with trusted execution context', + async (toolId) => { + const controller = new AbortController() + const response = await executeJiraTool(request(toolId, { signal: controller.signal })) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ success: true, output: { ok: true } }) + expect(OPERATIONS[toolId]).toHaveBeenCalledWith( + INPUTS[toolId], + expect.objectContaining({ + userId: 'user-1', + requestId: 'request-1', + signal: controller.signal, + }) + ) + } + ) + + it('preserves the distinct authentication envelopes', async () => { + const context = createExecutionContext({ workflowId: 'workflow-1' }) + const write = await executeJiraTool(request('jira_write', { context })) + const attachment = await executeJiraTool(request('jira_add_attachment', { context })) + + expect(write.status).toBe(401) + await expect(write.json()).resolves.toEqual({ error: 'Unauthorized' }) + expect(attachment.status).toBe(401) + await expect(attachment.json()).resolves.toEqual({ success: false, error: 'Unauthorized' }) + expect(mocks.write).not.toHaveBeenCalled() + expect(mocks.addAttachment).not.toHaveBeenCalled() + }) + + it('rejects invalid and oversized input before provider work', async () => { + const invalid = await executeJiraTool(request('jira_update', { input: { accessToken: '' } })) + expect(invalid.status).toBe(400) + await expect(invalid.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + + const oversized = await executeJiraTool( + request('jira_update', { + input: { + ...INPUTS.jira_update, + summary: 'x'.repeat(DEFAULT_MAX_JSON_BODY_BYTES + 1), + }, + }) + ) + expect(oversized.status).toBe(413) + expect(mocks.update).not.toHaveBeenCalled() + }) + + it('preserves provider status and route-compatible response bodies', async () => { + mocks.write.mockRejectedValue( + new JiraOperationError(429, { error: 'Rate limited', details: '{"errorMessages":[]}' }) + ) + + const response = await executeJiraTool(request('jira_write')) + + expect(response.status).toBe(429) + await expect(response.json()).resolves.toEqual({ + error: 'Rate limited', + details: '{"errorMessages":[]}', + }) + }) + + it('propagates cancellation before and after operation execution', async () => { + const before = new AbortController() + before.abort(new DOMException('cancelled', 'AbortError')) + await expect( + executeJiraTool(request('jira_write', { signal: before.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mocks.write).not.toHaveBeenCalled() + + const after = new AbortController() + mocks.write.mockImplementationOnce(async () => { + after.abort(new DOMException('cancelled', 'AbortError')) + return { success: true } + }) + await expect( + executeJiraTool(request('jira_write', { signal: after.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) +}) diff --git a/apps/sim/lib/internal/jira/execute-tool.ts b/apps/sim/lib/internal/jira/execute-tool.ts new file mode 100644 index 00000000000..1f2c0ec70f1 --- /dev/null +++ b/apps/sim/lib/internal/jira/execute-tool.ts @@ -0,0 +1,114 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { z } from 'zod' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { JiraOperationError } from '@/lib/internal/jira/errors' +import { + jiraAddAttachmentInputSchema, + jiraUpdateInputSchema, + jiraWriteInputSchema, +} from '@/lib/internal/jira/input' +import { + executeJiraAddAttachment, + executeJiraUpdate, + executeJiraWrite, + type JiraOperationContext, +} from '@/lib/internal/jira/operations' +import type { + InternalToolOperationCall, + InternalToolOperationHandler, +} from '@/lib/internal/tool-operations/types' + +const logger = createLogger('JiraToolExecution') + +function unauthorizedResponse(toolId: string): Response { + return toolId === 'jira_add_attachment' + ? Response.json({ success: false, error: 'Unauthorized' }, { status: 401 }) + : Response.json({ error: 'Unauthorized' }, { status: 401 }) +} + +function validateInput(schema: S, input: unknown): z.output | Response { + const parsed = schema.safeParse(input) + return parsed.success + ? parsed.data + : Response.json( + { error: 'Invalid request data', details: parsed.error.issues }, + { status: 400 } + ) +} + +async function dispatch( + request: InternalToolOperationCall, + context: JiraOperationContext +): Promise { + switch (request.toolId) { + case 'jira_write': { + const input = validateInput(jiraWriteInputSchema, request.input) + return input instanceof Response ? input : executeJiraWrite(input, context) + } + case 'jira_update': { + const input = validateInput(jiraUpdateInputSchema, request.input) + return input instanceof Response ? input : executeJiraUpdate(input, context) + } + case 'jira_add_attachment': { + const input = validateInput(jiraAddAttachmentInputSchema, request.input) + return input instanceof Response ? input : executeJiraAddAttachment(input, context) + } + default: + return Response.json( + { success: false, error: `Unsupported Jira tool: ${request.toolId}` }, + { status: 500 } + ) + } +} + +function unexpectedErrorResponse(request: InternalToolOperationCall, error: unknown): Response { + const message = getErrorMessage(error, 'Internal server error') + logger.error('Jira operation failed', { + error: message, + requestId: request.requestId, + toolId: request.toolId, + }) + const body = + request.toolId === 'jira_add_attachment' + ? { success: false, error: message } + : { error: message, success: false } + return Response.json(body, { status: isPayloadSizeLimitError(error) ? 413 : 500 }) +} + +export const executeJiraTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!request.context.userId) return unauthorizedResponse(request.toolId) + + let serializedInput: string + try { + serializedInput = JSON.stringify(request.input) ?? '' + } catch { + return Response.json({ error: 'Invalid request data' }, { status: 400 }) + } + if (Buffer.byteLength(serializedInput, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + + try { + const result = await dispatch(request, { + userId: request.context.userId, + requestId: request.requestId, + signal: request.signal, + }) + request.signal?.throwIfAborted() + return result instanceof Response ? result : Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof JiraOperationError) { + return Response.json(error.body, { status: error.status }) + } + return unexpectedErrorResponse(request, error) + } +} diff --git a/apps/sim/lib/internal/jira/input.ts b/apps/sim/lib/internal/jira/input.ts new file mode 100644 index 00000000000..7b37c927569 --- /dev/null +++ b/apps/sim/lib/internal/jira/input.ts @@ -0,0 +1,60 @@ +import { z } from 'zod' +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' + +export const jiraParentReferenceSchema = z.union([ + z.string().min(1), + z.object({ key: z.string().min(1) }).passthrough(), + z.object({ id: z.string().min(1) }).passthrough(), +]) + +export const jiraWriteInputSchema = z.object({ + domain: z.string({ error: 'Domain is required' }).min(1, 'Domain is required'), + accessToken: z.string({ error: 'Access token is required' }).min(1, 'Access token is required'), + projectId: z.string({ error: 'Project ID is required' }).min(1, 'Project ID is required'), + summary: z.string({ error: 'Summary is required' }).min(1, 'Summary is required'), + description: z.union([z.string(), z.record(z.string(), z.unknown())]).optional(), + priority: z.string().optional(), + assignee: z.string().optional(), + cloudId: z.string().optional(), + issueType: z.string().optional(), + parent: jiraParentReferenceSchema.optional(), + labels: z.array(z.string()).optional(), + duedate: z.string().optional(), + reporter: z.string().optional(), + environment: z.union([z.string(), z.record(z.string(), z.unknown())]).optional(), + customFieldId: z.string().optional(), + customFieldValue: z.string().optional(), + components: z.array(z.string()).optional(), + fixVersions: z.array(z.string()).optional(), +}) + +export const jiraUpdateInputSchema = z.object({ + domain: z.string().min(1, 'Domain is required'), + accessToken: z.string().min(1, 'Access token is required'), + issueKey: z.string().min(1, 'Issue key is required'), + summary: z.string().optional(), + description: z.union([z.string(), z.record(z.string(), z.unknown())]).optional(), + priority: z.string().optional(), + assignee: z.string().optional(), + labels: z.array(z.string()).optional(), + components: z.array(z.string()).optional(), + duedate: z.string().optional(), + fixVersions: z.array(z.string()).optional(), + environment: z.union([z.string(), z.record(z.string(), z.unknown())]).optional(), + customFieldId: z.string().optional(), + customFieldValue: z.string().optional(), + notifyUsers: z.boolean().optional(), + cloudId: z.string().optional(), +}) + +export const jiraAddAttachmentInputSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + domain: z.string().min(1, 'Domain is required'), + issueKey: z.string().min(1, 'Issue key is required'), + files: RawFileInputArraySchema, + cloudId: z.string().optional().nullable(), +}) + +export type JiraWriteInput = z.output +export type JiraUpdateInput = z.output +export type JiraAddAttachmentInput = z.output diff --git a/apps/sim/lib/internal/jira/operations.test.ts b/apps/sim/lib/internal/jira/operations.test.ts new file mode 100644 index 00000000000..d7583e4fb50 --- /dev/null +++ b/apps/sim/lib/internal/jira/operations.test.ts @@ -0,0 +1,226 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + assertAccess: vi.fn(), + createClient: vi.fn(), + downloadFile: vi.fn(), + processFiles: vi.fn(), + request: vi.fn(), +})) + +vi.mock('@/lib/internal/jira/client', () => ({ + createJiraClient: mocks.createClient, +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertAccess, +})) + +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + processFilesToUserFiles: mocks.processFiles, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mocks.downloadFile, +})) + +import type { JiraOperationError } from '@/lib/internal/jira/errors' +import { + executeJiraAddAttachment, + executeJiraUpdate, + executeJiraWrite, +} from '@/lib/internal/jira/operations' + +const client = { + cloudId: 'cloud-1', + issuePath: (path = '') => `https://api.atlassian.com/ex/jira/cloud-1/rest/api/3/issue${path}`, + request: mocks.request, +} + +const context = { + userId: 'user-1', + requestId: 'request-1', + signal: new AbortController().signal, +} + +describe('Jira operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.createClient.mockResolvedValue(client) + mocks.assertAccess.mockResolvedValue(null) + mocks.processFiles.mockReturnValue([ + { key: 'workspace/file.txt', name: 'file.txt', size: 4, type: 'text/plain' }, + ]) + mocks.downloadFile.mockResolvedValue({ + buffer: Buffer.from('test'), + contentType: 'text/plain', + }) + }) + + it('creates Jira fields, then preserves non-fatal assignment behavior', async () => { + mocks.request + .mockResolvedValueOnce({ + ok: true, + status: 201, + statusText: 'Created', + text: '{"id":"100","key":"PROJ-1","self":"jira-self","fields":{"summary":"Created"}}', + }) + .mockResolvedValueOnce({ + ok: false, + status: 403, + statusText: 'Forbidden', + text: 'denied', + }) + + const result = await executeJiraWrite( + { + accessToken: 'token', + domain: 'example.atlassian.net', + projectId: 'PROJ', + summary: 'Created', + issueType: 'Bug', + assignee: 'account-1', + priority: 'High', + labels: ['customer'], + }, + context + ) + + const createInit = mocks.request.mock.calls[0]?.[1] as RequestInit + expect(JSON.parse(String(createInit.body))).toEqual({ + fields: { + project: { key: 'PROJ' }, + issuetype: { name: 'Bug' }, + summary: 'Created', + priority: { name: 'High' }, + labels: ['customer'], + }, + }) + expect(mocks.request.mock.calls[1]?.[0]).toContain('/PROJ-1/assignee') + expect(result.output).toMatchObject({ + id: '100', + issueKey: 'PROJ-1', + summary: 'Created', + url: 'https://example.atlassian.net/browse/PROJ-1', + }) + expect(result.output).not.toHaveProperty('assigneeId') + }) + + it('preserves update query and provider error details', async () => { + mocks.request.mockResolvedValue({ + ok: false, + status: 429, + statusText: 'Too Many Requests', + text: '{"errorMessages":["Rate limited"]}', + }) + + await expect( + executeJiraUpdate( + { + accessToken: 'token', + domain: 'example.atlassian.net', + issueKey: 'PROJ-1', + summary: 'Updated', + notifyUsers: false, + }, + context + ) + ).rejects.toMatchObject>({ + status: 429, + body: { + error: expect.any(String), + details: '{"errorMessages":["Rate limited"]}', + }, + }) + expect(mocks.request.mock.calls[0]?.[0]).toContain('/PROJ-1?notifyUsers=false') + }) + + it('authorizes, bounds, and uploads attachment files sequentially', async () => { + mocks.request.mockResolvedValue({ + ok: true, + status: 200, + statusText: 'OK', + text: '[{"id":"10","filename":"file.txt","mimeType":"text/plain","size":4,"content":"url"}]', + }) + + const result = await executeJiraAddAttachment( + { + accessToken: 'token', + domain: 'example.atlassian.net', + issueKey: 'PROJ-1', + files: [{ key: 'workspace/file.txt', name: 'file.txt', size: 4 }], + }, + context + ) + + expect(mocks.assertAccess).toHaveBeenCalledWith( + 'workspace/file.txt', + 'user-1', + 'request-1', + expect.anything() + ) + expect(mocks.downloadFile).toHaveBeenCalledWith( + expect.objectContaining({ key: 'workspace/file.txt' }), + 'request-1', + expect.anything(), + expect.objectContaining({ signal: context.signal }) + ) + expect(mocks.request).toHaveBeenCalledWith( + expect.stringContaining('/PROJ-1/attachments'), + expect.objectContaining({ method: 'POST', body: expect.any(FormData) }), + context.signal + ) + expect(mocks.createClient).toHaveBeenCalledWith( + expect.objectContaining({ accessToken: 'token', domain: 'example.atlassian.net' }), + { + signal: context.signal, + validateCloudId: true, + } + ) + expect(result.output.attachmentIds).toEqual(['10']) + }) + + it('rejects unsafe attachment issue keys before file or provider work', async () => { + await expect( + executeJiraAddAttachment( + { + accessToken: 'token', + domain: 'example.atlassian.net', + issueKey: '../project', + files: [{ key: 'workspace/file.txt', name: 'file.txt', size: 4 }], + }, + context + ) + ).rejects.toMatchObject({ status: 400 }) + + expect(mocks.processFiles).not.toHaveBeenCalled() + expect(mocks.createClient).not.toHaveBeenCalled() + expect(mocks.request).not.toHaveBeenCalled() + }) + + it('fails closed when attachment access is denied', async () => { + mocks.assertAccess.mockResolvedValue( + Response.json({ success: false, error: 'File not found' }, { status: 404 }) + ) + + await expect( + executeJiraAddAttachment( + { + accessToken: 'token', + domain: 'example.atlassian.net', + issueKey: 'PROJ-1', + files: [{ key: 'workspace/file.txt', name: 'file.txt', size: 4 }], + }, + context + ) + ).rejects.toMatchObject({ + status: 404, + body: { success: false, error: 'File not found' }, + }) + expect(mocks.downloadFile).not.toHaveBeenCalled() + expect(mocks.request).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/jira/operations.ts b/apps/sim/lib/internal/jira/operations.ts new file mode 100644 index 00000000000..f776d998f15 --- /dev/null +++ b/apps/sim/lib/internal/jira/operations.ts @@ -0,0 +1,338 @@ +import type { Logger } from '@sim/logger' +import { createLogger } from '@sim/logger' +import { validateAlphanumericId, validateJiraIssueKey } from '@/lib/core/security/input-validation' +import { createJiraClient, type JiraClient } from '@/lib/internal/jira/client' +import { JiraOperationError } from '@/lib/internal/jira/errors' +import type { + JiraAddAttachmentInput, + JiraUpdateInput, + JiraWriteInput, +} from '@/lib/internal/jira/input' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' +import { assertToolFileAccess } from '@/app/api/files/authorization' +import { parseAtlassianErrorMessage, toAdf } from '@/tools/jira/utils' + +const logger = createLogger('JiraInternalOperation') + +export interface JiraOperationContext { + userId: string + requestId: string + signal?: AbortSignal +} + +type JsonObject = Record + +function asObject(value: unknown): JsonObject { + return value !== null && typeof value === 'object' && !Array.isArray(value) + ? (value as JsonObject) + : {} +} + +function parseObject(text: string): JsonObject { + return asObject(JSON.parse(text)) +} + +function optionalObject(text: string): JsonObject { + if (!text) return {} + try { + return parseObject(text) + } catch { + return {} + } +} + +function nestedString(value: unknown, key: string): string | undefined { + const nested = asObject(value)[key] + return typeof nested === 'string' ? nested : undefined +} + +function providerError( + response: { status: number; statusText: string; text: string }, + includeDetails: boolean +): JiraOperationError { + const message = parseAtlassianErrorMessage(response.status, response.statusText, response.text) + return new JiraOperationError( + response.status, + includeDetails ? { error: message, details: response.text } : { success: false, error: message } + ) +} + +function putOptionalIssueFields( + fields: JsonObject, + input: Pick< + JiraUpdateInput, + | 'description' + | 'priority' + | 'assignee' + | 'labels' + | 'components' + | 'duedate' + | 'fixVersions' + | 'environment' + | 'customFieldId' + | 'customFieldValue' + >, + includeAssignee: boolean +): void { + if (input.description !== undefined && input.description !== '') { + fields.description = toAdf(input.description) + } + if (input.priority) { + fields.priority = /^\d+$/.test(input.priority) + ? { id: input.priority } + : { name: input.priority } + } + if (includeAssignee && input.assignee) fields.assignee = { accountId: input.assignee } + if (input.labels?.length) fields.labels = input.labels + if (input.components?.length) fields.components = input.components.map((name) => ({ name })) + if (input.duedate) fields.duedate = input.duedate + if (input.fixVersions?.length) { + fields.fixVersions = input.fixVersions.map((name) => ({ name })) + } + if (input.environment !== undefined && input.environment !== '') { + fields.environment = toAdf(input.environment) + } + if (input.customFieldId && input.customFieldValue) { + const fieldId = input.customFieldId.startsWith('customfield_') + ? input.customFieldId + : `customfield_${input.customFieldId}` + fields[fieldId] = input.customFieldValue + } +} + +async function assignCreatedIssue( + client: JiraClient, + issueKey: string, + assignee: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const response = await client.request( + client.issuePath(`/${issueKey}/assignee`), + { + method: 'PUT', + headers: { Accept: 'application/json', 'Content-Type': 'application/json' }, + body: JSON.stringify({ accountId: assignee }), + }, + signal + ) + if (!response.ok) { + logger.warn('Failed to assign issue after successful creation', { + status: response.status, + error: response.text, + }) + return false + } + return true +} + +export async function executeJiraWrite(input: JiraWriteInput, context: JiraOperationContext) { + context.signal?.throwIfAborted() + const client = await createJiraClient(input, { + signal: context.signal, + validateCloudId: true, + }) + const projectIdValidation = validateAlphanumericId(input.projectId, 'projectId', 100) + if (!projectIdValidation.isValid) { + throw new JiraOperationError(400, { error: projectIdValidation.error }) + } + + const fields: JsonObject = { + project: /^\d+$/.test(input.projectId) ? { id: input.projectId } : { key: input.projectId }, + issuetype: { name: input.issueType || 'Task' }, + summary: input.summary, + } + putOptionalIssueFields(fields, input, false) + if (input.parent) { + fields.parent = + typeof input.parent === 'string' + ? /^\d+$/.test(input.parent) + ? { id: input.parent } + : { key: input.parent } + : input.parent + } + if (input.reporter) fields.reporter = { accountId: input.reporter } + + const response = await client.request( + client.issuePath(), + { + method: 'POST', + headers: { Accept: 'application/json', 'Content-Type': 'application/json' }, + body: JSON.stringify({ fields }), + }, + context.signal + ) + if (!response.ok) throw providerError(response, true) + + const data = parseObject(response.text) + const issueKey = typeof data.key === 'string' ? data.key : 'unknown' + const assigneeId = + input.assignee && (await assignCreatedIssue(client, issueKey, input.assignee, context.signal)) + ? input.assignee + : undefined + context.signal?.throwIfAborted() + + return { + success: true, + output: { + ts: new Date().toISOString(), + id: typeof data.id === 'string' ? data.id : '', + issueKey, + self: typeof data.self === 'string' ? data.self : '', + summary: nestedString(data.fields, 'summary') || input.summary || 'Issue created', + success: true, + url: `https://${input.domain}/browse/${issueKey}`, + ...(assigneeId ? { assigneeId } : {}), + }, + } +} + +export async function executeJiraUpdate(input: JiraUpdateInput, context: JiraOperationContext) { + context.signal?.throwIfAborted() + const client = await createJiraClient(input, { + signal: context.signal, + validateCloudId: true, + }) + const issueKeyValidation = validateJiraIssueKey(input.issueKey, 'issueKey') + if (!issueKeyValidation.isValid) { + throw new JiraOperationError(400, { error: issueKeyValidation.error }) + } + + const fields: JsonObject = {} + if (input.summary) fields.summary = input.summary + putOptionalIssueFields(fields, input, true) + const notifyParam = + input.notifyUsers === false + ? '?notifyUsers=false' + : input.notifyUsers === true + ? '?notifyUsers=true' + : '' + const response = await client.request( + `${client.issuePath(`/${input.issueKey}`)}${notifyParam}`, + { + method: 'PUT', + headers: { Accept: 'application/json', 'Content-Type': 'application/json' }, + body: JSON.stringify({ fields }), + }, + context.signal + ) + if (!response.ok) throw providerError(response, true) + + const data = optionalObject(response.text) + return { + success: true, + output: { + ts: new Date().toISOString(), + issueKey: typeof data.key === 'string' ? data.key : input.issueKey, + summary: nestedString(data.fields, 'summary') || input.summary || 'Issue updated', + success: true, + }, + } +} + +async function throwResponse(response: Response): Promise { + let body: unknown + try { + body = await response.json() + } catch { + body = { success: false, error: response.statusText || 'Jira operation failed' } + } + throw new JiraOperationError(response.status, body) +} + +function attachmentObject(value: unknown) { + const object = asObject(value) + return { + id: typeof object.id === 'string' ? object.id : '', + filename: typeof object.filename === 'string' ? object.filename : '', + mimeType: typeof object.mimeType === 'string' ? object.mimeType : '', + size: typeof object.size === 'number' ? object.size : 0, + content: typeof object.content === 'string' ? object.content : '', + } +} + +export async function executeJiraAddAttachment( + input: JiraAddAttachmentInput, + context: JiraOperationContext, + routeLogger: Logger = logger +) { + context.signal?.throwIfAborted() + const issueKeyValidation = validateJiraIssueKey(input.issueKey, 'issueKey') + if (!issueKeyValidation.isValid) { + throw new JiraOperationError(400, { error: issueKeyValidation.error }) + } + const userFiles = processFilesToUserFiles(input.files, context.requestId, routeLogger) + if (userFiles.length === 0) { + throw new JiraOperationError(400, { + success: false, + error: 'No valid files provided for upload', + }) + } + + const client = await createJiraClient(input, { + signal: context.signal, + validateCloudId: true, + }) + const formData = new FormData() + let remainingBytes = MAX_BUFFERED_TRANSFER_BYTES + + for (const file of userFiles) { + context.signal?.throwIfAborted() + const denied = await assertToolFileAccess( + file.key, + context.userId, + context.requestId, + routeLogger + ) + context.signal?.throwIfAborted() + if (denied) await throwResponse(denied) + + let downloaded: Awaited> + try { + downloaded = await downloadServableFileFromStorage(file, context.requestId, routeLogger, { + maxBytes: remainingBytes, + signal: context.signal, + }) + } catch (error) { + const notReady = docNotReadyResponse(error) + if (notReady) await throwResponse(notReady) + throw error + } + remainingBytes -= downloaded.buffer.length + formData.append( + 'file', + new Blob([new Uint8Array(downloaded.buffer)], { + type: downloaded.contentType || file.type || 'application/octet-stream', + }), + file.name + ) + } + + const response = await client.request( + client.issuePath(`/${input.issueKey}/attachments`), + { + method: 'POST', + headers: { 'X-Atlassian-Token': 'no-check' }, + body: formData, + }, + context.signal + ) + if (!response.ok) throw providerError(response, false) + + const parsed = JSON.parse(response.text) as unknown + const values = Array.isArray(parsed) ? parsed : [] + const attachments = values.map(attachmentObject) + return { + success: true, + output: { + ts: new Date().toISOString(), + issueKey: input.issueKey, + attachments, + attachmentIds: attachments.map((attachment) => attachment.id).filter(Boolean), + files: userFiles, + }, + } +} diff --git a/apps/sim/lib/internal/jsm/assets.ts b/apps/sim/lib/internal/jsm/assets.ts new file mode 100644 index 00000000000..283cced6f08 --- /dev/null +++ b/apps/sim/lib/internal/jsm/assets.ts @@ -0,0 +1,208 @@ +import type { ContractBody } from '@/lib/api/contracts' +import type { + jsmCreateObjectContract, + jsmDeleteObjectContract, + jsmGetObjectContract, + jsmGetObjectSchemaContract, + jsmListObjectSchemasContract, + jsmListObjectTypesContract, + jsmObjectTypeAttributesContract, + jsmSearchObjectsAqlContract, + jsmUpdateObjectContract, +} from '@/lib/api/contracts/selectors/jsm' +import { asArray, createJsmAssetsClient } from '@/lib/internal/jsm/client' +import { mapAssetObject } from '@/tools/jsm/utils' + +type ListObjectSchemasInput = ContractBody +type GetObjectSchemaInput = ContractBody +type ListObjectTypesInput = ContractBody +type ObjectTypeAttributesInput = ContractBody +type SearchObjectsAqlInput = ContractBody +type GetObjectInput = ContractBody +type CreateObjectInput = ContractBody +type UpdateObjectInput = ContractBody +type DeleteObjectInput = ContractBody + +function jsonBody(method: 'POST' | 'PUT', body: unknown): RequestInit { + return { method, body: JSON.stringify(body) } +} + +function toNumber(value: string | number | undefined, fallback: number): number { + if (value === undefined) return fallback + const parsed = typeof value === 'number' ? value : Number(value) + return Number.isFinite(parsed) ? parsed : fallback +} + +export async function executeJsmListObjectSchemas( + input: ListObjectSchemasInput, + signal?: AbortSignal +) { + const client = await createJsmAssetsClient(input, signal) + const query = new URLSearchParams() + if (input.startAt !== undefined) query.append('startAt', String(input.startAt)) + if (input.maxResults !== undefined) query.append('maxResults', String(input.maxResults)) + if (input.includeCounts !== undefined) { + query.append('includeCounts', String(input.includeCounts)) + } + const data = await client.json( + client.assets(`/objectschema/list${query.size ? `?${query}` : ''}`), + {}, + signal, + true + ) + const values = asArray(data.values) + return { + success: true, + output: { + ts: new Date().toISOString(), + schemas: values, + total: data.total ?? values.length, + isLast: data.isLast ?? data.last ?? true, + }, + } +} + +export async function executeJsmGetObjectSchema(input: GetObjectSchemaInput, signal?: AbortSignal) { + const client = await createJsmAssetsClient(input, signal) + const schema = await client.value( + client.assets(`/objectschema/${encodeURIComponent(input.schemaId)}`), + {}, + signal, + true + ) + return { success: true, output: { ts: new Date().toISOString(), schema: schema ?? null } } +} + +export async function executeJsmListObjectTypes(input: ListObjectTypesInput, signal?: AbortSignal) { + const client = await createJsmAssetsClient(input, signal) + const query = new URLSearchParams() + if (input.excludeAbstract !== undefined) { + query.append('excludeAbstract', String(input.excludeAbstract)) + } + const value = await client.value( + client.assets( + `/objectschema/${encodeURIComponent(input.schemaId)}/objecttypes${query.size ? `?${query}` : ''}` + ), + {}, + signal, + true + ) + const data = Array.isArray(value) ? {} : (value as Record) + const objectTypes = Array.isArray(value) ? value : asArray(data.values) + return { + success: true, + output: { ts: new Date().toISOString(), objectTypes, total: objectTypes.length }, + } +} + +export async function executeJsmGetObjectTypeAttributes( + input: ObjectTypeAttributesInput, + signal?: AbortSignal +) { + const client = await createJsmAssetsClient(input, signal) + const query = new URLSearchParams() + if (input.onlyValueEditable !== undefined) { + query.append('onlyValueEditable', String(input.onlyValueEditable)) + } + if (input.query) query.append('query', input.query) + const value = await client.value( + client.assets( + `/objecttype/${encodeURIComponent(input.objectTypeId)}/attributes${query.size ? `?${query}` : ''}` + ), + {}, + signal, + true + ) + const data = Array.isArray(value) ? {} : (value as Record) + const attributes = Array.isArray(value) ? value : asArray(data.values) + return { + success: true, + output: { ts: new Date().toISOString(), attributes, total: attributes.length }, + } +} + +export async function executeJsmSearchObjectsAql( + input: SearchObjectsAqlInput, + signal?: AbortSignal +) { + const client = await createJsmAssetsClient(input, signal) + const includeAttributes = + input.includeAttributes === undefined ? true : String(input.includeAttributes) === 'true' + const body: Record = { + qlQuery: input.qlQuery, + page: toNumber(input.page, 1), + resultsPerPage: toNumber(input.resultsPerPage, 25), + includeAttributes, + } + if (input.objectTypeId) body.objectTypeId = input.objectTypeId + if (input.objectSchemaId) body.objectSchemaId = input.objectSchemaId + const data = await client.json(client.assets('/object/aql'), jsonBody('POST', body), signal, true) + const entries = asArray(data.objectEntries) + return { + success: true, + output: { + ts: new Date().toISOString(), + objects: entries.map((entry) => + mapAssetObject(entry as Parameters[0]) + ), + total: data.totalFilterCount ?? entries.length, + pageNumber: data.pageNumber ?? 1, + pageSize: data.pageSize ?? entries.length, + }, + } +} + +async function assetObject( + input: GetObjectInput | CreateObjectInput | UpdateObjectInput, + path: string, + init: RequestInit, + signal?: AbortSignal +) { + const client = await createJsmAssetsClient(input, signal) + const data = await client.json(client.assets(path), init, signal, true) + return { + success: true, + output: { + ts: new Date().toISOString(), + object: mapAssetObject(data as typeof data & Parameters[0]), + }, + } +} + +export function executeJsmGetObject(input: GetObjectInput, signal?: AbortSignal) { + return assetObject(input, `/object/${encodeURIComponent(input.objectId)}`, {}, signal) +} + +export function executeJsmCreateObject(input: CreateObjectInput, signal?: AbortSignal) { + return assetObject( + input, + '/object/create', + jsonBody('POST', { objectTypeId: input.objectTypeId, attributes: input.attributes }), + signal + ) +} + +export function executeJsmUpdateObject(input: UpdateObjectInput, signal?: AbortSignal) { + const body: Record = { attributes: input.attributes } + if (input.objectTypeId) body.objectTypeId = input.objectTypeId + return assetObject( + input, + `/object/${encodeURIComponent(input.objectId)}`, + jsonBody('PUT', body), + signal + ) +} + +export async function executeJsmDeleteObject(input: DeleteObjectInput, signal?: AbortSignal) { + const client = await createJsmAssetsClient(input, signal) + await client.empty( + client.assets(`/object/${encodeURIComponent(input.objectId)}`), + { method: 'DELETE' }, + signal, + true + ) + return { + success: true, + output: { ts: new Date().toISOString(), objectId: input.objectId, deleted: true }, + } +} diff --git a/apps/sim/lib/internal/jsm/client.test.ts b/apps/sim/lib/internal/jsm/client.test.ts new file mode 100644 index 00000000000..62b40ef807d --- /dev/null +++ b/apps/sim/lib/internal/jsm/client.test.ts @@ -0,0 +1,67 @@ +/** + * @vitest-environment node + */ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { JsmClient } from '@/lib/internal/jsm/client' +import type { JsmOperationError } from '@/lib/internal/jsm/errors' + +describe('JsmClient', () => { + afterEach(() => vi.unstubAllGlobals()) + + it('forwards authorization, experimental opt-in, and cancellation to fetch', async () => { + const controller = new AbortController() + const fetchMock = vi.fn().mockResolvedValue(new Response(JSON.stringify({ ok: true }))) + vi.stubGlobal('fetch', fetchMock) + const client = new JsmClient('cloud-id', 'secret-token') + + await client.json('https://api.atlassian.com/resource', { method: 'GET' }, controller.signal) + + const init = fetchMock.mock.calls[0]?.[1] as RequestInit + expect(init.signal).toBe(controller.signal) + expect(init.headers).toMatchObject({ + Authorization: 'Bearer secret-token', + Accept: 'application/json', + 'Content-Type': 'application/json', + 'X-ExperimentalApi': 'opt-in', + }) + }) + + it('preserves Atlassian status and provider details when requested', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + new Response('{"errorMessages":["Too many requests"]}', { + status: 429, + statusText: 'Too Many Requests', + }) + ) + ) + const client = new JsmClient('cloud-id', 'secret-token') + + await expect( + client.json('https://api.atlassian.com/resource', {}, undefined, true) + ).rejects.toMatchObject({ + status: 429, + body: { + error: expect.any(String), + details: '{"errorMessages":["Too many requests"]}', + }, + }) + }) + + it('does not start a request with an already-aborted signal', async () => { + const fetchMock = vi.fn() + vi.stubGlobal('fetch', fetchMock) + const controller = new AbortController() + controller.abort() + + await expect( + new JsmClient('cloud-id', 'secret-token').json( + 'https://api.atlassian.com/resource', + {}, + controller.signal + ) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(fetchMock).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/jsm/client.ts b/apps/sim/lib/internal/jsm/client.ts new file mode 100644 index 00000000000..6a2ada6c6aa --- /dev/null +++ b/apps/sim/lib/internal/jsm/client.ts @@ -0,0 +1,187 @@ +import { + validateAssetsWorkspaceId, + validateJiraCloudId, +} from '@/lib/core/security/input-validation' +import { JsmOperationError } from '@/lib/internal/jsm/errors' +import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' +import { resolveAssetsContext } from '@/tools/jsm/utils' + +export interface JsmConnectionConfig { + domain: string + accessToken: string + cloudId?: string +} + +export interface JsmAssetsConnectionConfig extends JsmConnectionConfig { + workspaceId?: string +} + +export type JsonObject = Record + +export function asObject(value: unknown): JsonObject { + return value && typeof value === 'object' && !Array.isArray(value) ? (value as JsonObject) : {} +} + +export function asArray(value: unknown): unknown[] { + return Array.isArray(value) ? value : [] +} + +export function nested(object: JsonObject, ...keys: string[]): unknown { + let value: unknown = object + for (const key of keys) value = asObject(value)[key] + return value +} + +function waitForDiscovery(promise: Promise, signal?: AbortSignal): Promise { + if (!signal) return promise + signal.throwIfAborted() + return new Promise((resolve, reject) => { + const cleanup = () => signal.removeEventListener('abort', onAbort) + const onAbort = () => { + cleanup() + reject(signal.reason) + } + signal.addEventListener('abort', onAbort, { once: true }) + promise.then( + (result) => { + cleanup() + resolve(result) + }, + (error: unknown) => { + cleanup() + reject(error) + } + ) + }) +} + +export class JsmClient { + constructor( + readonly cloudId: string, + private readonly accessToken: string, + readonly workspaceId?: string + ) {} + + service(path: string): string { + return `https://api.atlassian.com/ex/jira/${this.cloudId}/rest/servicedeskapi${path}` + } + + forms(path: string): string { + return `https://api.atlassian.com/ex/jira/${this.cloudId}/forms${path}` + } + + assets(path: string): string { + if (!this.workspaceId) throw new Error('JSM Assets client requires a workspace ID') + return `https://api.atlassian.com/ex/jira/${this.cloudId}/jsm/assets/workspace/${this.workspaceId}/v1${path}` + } + + async fetch(path: string, init: RequestInit = {}, signal?: AbortSignal): Promise { + signal?.throwIfAborted() + return fetch(path, { + ...init, + headers: { + Authorization: `Bearer ${this.accessToken}`, + Accept: 'application/json', + 'Content-Type': 'application/json', + 'X-ExperimentalApi': 'opt-in', + ...init.headers, + }, + signal, + }) + } + + async json( + path: string, + init: RequestInit = {}, + signal?: AbortSignal, + includeProviderDetails = false + ): Promise { + return asObject(await this.value(path, init, signal, includeProviderDetails)) + } + + async value( + path: string, + init: RequestInit = {}, + signal?: AbortSignal, + includeProviderDetails = false + ): Promise { + const response = await this.fetch(path, init, signal) + if (!response.ok) await throwJsmResponseError(response, includeProviderDetails) + return response.json() + } + + async empty( + path: string, + init: RequestInit, + signal?: AbortSignal, + includeProviderDetails = false + ): Promise { + const response = await this.fetch(path, init, signal) + if (!response.ok) await throwJsmResponseError(response, includeProviderDetails) + await response.arrayBuffer() + } + + async optionalJson( + path: string, + init: RequestInit, + signal?: AbortSignal, + includeProviderDetails = false + ): Promise { + const response = await this.fetch(path, init, signal) + if (!response.ok) await throwJsmResponseError(response, includeProviderDetails) + const text = await response.text() + return text ? asObject(JSON.parse(text)) : {} + } +} + +export async function throwJsmResponseError( + response: Response, + includeProviderDetails = false +): Promise { + const errorText = await response.text() + const message = parseAtlassianErrorMessage(response.status, response.statusText, errorText) + throw new JsmOperationError( + message, + response.status, + includeProviderDetails ? { error: message, details: errorText } : undefined + ) +} + +export async function createJsmClient( + config: JsmConnectionConfig, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const cloudId = + config.cloudId || + (await waitForDiscovery(getJiraCloudId(config.domain, config.accessToken), signal)) + signal?.throwIfAborted() + const validation = validateJiraCloudId(cloudId, 'cloudId') + if (!validation.isValid) { + throw new JsmOperationError(validation.error || 'Invalid cloudId', 400) + } + return new JsmClient(validation.sanitized ?? cloudId, config.accessToken) +} + +export async function createJsmAssetsClient( + config: JsmAssetsConnectionConfig, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const context = await waitForDiscovery( + resolveAssetsContext(config.domain, config.accessToken, config.cloudId, config.workspaceId), + signal + ) + signal?.throwIfAborted() + const cloudId = validateJiraCloudId(context.cloudId, 'cloudId') + if (!cloudId.isValid) throw new JsmOperationError(cloudId.error || 'Invalid cloudId', 400) + const workspaceId = validateAssetsWorkspaceId(context.workspaceId, 'workspaceId') + if (!workspaceId.isValid) { + throw new JsmOperationError(workspaceId.error || 'Invalid workspaceId', 400) + } + return new JsmClient( + cloudId.sanitized ?? context.cloudId, + config.accessToken, + workspaceId.sanitized ?? context.workspaceId + ) +} diff --git a/apps/sim/lib/internal/jsm/errors.ts b/apps/sim/lib/internal/jsm/errors.ts new file mode 100644 index 00000000000..ba9c38622df --- /dev/null +++ b/apps/sim/lib/internal/jsm/errors.ts @@ -0,0 +1,10 @@ +export class JsmOperationError extends Error { + constructor( + message: string, + readonly status: number, + readonly body?: Record + ) { + super(message) + this.name = 'JsmOperationError' + } +} diff --git a/apps/sim/lib/internal/jsm/execute-tool.test.ts b/apps/sim/lib/internal/jsm/execute-tool.test.ts new file mode 100644 index 00000000000..33c8a23195b --- /dev/null +++ b/apps/sim/lib/internal/jsm/execute-tool.test.ts @@ -0,0 +1,364 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { JsmOperationError } from '@/lib/internal/jsm/errors' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' +import type { ExecutionContext } from '@/executor/types' + +const operations = vi.hoisted(() => ({ + executeJsmAddComment: vi.fn(), + executeJsmAddCustomer: vi.fn(), + executeJsmAddOrganization: vi.fn(), + executeJsmAddParticipants: vi.fn(), + executeJsmAnswerApproval: vi.fn(), + executeJsmAttachForm: vi.fn(), + executeJsmCopyForms: vi.fn(), + executeJsmCreateObject: vi.fn(), + executeJsmCreateOrganization: vi.fn(), + executeJsmCreateRequest: vi.fn(), + executeJsmDeleteForm: vi.fn(), + executeJsmDeleteObject: vi.fn(), + executeJsmExternaliseForm: vi.fn(), + executeJsmGetApprovals: vi.fn(), + executeJsmGetComments: vi.fn(), + executeJsmGetCustomers: vi.fn(), + executeJsmGetForm: vi.fn(), + executeJsmGetFormAnswers: vi.fn(), + executeJsmGetFormStructure: vi.fn(), + executeJsmGetFormTemplates: vi.fn(), + executeJsmGetIssueForms: vi.fn(), + executeJsmGetObject: vi.fn(), + executeJsmGetObjectSchema: vi.fn(), + executeJsmGetObjectTypeAttributes: vi.fn(), + executeJsmGetOrganizations: vi.fn(), + executeJsmGetParticipants: vi.fn(), + executeJsmGetQueues: vi.fn(), + executeJsmGetRequest: vi.fn(), + executeJsmGetRequests: vi.fn(), + executeJsmGetRequestTypeFields: vi.fn(), + executeJsmGetRequestTypes: vi.fn(), + executeJsmGetServiceDesks: vi.fn(), + executeJsmGetSla: vi.fn(), + executeJsmGetTransitions: vi.fn(), + executeJsmInternaliseForm: vi.fn(), + executeJsmListObjectSchemas: vi.fn(), + executeJsmListObjectTypes: vi.fn(), + executeJsmReopenForm: vi.fn(), + executeJsmSaveFormAnswers: vi.fn(), + executeJsmSearchObjectsAql: vi.fn(), + executeJsmSubmitForm: vi.fn(), + executeJsmTransitionRequest: vi.fn(), + executeJsmUpdateObject: vi.fn(), +})) + +vi.mock('@/lib/internal/jsm/assets', () => ({ + executeJsmCreateObject: operations.executeJsmCreateObject, + executeJsmDeleteObject: operations.executeJsmDeleteObject, + executeJsmGetObject: operations.executeJsmGetObject, + executeJsmGetObjectSchema: operations.executeJsmGetObjectSchema, + executeJsmGetObjectTypeAttributes: operations.executeJsmGetObjectTypeAttributes, + executeJsmListObjectSchemas: operations.executeJsmListObjectSchemas, + executeJsmListObjectTypes: operations.executeJsmListObjectTypes, + executeJsmSearchObjectsAql: operations.executeJsmSearchObjectsAql, + executeJsmUpdateObject: operations.executeJsmUpdateObject, +})) + +vi.mock('@/lib/internal/jsm/forms', () => ({ + executeJsmAttachForm: operations.executeJsmAttachForm, + executeJsmCopyForms: operations.executeJsmCopyForms, + executeJsmDeleteForm: operations.executeJsmDeleteForm, + executeJsmExternaliseForm: operations.executeJsmExternaliseForm, + executeJsmGetForm: operations.executeJsmGetForm, + executeJsmGetFormAnswers: operations.executeJsmGetFormAnswers, + executeJsmGetFormStructure: operations.executeJsmGetFormStructure, + executeJsmGetFormTemplates: operations.executeJsmGetFormTemplates, + executeJsmGetIssueForms: operations.executeJsmGetIssueForms, + executeJsmInternaliseForm: operations.executeJsmInternaliseForm, + executeJsmReopenForm: operations.executeJsmReopenForm, + executeJsmSaveFormAnswers: operations.executeJsmSaveFormAnswers, + executeJsmSubmitForm: operations.executeJsmSubmitForm, +})) + +vi.mock('@/lib/internal/jsm/service-desk', () => ({ + executeJsmAddComment: operations.executeJsmAddComment, + executeJsmAddCustomer: operations.executeJsmAddCustomer, + executeJsmAddOrganization: operations.executeJsmAddOrganization, + executeJsmAddParticipants: operations.executeJsmAddParticipants, + executeJsmAnswerApproval: operations.executeJsmAnswerApproval, + executeJsmCreateOrganization: operations.executeJsmCreateOrganization, + executeJsmCreateRequest: operations.executeJsmCreateRequest, + executeJsmGetApprovals: operations.executeJsmGetApprovals, + executeJsmGetComments: operations.executeJsmGetComments, + executeJsmGetCustomers: operations.executeJsmGetCustomers, + executeJsmGetOrganizations: operations.executeJsmGetOrganizations, + executeJsmGetParticipants: operations.executeJsmGetParticipants, + executeJsmGetQueues: operations.executeJsmGetQueues, + executeJsmGetRequest: operations.executeJsmGetRequest, + executeJsmGetRequests: operations.executeJsmGetRequests, + executeJsmGetRequestTypeFields: operations.executeJsmGetRequestTypeFields, + executeJsmGetRequestTypes: operations.executeJsmGetRequestTypes, + executeJsmGetServiceDesks: operations.executeJsmGetServiceDesks, + executeJsmGetSla: operations.executeJsmGetSla, + executeJsmGetTransitions: operations.executeJsmGetTransitions, + executeJsmTransitionRequest: operations.executeJsmTransitionRequest, +})) + +import { executeJsmTool } from '@/lib/internal/jsm/execute-tool' + +const BASE = { + domain: 'example.atlassian.net', + accessToken: 'token', + cloudId: '12345678-1234-1234-1234-123456789012', +} +const ISSUE = { ...BASE, issueIdOrKey: 'HELP-1' } +const FORM = { ...ISSUE, formId: '12345678-1234-1234-1234-123456789012' } +const ASSETS = { ...BASE, workspaceId: '12345678-1234-1234-1234-123456789012' } + +type OperationName = keyof typeof operations + +interface DispatchCase { + toolId: string + operation: OperationName + input: Record +} + +const CASES: DispatchCase[] = [ + { toolId: 'jsm_get_service_desks', operation: 'executeJsmGetServiceDesks', input: BASE }, + { + toolId: 'jsm_get_queues', + operation: 'executeJsmGetQueues', + input: { ...BASE, serviceDeskId: '1' }, + }, + { + toolId: 'jsm_get_request_types', + operation: 'executeJsmGetRequestTypes', + input: { ...BASE, serviceDeskId: '1' }, + }, + { + toolId: 'jsm_get_request_type_fields', + operation: 'executeJsmGetRequestTypeFields', + input: { ...BASE, serviceDeskId: '1', requestTypeId: '2' }, + }, + { toolId: 'jsm_get_requests', operation: 'executeJsmGetRequests', input: BASE }, + { + toolId: 'jsm_create_request', + operation: 'executeJsmCreateRequest', + input: { ...BASE, serviceDeskId: '1', requestTypeId: '2', summary: 'Help' }, + }, + { toolId: 'jsm_get_request', operation: 'executeJsmGetRequest', input: ISSUE }, + { + toolId: 'jsm_add_comment', + operation: 'executeJsmAddComment', + input: { ...ISSUE, body: 'Update' }, + }, + { toolId: 'jsm_get_comments', operation: 'executeJsmGetComments', input: ISSUE }, + { + toolId: 'jsm_transition_request', + operation: 'executeJsmTransitionRequest', + input: { ...ISSUE, transitionId: '3' }, + }, + { toolId: 'jsm_get_transitions', operation: 'executeJsmGetTransitions', input: ISSUE }, + { toolId: 'jsm_get_sla', operation: 'executeJsmGetSla', input: ISSUE }, + { + toolId: 'jsm_get_approvals', + operation: 'executeJsmGetApprovals', + input: { ...ISSUE, action: 'get' }, + }, + { + toolId: 'jsm_answer_approval', + operation: 'executeJsmAnswerApproval', + input: { ...ISSUE, action: 'answer', approvalId: '4', decision: 'approve' }, + }, + { + toolId: 'jsm_get_participants', + operation: 'executeJsmGetParticipants', + input: { ...ISSUE, action: 'get' }, + }, + { + toolId: 'jsm_add_participants', + operation: 'executeJsmAddParticipants', + input: { ...ISSUE, action: 'add', accountIds: ['account-1'] }, + }, + { + toolId: 'jsm_get_customers', + operation: 'executeJsmGetCustomers', + input: { ...BASE, serviceDeskId: '1' }, + }, + { + toolId: 'jsm_add_customer', + operation: 'executeJsmAddCustomer', + input: { ...BASE, serviceDeskId: '1', accountIds: ['account-1'] }, + }, + { + toolId: 'jsm_get_organizations', + operation: 'executeJsmGetOrganizations', + input: { ...BASE, serviceDeskId: '1' }, + }, + { + toolId: 'jsm_create_organization', + operation: 'executeJsmCreateOrganization', + input: { ...BASE, action: 'create', name: 'Acme' }, + }, + { + toolId: 'jsm_add_organization', + operation: 'executeJsmAddOrganization', + input: { ...BASE, action: 'add_to_service_desk', serviceDeskId: '1', organizationId: '2' }, + }, + { toolId: 'jsm_get_issue_forms', operation: 'executeJsmGetIssueForms', input: ISSUE }, + { + toolId: 'jsm_attach_form', + operation: 'executeJsmAttachForm', + input: { ...ISSUE, formTemplateId: FORM.formId }, + }, + { toolId: 'jsm_get_form', operation: 'executeJsmGetForm', input: FORM }, + { toolId: 'jsm_submit_form', operation: 'executeJsmSubmitForm', input: FORM }, + { toolId: 'jsm_delete_form', operation: 'executeJsmDeleteForm', input: FORM }, + { toolId: 'jsm_externalise_form', operation: 'executeJsmExternaliseForm', input: FORM }, + { toolId: 'jsm_internalise_form', operation: 'executeJsmInternaliseForm', input: FORM }, + { toolId: 'jsm_reopen_form', operation: 'executeJsmReopenForm', input: FORM }, + { + toolId: 'jsm_save_form_answers', + operation: 'executeJsmSaveFormAnswers', + input: { ...FORM, answers: { q1: 'Yes' } }, + }, + { toolId: 'jsm_get_form_answers', operation: 'executeJsmGetFormAnswers', input: FORM }, + { + toolId: 'jsm_get_form_templates', + operation: 'executeJsmGetFormTemplates', + input: { ...BASE, projectIdOrKey: 'HELP' }, + }, + { + toolId: 'jsm_get_form_structure', + operation: 'executeJsmGetFormStructure', + input: { ...BASE, projectIdOrKey: 'HELP', formId: FORM.formId }, + }, + { + toolId: 'jsm_copy_forms', + operation: 'executeJsmCopyForms', + input: { ...BASE, sourceIssueIdOrKey: 'HELP-1', targetIssueIdOrKey: 'HELP-2' }, + }, + { toolId: 'jsm_list_object_schemas', operation: 'executeJsmListObjectSchemas', input: ASSETS }, + { + toolId: 'jsm_get_object_schema', + operation: 'executeJsmGetObjectSchema', + input: { ...ASSETS, schemaId: '1' }, + }, + { + toolId: 'jsm_list_object_types', + operation: 'executeJsmListObjectTypes', + input: { ...ASSETS, schemaId: '1' }, + }, + { + toolId: 'jsm_get_object_type_attributes', + operation: 'executeJsmGetObjectTypeAttributes', + input: { ...ASSETS, objectTypeId: '1' }, + }, + { + toolId: 'jsm_search_objects_aql', + operation: 'executeJsmSearchObjectsAql', + input: { ...ASSETS, qlQuery: 'objectType = Host' }, + }, + { + toolId: 'jsm_get_object', + operation: 'executeJsmGetObject', + input: { ...ASSETS, objectId: '1' }, + }, + { + toolId: 'jsm_create_object', + operation: 'executeJsmCreateObject', + input: { + ...ASSETS, + objectTypeId: '1', + attributes: [{ objectTypeAttributeId: '2', objectAttributeValues: [{ value: 'host' }] }], + }, + }, + { + toolId: 'jsm_update_object', + operation: 'executeJsmUpdateObject', + input: { + ...ASSETS, + objectId: '1', + attributes: [{ objectTypeAttributeId: '2', objectAttributeValues: [{ value: 'host' }] }], + }, + }, + { + toolId: 'jsm_delete_object', + operation: 'executeJsmDeleteObject', + input: { ...ASSETS, objectId: '1' }, + }, +] + +function request( + toolId: string, + input: Record, + signal?: AbortSignal +): InternalToolOperationCall { + return { + toolId, + input, + headers: new Headers(), + context: { workflowId: 'workflow-1', userId: 'user-1' } as ExecutionContext, + requestId: 'request-1', + signal, + } +} + +describe('executeJsmTool', () => { + beforeEach(() => { + vi.clearAllMocks() + for (const [operation, mock] of Object.entries(operations)) { + mock.mockResolvedValue({ operation }) + } + }) + + it('covers all 43 canonical JSM tools', () => { + expect(CASES).toHaveLength(43) + expect(new Set(CASES.map(({ toolId }) => toolId)).size).toBe(43) + }) + + it.each(CASES)('dispatches $toolId through $operation', async ({ toolId, operation, input }) => { + const response = await executeJsmTool(request(toolId, input)) + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ operation }) + expect(operations[operation]).toHaveBeenCalledWith(expect.objectContaining(input), undefined) + }) + + it('preserves typed provider errors', async () => { + operations.executeJsmGetServiceDesks.mockRejectedValueOnce( + new JsmOperationError('Rejected', 429, { error: 'Rejected', details: 'rate limited' }) + ) + const response = await executeJsmTool(request('jsm_get_service_desks', BASE)) + expect(response.status).toBe(429) + expect(await response.json()).toEqual({ error: 'Rejected', details: 'rate limited' }) + }) + + it('returns canonical input validation errors', async () => { + const invalidInput = request('jsm_get_service_desks', BASE) + invalidInput.input = '{' + const invalidInputResponse = await executeJsmTool(invalidInput) + expect(invalidInputResponse.status).toBe(400) + expect(await invalidInputResponse.json()).toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + const validation = await executeJsmTool(request('jsm_get_service_desks', {})) + expect(validation.status).toBe(400) + expect(await validation.json()).toMatchObject({ error: 'Invalid request data' }) + }) + + it('stops before dispatch when cancelled', async () => { + const controller = new AbortController() + controller.abort() + await expect( + executeJsmTool(request('jsm_get_service_desks', BASE, controller.signal)) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(operations.executeJsmGetServiceDesks).not.toHaveBeenCalled() + }) + + it('returns an explicit error for unknown tools', async () => { + const response = await executeJsmTool(request('jsm_unknown', BASE)) + expect(response.status).toBe(500) + expect(await response.json()).toEqual({ error: 'Unsupported JSM tool: jsm_unknown' }) + }) +}) diff --git a/apps/sim/lib/internal/jsm/execute-tool.ts b/apps/sim/lib/internal/jsm/execute-tool.ts new file mode 100644 index 00000000000..340bc738d61 --- /dev/null +++ b/apps/sim/lib/internal/jsm/execute-tool.ts @@ -0,0 +1,243 @@ +import { getErrorMessage } from '@sim/utils/errors' +import type { AnyApiRouteContract, ContractBody } from '@/lib/api/contracts' +import { + jsmApprovalsContract, + jsmAttachFormContract, + jsmCommentContract, + jsmCommentsContract, + jsmCopyFormsContract, + jsmCreateObjectContract, + jsmCustomersContract, + jsmDeleteFormContract, + jsmDeleteObjectContract, + jsmExternaliseFormContract, + jsmFormAnswersContract, + jsmGetFormContract, + jsmGetObjectContract, + jsmGetObjectSchemaContract, + jsmInternaliseFormContract, + jsmIssueFormsContract, + jsmListObjectSchemasContract, + jsmListObjectTypesContract, + jsmObjectTypeAttributesContract, + jsmOrganizationContract, + jsmOrganizationsContract, + jsmParticipantsContract, + jsmProjectFormStructureContract, + jsmProjectFormTemplatesContract, + jsmQueuesContract, + jsmReopenFormContract, + jsmRequestContract, + jsmRequestsContract, + jsmRequestTypeFieldsContract, + jsmRequestTypesContract, + jsmSaveFormAnswersContract, + jsmSearchObjectsAqlContract, + jsmServiceDesksContract, + jsmSlaContract, + jsmSubmitFormContract, + jsmTransitionContract, + jsmTransitionsContract, + jsmUpdateObjectContract, +} from '@/lib/api/contracts/selectors/jsm' +import { + executeJsmCreateObject, + executeJsmDeleteObject, + executeJsmGetObject, + executeJsmGetObjectSchema, + executeJsmGetObjectTypeAttributes, + executeJsmListObjectSchemas, + executeJsmListObjectTypes, + executeJsmSearchObjectsAql, + executeJsmUpdateObject, +} from '@/lib/internal/jsm/assets' +import { JsmOperationError } from '@/lib/internal/jsm/errors' +import { + executeJsmAttachForm, + executeJsmCopyForms, + executeJsmDeleteForm, + executeJsmExternaliseForm, + executeJsmGetForm, + executeJsmGetFormAnswers, + executeJsmGetFormStructure, + executeJsmGetFormTemplates, + executeJsmGetIssueForms, + executeJsmInternaliseForm, + executeJsmReopenForm, + executeJsmSaveFormAnswers, + executeJsmSubmitForm, +} from '@/lib/internal/jsm/forms' +import { + executeJsmAddComment, + executeJsmAddCustomer, + executeJsmAddOrganization, + executeJsmAddParticipants, + executeJsmAnswerApproval, + executeJsmCreateOrganization, + executeJsmCreateRequest, + executeJsmGetApprovals, + executeJsmGetComments, + executeJsmGetCustomers, + executeJsmGetOrganizations, + executeJsmGetParticipants, + executeJsmGetQueues, + executeJsmGetRequest, + executeJsmGetRequests, + executeJsmGetRequestTypeFields, + executeJsmGetRequestTypes, + executeJsmGetServiceDesks, + executeJsmGetSla, + executeJsmGetTransitions, + executeJsmTransitionRequest, +} from '@/lib/internal/jsm/service-desk' +import { parseInternalToolInput } from '@/lib/internal/tool-operations/parse-input' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +async function executeOperation( + contract: C, + input: unknown, + execute: (input: ContractBody, signal?: AbortSignal) => Promise, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const parsed = parseInternalToolInput(contract, input) + if (!parsed.success) return parsed.response + try { + const result = await execute(parsed.data, signal) + signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + signal?.throwIfAborted() + if (error instanceof JsmOperationError) { + return Response.json(error.body ?? { error: error.message }, { status: error.status }) + } + return Response.json( + { error: getErrorMessage(error, 'Internal server error'), success: false }, + { status: 500 } + ) + } +} + +export const executeJsmTool: InternalToolOperationHandler = async ({ toolId, input, signal }) => { + signal?.throwIfAborted() + switch (toolId) { + case 'jsm_get_service_desks': + return executeOperation(jsmServiceDesksContract, input, executeJsmGetServiceDesks, signal) + case 'jsm_get_queues': + return executeOperation(jsmQueuesContract, input, executeJsmGetQueues, signal) + case 'jsm_get_request_types': + return executeOperation(jsmRequestTypesContract, input, executeJsmGetRequestTypes, signal) + case 'jsm_get_request_type_fields': + return executeOperation( + jsmRequestTypeFieldsContract, + input, + executeJsmGetRequestTypeFields, + signal + ) + case 'jsm_get_requests': + return executeOperation(jsmRequestsContract, input, executeJsmGetRequests, signal) + case 'jsm_create_request': + return executeOperation(jsmRequestContract, input, executeJsmCreateRequest, signal) + case 'jsm_get_request': + return executeOperation(jsmRequestContract, input, executeJsmGetRequest, signal) + case 'jsm_add_comment': + return executeOperation(jsmCommentContract, input, executeJsmAddComment, signal) + case 'jsm_get_comments': + return executeOperation(jsmCommentsContract, input, executeJsmGetComments, signal) + case 'jsm_transition_request': + return executeOperation(jsmTransitionContract, input, executeJsmTransitionRequest, signal) + case 'jsm_get_transitions': + return executeOperation(jsmTransitionsContract, input, executeJsmGetTransitions, signal) + case 'jsm_get_sla': + return executeOperation(jsmSlaContract, input, executeJsmGetSla, signal) + case 'jsm_get_approvals': + return executeOperation(jsmApprovalsContract, input, executeJsmGetApprovals, signal) + case 'jsm_answer_approval': + return executeOperation(jsmApprovalsContract, input, executeJsmAnswerApproval, signal) + case 'jsm_get_participants': + return executeOperation(jsmParticipantsContract, input, executeJsmGetParticipants, signal) + case 'jsm_add_participants': + return executeOperation(jsmParticipantsContract, input, executeJsmAddParticipants, signal) + case 'jsm_get_customers': + return executeOperation(jsmCustomersContract, input, executeJsmGetCustomers, signal) + case 'jsm_add_customer': + return executeOperation(jsmCustomersContract, input, executeJsmAddCustomer, signal) + case 'jsm_get_organizations': + return executeOperation(jsmOrganizationsContract, input, executeJsmGetOrganizations, signal) + case 'jsm_create_organization': + return executeOperation(jsmOrganizationContract, input, executeJsmCreateOrganization, signal) + case 'jsm_add_organization': + return executeOperation(jsmOrganizationContract, input, executeJsmAddOrganization, signal) + case 'jsm_get_issue_forms': + return executeOperation(jsmIssueFormsContract, input, executeJsmGetIssueForms, signal) + case 'jsm_attach_form': + return executeOperation(jsmAttachFormContract, input, executeJsmAttachForm, signal) + case 'jsm_get_form': + return executeOperation(jsmGetFormContract, input, executeJsmGetForm, signal) + case 'jsm_submit_form': + return executeOperation(jsmSubmitFormContract, input, executeJsmSubmitForm, signal) + case 'jsm_delete_form': + return executeOperation(jsmDeleteFormContract, input, executeJsmDeleteForm, signal) + case 'jsm_externalise_form': + return executeOperation(jsmExternaliseFormContract, input, executeJsmExternaliseForm, signal) + case 'jsm_internalise_form': + return executeOperation(jsmInternaliseFormContract, input, executeJsmInternaliseForm, signal) + case 'jsm_reopen_form': + return executeOperation(jsmReopenFormContract, input, executeJsmReopenForm, signal) + case 'jsm_save_form_answers': + return executeOperation(jsmSaveFormAnswersContract, input, executeJsmSaveFormAnswers, signal) + case 'jsm_get_form_answers': + return executeOperation(jsmFormAnswersContract, input, executeJsmGetFormAnswers, signal) + case 'jsm_get_form_templates': + return executeOperation( + jsmProjectFormTemplatesContract, + input, + executeJsmGetFormTemplates, + signal + ) + case 'jsm_get_form_structure': + return executeOperation( + jsmProjectFormStructureContract, + input, + executeJsmGetFormStructure, + signal + ) + case 'jsm_copy_forms': + return executeOperation(jsmCopyFormsContract, input, executeJsmCopyForms, signal) + case 'jsm_list_object_schemas': + return executeOperation( + jsmListObjectSchemasContract, + input, + executeJsmListObjectSchemas, + signal + ) + case 'jsm_get_object_schema': + return executeOperation(jsmGetObjectSchemaContract, input, executeJsmGetObjectSchema, signal) + case 'jsm_list_object_types': + return executeOperation(jsmListObjectTypesContract, input, executeJsmListObjectTypes, signal) + case 'jsm_get_object_type_attributes': + return executeOperation( + jsmObjectTypeAttributesContract, + input, + executeJsmGetObjectTypeAttributes, + signal + ) + case 'jsm_search_objects_aql': + return executeOperation( + jsmSearchObjectsAqlContract, + input, + executeJsmSearchObjectsAql, + signal + ) + case 'jsm_get_object': + return executeOperation(jsmGetObjectContract, input, executeJsmGetObject, signal) + case 'jsm_create_object': + return executeOperation(jsmCreateObjectContract, input, executeJsmCreateObject, signal) + case 'jsm_update_object': + return executeOperation(jsmUpdateObjectContract, input, executeJsmUpdateObject, signal) + case 'jsm_delete_object': + return executeOperation(jsmDeleteObjectContract, input, executeJsmDeleteObject, signal) + default: + return Response.json({ error: `Unsupported JSM tool: ${toolId}` }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/jsm/forms.ts b/apps/sim/lib/internal/jsm/forms.ts new file mode 100644 index 00000000000..9b7efd15d5c --- /dev/null +++ b/apps/sim/lib/internal/jsm/forms.ts @@ -0,0 +1,319 @@ +import type { + JsmAttachFormBody, + JsmCopyFormsBody, + JsmDeleteFormBody, + JsmExternaliseFormBody, + JsmFormAnswersBody, + JsmGetFormBody, + JsmInternaliseFormBody, + JsmIssueFormsBody, + JsmProjectFormStructureBody, + JsmProjectFormTemplatesBody, + JsmReopenFormBody, + JsmSaveFormAnswersBody, + JsmSubmitFormBody, +} from '@/lib/api/contracts/selectors/jsm' +import { validateJiraCloudId, validateJiraIssueKey } from '@/lib/core/security/input-validation' +import { asArray, asObject, createJsmClient, nested } from '@/lib/internal/jsm/client' +import { JsmOperationError } from '@/lib/internal/jsm/errors' + +type IssueFormInput = + | JsmGetFormBody + | JsmSubmitFormBody + | JsmDeleteFormBody + | JsmExternaliseFormBody + | JsmInternaliseFormBody + | JsmReopenFormBody + | JsmSaveFormAnswersBody + | JsmFormAnswersBody + +function validateIssueKey(value: string, field: string): void { + const validation = validateJiraIssueKey(value, field) + if (!validation.isValid) throw new JsmOperationError(validation.error || `Invalid ${field}`, 400) +} + +function validateFormId(value: string, field = 'formId'): void { + const validation = validateJiraCloudId(value, field) + if (!validation.isValid) throw new JsmOperationError(validation.error || `Invalid ${field}`, 400) +} + +async function issueFormClient(input: IssueFormInput, signal?: AbortSignal) { + validateIssueKey(input.issueIdOrKey, 'issueIdOrKey') + validateFormId(input.formId) + return createJsmClient(input, signal) +} + +function issueFormPath(issueIdOrKey: string, formId?: string): string { + const issue = encodeURIComponent(issueIdOrKey) + return formId ? `/issue/${issue}/form/${encodeURIComponent(formId)}` : `/issue/${issue}/form` +} + +export async function executeJsmGetIssueForms(input: JsmIssueFormsBody, signal?: AbortSignal) { + validateIssueKey(input.issueIdOrKey, 'issueIdOrKey') + const client = await createJsmClient(input, signal) + const value = await client.value( + client.forms(issueFormPath(input.issueIdOrKey)), + {}, + signal, + true + ) + const data = asObject(value) + const forms = Array.isArray(value) ? value : asArray(data.values ?? data.forms) + return { + success: true, + output: { + ts: new Date().toISOString(), + issueIdOrKey: input.issueIdOrKey, + forms: forms.map((entry) => { + const form = asObject(entry) + return { + id: form.id ?? null, + name: form.name ?? null, + updated: form.updated ?? null, + submitted: form.submitted ?? false, + lock: form.lock ?? false, + internal: form.internal ?? null, + formTemplateId: nested(form, 'formTemplate', 'id') ?? null, + } + }), + total: forms.length, + }, + } +} + +export async function executeJsmAttachForm(input: JsmAttachFormBody, signal?: AbortSignal) { + validateIssueKey(input.issueIdOrKey, 'issueIdOrKey') + validateFormId(input.formTemplateId, 'formTemplateId') + const client = await createJsmClient(input, signal) + const data = await client.json( + client.forms(issueFormPath(input.issueIdOrKey)), + { method: 'POST', body: JSON.stringify({ formTemplate: { id: input.formTemplateId } }) }, + signal, + true + ) + return { + success: true, + output: { + ts: new Date().toISOString(), + issueIdOrKey: input.issueIdOrKey, + id: data.id ?? null, + name: data.name ?? null, + updated: data.updated ?? null, + submitted: data.submitted ?? false, + lock: data.lock ?? false, + internal: data.internal ?? null, + formTemplateId: nested(data, 'formTemplate', 'id') ?? null, + }, + } +} + +export async function executeJsmGetForm(input: JsmGetFormBody, signal?: AbortSignal) { + const client = await issueFormClient(input, signal) + const data = await client.json( + client.forms(issueFormPath(input.issueIdOrKey, input.formId)), + {}, + signal, + true + ) + return { + success: true, + output: { + ts: new Date().toISOString(), + issueIdOrKey: input.issueIdOrKey, + formId: input.formId, + design: data.design ?? null, + state: data.state ?? null, + updated: data.updated ?? null, + }, + } +} + +async function executeFormAction( + input: IssueFormInput, + action: string, + fallbackField: 'status' | 'visibility', + fallbackValue: string, + signal?: AbortSignal +) { + const client = await issueFormClient(input, signal) + const data = await client.optionalJson( + client.forms(`${issueFormPath(input.issueIdOrKey, input.formId)}/action/${action}`), + { method: 'PUT' }, + signal, + true + ) + return { + success: true, + output: { + ts: new Date().toISOString(), + issueIdOrKey: input.issueIdOrKey, + formId: input.formId, + [fallbackField]: data[fallbackField] ?? fallbackValue, + }, + } +} + +export function executeJsmSubmitForm(input: JsmSubmitFormBody, signal?: AbortSignal) { + return executeFormAction(input, 'submit', 'status', 'submitted', signal) +} + +export async function executeJsmDeleteForm(input: JsmDeleteFormBody, signal?: AbortSignal) { + const client = await issueFormClient(input, signal) + await client.empty( + client.forms(issueFormPath(input.issueIdOrKey, input.formId)), + { method: 'DELETE' }, + signal, + true + ) + return { + success: true, + output: { + ts: new Date().toISOString(), + issueIdOrKey: input.issueIdOrKey, + formId: input.formId, + deleted: true, + }, + } +} + +export function executeJsmExternaliseForm(input: JsmExternaliseFormBody, signal?: AbortSignal) { + return executeFormAction(input, 'external', 'visibility', 'external', signal) +} + +export function executeJsmInternaliseForm(input: JsmInternaliseFormBody, signal?: AbortSignal) { + return executeFormAction(input, 'internal', 'visibility', 'internal', signal) +} + +export function executeJsmReopenForm(input: JsmReopenFormBody, signal?: AbortSignal) { + return executeFormAction(input, 'reopen', 'status', 'open', signal) +} + +export async function executeJsmSaveFormAnswers( + input: JsmSaveFormAnswersBody, + signal?: AbortSignal +) { + const client = await issueFormClient(input, signal) + const data = await client.json( + client.forms(issueFormPath(input.issueIdOrKey, input.formId)), + { method: 'PUT', body: JSON.stringify({ answers: input.answers }) }, + signal, + true + ) + return { + success: true, + output: { + ts: new Date().toISOString(), + issueIdOrKey: input.issueIdOrKey, + formId: input.formId, + state: data.state ?? null, + updated: data.updated ?? null, + }, + } +} + +export async function executeJsmGetFormAnswers(input: JsmFormAnswersBody, signal?: AbortSignal) { + const client = await issueFormClient(input, signal) + const answers = await client.value( + client.forms(`${issueFormPath(input.issueIdOrKey, input.formId)}/format/answers`), + {}, + signal, + true + ) + return { + success: true, + output: { + ts: new Date().toISOString(), + issueIdOrKey: input.issueIdOrKey, + formId: input.formId, + answers: answers ?? null, + }, + } +} + +export async function executeJsmGetFormTemplates( + input: JsmProjectFormTemplatesBody, + signal?: AbortSignal +) { + validateIssueKey(input.projectIdOrKey, 'projectIdOrKey') + const client = await createJsmClient(input, signal) + const value = await client.value( + client.forms(`/project/${encodeURIComponent(input.projectIdOrKey)}/form`), + {}, + signal, + true + ) + const data = asObject(value) + const templates = Array.isArray(value) ? value : asArray(data.values) + return { + success: true, + output: { + ts: new Date().toISOString(), + projectIdOrKey: input.projectIdOrKey, + templates: templates.map((entry) => { + const template = asObject(entry) + return { + id: template.id ?? null, + name: template.name ?? null, + updated: template.updated ?? null, + issueCreateIssueTypeIds: template.issueCreateIssueTypeIds ?? [], + issueCreateRequestTypeIds: template.issueCreateRequestTypeIds ?? [], + portalRequestTypeIds: template.portalRequestTypeIds ?? [], + recommendedIssueRequestTypeIds: template.recommendedIssueRequestTypeIds ?? [], + } + }), + total: templates.length, + }, + } +} + +export async function executeJsmGetFormStructure( + input: JsmProjectFormStructureBody, + signal?: AbortSignal +) { + validateIssueKey(input.projectIdOrKey, 'projectIdOrKey') + validateFormId(input.formId) + const client = await createJsmClient(input, signal) + const data = await client.json( + client.forms( + `/project/${encodeURIComponent(input.projectIdOrKey)}/form/${encodeURIComponent(input.formId)}` + ), + {}, + signal, + true + ) + return { + success: true, + output: { + ts: new Date().toISOString(), + projectIdOrKey: input.projectIdOrKey, + formId: input.formId, + design: data.design ?? null, + updated: data.updated ?? null, + publish: data.publish ?? null, + }, + } +} + +export async function executeJsmCopyForms(input: JsmCopyFormsBody, signal?: AbortSignal) { + validateIssueKey(input.sourceIssueIdOrKey, 'sourceIssueIdOrKey') + validateIssueKey(input.targetIssueIdOrKey, 'targetIssueIdOrKey') + const client = await createJsmClient(input, signal) + const data = await client.json( + client.forms( + `/issue/${encodeURIComponent(input.sourceIssueIdOrKey)}/form/copy/${encodeURIComponent(input.targetIssueIdOrKey)}` + ), + { method: 'POST', body: JSON.stringify(input.formIds?.length ? { ids: input.formIds } : {}) }, + signal, + true + ) + return { + success: true, + output: { + ts: new Date().toISOString(), + sourceIssueIdOrKey: input.sourceIssueIdOrKey, + targetIssueIdOrKey: input.targetIssueIdOrKey, + copiedForms: data.copiedForms ?? [], + errors: data.errors ?? [], + }, + } +} diff --git a/apps/sim/lib/internal/jsm/operations.test.ts b/apps/sim/lib/internal/jsm/operations.test.ts new file mode 100644 index 00000000000..f7b89934037 --- /dev/null +++ b/apps/sim/lib/internal/jsm/operations.test.ts @@ -0,0 +1,148 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => { + const client = { + service: vi.fn((path: string) => `service:${path}`), + forms: vi.fn((path: string) => `forms:${path}`), + assets: vi.fn((path: string) => `assets:${path}`), + json: vi.fn(), + value: vi.fn(), + empty: vi.fn(), + optionalJson: vi.fn(), + } + return { + client, + createJsmClient: vi.fn(async () => client), + createJsmAssetsClient: vi.fn(async () => client), + mapAssetObject: vi.fn((value: unknown) => value), + } +}) + +vi.mock('@/lib/internal/jsm/client', () => ({ + asArray: (value: unknown) => (Array.isArray(value) ? value : []), + asObject: (value: unknown) => + value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : {}, + nested: (value: unknown, ...keys: string[]) => { + let current = value + for (const key of keys) { + current = + current && typeof current === 'object' && !Array.isArray(current) + ? (current as Record)[key] + : undefined + } + return current + }, + createJsmClient: mocks.createJsmClient, + createJsmAssetsClient: mocks.createJsmAssetsClient, +})) + +vi.mock('@/tools/jsm/utils', () => ({ mapAssetObject: mocks.mapAssetObject })) + +import { executeJsmSearchObjectsAql } from '@/lib/internal/jsm/assets' +import { executeJsmSubmitForm } from '@/lib/internal/jsm/forms' +import { executeJsmCreateRequest, listJsmServiceDeskOptions } from '@/lib/internal/jsm/service-desk' + +const BASE = { + domain: 'example.atlassian.net', + accessToken: 'token', + cloudId: '12345678-1234-1234-1234-123456789012', +} + +describe('JSM operations', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('drains selector pages using the number of returned rows as the next offset', async () => { + mocks.client.json + .mockResolvedValueOnce({ + values: [{ id: '1', projectName: 'One' }], + _links: { next: 'next' }, + }) + .mockResolvedValueOnce({ values: [{ id: '2', projectName: 'Two' }], isLastPage: true }) + + await expect(listJsmServiceDeskOptions(BASE)).resolves.toEqual([ + { id: '1', name: 'One' }, + { id: '2', name: 'Two' }, + ]) + expect(mocks.client.json).toHaveBeenNthCalledWith( + 1, + 'service:/servicedesk?start=0&limit=100', + {}, + undefined + ) + expect(mocks.client.json).toHaveBeenNthCalledWith( + 2, + 'service:/servicedesk?start=1&limit=100', + {}, + undefined + ) + }) + + it('keeps form answers separate from explicitly supplied request field values', async () => { + mocks.client.json.mockResolvedValueOnce({ issueKey: 'HELP-1' }) + await executeJsmCreateRequest({ + ...BASE, + serviceDeskId: '1', + requestTypeId: '2', + summary: 'Do not duplicate this linked form field', + formAnswers: { q1: 'yes' }, + requestFieldValues: { customfield_1: 'safe' }, + }) + const init = mocks.client.json.mock.calls[0]?.[1] as RequestInit + expect(JSON.parse(String(init.body))).toEqual({ + serviceDeskId: '1', + requestTypeId: '2', + form: { answers: { q1: 'yes' } }, + requestFieldValues: { customfield_1: 'safe' }, + }) + }) + + it('uses the Forms action endpoint and preserves its empty-body default', async () => { + mocks.client.optionalJson.mockResolvedValueOnce({}) + const result = await executeJsmSubmitForm({ + ...BASE, + issueIdOrKey: 'HELP-1', + formId: '12345678-1234-1234-1234-123456789012', + }) + expect(mocks.client.optionalJson).toHaveBeenCalledWith( + 'forms:/issue/HELP-1/form/12345678-1234-1234-1234-123456789012/action/submit', + { method: 'PUT' }, + undefined, + true + ) + expect(result.output.status).toBe('submitted') + }) + + it('normalizes Assets AQL pagination and forwards the execution signal', async () => { + const controller = new AbortController() + mocks.client.json.mockResolvedValueOnce({ objectEntries: [{ id: 'object-1' }] }) + await executeJsmSearchObjectsAql( + { ...BASE, qlQuery: 'objectType = Host', page: '2', resultsPerPage: '10' }, + controller.signal + ) + expect(mocks.createJsmAssetsClient).toHaveBeenCalledWith( + expect.objectContaining({ qlQuery: 'objectType = Host' }), + controller.signal + ) + expect(mocks.client.json).toHaveBeenCalledWith( + 'assets:/object/aql', + { + method: 'POST', + body: JSON.stringify({ + qlQuery: 'objectType = Host', + page: 2, + resultsPerPage: 10, + includeAttributes: true, + }), + }, + controller.signal, + true + ) + }) +}) diff --git a/apps/sim/lib/internal/jsm/service-desk.ts b/apps/sim/lib/internal/jsm/service-desk.ts new file mode 100644 index 00000000000..711bc6b7996 --- /dev/null +++ b/apps/sim/lib/internal/jsm/service-desk.ts @@ -0,0 +1,717 @@ +import { createLogger } from '@sim/logger' +import type { + JsmApprovalsBody, + JsmCommentBody, + JsmCommentsBody, + JsmCustomersBody, + JsmOrganizationBody, + JsmOrganizationsBody, + JsmParticipantsBody, + JsmQueuesBody, + JsmRequestBody, + JsmRequestsBody, + JsmRequestTypeFieldsBody, + JsmRequestTypesBody, + JsmServiceDesksBody, + JsmSlaBody, + JsmTransitionBody, + JsmTransitionsBody, +} from '@/lib/api/contracts/selectors/jsm' +import { + validateAlphanumericId, + validateEnum, + validateJiraIssueKey, +} from '@/lib/core/security/input-validation' +import { asArray, asObject, createJsmClient } from '@/lib/internal/jsm/client' +import { JsmOperationError } from '@/lib/internal/jsm/errors' + +const logger = createLogger('JsmServiceDeskOperations') +const SELECTOR_PAGE_SIZE = 100 +const SELECTOR_MAX_PAGES = 50 + +function validateId(value: string, field: string): void { + const validation = validateAlphanumericId(value, field) + if (!validation.isValid) throw new JsmOperationError(validation.error || `Invalid ${field}`, 400) +} + +function validateIssue(value: string): void { + const validation = validateJiraIssueKey(value, 'issueIdOrKey') + if (!validation.isValid) { + throw new JsmOperationError(validation.error || 'Invalid issueIdOrKey', 400) + } +} + +function append(query: URLSearchParams, key: string, value: unknown): void { + if (value !== undefined && value !== null && value !== '') query.append(key, String(value)) +} + +function pagedOutput(data: Record, key: string, context = {}) { + return { + ts: new Date().toISOString(), + ...context, + [key]: data.values || [], + total: data.size || 0, + isLastPage: data.isLastPage ?? true, + } +} + +function csv(value: string | string[] | undefined): string[] { + if (!value) return [] + return Array.isArray(value) + ? value + : value + .split(',') + .map((item) => item.trim()) + .filter(Boolean) +} + +function serviceDeskPath(serviceDeskId: string, suffix: string): string { + return `/servicedesk/${encodeURIComponent(serviceDeskId)}${suffix}` +} + +export async function executeJsmGetServiceDesks(input: JsmServiceDesksBody, signal?: AbortSignal) { + const client = await createJsmClient(input, signal) + const query = new URLSearchParams() + append(query, 'expand', input.expand) + append(query, 'start', input.start) + append(query, 'limit', input.limit) + const data = await client.json( + client.service(`/servicedesk${query.size ? `?${query}` : ''}`), + {}, + signal, + true + ) + return { success: true, output: pagedOutput(data, 'serviceDesks') } +} + +export async function executeJsmGetQueues(input: JsmQueuesBody, signal?: AbortSignal) { + validateId(input.serviceDeskId, 'serviceDeskId') + const client = await createJsmClient(input, signal) + const query = new URLSearchParams() + append(query, 'includeCount', input.includeCount) + append(query, 'start', input.start) + append(query, 'limit', input.limit) + const data = await client.json( + client.service( + `${serviceDeskPath(input.serviceDeskId, '/queue')}${query.size ? `?${query}` : ''}` + ), + {}, + signal, + true + ) + return { success: true, output: pagedOutput(data, 'queues') } +} + +export async function executeJsmGetRequestTypes(input: JsmRequestTypesBody, signal?: AbortSignal) { + validateId(input.serviceDeskId, 'serviceDeskId') + const client = await createJsmClient(input, signal) + const query = new URLSearchParams() + append(query, 'searchQuery', input.searchQuery) + append(query, 'groupId', input.groupId) + append(query, 'expand', input.expand) + append(query, 'start', input.start) + append(query, 'limit', input.limit) + const data = await client.json( + client.service( + `${serviceDeskPath(input.serviceDeskId, '/requesttype')}${query.size ? `?${query}` : ''}` + ), + {}, + signal, + true + ) + return { success: true, output: pagedOutput(data, 'requestTypes') } +} + +export async function executeJsmGetRequestTypeFields( + input: JsmRequestTypeFieldsBody, + signal?: AbortSignal +) { + validateId(input.serviceDeskId, 'serviceDeskId') + validateId(input.requestTypeId, 'requestTypeId') + const client = await createJsmClient(input, signal) + const data = await client.json( + client.service( + serviceDeskPath( + input.serviceDeskId, + `/requesttype/${encodeURIComponent(input.requestTypeId)}/field` + ) + ), + {}, + signal, + true + ) + return { + success: true, + output: { + ts: new Date().toISOString(), + serviceDeskId: input.serviceDeskId, + requestTypeId: input.requestTypeId, + canAddRequestParticipants: data.canAddRequestParticipants ?? false, + canRaiseOnBehalfOf: data.canRaiseOnBehalfOf ?? false, + requestTypeFields: asArray(data.requestTypeFields).map((entry) => { + const field = asObject(entry) + return { + fieldId: field.fieldId ?? null, + name: field.name ?? null, + description: field.description ?? null, + required: field.required ?? false, + visible: field.visible ?? true, + validValues: field.validValues ?? [], + presetValues: field.presetValues ?? [], + defaultValues: field.defaultValues ?? [], + jiraSchema: field.jiraSchema ?? null, + } + }), + }, + } +} + +const REQUEST_OWNERSHIP = [ + 'OWNED_REQUESTS', + 'PARTICIPATED_REQUESTS', + 'APPROVER', + 'ALL_REQUESTS', +] as const +const REQUEST_STATUS = ['OPEN_REQUESTS', 'CLOSED_REQUESTS', 'ALL_REQUESTS'] as const + +function requireAction( + action: string, + allowed: T, + expected: T[number] +): void { + const validation = validateEnum(action, allowed, 'action') + if (!validation.isValid) { + throw new JsmOperationError(validation.error || 'Invalid action', 400) + } + if (action !== expected) throw new JsmOperationError('Invalid action', 400) +} + +export async function executeJsmGetRequests(input: JsmRequestsBody, signal?: AbortSignal) { + if (input.serviceDeskId) validateId(input.serviceDeskId, 'serviceDeskId') + if (input.requestOwnership) { + const result = validateEnum(input.requestOwnership, REQUEST_OWNERSHIP, 'requestOwnership') + if (!result.isValid) + throw new JsmOperationError(result.error || 'Invalid requestOwnership', 400) + } + if (input.requestStatus) { + const result = validateEnum(input.requestStatus, REQUEST_STATUS, 'requestStatus') + if (!result.isValid) throw new JsmOperationError(result.error || 'Invalid requestStatus', 400) + } + const client = await createJsmClient(input, signal) + const query = new URLSearchParams() + append(query, 'serviceDeskId', input.serviceDeskId) + append(query, 'requestOwnership', input.requestOwnership) + append(query, 'requestStatus', input.requestStatus) + append(query, 'requestTypeId', input.requestTypeId) + append(query, 'searchTerm', input.searchTerm) + append(query, 'expand', input.expand) + append(query, 'start', input.start) + append(query, 'limit', input.limit) + const data = await client.json( + client.service(`/request${query.size ? `?${query}` : ''}`), + {}, + signal, + true + ) + return { success: true, output: pagedOutput(data, 'requests') } +} + +function currentStatus(data: Record) { + const value = asObject(data.currentStatus) + return data.currentStatus + ? { + status: value.status ?? null, + statusCategory: value.statusCategory ?? null, + statusDate: value.statusDate ?? null, + } + : null +} + +function reporter(data: Record, includeActive: boolean) { + if (!data.reporter) return null + const value = asObject(data.reporter) + return { + accountId: value.accountId ?? null, + displayName: value.displayName ?? null, + emailAddress: value.emailAddress ?? null, + ...(includeActive ? { active: value.active ?? true } : {}), + } +} + +export async function executeJsmCreateRequest(input: JsmRequestBody, signal?: AbortSignal) { + if (!input.serviceDeskId || !input.requestTypeId || (!input.summary && !input.formAnswers)) { + throw new JsmOperationError( + 'Service Desk ID, Request Type ID, and summary or form answers are required', + 400 + ) + } + validateId(input.serviceDeskId, 'serviceDeskId') + validateId(input.requestTypeId, 'requestTypeId') + const client = await createJsmClient(input, signal) + const body: Record = { + serviceDeskId: input.serviceDeskId, + requestTypeId: input.requestTypeId, + } + if (input.formAnswers) { + body.form = { answers: input.formAnswers } + if (input.requestFieldValues) body.requestFieldValues = input.requestFieldValues + } else if (input.summary || input.description || input.requestFieldValues) { + body.requestFieldValues = input.requestFieldValues + ? { + ...(!input.requestFieldValues.summary && input.summary ? { summary: input.summary } : {}), + ...(!input.requestFieldValues.description && input.description + ? { description: input.description } + : {}), + ...input.requestFieldValues, + } + : { + ...(input.summary ? { summary: input.summary } : {}), + ...(input.description ? { description: input.description } : {}), + } + } + if (input.raiseOnBehalfOf) body.raiseOnBehalfOf = input.raiseOnBehalfOf + if (input.requestParticipants) body.requestParticipants = csv(input.requestParticipants) + if (input.channel) body.channel = input.channel + const data = await client.json( + client.service('/request'), + { method: 'POST', body: JSON.stringify(body) }, + signal, + true + ) + return { + success: true, + output: { + ts: new Date().toISOString(), + issueId: data.issueId, + issueKey: data.issueKey, + requestTypeId: data.requestTypeId, + serviceDeskId: data.serviceDeskId, + createdDate: data.createdDate ?? null, + currentStatus: currentStatus(data), + reporter: reporter(data, false), + success: true, + url: `https://${input.domain}/browse/${String(data.issueKey)}`, + }, + } +} + +export async function executeJsmGetRequest(input: JsmRequestBody, signal?: AbortSignal) { + if (!input.issueIdOrKey) throw new JsmOperationError('Issue ID or key is required', 400) + validateIssue(input.issueIdOrKey) + const client = await createJsmClient(input, signal) + const query = new URLSearchParams() + append(query, 'expand', input.expand) + const data = await client.json( + client.service( + `/request/${encodeURIComponent(input.issueIdOrKey)}${query.size ? `?${query}` : ''}` + ), + {}, + signal, + true + ) + return { + success: true, + output: { + ts: new Date().toISOString(), + issueId: data.issueId ?? null, + issueKey: data.issueKey ?? null, + requestTypeId: data.requestTypeId ?? null, + serviceDeskId: data.serviceDeskId ?? null, + createdDate: data.createdDate ?? null, + currentStatus: currentStatus(data), + reporter: reporter(data, true), + requestFieldValues: asArray(data.requestFieldValues).map((entry) => { + const field = asObject(entry) + return { + fieldId: field.fieldId ?? null, + label: field.label ?? null, + value: field.value ?? null, + } + }), + url: `https://${input.domain}/browse/${String(data.issueKey)}`, + request: data, + }, + } +} + +export async function executeJsmAddComment(input: JsmCommentBody, signal?: AbortSignal) { + validateIssue(input.issueIdOrKey) + const client = await createJsmClient(input, signal) + const data = await client.json( + client.service(`/request/${encodeURIComponent(input.issueIdOrKey)}/comment`), + { method: 'POST', body: JSON.stringify({ body: input.body, public: input.isPublic ?? true }) }, + signal, + true + ) + const author = asObject(data.author) + return { + success: true, + output: { + ts: new Date().toISOString(), + issueIdOrKey: input.issueIdOrKey, + commentId: data.id, + body: data.body, + isPublic: data.public, + author: data.author + ? { + accountId: author.accountId ?? null, + displayName: author.displayName ?? null, + emailAddress: author.emailAddress ?? null, + } + : null, + createdDate: data.created ?? null, + success: true, + }, + } +} + +async function issuePage( + input: JsmCommentsBody | JsmSlaBody | JsmTransitionsBody, + suffix: string, + key: string, + extra: Record, + signal?: AbortSignal +) { + validateIssue(input.issueIdOrKey) + const client = await createJsmClient(input, signal) + const query = new URLSearchParams() + if ('isPublic' in input) append(query, 'public', input.isPublic) + if ('internal' in input) append(query, 'internal', input.internal) + if ('expand' in input) append(query, 'expand', input.expand) + append(query, 'start', input.start) + append(query, 'limit', input.limit) + const data = await client.json( + client.service( + `/request/${encodeURIComponent(input.issueIdOrKey)}/${suffix}${query.size ? `?${query}` : ''}` + ), + {}, + signal, + true + ) + return { + success: true, + output: pagedOutput(data, key, { issueIdOrKey: input.issueIdOrKey, ...extra }), + } +} + +export function executeJsmGetComments(input: JsmCommentsBody, signal?: AbortSignal) { + return issuePage(input, 'comment', 'comments', {}, signal) +} + +export async function executeJsmTransitionRequest(input: JsmTransitionBody, signal?: AbortSignal) { + validateIssue(input.issueIdOrKey) + validateId(input.transitionId, 'transitionId') + const client = await createJsmClient(input, signal) + const body: Record = { id: input.transitionId } + if (input.comment) body.additionalComment = { body: input.comment } + await client.empty( + client.service(`/request/${encodeURIComponent(input.issueIdOrKey)}/transition`), + { method: 'POST', body: JSON.stringify(body) }, + signal, + true + ) + return { + success: true, + output: { + ts: new Date().toISOString(), + issueIdOrKey: input.issueIdOrKey, + transitionId: input.transitionId, + success: true, + }, + } +} + +export function executeJsmGetTransitions(input: JsmTransitionsBody, signal?: AbortSignal) { + return issuePage(input, 'transition', 'transitions', {}, signal) +} + +export function executeJsmGetSla(input: JsmSlaBody, signal?: AbortSignal) { + return issuePage(input, 'sla', 'slas', {}, signal) +} + +export async function executeJsmGetApprovals(input: JsmApprovalsBody, signal?: AbortSignal) { + requireAction(input.action, ['get', 'answer'] as const, 'get') + validateIssue(input.issueIdOrKey) + const client = await createJsmClient(input, signal) + const query = new URLSearchParams() + append(query, 'start', input.start) + append(query, 'limit', input.limit) + const data = await client.json( + client.service( + `/request/${encodeURIComponent(input.issueIdOrKey)}/approval${query.size ? `?${query}` : ''}` + ), + {}, + signal, + true + ) + return { + success: true, + output: pagedOutput(data, 'approvals', { issueIdOrKey: input.issueIdOrKey }), + } +} + +export async function executeJsmAnswerApproval(input: JsmApprovalsBody, signal?: AbortSignal) { + requireAction(input.action, ['get', 'answer'] as const, 'answer') + validateIssue(input.issueIdOrKey) + if (!input.approvalId) throw new JsmOperationError('Approval ID is required', 400) + validateId(input.approvalId, 'approvalId') + const decision = validateEnum(input.decision, ['approve', 'decline'] as const, 'decision') + if (!decision.isValid) throw new JsmOperationError(decision.error || 'Invalid decision', 400) + const client = await createJsmClient(input, signal) + const data = await client.json( + client.service( + `/request/${encodeURIComponent(input.issueIdOrKey)}/approval/${encodeURIComponent(input.approvalId)}` + ), + { method: 'POST', body: JSON.stringify({ decision: input.decision }) }, + signal, + true + ) + return { + success: true, + output: { + ts: new Date().toISOString(), + issueIdOrKey: input.issueIdOrKey, + approvalId: input.approvalId, + decision: input.decision, + id: data.id ?? null, + name: data.name ?? null, + finalDecision: data.finalDecision ?? null, + canAnswerApproval: data.canAnswerApproval ?? null, + approvers: asArray(data.approvers).map((entry) => { + const item = asObject(entry) + const approver = asObject(item.approver) + return { + approver: { + accountId: approver.accountId ?? null, + displayName: approver.displayName ?? null, + emailAddress: approver.emailAddress ?? null, + active: approver.active ?? null, + }, + approverDecision: item.approverDecision ?? null, + } + }), + createdDate: data.createdDate ?? null, + completedDate: data.completedDate ?? null, + approval: data, + success: true, + }, + } +} + +export async function executeJsmGetParticipants(input: JsmParticipantsBody, signal?: AbortSignal) { + requireAction(input.action, ['get', 'add'] as const, 'get') + validateIssue(input.issueIdOrKey) + const client = await createJsmClient(input, signal) + const query = new URLSearchParams() + append(query, 'start', input.start) + append(query, 'limit', input.limit) + const data = await client.json( + client.service( + `/request/${encodeURIComponent(input.issueIdOrKey)}/participant${query.size ? `?${query}` : ''}` + ), + {}, + signal, + true + ) + return { + success: true, + output: pagedOutput(data, 'participants', { issueIdOrKey: input.issueIdOrKey }), + } +} + +export async function executeJsmAddParticipants(input: JsmParticipantsBody, signal?: AbortSignal) { + requireAction(input.action, ['get', 'add'] as const, 'add') + validateIssue(input.issueIdOrKey) + if (!input.accountIds) throw new JsmOperationError('Account IDs are required', 400) + const client = await createJsmClient(input, signal) + const data = await client.json( + client.service(`/request/${encodeURIComponent(input.issueIdOrKey)}/participant`), + { method: 'POST', body: JSON.stringify({ accountIds: csv(input.accountIds) }) }, + signal, + true + ) + return { + success: true, + output: { + ts: new Date().toISOString(), + issueIdOrKey: input.issueIdOrKey, + participants: data.values || [], + success: true, + }, + } +} + +async function customerClient(input: JsmCustomersBody, signal?: AbortSignal) { + validateId(input.serviceDeskId, 'serviceDeskId') + return createJsmClient(input, signal) +} + +export async function executeJsmGetCustomers(input: JsmCustomersBody, signal?: AbortSignal) { + if (input.emails !== undefined) + throw new JsmOperationError( + 'The `emails` parameter is no longer supported. Use `accountIds` (Atlassian account IDs) instead.', + 400 + ) + const client = await customerClient(input, signal) + const query = new URLSearchParams() + append(query, 'query', input.query) + append(query, 'start', input.start) + append(query, 'limit', input.limit) + const data = await client.json( + client.service( + `${serviceDeskPath(input.serviceDeskId, '/customer')}${query.size ? `?${query}` : ''}` + ), + {}, + signal, + true + ) + return { success: true, output: pagedOutput(data, 'customers') } +} + +export async function executeJsmAddCustomer(input: JsmCustomersBody, signal?: AbortSignal) { + if (input.emails !== undefined) + throw new JsmOperationError( + 'The `emails` parameter is no longer supported. Use `accountIds` (Atlassian account IDs) instead.', + 400 + ) + const accountIds = csv(input.accountIds) + if (!accountIds.length) throw new JsmOperationError('Account IDs are required', 400) + const client = await customerClient(input, signal) + await client.empty( + client.service(serviceDeskPath(input.serviceDeskId, '/customer')), + { method: 'POST', body: JSON.stringify({ accountIds }) }, + signal, + true + ) + return { + success: true, + output: { ts: new Date().toISOString(), serviceDeskId: input.serviceDeskId, success: true }, + } +} + +export async function executeJsmGetOrganizations( + input: JsmOrganizationsBody, + signal?: AbortSignal +) { + validateId(input.serviceDeskId, 'serviceDeskId') + const client = await createJsmClient(input, signal) + const query = new URLSearchParams() + append(query, 'start', input.start) + append(query, 'limit', input.limit) + const data = await client.json( + client.service( + `${serviceDeskPath(input.serviceDeskId, '/organization')}${query.size ? `?${query}` : ''}` + ), + {}, + signal, + true + ) + return { success: true, output: pagedOutput(data, 'organizations') } +} + +export async function executeJsmCreateOrganization( + input: JsmOrganizationBody, + signal?: AbortSignal +) { + requireAction(input.action, ['create', 'add_to_service_desk'] as const, 'create') + if (!input.name) throw new JsmOperationError('Organization name is required', 400) + const client = await createJsmClient(input, signal) + const data = await client.json( + client.service('/organization'), + { method: 'POST', body: JSON.stringify({ name: input.name }) }, + signal, + true + ) + return { + success: true, + output: { + ts: new Date().toISOString(), + organizationId: data.id, + name: data.name, + success: true, + }, + } +} + +export async function executeJsmAddOrganization(input: JsmOrganizationBody, signal?: AbortSignal) { + requireAction(input.action, ['create', 'add_to_service_desk'] as const, 'add_to_service_desk') + if (!input.serviceDeskId) throw new JsmOperationError('Service Desk ID is required', 400) + if (!input.organizationId) throw new JsmOperationError('Organization ID is required', 400) + validateId(input.serviceDeskId, 'serviceDeskId') + validateId(input.organizationId, 'organizationId') + const organizationId = Number.parseInt(input.organizationId.trim(), 10) + if (!Number.isFinite(organizationId) || organizationId <= 0) { + throw new JsmOperationError('organizationId must be a positive integer', 400) + } + const client = await createJsmClient(input, signal) + await client.empty( + client.service(serviceDeskPath(input.serviceDeskId, '/organization')), + { method: 'POST', body: JSON.stringify({ organizationId }) }, + signal, + true + ) + return { + success: true, + output: { + ts: new Date().toISOString(), + serviceDeskId: input.serviceDeskId, + organizationId: input.organizationId, + success: true, + }, + } +} + +interface SelectorConnectionInput { + domain: string + accessToken: string +} + +async function collectSelectorValues( + input: SelectorConnectionInput, + path: string, + signal?: AbortSignal +): Promise[]> { + const client = await createJsmClient(input, signal) + const values: Record[] = [] + let start = 0 + for (let page = 0; page < SELECTOR_MAX_PAGES; page++) { + signal?.throwIfAborted() + const data = await client.json( + client.service(`${path}?start=${start}&limit=${SELECTOR_PAGE_SIZE}`), + {}, + signal + ) + const pageValues = asArray(data.values).map(asObject) + values.push(...pageValues) + const links = asObject(data._links) + if (data.isLastPage === true || !links.next || pageValues.length === 0) return values + start += pageValues.length + } + logger.warn('JSM selector hit pagination cap; list may be incomplete', { + pages: SELECTOR_MAX_PAGES, + collected: values.length, + path, + }) + return values +} + +export async function listJsmServiceDeskOptions( + input: SelectorConnectionInput, + signal?: AbortSignal +) { + const values = await collectSelectorValues(input, '/servicedesk', signal) + return values.map((value) => ({ id: String(value.id), name: String(value.projectName) })) +} + +export async function listJsmRequestTypeOptions( + input: SelectorConnectionInput & { serviceDeskId: string }, + signal?: AbortSignal +) { + validateId(input.serviceDeskId, 'serviceDeskId') + const values = await collectSelectorValues( + input, + serviceDeskPath(input.serviceDeskId, '/requesttype'), + signal + ) + return values.map((value) => ({ id: String(value.id), name: String(value.name) })) +} diff --git a/apps/sim/lib/internal/jupyter/client.test.ts b/apps/sim/lib/internal/jupyter/client.test.ts new file mode 100644 index 00000000000..c88f3694337 --- /dev/null +++ b/apps/sim/lib/internal/jupyter/client.test.ts @@ -0,0 +1,112 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const securityMocks = vi.hoisted(() => ({ + validateUrlWithDNS: vi.fn(), + secureFetchWithPinnedIP: vi.fn(), +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + MAX_JSON_API_RESPONSE_BYTES: 10 * 1024 * 1024, + validateUrlWithDNS: securityMocks.validateUrlWithDNS, + secureFetchWithPinnedIP: securityMocks.secureFetchWithPinnedIP, +})) + +import { InvalidJupyterTargetError, requestJupyterApi } from '@/lib/internal/jupyter/client' + +describe('Jupyter client', () => { + beforeEach(() => { + vi.clearAllMocks() + securityMocks.validateUrlWithDNS.mockResolvedValue({ + isValid: true, + resolvedIP: '192.0.2.10', + }) + securityMocks.secureFetchWithPinnedIP.mockResolvedValue({ ok: true, status: 200 }) + }) + + it('sends one bounded, non-redirecting request with token auth and cancellation', async () => { + const controller = new AbortController() + + await requestJupyterApi( + { + serverUrl: 'jupyter.example.com:8888/base/', + token: 'secret-token', + method: 'POST', + path: 'kernels', + body: { name: 'python3' }, + }, + controller.signal + ) + + expect(securityMocks.validateUrlWithDNS).toHaveBeenCalledWith( + 'http://jupyter.example.com:8888/base/api/kernels', + 'serverUrl', + { allowHttp: true } + ) + expect(securityMocks.secureFetchWithPinnedIP).toHaveBeenCalledWith( + 'http://jupyter.example.com:8888/base/api/kernels', + '192.0.2.10', + { + method: 'POST', + headers: { + Authorization: 'token secret-token', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ name: 'python3' }), + allowHttp: true, + maxRedirects: 0, + maxResponseBytes: 10 * 1024 * 1024, + signal: controller.signal, + } + ) + }) + + it('rejects an invalid DNS-resolved target before starting network work', async () => { + securityMocks.validateUrlWithDNS.mockResolvedValue({ + isValid: false, + error: 'host is blocked', + }) + + await expect( + requestJupyterApi({ + serverUrl: 'blocked.example.com', + token: 'token', + method: 'GET', + path: 'kernels', + }) + ).rejects.toEqual(new InvalidJupyterTargetError('Invalid Jupyter serverUrl: host is blocked')) + expect(securityMocks.secureFetchWithPinnedIP).not.toHaveBeenCalled() + }) + + it('classifies malformed server URLs before starting DNS work', async () => { + await expect( + requestJupyterApi({ + serverUrl: 'http://[invalid', + token: 'token', + method: 'GET', + path: 'kernels', + }) + ).rejects.toEqual(new InvalidJupyterTargetError('Invalid Jupyter server URL: http://[invalid')) + expect(securityMocks.validateUrlWithDNS).not.toHaveBeenCalled() + }) + + it('does not start DNS work after cancellation', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + requestJupyterApi( + { + serverUrl: 'jupyter.example.com', + token: 'token', + method: 'GET', + path: 'sessions', + }, + controller.signal + ) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(securityMocks.validateUrlWithDNS).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/jupyter/client.ts b/apps/sim/lib/internal/jupyter/client.ts new file mode 100644 index 00000000000..b6f88b02358 --- /dev/null +++ b/apps/sim/lib/internal/jupyter/client.ts @@ -0,0 +1,65 @@ +import type { JupyterProxyBody } from '@/lib/api/contracts/tools/jupyter' +import { + MAX_JSON_API_RESPONSE_BYTES, + type SecureFetchResponse, + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { + buildJupyterAuthHeaders, + InvalidJupyterServerUrlError, + normalizeJupyterServerUrl, +} from '@/lib/internal/jupyter/protocol' + +export class InvalidJupyterTargetError extends Error { + constructor(message: string) { + super(message) + this.name = 'InvalidJupyterTargetError' + } +} + +export interface JupyterApiRequest { + serverUrl: string + token: string + method: JupyterProxyBody['method'] + path: string + body?: unknown +} + +/** Sends one bounded, DNS-pinned request to a user-supplied Jupyter server. */ +export async function requestJupyterApi( + input: JupyterApiRequest, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + let base: string + try { + base = normalizeJupyterServerUrl(input.serverUrl) + } catch (error) { + if (error instanceof InvalidJupyterServerUrlError) { + throw new InvalidJupyterTargetError(error.message) + } + throw error + } + const url = `${base}/api/${input.path}` + + const urlValidation = await validateUrlWithDNS(url, 'serverUrl', { allowHttp: true }) + signal?.throwIfAborted() + if (!urlValidation.isValid || !urlValidation.resolvedIP) { + throw new InvalidJupyterTargetError(`Invalid Jupyter serverUrl: ${urlValidation.error}`) + } + + const hasBody = input.body !== undefined && input.body !== null + return secureFetchWithPinnedIP(url, urlValidation.resolvedIP, { + method: input.method, + headers: { + ...buildJupyterAuthHeaders(input.token), + ...(hasBody ? { 'Content-Type': 'application/json' } : {}), + }, + body: hasBody ? JSON.stringify(input.body) : undefined, + allowHttp: true, + maxRedirects: 0, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + signal, + }) +} diff --git a/apps/sim/lib/internal/jupyter/execute-tool.test.ts b/apps/sim/lib/internal/jupyter/execute-tool.test.ts new file mode 100644 index 00000000000..e956ae7e1d5 --- /dev/null +++ b/apps/sim/lib/internal/jupyter/execute-tool.test.ts @@ -0,0 +1,132 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operationMocks = vi.hoisted(() => ({ + executeJupyterProxy: vi.fn(), + executeJupyterUpload: vi.fn(), +})) + +vi.mock('@/lib/internal/jupyter/operations', () => operationMocks) + +import { executeJupyterTool, JUPYTER_PROXY_TOOL_IDS } from '@/lib/internal/jupyter/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const PROXY_BODY = { + serverUrl: 'http://jupyter.example.com:8888', + token: 'token', + method: 'GET' as const, + path: 'kernels', +} + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'jupyter_list_kernels', + input: PROXY_BODY, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeJupyterTool', () => { + beforeEach(() => { + vi.clearAllMocks() + operationMocks.executeJupyterProxy.mockImplementation(async () => + Response.json([{ id: 'kernel-1' }]) + ) + operationMocks.executeJupyterUpload.mockImplementation(async () => + Response.json({ success: true, output: { name: 'file.txt', path: 'file.txt' } }) + ) + }) + + it.each(JUPYTER_PROXY_TOOL_IDS)('recognizes proxy tool ID %s', async (toolId) => { + const response = await executeJupyterTool(createRequest({ toolId })) + + expect(response.status).toBe(200) + expect(operationMocks.executeJupyterProxy).toHaveBeenCalledWith(PROXY_BODY, { + requestId: 'request-1', + signal: undefined, + }) + }) + + it('validates the canonical proxy contract before provider work', async () => { + const response = await executeJupyterTool(createRequest({ input: { ...PROXY_BODY, path: '' } })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(operationMocks.executeJupyterProxy).not.toHaveBeenCalled() + }) + + it('dispatches upload with trusted user context and cancellation', async () => { + const controller = new AbortController() + const input = { + serverUrl: 'http://jupyter.example.com:8888', + token: 'token', + fileContent: Buffer.from('hello').toString('base64'), + fileName: 'hello.txt', + } + + const response = await executeJupyterTool( + createRequest({ + toolId: 'jupyter_upload_file', + input, + signal: controller.signal, + }) + ) + + expect(response.status).toBe(200) + expect(operationMocks.executeJupyterUpload).toHaveBeenCalledWith(input, { + userId: 'user-1', + requestId: 'request-1', + signal: controller.signal, + }) + }) + + it('fails upload closed without a trusted execution user', async () => { + const response = await executeJupyterTool( + createRequest({ + toolId: 'jupyter_upload_file', + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + }, + }) + ) + + expect(response.status).toBe(401) + await expect(response.json()).resolves.toEqual({ success: false, error: 'Unauthorized' }) + expect(operationMocks.executeJupyterUpload).not.toHaveBeenCalled() + }) + + it('propagates cancellation before provider work starts', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeJupyterTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(operationMocks.executeJupyterProxy).not.toHaveBeenCalled() + }) + + it('returns a deterministic error for unsupported IDs', async () => { + const response = await executeJupyterTool(createRequest({ toolId: 'jupyter_unknown' })) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'Unsupported Jupyter tool: jupyter_unknown', + }) + }) +}) diff --git a/apps/sim/lib/internal/jupyter/execute-tool.ts b/apps/sim/lib/internal/jupyter/execute-tool.ts new file mode 100644 index 00000000000..26ba77d704b --- /dev/null +++ b/apps/sim/lib/internal/jupyter/execute-tool.ts @@ -0,0 +1,97 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { AnyApiRouteContract } from '@/lib/api/contracts' +import { jupyterUploadContract } from '@/lib/api/contracts/storage-transfer' +import { jupyterProxyContract } from '@/lib/api/contracts/tools/jupyter' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { executeJupyterProxy, executeJupyterUpload } from '@/lib/internal/jupyter/operations' +import { parseInternalToolInput } from '@/lib/internal/tool-operations/parse-input' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const proxyLogger = createLogger('JupyterProxyAPI') +const uploadLogger = createLogger('JupyterUploadAPI') + +export const JUPYTER_PROXY_TOOL_IDS = [ + 'jupyter_copy_content', + 'jupyter_create_file', + 'jupyter_create_session', + 'jupyter_delete_content', + 'jupyter_delete_session', + 'jupyter_get_content', + 'jupyter_interrupt_kernel', + 'jupyter_list_contents', + 'jupyter_list_kernels', + 'jupyter_list_kernelspecs', + 'jupyter_list_sessions', + 'jupyter_rename_content', + 'jupyter_restart_kernel', + 'jupyter_start_kernel', + 'jupyter_stop_kernel', +] as const + +const JUPYTER_PROXY_TOOL_ID_SET = new Set(JUPYTER_PROXY_TOOL_IDS) + +function parseJupyterBody(contract: C, input: unknown) { + return parseInternalToolInput(contract, input, { + maxInputBytes: DEFAULT_MAX_JSON_BODY_BYTES, + }) +} + +function unexpectedErrorResponse( + scope: 'proxy' | 'upload', + requestId: string, + error: unknown, + signal?: AbortSignal +): Response { + signal?.throwIfAborted() + const logger = scope === 'proxy' ? proxyLogger : uploadLogger + logger.error(`[${requestId}] Unexpected error:`, error) + return Response.json( + { success: false, error: getErrorMessage(error, 'Unknown error') }, + { status: 500 } + ) +} + +/** Executes every Jupyter tool without routing through the application's HTTP listener. */ +export const executeJupyterTool: InternalToolOperationHandler = async ({ + toolId, + input, + context, + requestId, + signal, +}) => { + signal?.throwIfAborted() + + if (JUPYTER_PROXY_TOOL_ID_SET.has(toolId)) { + const parsed = parseJupyterBody(jupyterProxyContract, input) + if (!parsed.success) return parsed.response + try { + const response = await executeJupyterProxy(parsed.data, { requestId, signal }) + signal?.throwIfAborted() + return response + } catch (error) { + return unexpectedErrorResponse('proxy', requestId, error, signal) + } + } + + if (toolId === 'jupyter_upload_file') { + if (!context.userId) { + return Response.json({ success: false, error: 'Unauthorized' }, { status: 401 }) + } + const parsed = parseJupyterBody(jupyterUploadContract, input) + if (!parsed.success) return parsed.response + try { + const response = await executeJupyterUpload(parsed.data, { + userId: context.userId, + requestId, + signal, + }) + signal?.throwIfAborted() + return response + } catch (error) { + return unexpectedErrorResponse('upload', requestId, error, signal) + } + } + + return Response.json({ error: `Unsupported Jupyter tool: ${toolId}` }, { status: 500 }) +} diff --git a/apps/sim/lib/internal/jupyter/file-input.test.ts b/apps/sim/lib/internal/jupyter/file-input.test.ts new file mode 100644 index 00000000000..ebc6eaf4f4d --- /dev/null +++ b/apps/sim/lib/internal/jupyter/file-input.test.ts @@ -0,0 +1,164 @@ +/** + * @vitest-environment node + */ +import { createLogger } from '@sim/logger' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const fileMocks = vi.hoisted(() => ({ + processFilesToUserFiles: vi.fn(), + downloadServableFileFromStorage: vi.fn(), + assertToolFileAccess: vi.fn(), + docNotReadyResponse: vi.fn(), + isPayloadSizeLimitError: vi.fn(), +})) + +vi.mock('@/lib/uploads/shared/types', () => ({ + MAX_BUFFERED_TRANSFER_BYTES: 50 * 1024 * 1024, +})) +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + processFilesToUserFiles: fileMocks.processFilesToUserFiles, +})) +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: fileMocks.downloadServableFileFromStorage, +})) +vi.mock('@/lib/uploads/utils/servable-file-response', () => ({ + docNotReadyResponse: fileMocks.docNotReadyResponse, +})) +vi.mock('@/lib/core/utils/stream-limits', () => ({ + isPayloadSizeLimitError: fileMocks.isPayloadSizeLimitError, +})) +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: fileMocks.assertToolFileAccess, +})) + +import { resolveJupyterUploadFile } from '@/lib/internal/jupyter/file-input' + +const logger = createLogger('JupyterFileInputTest') +const FILE = { + id: 'file-1', + name: 'source.txt', + url: '/api/files/serve/workspace/file-1', + size: 5, + type: 'text/plain', + key: 'workspace-1/file-1', +} + +describe('Jupyter upload file resolution', () => { + beforeEach(() => { + vi.clearAllMocks() + fileMocks.processFilesToUserFiles.mockReturnValue([FILE]) + fileMocks.assertToolFileAccess.mockResolvedValue(null) + fileMocks.downloadServableFileFromStorage.mockResolvedValue({ + buffer: Buffer.from('hello'), + contentType: 'text/plain', + }) + fileMocks.docNotReadyResponse.mockReturnValue(null) + fileMocks.isPayloadSizeLimitError.mockReturnValue(false) + }) + + it('authorizes and resolves protected Sim files under the transfer byte cap', async () => { + const controller = new AbortController() + const result = await resolveJupyterUploadFile( + { + serverUrl: 'http://jupyter.example.com', + token: 'token', + file: FILE, + fileName: 'renamed.txt', + }, + { + userId: 'user-1', + requestId: 'request-1', + logger, + signal: controller.signal, + } + ) + + expect(result).toEqual({ + success: true, + buffer: Buffer.from('hello'), + fileName: 'renamed.txt', + }) + expect(fileMocks.assertToolFileAccess).toHaveBeenCalledWith( + FILE.key, + 'user-1', + 'request-1', + logger + ) + expect(fileMocks.downloadServableFileFromStorage).toHaveBeenCalledWith( + FILE, + 'request-1', + logger, + { + maxBytes: 50 * 1024 * 1024, + signal: controller.signal, + } + ) + }) + + it('returns file authorization denials without downloading bytes', async () => { + const denied = Response.json({ success: false, error: 'Forbidden' }, { status: 403 }) + fileMocks.assertToolFileAccess.mockResolvedValue(denied) + + const result = await resolveJupyterUploadFile( + { + serverUrl: 'http://jupyter.example.com', + token: 'token', + file: FILE, + }, + { userId: 'user-1', requestId: 'request-1', logger } + ) + + expect(result).toEqual({ success: false, response: denied }) + expect(fileMocks.downloadServableFileFromStorage).not.toHaveBeenCalled() + }) + + it('preserves the payload-too-large response for protected files', async () => { + fileMocks.downloadServableFileFromStorage.mockRejectedValue(new Error('file too large')) + fileMocks.isPayloadSizeLimitError.mockReturnValue(true) + + const result = await resolveJupyterUploadFile( + { + serverUrl: 'http://jupyter.example.com', + token: 'token', + file: FILE, + }, + { userId: 'user-1', requestId: 'request-1', logger } + ) + + expect(result.success).toBe(false) + if (result.success) throw new Error('Expected a file resolution error') + expect(result.response.status).toBe(413) + await expect(result.response.json()).resolves.toEqual({ + success: false, + error: 'file too large', + }) + }) + + it('keeps legacy inline base64 support and the missing-file envelope', async () => { + const inline = await resolveJupyterUploadFile( + { + serverUrl: 'http://jupyter.example.com', + token: 'token', + fileContent: Buffer.from('hello').toString('base64'), + }, + { userId: 'user-1', requestId: 'request-1', logger } + ) + expect(inline).toEqual({ + success: true, + buffer: Buffer.from('hello'), + fileName: 'file', + }) + + const missing = await resolveJupyterUploadFile( + { serverUrl: 'http://jupyter.example.com', token: 'token' }, + { userId: 'user-1', requestId: 'request-1', logger } + ) + expect(missing.success).toBe(false) + if (missing.success) throw new Error('Expected a missing-file response') + expect(missing.response.status).toBe(400) + await expect(missing.response.json()).resolves.toEqual({ + success: false, + error: 'File is required', + }) + }) +}) diff --git a/apps/sim/lib/internal/jupyter/file-input.ts b/apps/sim/lib/internal/jupyter/file-input.ts new file mode 100644 index 00000000000..050a12e91bc --- /dev/null +++ b/apps/sim/lib/internal/jupyter/file-input.ts @@ -0,0 +1,82 @@ +import type { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { JupyterUploadBody } from '@/lib/api/contracts/storage-transfer' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +type JupyterLogger = ReturnType + +export type JupyterUploadFileResolution = + | { success: true; buffer: Buffer; fileName: string } + | { success: false; response: Response } + +/** Resolves either a protected Sim file or legacy inline base64 content for upload. */ +export async function resolveJupyterUploadFile( + input: JupyterUploadBody, + context: { + userId: string + requestId: string + logger: JupyterLogger + signal?: AbortSignal + } +): Promise { + const { userId, requestId, logger, signal } = context + signal?.throwIfAborted() + + if (input.file) { + const userFiles = processFilesToUserFiles([input.file as RawFileInput], requestId, logger) + if (userFiles.length === 0) { + return { + success: false, + response: Response.json({ success: false, error: 'Invalid file input' }, { status: 400 }), + } + } + const userFile = userFiles[0] + + const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) + signal?.throwIfAborted() + if (denied) return { success: false, response: denied } + + try { + signal?.throwIfAborted() + const result = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + signal, + }) + signal?.throwIfAborted() + return { + success: true, + buffer: result.buffer, + fileName: input.fileName || userFile.name, + } + } catch (error) { + signal?.throwIfAborted() + const notReady = docNotReadyResponse(error) + if (notReady) return { success: false, response: notReady } + return { + success: false, + response: Response.json( + { success: false, error: getErrorMessage(error, 'Failed to download file') }, + { status: isPayloadSizeLimitError(error) ? 413 : 500 } + ), + } + } + } + + if (input.fileContent) { + return { + success: true, + buffer: Buffer.from(input.fileContent, 'base64'), + fileName: input.fileName || 'file', + } + } + + return { + success: false, + response: Response.json({ success: false, error: 'File is required' }, { status: 400 }), + } +} diff --git a/apps/sim/lib/internal/jupyter/operations.test.ts b/apps/sim/lib/internal/jupyter/operations.test.ts new file mode 100644 index 00000000000..b8a8d33ca7b --- /dev/null +++ b/apps/sim/lib/internal/jupyter/operations.test.ts @@ -0,0 +1,197 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const clientMocks = vi.hoisted(() => { + class InvalidJupyterTargetError extends Error {} + return { + InvalidJupyterTargetError, + requestJupyterApi: vi.fn(), + } +}) +const fileInputMocks = vi.hoisted(() => ({ + resolveJupyterUploadFile: vi.fn(), +})) + +vi.mock('@/lib/internal/jupyter/client', () => clientMocks) +vi.mock('@/lib/internal/jupyter/file-input', () => fileInputMocks) + +import { executeJupyterProxy, executeJupyterUpload } from '@/lib/internal/jupyter/operations' + +function jupyterResponse(options: { + ok?: boolean + status?: number + contentType?: string | null + text?: string + json?: unknown +}) { + return { + ok: options.ok ?? true, + status: options.status ?? 200, + headers: { + get: vi.fn().mockReturnValue(options.contentType ?? null), + }, + text: vi.fn().mockResolvedValue(options.text ?? ''), + json: vi.fn().mockResolvedValue(options.json), + } +} + +describe('Jupyter operations', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('mirrors upstream proxy status, body, and content type exactly', async () => { + const controller = new AbortController() + clientMocks.requestJupyterApi.mockResolvedValue( + jupyterResponse({ + status: 503, + contentType: 'application/problem+json', + text: '{"message":"busy"}', + }) + ) + + const input = { + serverUrl: 'http://jupyter.example.com', + token: 'token', + method: 'GET' as const, + path: 'kernels', + } + const response = await executeJupyterProxy(input, { + requestId: 'request-1', + signal: controller.signal, + }) + + expect(response.status).toBe(503) + expect(response.headers.get('content-type')).toBe('application/problem+json') + await expect(response.text()).resolves.toBe('{"message":"busy"}') + expect(clientMocks.requestJupyterApi).toHaveBeenCalledWith(input, controller.signal) + }) + + it('rejects traversal before contacting Jupyter', async () => { + const response = await executeJupyterProxy( + { + serverUrl: 'http://jupyter.example.com', + token: 'token', + method: 'GET', + path: 'contents/a%2f..%2fsecret', + }, + { requestId: 'request-1' } + ) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Invalid Jupyter path: contents/a%2f..%2fsecret', + }) + expect(clientMocks.requestJupyterApi).not.toHaveBeenCalled() + }) + + it('maps DNS target validation to the compatibility response', async () => { + clientMocks.requestJupyterApi.mockRejectedValue( + new clientMocks.InvalidJupyterTargetError( + 'Invalid Jupyter serverUrl: private target is blocked' + ) + ) + + const response = await executeJupyterProxy( + { + serverUrl: 'http://blocked.example.com', + token: 'token', + method: 'GET', + path: 'kernels', + }, + { requestId: 'request-1' } + ) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Invalid Jupyter serverUrl: private target is blocked', + }) + }) + + it('uploads resolved bytes and preserves the upload response envelope', async () => { + const controller = new AbortController() + fileInputMocks.resolveJupyterUploadFile.mockResolvedValue({ + success: true, + buffer: Buffer.from('hello'), + fileName: 'hello world.txt', + }) + clientMocks.requestJupyterApi.mockResolvedValue( + jupyterResponse({ + json: { + name: 'hello world.txt', + path: 'docs/hello world.txt', + size: 5, + last_modified: '2026-08-27T10:00:00Z', + }, + }) + ) + + const input = { + serverUrl: 'http://jupyter.example.com', + token: 'token', + directory: 'docs/', + fileContent: Buffer.from('ignored').toString('base64'), + } + const response = await executeJupyterUpload(input, { + userId: 'user-1', + requestId: 'request-1', + signal: controller.signal, + }) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: true, + output: { + name: 'hello world.txt', + path: 'docs/hello world.txt', + size: 5, + lastModified: '2026-08-27T10:00:00Z', + }, + }) + expect(clientMocks.requestJupyterApi).toHaveBeenCalledWith( + { + serverUrl: input.serverUrl, + token: input.token, + method: 'PUT', + path: 'contents/docs/hello%20world.txt', + body: { + type: 'file', + format: 'base64', + content: Buffer.from('hello').toString('base64'), + }, + }, + controller.signal + ) + }) + + it('preserves upstream upload failures without retrying them', async () => { + fileInputMocks.resolveJupyterUploadFile.mockResolvedValue({ + success: true, + buffer: Buffer.from('hello'), + fileName: 'hello.txt', + }) + clientMocks.requestJupyterApi.mockResolvedValue( + jupyterResponse({ ok: false, status: 409, text: 'already exists' }) + ) + + const response = await executeJupyterUpload( + { + serverUrl: 'http://jupyter.example.com', + token: 'token', + fileContent: Buffer.from('hello').toString('base64'), + }, + { userId: 'user-1', requestId: 'request-1' } + ) + + expect(response.status).toBe(409) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Jupyter API error: 409 already exists', + }) + expect(clientMocks.requestJupyterApi).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/sim/lib/internal/jupyter/operations.ts b/apps/sim/lib/internal/jupyter/operations.ts new file mode 100644 index 00000000000..a42129cd911 --- /dev/null +++ b/apps/sim/lib/internal/jupyter/operations.ts @@ -0,0 +1,153 @@ +import { createLogger } from '@sim/logger' +import type { JupyterUploadBody } from '@/lib/api/contracts/storage-transfer' +import type { JupyterProxyBody } from '@/lib/api/contracts/tools/jupyter' +import { InvalidJupyterTargetError, requestJupyterApi } from '@/lib/internal/jupyter/client' +import { resolveJupyterUploadFile } from '@/lib/internal/jupyter/file-input' +import { + assertSafeJupyterProxyPath, + encodeJupyterPath, + parseJupyterContentModel, + UnsafeJupyterPathError, +} from '@/lib/internal/jupyter/protocol' + +const uploadLogger = createLogger('JupyterUploadAPI') + +export interface JupyterOperationContext { + requestId: string + signal?: AbortSignal +} + +export interface JupyterUploadOperationContext extends JupyterOperationContext { + userId: string +} + +function validationErrorResponse(error: UnsafeJupyterPathError | InvalidJupyterTargetError) { + return Response.json({ success: false, error: error.message }, { status: 400 }) +} + +/** Executes the shared Jupyter proxy contract and mirrors the upstream response verbatim. */ +export async function executeJupyterProxy( + input: JupyterProxyBody, + context: JupyterOperationContext +): Promise { + context.signal?.throwIfAborted() + try { + assertSafeJupyterProxyPath(input.path) + } catch (error) { + if (error instanceof UnsafeJupyterPathError) return validationErrorResponse(error) + throw error + } + + let upstream + try { + upstream = await requestJupyterApi(input, context.signal) + } catch (error) { + if (error instanceof InvalidJupyterTargetError) return validationErrorResponse(error) + throw error + } + + const text = await upstream.text() + context.signal?.throwIfAborted() + return new Response(text.length > 0 ? text : null, { + status: upstream.status, + headers: { 'Content-Type': upstream.headers.get('content-type') || 'application/json' }, + }) +} + +/** Resolves and uploads a file through Jupyter's Contents API. */ +export async function executeJupyterUpload( + input: JupyterUploadBody, + context: JupyterUploadOperationContext +): Promise { + const { requestId, signal } = context + signal?.throwIfAborted() + + const file = await resolveJupyterUploadFile(input, { + userId: context.userId, + requestId, + logger: uploadLogger, + signal, + }) + if (!file.success) return file.response + + if (/[/\\]/.test(file.fileName)) { + return Response.json( + { success: false, error: 'File name must not contain path separators' }, + { status: 400 } + ) + } + + const destinationDirectory = (input.directory ?? '').replace(/\/+$/, '') + const destinationPath = destinationDirectory + ? `${destinationDirectory}/${file.fileName}` + : file.fileName + + let encodedDestinationPath: string + try { + encodedDestinationPath = encodeJupyterPath(destinationPath) + } catch (error) { + if (error instanceof UnsafeJupyterPathError) return validationErrorResponse(error) + throw error + } + + let response + try { + response = await requestJupyterApi( + { + serverUrl: input.serverUrl, + token: input.token, + method: 'PUT', + path: `contents/${encodedDestinationPath}`, + body: { + type: 'file', + format: 'base64', + content: file.buffer.toString('base64'), + }, + }, + signal + ) + } catch (error) { + if (error instanceof InvalidJupyterTargetError) return validationErrorResponse(error) + throw error + } + + if (!response.ok) { + const errorText = await response.text() + signal?.throwIfAborted() + uploadLogger.error(`[${requestId}] Jupyter API error:`, { + status: response.status, + errorText, + }) + return Response.json( + { success: false, error: `Jupyter API error: ${response.status} ${errorText}` }, + { status: response.status } + ) + } + + const uploadedValue: unknown = await response.json() + signal?.throwIfAborted() + const uploaded = parseJupyterContentModel(uploadedValue) + if (!uploaded) { + uploadLogger.error(`[${requestId}] Jupyter returned an invalid upload response`) + return Response.json( + { success: false, error: 'Jupyter returned an invalid upload response' }, + { status: 502 } + ) + } + + const uploadedName = uploaded.name ?? file.fileName + const uploadedPath = uploaded.path ?? destinationPath + const uploadedSize = uploaded.size ?? file.buffer.length + const lastModified = uploaded.lastModified ?? null + + uploadLogger.info(`[${requestId}] File uploaded to Jupyter: ${uploadedPath}`) + return Response.json({ + success: true, + output: { + name: uploadedName, + path: uploadedPath, + size: uploadedSize, + lastModified, + }, + }) +} diff --git a/apps/sim/lib/internal/jupyter/protocol.test.ts b/apps/sim/lib/internal/jupyter/protocol.test.ts new file mode 100644 index 00000000000..431e8294ca7 --- /dev/null +++ b/apps/sim/lib/internal/jupyter/protocol.test.ts @@ -0,0 +1,77 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + assertSafeJupyterProxyPath, + encodeJupyterPath, + normalizeJupyterServerUrl, + parseJupyterContentModel, + UnsafeJupyterPathError, +} from '@/lib/internal/jupyter/protocol' + +describe('Jupyter protocol', () => { + it('normalizes a Jupyter Contents API model without changing its values', () => { + const content = { cells: [] } + + expect( + parseJupyterContentModel({ + name: 'analysis.ipynb', + path: 'notebooks/analysis.ipynb', + type: 'notebook', + writable: true, + created: '2026-07-09T10:00:00Z', + last_modified: '2026-07-09T11:00:00Z', + size: 42, + mimetype: 'application/x-ipynb+json', + format: 'json', + content, + }) + ).toEqual({ + name: 'analysis.ipynb', + path: 'notebooks/analysis.ipynb', + type: 'notebook', + writable: true, + created: '2026-07-09T10:00:00Z', + lastModified: '2026-07-09T11:00:00Z', + size: 42, + mimetype: 'application/x-ipynb+json', + format: 'json', + content, + }) + }) + + it('rejects non-object models and omits fields with invalid types', () => { + expect(parseJupyterContentModel(null)).toBeNull() + expect( + parseJupyterContentModel({ + name: 42, + path: 'valid/path', + size: '42', + content: null, + }) + ).toEqual({ + path: 'valid/path', + content: null, + }) + }) + + it('preserves base paths while normalizing server URLs', () => { + expect(normalizeJupyterServerUrl('jupyter.internal:8888/base/')).toBe( + 'http://jupyter.internal:8888/base' + ) + }) + + it('encodes contents paths without encoding their separators', () => { + expect(encodeJupyterPath('folder name/analysis #1.ipynb')).toBe( + 'folder%20name/analysis%20%231.ipynb' + ) + }) + + it('rejects literal and encoded traversal in proxy paths', () => { + expect(() => assertSafeJupyterProxyPath('contents/a/../secret')).toThrow(UnsafeJupyterPathError) + expect(() => assertSafeJupyterProxyPath('contents/a%2f..%2fsecret?content=1')).toThrow( + UnsafeJupyterPathError + ) + }) +}) diff --git a/apps/sim/lib/internal/jupyter/protocol.ts b/apps/sim/lib/internal/jupyter/protocol.ts new file mode 100644 index 00000000000..4766256efe2 --- /dev/null +++ b/apps/sim/lib/internal/jupyter/protocol.ts @@ -0,0 +1,169 @@ +import { isPlainRecord } from '@sim/utils/object' + +const PROTOCOL_PATTERN = /^https?:\/\//i + +/** Error thrown when a Jupyter server URL cannot be normalized to an HTTP(S) origin. */ +export class InvalidJupyterServerUrlError extends Error { + constructor(rawUrl: string) { + super(`Invalid Jupyter server URL: ${rawUrl}`) + this.name = 'InvalidJupyterServerUrlError' + } +} + +/** Normalizes a user-supplied Jupyter server URL without changing its base path. */ +export function normalizeJupyterServerUrl(rawUrl: unknown): string { + const raw = typeof rawUrl === 'string' ? rawUrl.trim() : '' + if (!raw) throw new InvalidJupyterServerUrlError(String(rawUrl)) + + const withProtocol = PROTOCOL_PATTERN.test(raw) ? raw : `http://${raw}` + + let parsed: URL + try { + parsed = new URL(withProtocol) + } catch { + throw new InvalidJupyterServerUrlError(raw) + } + + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + throw new InvalidJupyterServerUrlError(raw) + } + + return `${parsed.origin}${parsed.pathname.replace(/\/+$/, '')}` +} + +/** Builds the token authorization header expected by Jupyter Server. */ +export function buildJupyterAuthHeaders(token: string): Record { + return { Authorization: `token ${token}` } +} + +/** Error thrown when a Jupyter path contains a traversal segment. */ +export class UnsafeJupyterPathError extends Error { + constructor(rawPath: string) { + super(`Invalid Jupyter path: ${rawPath}`) + this.name = 'UnsafeJupyterPathError' + } +} + +function assertNoJupyterPathTraversal(path: string | undefined): string[] { + const raw = path ?? '' + + let decoded: string + try { + decoded = decodeURIComponent(raw) + } catch { + decoded = raw + } + + if (decoded.split('/').some((segment) => segment === '.' || segment === '..')) { + throw new UnsafeJupyterPathError(raw) + } + + return raw.split('/').filter((segment) => segment.length > 0) +} + +/** Encodes a Jupyter contents path segment-by-segment while preserving separators. */ +export function encodeJupyterPath(path: string | undefined): string { + return assertNoJupyterPathTraversal(path).map(encodeURIComponent).join('/') +} + +/** Validates a Jupyter contents path that will be sent in a JSON body. */ +export function assertSafeJupyterPath(path: string): string { + assertNoJupyterPathTraversal(path) + return path +} + +/** Validates an encoded relative path beneath Jupyter's `/api/` prefix. */ +export function assertSafeJupyterProxyPath(rawPath: string): void { + const [pathname] = rawPath.split('?') + assertNoJupyterPathTraversal(pathname) +} + +export interface JupyterContentModel { + name?: string + path?: string + type?: 'directory' | 'file' | 'notebook' + writable?: boolean + created?: string + lastModified?: string + size?: number + mimetype?: string + format?: 'json' | 'text' | 'base64' + content?: unknown +} + +/** Parses the shared model returned by Jupyter's Contents API. */ +export function parseJupyterContentModel(value: unknown): JupyterContentModel | null { + if (!isPlainRecord(value)) return null + + const type = + value.type === 'directory' || value.type === 'file' || value.type === 'notebook' + ? value.type + : undefined + const format = + value.format === 'json' || value.format === 'text' || value.format === 'base64' + ? value.format + : undefined + + return { + ...(typeof value.name === 'string' ? { name: value.name } : {}), + ...(typeof value.path === 'string' ? { path: value.path } : {}), + ...(type ? { type } : {}), + ...(typeof value.writable === 'boolean' ? { writable: value.writable } : {}), + ...(typeof value.created === 'string' ? { created: value.created } : {}), + ...(typeof value.last_modified === 'string' ? { lastModified: value.last_modified } : {}), + ...(typeof value.size === 'number' ? { size: value.size } : {}), + ...(typeof value.mimetype === 'string' ? { mimetype: value.mimetype } : {}), + ...(format ? { format } : {}), + ...('content' in value ? { content: value.content } : {}), + } +} + +interface RawJupyterKernel { + id?: string + name?: string + last_activity?: string + execution_state?: string + connections?: number +} + +/** Maps a raw Jupyter kernel model to Sim's tool output shape. */ +export function mapJupyterKernel(raw: RawJupyterKernel): { + id: string + name: string + lastActivity: string | null + executionState: string | null + connections: number | null +} { + return { + id: raw.id ?? '', + name: raw.name ?? '', + lastActivity: raw.last_activity ?? null, + executionState: raw.execution_state ?? null, + connections: raw.connections ?? null, + } +} + +interface RawJupyterSession { + id?: string + path?: string + name?: string + type?: string + kernel?: RawJupyterKernel | null +} + +/** Maps a raw Jupyter session model to Sim's tool output shape. */ +export function mapJupyterSession(raw: RawJupyterSession): { + id: string + path: string + name: string + type: string + kernel: ReturnType | null +} { + return { + id: raw.id ?? '', + path: raw.path ?? '', + name: raw.name ?? '', + type: raw.type ?? '', + kernel: raw.kernel ? mapJupyterKernel(raw.kernel) : null, + } +} diff --git a/apps/sim/lib/internal/knowledge/execute-tool.test.ts b/apps/sim/lib/internal/knowledge/execute-tool.test.ts new file mode 100644 index 00000000000..c8a07941ed5 --- /dev/null +++ b/apps/sim/lib/internal/knowledge/execute-tool.test.ts @@ -0,0 +1,184 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { InvalidInternalDelegationBindingError } from '@/lib/auth/internal-delegation' + +const mocks = vi.hoisted(() => ({ + createExecutorPrincipalFromExecutionContext: vi.fn(), + createChunkOperation: vi.fn(), + createDocumentsOperation: vi.fn(), + deleteChunkOperation: vi.fn(), + deleteDocumentOperation: vi.fn(), + listChunksOperation: vi.fn(), + listConnectorsOperation: vi.fn(), + listDocumentsOperation: vi.fn(), + listTagsOperation: vi.fn(), + readConnectorOperation: vi.fn(), + readDocumentOperation: vi.fn(), + searchOperation: vi.fn(), + syncConnectorOperation: vi.fn(), + updateChunkOperation: vi.fn(), + upsertDocumentOperation: vi.fn(), +})) + +vi.mock('@/lib/internal/principals/executor', () => ({ + createExecutorPrincipalFromExecutionContext: mocks.createExecutorPrincipalFromExecutionContext, +})) + +vi.mock('@/lib/internal/knowledge/operations', () => ({ + createChunkOperation: mocks.createChunkOperation, + createDocumentsOperation: mocks.createDocumentsOperation, + deleteChunkOperation: mocks.deleteChunkOperation, + deleteDocumentOperation: mocks.deleteDocumentOperation, + listChunksOperation: mocks.listChunksOperation, + listConnectorsOperation: mocks.listConnectorsOperation, + listDocumentsOperation: mocks.listDocumentsOperation, + listTagsOperation: mocks.listTagsOperation, + readConnectorOperation: mocks.readConnectorOperation, + readDocumentOperation: mocks.readDocumentOperation, + searchOperation: mocks.searchOperation, + syncConnectorOperation: mocks.syncConnectorOperation, + updateChunkOperation: mocks.updateChunkOperation, + upsertDocumentOperation: mocks.upsertDocumentOperation, +})) + +import { executeKnowledgeTool, KNOWLEDGE_TOOL_IDS } from '@/lib/internal/knowledge/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const principal = { + kind: 'delegated' as const, + serviceId: 'executor' as const, + subjectUserId: 'trusted-user', + workspaceId: 'workspace-1', + delegationId: 'delegation-1', + audience: 'sim:knowledge', + issuedAt: new Date('2026-01-01T00:00:00.000Z'), + expiresAt: new Date('2026-01-01T00:05:00.000Z'), + delegationContext: { kind: 'workflow_execution' as const, workflowId: 'workflow-1' }, +} + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'knowledge_list_tags', + input: { knowledgeBaseId: 'kb-1' }, + headers: new Headers(), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + userId: 'trusted-user', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeKnowledgeTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.createExecutorPrincipalFromExecutionContext.mockResolvedValue(principal) + mocks.listTagsOperation.mockResolvedValue({ + body: { + success: true, + data: [ + { + id: 'tag-1', + tagSlot: 'tag1', + displayName: 'Team', + fieldType: 'text', + createdAt: '2026-01-01T00:00:00.000Z', + updatedAt: '2026-01-01T00:00:00.000Z', + }, + ], + }, + }) + }) + + it('validates operation input and calls the direct operation with trusted scope', async () => { + const controller = new AbortController() + const request = createRequest({ signal: controller.signal }) + + const response = await executeKnowledgeTool(request) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ success: true }) + expect(mocks.createExecutorPrincipalFromExecutionContext).toHaveBeenCalledWith({ + context: request.context, + audience: 'sim:knowledge', + }) + expect(mocks.listTagsOperation).toHaveBeenCalledWith( + 'kb-1', + expect.objectContaining({ + principal, + headers: request.headers, + signal: controller.signal, + }) + ) + }) + + it('rejects malformed operation input before application work', async () => { + const response = await executeKnowledgeTool(createRequest({ input: {} })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Validation error', + details: expect.any(Array), + }) + expect(mocks.listTagsOperation).not.toHaveBeenCalled() + }) + + it('returns canonical validation errors for invalid query input', async () => { + const response = await executeKnowledgeTool( + createRequest({ + toolId: 'knowledge_list_documents', + input: { knowledgeBaseId: 'kb-1', limit: '101' }, + }) + ) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Validation error', + details: expect.any(Array), + }) + expect(mocks.listDocumentsOperation).not.toHaveBeenCalled() + }) + + it('propagates cancellation before principal construction', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeKnowledgeTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mocks.createExecutorPrincipalFromExecutionContext).not.toHaveBeenCalled() + }) + + it('preserves the internal auth error when delegation no longer binds', async () => { + mocks.createExecutorPrincipalFromExecutionContext.mockRejectedValue( + new InvalidInternalDelegationBindingError() + ) + + const response = await executeKnowledgeTool(createRequest()) + + expect(response.status).toBe(401) + await expect(response.json()).resolves.toEqual({ error: 'Authentication required' }) + expect(mocks.listTagsOperation).not.toHaveBeenCalled() + }) + + it('declares the complete canonical tool ID set', () => { + expect(KNOWLEDGE_TOOL_IDS).toHaveLength(14) + expect(new Set(KNOWLEDGE_TOOL_IDS).size).toBe(KNOWLEDGE_TOOL_IDS.length) + }) + + it('returns a deterministic error for unsupported Knowledge tools', async () => { + const response = await executeKnowledgeTool(createRequest({ toolId: 'knowledge_unknown' })) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'Unsupported Knowledge tool: knowledge_unknown', + }) + }) +}) diff --git a/apps/sim/lib/internal/knowledge/execute-tool.ts b/apps/sim/lib/internal/knowledge/execute-tool.ts new file mode 100644 index 00000000000..15f59baeb92 --- /dev/null +++ b/apps/sim/lib/internal/knowledge/execute-tool.ts @@ -0,0 +1,303 @@ +import { createLogger } from '@sim/logger' +import type { AnyApiRouteContract } from '@/lib/api/contracts' +import { + createKnowledgeChunkContract, + createKnowledgeDocumentsContract, + deleteKnowledgeChunkContract, + deleteKnowledgeDocumentContract, + getKnowledgeConnectorContract, + getKnowledgeDocumentContract, + internalKnowledgeSearchContract, + listKnowledgeChunksContract, + listKnowledgeConnectorsContract, + listKnowledgeDocumentsContract, + listTagDefinitionsContract, + triggerKnowledgeConnectorSyncContract, + updateKnowledgeChunkContract, + upsertKnowledgeDocumentContract, +} from '@/lib/api/contracts/knowledge' +import type { JsonErrorResponseDescriptor } from '@/lib/api/server/routes/types' +import { InvalidInternalDelegationBindingError } from '@/lib/auth/internal-delegation' +import { + createChunkOperation, + createDocumentsOperation, + deleteChunkOperation, + deleteDocumentOperation, + type KnowledgeOperationResponse, + listChunksOperation, + listConnectorsOperation, + listDocumentsOperation, + listTagsOperation, + readConnectorOperation, + readDocumentOperation, + searchOperation, + syncConnectorOperation, + updateChunkOperation, + upsertDocumentOperation, +} from '@/lib/internal/knowledge/operations' +import { createExecutorPrincipalFromExecutionContext } from '@/lib/internal/principals/executor' +import { parseInternalContractInput } from '@/lib/internal/tool-operations/parse-contract-input' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' +import { KNOWLEDGE_DELEGATION_AUDIENCE } from '@/lib/knowledge/application/authorization' + +const logger = createLogger('KnowledgeToolExecution') +const MAX_KNOWLEDGE_BODY_BYTES = 2 * 1024 * 1024 + +export const KNOWLEDGE_TOOL_IDS = [ + 'knowledge_create_document', + 'knowledge_delete_chunk', + 'knowledge_delete_document', + 'knowledge_get_connector', + 'knowledge_get_document', + 'knowledge_list_chunks', + 'knowledge_list_connectors', + 'knowledge_list_documents', + 'knowledge_list_tags', + 'knowledge_search', + 'knowledge_trigger_sync', + 'knowledge_update_chunk', + 'knowledge_upload_chunk', + 'knowledge_upsert_document', +] as const + +type KnowledgeErrorPolicy = + (typeof internalKnowledgeErrorPolicies)[keyof typeof internalKnowledgeErrorPolicies] + +function normalizeKnowledgeInput(input: unknown): unknown { + if (typeof input !== 'object' || input === null || Array.isArray(input)) return input + const record = input as Record + return { ...record, id: record.knowledgeBaseId } +} + +function descriptorResponse(descriptor: JsonErrorResponseDescriptor): Response { + return Response.json(descriptor.body, { + status: descriptor.status, + headers: descriptor.headers, + }) +} + +function projectError( + policy: KnowledgeErrorPolicy, + error: unknown, + requestId: string, + signal?: AbortSignal +): Response { + signal?.throwIfAborted() + const projected = policy.project(error) + if (projected) return descriptorResponse(projected) + logger.error(`[${requestId}] Knowledge tool execution failed`, { error }) + return descriptorResponse( + policy.unhandled?.() ?? { status: 500, body: { error: 'Internal server error' } } + ) +} + +function successResponse( + contract: C, + result: KnowledgeOperationResponse +): Response { + if (contract.response.mode !== 'json') { + throw new Error('Knowledge tool contract must return JSON') + } + const validated = contract.response.schema.parse(result.body) as Record + return Response.json({ ...validated, ...result.bodyFields }, { headers: result.headers }) +} + +/** Executes every Knowledge tool through the same authorized application use cases as HTTP. */ +export const executeKnowledgeTool: InternalToolOperationHandler = async (request) => { + const { toolId, signal, requestId } = request + signal?.throwIfAborted() + + let policy: KnowledgeErrorPolicy = internalKnowledgeErrorPolicies.documents + try { + let principal + try { + principal = await createExecutorPrincipalFromExecutionContext({ + context: request.context, + audience: KNOWLEDGE_DELEGATION_AUDIENCE, + }) + } catch (error) { + if ( + error instanceof InvalidInternalDelegationBindingError || + (error instanceof Error && error.message === 'Authentication required') + ) { + return Response.json({ error: 'Authentication required' }, { status: 401 }) + } + throw error + } + signal?.throwIfAborted() + const context = { principal, headers: request.headers, signal } + const input = normalizeKnowledgeInput(request.input) + + switch (toolId) { + case 'knowledge_create_document': { + policy = internalKnowledgeErrorPolicies.uploads + const parsed = parseInternalContractInput(createKnowledgeDocumentsContract, input) + if (!parsed.success) return parsed.response + return successResponse( + createKnowledgeDocumentsContract, + await createDocumentsOperation(parsed.data.params.id, parsed.data.body, context) + ) + } + case 'knowledge_delete_chunk': { + policy = internalKnowledgeErrorPolicies.chunks + const parsed = parseInternalContractInput(deleteKnowledgeChunkContract, input) + if (!parsed.success) return parsed.response + return successResponse( + deleteKnowledgeChunkContract, + await deleteChunkOperation( + parsed.data.params.id, + parsed.data.params.documentId, + parsed.data.params.chunkId, + context + ) + ) + } + case 'knowledge_delete_document': { + policy = internalKnowledgeErrorPolicies.documents + const parsed = parseInternalContractInput(deleteKnowledgeDocumentContract, input) + if (!parsed.success) return parsed.response + return successResponse( + deleteKnowledgeDocumentContract, + await deleteDocumentOperation( + parsed.data.params.id, + parsed.data.params.documentId, + context + ) + ) + } + case 'knowledge_get_connector': { + policy = internalKnowledgeErrorPolicies.connectors + const parsed = parseInternalContractInput(getKnowledgeConnectorContract, input) + if (!parsed.success) return parsed.response + return successResponse( + getKnowledgeConnectorContract, + await readConnectorOperation( + parsed.data.params.id, + parsed.data.params.connectorId, + context + ) + ) + } + case 'knowledge_get_document': { + policy = internalKnowledgeErrorPolicies.documents + const parsed = parseInternalContractInput(getKnowledgeDocumentContract, input) + if (!parsed.success) return parsed.response + return successResponse( + getKnowledgeDocumentContract, + await readDocumentOperation(parsed.data.params.id, parsed.data.params.documentId, context) + ) + } + case 'knowledge_list_chunks': { + policy = internalKnowledgeErrorPolicies.chunkList + const parsed = parseInternalContractInput(listKnowledgeChunksContract, input) + if (!parsed.success) return parsed.response + return successResponse( + listKnowledgeChunksContract, + await listChunksOperation( + parsed.data.params.id, + parsed.data.params.documentId, + parsed.data.query, + context + ) + ) + } + case 'knowledge_list_connectors': { + policy = internalKnowledgeErrorPolicies.connectors + const parsed = parseInternalContractInput(listKnowledgeConnectorsContract, input) + if (!parsed.success) return parsed.response + return successResponse( + listKnowledgeConnectorsContract, + await listConnectorsOperation(parsed.data.params.id, context) + ) + } + case 'knowledge_list_documents': { + policy = internalKnowledgeErrorPolicies.documents + const parsed = parseInternalContractInput(listKnowledgeDocumentsContract, input) + if (!parsed.success) return parsed.response + return successResponse( + listKnowledgeDocumentsContract, + await listDocumentsOperation(parsed.data.params.id, parsed.data.query, context) + ) + } + case 'knowledge_list_tags': { + policy = internalKnowledgeErrorPolicies.tags + const parsed = parseInternalContractInput(listTagDefinitionsContract, input) + if (!parsed.success) return parsed.response + return successResponse( + listTagDefinitionsContract, + await listTagsOperation(parsed.data.params.id, context) + ) + } + case 'knowledge_search': { + policy = internalKnowledgeErrorPolicies.search + const parsed = parseInternalContractInput(internalKnowledgeSearchContract, input, { + maxInputBytes: MAX_KNOWLEDGE_BODY_BYTES, + }) + if (!parsed.success) return parsed.response + return successResponse( + internalKnowledgeSearchContract, + await searchOperation(parsed.data.body, context) + ) + } + case 'knowledge_trigger_sync': { + policy = internalKnowledgeErrorPolicies.connectors + const parsed = parseInternalContractInput(triggerKnowledgeConnectorSyncContract, input) + if (!parsed.success) return parsed.response + return successResponse( + triggerKnowledgeConnectorSyncContract, + await syncConnectorOperation( + parsed.data.params.id, + parsed.data.params.connectorId, + parsed.data.query.rehydrate, + context + ) + ) + } + case 'knowledge_update_chunk': { + policy = internalKnowledgeErrorPolicies.chunks + const parsed = parseInternalContractInput(updateKnowledgeChunkContract, input) + if (!parsed.success) return parsed.response + return successResponse( + updateKnowledgeChunkContract, + await updateChunkOperation( + parsed.data.params.id, + parsed.data.params.documentId, + parsed.data.params.chunkId, + parsed.data.body, + context + ) + ) + } + case 'knowledge_upload_chunk': { + policy = internalKnowledgeErrorPolicies.chunks + const parsed = parseInternalContractInput(createKnowledgeChunkContract, input) + if (!parsed.success) return parsed.response + return successResponse( + createKnowledgeChunkContract, + await createChunkOperation( + parsed.data.params.id, + parsed.data.params.documentId, + parsed.data.body, + context + ) + ) + } + case 'knowledge_upsert_document': { + policy = internalKnowledgeErrorPolicies.upsert + const parsed = parseInternalContractInput(upsertKnowledgeDocumentContract, input, { + maxInputBytes: MAX_KNOWLEDGE_BODY_BYTES, + }) + if (!parsed.success) return parsed.response + return successResponse( + upsertKnowledgeDocumentContract, + await upsertDocumentOperation(parsed.data.params.id, parsed.data.body, context) + ) + } + default: + return Response.json({ error: `Unsupported Knowledge tool: ${toolId}` }, { status: 500 }) + } + } catch (error) { + return projectError(policy, error, requestId, signal) + } +} diff --git a/apps/sim/lib/internal/knowledge/list-tags.ts b/apps/sim/lib/internal/knowledge/list-tags.ts new file mode 100644 index 00000000000..f2db525ceac --- /dev/null +++ b/apps/sim/lib/internal/knowledge/list-tags.ts @@ -0,0 +1,26 @@ +import { createExecutorPrincipalFromExecutionContext } from '@/lib/internal/principals/executor' +import type { InternalToolOperationContext } from '@/lib/internal/tool-operations/types' +import { KNOWLEDGE_DELEGATION_AUDIENCE } from '@/lib/knowledge/application/authorization' +import { listKnowledgeTags } from '@/lib/knowledge/application/tags' + +export interface ListKnowledgeTagsAsExecutorInput { + knowledgeBaseId: string + workspaceId: string + context: InternalToolOperationContext +} + +export async function listKnowledgeTagsAsExecutor({ + knowledgeBaseId, + workspaceId, + context, +}: ListKnowledgeTagsAsExecutorInput) { + const principal = await createExecutorPrincipalFromExecutionContext({ + context, + audience: KNOWLEDGE_DELEGATION_AUDIENCE, + }) + const result = await listKnowledgeTags.execute({ + principal, + input: { knowledgeBaseId, assertedWorkspaceId: workspaceId }, + }) + return result.tagDefinitions +} diff --git a/apps/sim/lib/internal/knowledge/operations.test.ts b/apps/sim/lib/internal/knowledge/operations.test.ts new file mode 100644 index 00000000000..36e1db99a6e --- /dev/null +++ b/apps/sim/lib/internal/knowledge/operations.test.ts @@ -0,0 +1,159 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + requireWorkspaceBillingAttributionHeader: vi.fn(), + listKnowledgeTags: { execute: vi.fn() }, + syncKnowledgeConnector: { execute: vi.fn() }, + connectorSynced: vi.fn(), +})) + +vi.mock('@/lib/billing/core/billing-attribution', () => ({ + requireWorkspaceBillingAttributionHeader: mocks.requireWorkspaceBillingAttributionHeader, +})) + +vi.mock('@/lib/knowledge/api/internal-route', () => ({ + internalKnowledgeProvenanceUserId: (_headers: Headers, principal: { subjectUserId?: string }) => + principal.subjectUserId ?? 'billing-owner', + internalKnowledgeAnalytics: { + connectorSynced: mocks.connectorSynced, + documentDeleted: vi.fn(), + documentUpserted: vi.fn(), + documentsUploaded: vi.fn(), + }, + toInternalKnowledgeChunk: (value: unknown) => value, + toInternalKnowledgeConnector: (value: unknown) => value, + toInternalKnowledgeConnectorDetail: (value: unknown) => value, + toInternalKnowledgeDocument: (value: unknown) => value, + toInternalKnowledgeTag: (value: unknown) => value, +})) + +vi.mock('@/lib/knowledge/api/secret-provenance', () => ({ + finalizeKnowledgePersistedResponse: vi.fn().mockResolvedValue({}), + finalizeKnowledgeProvenanceResponse: vi.fn().mockResolvedValue({}), + finalizeKnowledgeRegistryResponse: vi.fn().mockReturnValue({}), + resolveKnowledgeDocumentWriteSecretProvenance: vi.fn().mockReturnValue({ success: true }), + resolveKnowledgeWriteSecretProvenance: vi.fn().mockReturnValue({ success: true }), +})) + +vi.mock('@/lib/knowledge/application/chunks', () => ({ + createKnowledgeChunk: { execute: vi.fn() }, + deleteKnowledgeChunk: { execute: vi.fn() }, + listKnowledgeChunks: { execute: vi.fn() }, + updateKnowledgeChunk: { execute: vi.fn() }, +})) + +vi.mock('@/lib/knowledge/application/connectors', () => ({ + listKnowledgeConnectors: { execute: vi.fn() }, + readKnowledgeConnector: { execute: vi.fn() }, + syncKnowledgeConnector: mocks.syncKnowledgeConnector, +})) + +vi.mock('@/lib/knowledge/application/documents', () => ({ + createKnowledgeDocuments: { execute: vi.fn() }, + deleteKnowledgeDocument: { execute: vi.fn() }, + listKnowledgeDocuments: { execute: vi.fn() }, + readKnowledgeDocument: { execute: vi.fn() }, + upsertKnowledgeDocument: { execute: vi.fn() }, +})) + +vi.mock('@/lib/knowledge/application/search', () => ({ + searchKnowledge: { execute: vi.fn() }, +})) + +vi.mock('@/lib/knowledge/application/tags', () => ({ + listKnowledgeTags: mocks.listKnowledgeTags, +})) + +vi.mock('@/lib/knowledge/model-input-provenance', () => ({ + prepareKnowledgeModelInputProvenance: vi.fn(), +})) + +vi.mock('@/lib/knowledge/secret-provenance', () => ({ + createKnowledgeDocumentSourceValue: vi.fn(), +})) + +import { + type KnowledgeOperationContext, + listTagsOperation, + syncConnectorOperation, +} from '@/lib/internal/knowledge/operations' + +const principal = { + kind: 'delegated' as const, + serviceId: 'executor' as const, + subjectUserId: 'trusted-user', + workspaceId: 'workspace-1', + delegationId: 'delegation-1', + audience: 'sim:knowledge', + issuedAt: new Date('2026-01-01T00:00:00.000Z'), + expiresAt: new Date('2026-01-01T00:05:00.000Z'), + delegationContext: { kind: 'workflow_execution' as const, workflowId: 'workflow-1' }, +} + +function createContext(): KnowledgeOperationContext { + return { principal, headers: new Headers({ 'x-billing': 'snapshot' }) } +} + +describe('Knowledge direct operations', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('calls the canonical tag use case with principal workspace assertion', async () => { + const tag = { + id: 'tag-1', + tagSlot: 'tag1', + displayName: 'Team', + fieldType: 'text', + createdAt: '2026-01-01T00:00:00.000Z', + updatedAt: '2026-01-01T00:00:00.000Z', + } + mocks.listKnowledgeTags.execute.mockResolvedValue({ tagDefinitions: [tag] }) + const context = createContext() + + const result = await listTagsOperation('kb-1', context) + + expect(mocks.listKnowledgeTags.execute).toHaveBeenCalledWith({ + principal, + input: { knowledgeBaseId: 'kb-1', assertedWorkspaceId: 'workspace-1' }, + request: { headers: context.headers }, + }) + expect(result.body).toEqual({ success: true, data: [tag] }) + }) + + it('restores exact billing attribution before the canonical connector sync use case', async () => { + const attribution = { actorUserId: 'trusted-user', workspaceId: 'workspace-1' } + mocks.requireWorkspaceBillingAttributionHeader.mockReturnValue(attribution) + mocks.syncKnowledgeConnector.execute.mockImplementation(async ({ input }) => { + await expect(input.resolveBillingAttribution('workspace-1')).resolves.toBe(attribution) + return { + knowledgeBaseId: 'kb-1', + workspaceId: 'workspace-1', + connectorId: 'connector-1', + connectorType: 'notion', + } + }) + const context = createContext() + + const result = await syncConnectorOperation('kb-1', 'connector-1', false, context) + + expect(mocks.requireWorkspaceBillingAttributionHeader).toHaveBeenCalledWith(context.headers, { + workspaceId: 'workspace-1', + }) + expect(mocks.syncKnowledgeConnector.execute).toHaveBeenCalledWith({ + principal, + input: expect.objectContaining({ + knowledgeBaseId: 'kb-1', + connectorId: 'connector-1', + assertedWorkspaceId: 'workspace-1', + source: 'ui', + }), + request: { headers: context.headers }, + }) + expect(mocks.connectorSynced).toHaveBeenCalledOnce() + expect(result.body).toEqual({ success: true, message: 'Sync triggered' }) + }) +}) diff --git a/apps/sim/lib/internal/knowledge/operations.ts b/apps/sim/lib/internal/knowledge/operations.ts new file mode 100644 index 00000000000..10d179bdece --- /dev/null +++ b/apps/sim/lib/internal/knowledge/operations.ts @@ -0,0 +1,647 @@ +import type { WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' +import type { z } from 'zod' +import { + type createChunkBodySchema, + type createKnowledgeDocumentsBodySchema, + type listKnowledgeChunksQuerySchema, + type listKnowledgeDocumentsQuerySchema, + parseDocumentTagFiltersParam, + type updateChunkBodySchema, + type upsertDocumentBodySchema, +} from '@/lib/api/contracts/knowledge' +import type { KnowledgeSearchBody } from '@/lib/api/contracts/knowledge/search' +import { AuthType } from '@/lib/auth/hybrid' +import { requireWorkspaceBillingAttributionHeader } from '@/lib/billing/core/billing-attribution' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { + internalKnowledgeAnalytics, + internalKnowledgeProvenanceUserId, + toInternalKnowledgeChunk, + toInternalKnowledgeConnector, + toInternalKnowledgeConnectorDetail, + toInternalKnowledgeDocument, + toInternalKnowledgeTag, +} from '@/lib/knowledge/api/internal-route' +import { + finalizeKnowledgePersistedResponse, + finalizeKnowledgeProvenanceResponse, + finalizeKnowledgeRegistryResponse, + resolveKnowledgeDocumentWriteSecretProvenance, + resolveKnowledgeWriteSecretProvenance, +} from '@/lib/knowledge/api/secret-provenance' +import { + createKnowledgeChunk, + deleteKnowledgeChunk, + listKnowledgeChunks, + updateKnowledgeChunk, +} from '@/lib/knowledge/application/chunks' +import { + listKnowledgeConnectors, + readKnowledgeConnector, + syncKnowledgeConnector, +} from '@/lib/knowledge/application/connectors' +import { + createKnowledgeDocuments, + deleteKnowledgeDocument, + listKnowledgeDocuments, + readKnowledgeDocument, + upsertKnowledgeDocument, +} from '@/lib/knowledge/application/documents' +import { searchKnowledge } from '@/lib/knowledge/application/search' +import { listKnowledgeTags } from '@/lib/knowledge/application/tags' +import { prepareKnowledgeModelInputProvenance } from '@/lib/knowledge/model-input-provenance' +import { createKnowledgeDocumentSourceValue } from '@/lib/knowledge/secret-provenance' + +export interface KnowledgeOperationContext { + principal: WorkflowExecutionDelegatedPrincipal + headers: Headers + signal?: AbortSignal +} + +export interface KnowledgeOperationResponse { + body: Record + headers?: HeadersInit + bodyFields?: Readonly> +} + +type CreateDocumentsBody = z.output +type UpsertDocumentBody = z.output +type CreateChunkBody = z.output +type UpdateChunkBody = z.output +type ListDocumentsQuery = z.output +type ListChunksQuery = z.output + +function throwIfAborted(context: KnowledgeOperationContext): void { + context.signal?.throwIfAborted() +} + +function billingAttribution(context: KnowledgeOperationContext, workspaceId: string) { + return requireWorkspaceBillingAttributionHeader(context.headers, { workspaceId }) +} + +function resolveChunkContentProvenance( + context: KnowledgeOperationContext, + payload: unknown, + workspaceId: string | undefined, + includeContent: boolean +) { + const resolved = resolveKnowledgeWriteSecretProvenance({ + headers: context.headers, + payload, + authType: AuthType.INTERNAL_JWT, + userId: internalKnowledgeProvenanceUserId(context.headers, context.principal, workspaceId), + ...(workspaceId ? { workspaceId } : {}), + selectionKeys: includeContent ? ['chunk-content'] : [], + }) + if (!resolved.success) { + throw new OrchestrationError('validation', 'Invalid knowledge secret provenance') + } + return resolved.provenances?.[0] +} + +export async function listDocumentsOperation( + knowledgeBaseId: string, + query: ListDocumentsQuery, + context: KnowledgeOperationContext +): Promise { + throwIfAborted(context) + let tagFilters + try { + tagFilters = parseDocumentTagFiltersParam(query.tagFilters) + } catch { + throw new OrchestrationError('validation', 'tagFilters must be a valid JSON array') + } + const result = await listKnowledgeDocuments.execute({ + principal: context.principal, + input: { + knowledgeBaseId, + assertedWorkspaceId: context.principal.workspaceId, + enabledFilter: query.enabledFilter, + search: query.search, + limit: query.limit, + offset: query.offset, + sortBy: query.sortBy, + sortOrder: query.sortOrder, + tagFilters, + }, + request: { headers: context.headers }, + }) + throwIfAborted(context) + const body = { + success: true, + data: { + documents: result.documents.map(toInternalKnowledgeDocument), + pagination: result.pagination, + }, + } + const finalization = await finalizeKnowledgePersistedResponse({ + headers: context.headers, + authType: AuthType.INTERNAL_JWT, + userId: internalKnowledgeProvenanceUserId( + context.headers, + context.principal, + result.workspaceId + ), + workspaceId: result.workspaceId, + body, + documents: result.documents.map((document) => ({ + id: document.id, + source: createKnowledgeDocumentSourceValue(document), + value: document, + })), + }) + return { body, ...finalization } +} + +export async function createDocumentsOperation( + knowledgeBaseId: string, + bodyInput: CreateDocumentsBody, + context: KnowledgeOperationContext +): Promise { + throwIfAborted(context) + const documents = bodyInput.bulk ? bodyInput.documents : [bodyInput] + const input = { + knowledgeBaseId, + assertedWorkspaceId: context.principal.workspaceId, + documents, + bulk: bodyInput.bulk, + processingOptions: bodyInput.bulk ? bodyInput.processingOptions : undefined, + resolveBillingAttribution: (workspaceId: string) => + Promise.resolve(billingAttribution(context, workspaceId)), + resolveSecretProvenances: ({ + userId, + workspaceId, + }: { + userId: string + workspaceId?: string + }) => { + const resolved = resolveKnowledgeDocumentWriteSecretProvenance({ + headers: context.headers, + payload: bodyInput, + authType: AuthType.INTERNAL_JWT, + userId, + workspaceId, + documents, + }) + if (!resolved.success) { + throw new OrchestrationError('validation', 'Invalid knowledge secret provenance') + } + return resolved.provenances + }, + source: 'ui' as const, + } + const result = await createKnowledgeDocuments.execute({ + principal: context.principal, + input, + request: { headers: context.headers }, + }) + internalKnowledgeAnalytics.documentsUploaded({ principal: context.principal, input, result }) + throwIfAborted(context) + const body = { + success: true, + data: result.kind === 'bulk' ? result.data : toInternalKnowledgeDocument(result.data), + } + const finalization = await finalizeKnowledgeProvenanceResponse({ + headers: context.headers, + authType: AuthType.INTERNAL_JWT, + userId: result.userId, + workspaceId: result.workspaceId, + provenances: + result.secretProvenances?.flatMap((provenance) => [ + provenance.filename, + ...provenance.tags.map((tag) => tag.provenance), + ]) ?? [], + body, + }) + return { body, ...finalization } +} + +export async function readDocumentOperation( + knowledgeBaseId: string, + documentId: string, + context: KnowledgeOperationContext +): Promise { + throwIfAborted(context) + const result = await readKnowledgeDocument.execute({ + principal: context.principal, + input: { + knowledgeBaseId, + documentId, + assertedWorkspaceId: context.principal.workspaceId, + }, + request: { headers: context.headers }, + }) + throwIfAborted(context) + const body = { success: true, data: toInternalKnowledgeDocument(result.document) } + const finalization = await finalizeKnowledgePersistedResponse({ + headers: context.headers, + authType: AuthType.INTERNAL_JWT, + userId: internalKnowledgeProvenanceUserId( + context.headers, + context.principal, + result.workspaceId + ), + workspaceId: result.workspaceId, + body, + documents: [ + { + id: result.document.id, + source: createKnowledgeDocumentSourceValue(result.document), + value: result.document, + }, + ], + }) + return { body, ...finalization } +} + +export async function deleteDocumentOperation( + knowledgeBaseId: string, + documentId: string, + context: KnowledgeOperationContext +): Promise { + throwIfAborted(context) + const input = { + knowledgeBaseId, + documentId, + assertedWorkspaceId: context.principal.workspaceId, + source: 'ui', + } + const result = await deleteKnowledgeDocument.execute({ + principal: context.principal, + input, + request: { headers: context.headers }, + }) + internalKnowledgeAnalytics.documentDeleted({ principal: context.principal, result }) + throwIfAborted(context) + return { + body: { + success: true, + data: { success: true, message: 'Document deleted successfully' }, + }, + } +} + +export async function upsertDocumentOperation( + knowledgeBaseId: string, + bodyInput: UpsertDocumentBody, + context: KnowledgeOperationContext +): Promise { + throwIfAborted(context) + const input = { + knowledgeBaseId, + assertedWorkspaceId: context.principal.workspaceId, + documentId: bodyInput.documentId, + filename: bodyInput.filename, + fileUrl: bodyInput.fileUrl, + fileSize: bodyInput.fileSize, + mimeType: bodyInput.mimeType, + documentTagsData: bodyInput.documentTagsData, + processingOptions: bodyInput.processingOptions, + resolveBillingAttribution: (workspaceId: string) => + Promise.resolve(billingAttribution(context, workspaceId)), + resolveSecretProvenances: ({ + userId, + workspaceId, + }: { + userId: string + workspaceId?: string + }) => { + const resolved = resolveKnowledgeDocumentWriteSecretProvenance({ + headers: context.headers, + payload: bodyInput, + authType: AuthType.INTERNAL_JWT, + userId, + workspaceId, + documents: [bodyInput], + }) + if (!resolved.success) { + throw new OrchestrationError('validation', 'Invalid knowledge secret provenance') + } + return resolved.provenances + }, + } + const result = await upsertKnowledgeDocument.execute({ + principal: context.principal, + input, + request: { headers: context.headers }, + }) + internalKnowledgeAnalytics.documentUpserted({ principal: context.principal, input, result }) + throwIfAborted(context) + const body = { + success: true, + data: { + documentsCreated: [ + { + documentId: result.document.documentId, + filename: result.document.filename, + status: 'pending' as const, + }, + ], + isUpdate: result.isUpdate, + previousDocumentId: result.previousDocumentId, + processingMethod: 'background' as const, + processingConfig: result.processingConfig, + }, + } + const finalization = await finalizeKnowledgeProvenanceResponse({ + headers: context.headers, + authType: AuthType.INTERNAL_JWT, + userId: result.userId, + workspaceId: result.workspaceId, + body, + provenances: + result.secretProvenances?.flatMap((provenance) => [ + provenance.filename, + ...provenance.tags.map((tag) => tag.provenance), + ]) ?? [], + }) + return { body, ...finalization } +} + +export async function listChunksOperation( + knowledgeBaseId: string, + documentId: string, + query: ListChunksQuery, + context: KnowledgeOperationContext +): Promise { + throwIfAborted(context) + const result = await listKnowledgeChunks.execute({ + principal: context.principal, + input: { + knowledgeBaseId, + documentId, + assertedWorkspaceId: context.principal.workspaceId, + ...query, + }, + request: { headers: context.headers }, + }) + throwIfAborted(context) + const body = { + success: true, + data: result.chunks.map(toInternalKnowledgeChunk), + pagination: result.pagination, + } + const finalization = await finalizeKnowledgePersistedResponse({ + headers: context.headers, + authType: AuthType.INTERNAL_JWT, + userId: internalKnowledgeProvenanceUserId( + context.headers, + context.principal, + result.workspaceId + ), + workspaceId: result.workspaceId, + body, + chunks: result.chunks.map((chunk) => ({ + id: chunk.id, + documentId: result.documentId, + content: chunk.content, + value: chunk, + })), + }) + return { body, ...finalization } +} + +export async function createChunkOperation( + knowledgeBaseId: string, + documentId: string, + bodyInput: CreateChunkBody, + context: KnowledgeOperationContext +): Promise { + throwIfAborted(context) + const result = await createKnowledgeChunk.execute({ + principal: context.principal, + input: { + knowledgeBaseId, + documentId, + assertedWorkspaceId: context.principal.workspaceId, + content: bodyInput.content, + enabled: bodyInput.enabled, + resolveContentProvenance: ({ workspaceId }) => + resolveChunkContentProvenance(context, bodyInput, workspaceId, true), + }, + request: { headers: context.headers }, + }) + throwIfAborted(context) + const body = { success: true, data: toInternalKnowledgeChunk(result.chunk) } + const finalization = await finalizeKnowledgeProvenanceResponse({ + headers: context.headers, + authType: AuthType.INTERNAL_JWT, + userId: result.userId, + workspaceId: result.workspaceId, + body, + provenances: result.provenance ? [result.provenance] : [], + }) + return { body, ...finalization } +} + +export async function updateChunkOperation( + knowledgeBaseId: string, + documentId: string, + chunkId: string, + bodyInput: UpdateChunkBody, + context: KnowledgeOperationContext +): Promise { + throwIfAborted(context) + const result = await updateKnowledgeChunk.execute({ + principal: context.principal, + input: { + knowledgeBaseId, + documentId, + chunkId, + assertedWorkspaceId: context.principal.workspaceId, + content: bodyInput.content, + enabled: bodyInput.enabled, + resolveContentProvenance: ({ workspaceId }) => + resolveChunkContentProvenance( + context, + bodyInput, + workspaceId, + bodyInput.content !== undefined + ), + }, + request: { headers: context.headers }, + }) + throwIfAborted(context) + const body = { success: true, data: toInternalKnowledgeChunk(result.chunk) } + const finalization = await finalizeKnowledgePersistedResponse({ + headers: context.headers, + authType: AuthType.INTERNAL_JWT, + userId: internalKnowledgeProvenanceUserId( + context.headers, + context.principal, + result.workspaceId + ), + workspaceId: result.workspaceId, + body, + chunks: [ + { + id: result.chunk.id, + documentId: result.documentId, + content: result.chunk.content, + value: result.chunk, + }, + ], + }) + return { body, ...finalization } +} + +export async function deleteChunkOperation( + knowledgeBaseId: string, + documentId: string, + chunkId: string, + context: KnowledgeOperationContext +): Promise { + throwIfAborted(context) + await deleteKnowledgeChunk.execute({ + principal: context.principal, + input: { + knowledgeBaseId, + documentId, + chunkId, + assertedWorkspaceId: context.principal.workspaceId, + }, + request: { headers: context.headers }, + }) + throwIfAborted(context) + return { body: { success: true, data: { message: 'Chunk deleted successfully' } } } +} + +export async function listConnectorsOperation( + knowledgeBaseId: string, + context: KnowledgeOperationContext +): Promise { + throwIfAborted(context) + const result = await listKnowledgeConnectors.execute({ + principal: context.principal, + input: { knowledgeBaseId, assertedWorkspaceId: context.principal.workspaceId }, + request: { headers: context.headers }, + }) + throwIfAborted(context) + return { + body: { success: true, data: result.connectors.map(toInternalKnowledgeConnector) }, + } +} + +export async function readConnectorOperation( + knowledgeBaseId: string, + connectorId: string, + context: KnowledgeOperationContext +): Promise { + throwIfAborted(context) + const result = await readKnowledgeConnector.execute({ + principal: context.principal, + input: { + knowledgeBaseId, + connectorId, + assertedWorkspaceId: context.principal.workspaceId, + }, + request: { headers: context.headers }, + }) + throwIfAborted(context) + return { body: { success: true, data: toInternalKnowledgeConnectorDetail(result.connector) } } +} + +export async function syncConnectorOperation( + knowledgeBaseId: string, + connectorId: string, + rehydrate: boolean, + context: KnowledgeOperationContext +): Promise { + throwIfAborted(context) + const input = { + knowledgeBaseId, + connectorId, + assertedWorkspaceId: context.principal.workspaceId, + rehydrate, + resolveBillingAttribution: (workspaceId: string) => + Promise.resolve(billingAttribution(context, workspaceId)), + source: 'ui' as const, + } + const result = await syncKnowledgeConnector.execute({ + principal: context.principal, + input, + request: { headers: context.headers }, + }) + internalKnowledgeAnalytics.connectorSynced({ principal: context.principal, input, result }) + throwIfAborted(context) + return { body: { success: true, message: 'Sync triggered' } } +} + +export async function listTagsOperation( + knowledgeBaseId: string, + context: KnowledgeOperationContext +): Promise { + throwIfAborted(context) + const result = await listKnowledgeTags.execute({ + principal: context.principal, + input: { knowledgeBaseId, assertedWorkspaceId: context.principal.workspaceId }, + request: { headers: context.headers }, + }) + throwIfAborted(context) + return { + body: { success: true, data: result.tagDefinitions.map(toInternalKnowledgeTag) }, + } +} + +export async function searchOperation( + bodyInput: KnowledgeSearchBody & { skipUsageBilling?: boolean }, + context: KnowledgeOperationContext +): Promise { + throwIfAborted(context) + const result = await searchKnowledge.execute({ + principal: context.principal, + input: { + workspaceId: context.principal.workspaceId, + knowledgeBaseIds: Array.isArray(bodyInput.knowledgeBaseIds) + ? bodyInput.knowledgeBaseIds + : [bodyInput.knowledgeBaseIds], + query: bodyInput.query, + topK: bodyInput.topK, + tagFilters: bodyInput.tagFilters, + searchMode: bodyInput.searchMode, + rerankerEnabled: bodyInput.rerankerEnabled, + rerankerModel: bodyInput.rerankerModel, + rerankerInputCount: bodyInput.rerankerInputCount, + rerankerApiKey: bodyInput.rerankerApiKey, + skipUsageBilling: bodyInput.skipUsageBilling, + resolveBillingAttribution: (workspaceId: string) => + Promise.resolve(billingAttribution(context, workspaceId)), + prepareModelInputProvenance: async ({ userId, workspaceId }) => { + const prepared = await prepareKnowledgeModelInputProvenance({ + headers: context.headers, + payload: bodyInput, + isInternalRequest: true, + userId, + workspaceId, + modelInput: bodyInput.query, + }) + if (!prepared.success) throw new OrchestrationError('validation', prepared.error) + return prepared.registry + }, + }, + request: { headers: context.headers }, + }) + throwIfAborted(context) + const body = { + success: true, + data: { + results: result.results.map(({ embeddingId: _embeddingId, ...item }) => item), + query: result.query, + knowledgeBaseIds: result.knowledgeBaseIds, + knowledgeBaseId: result.knowledgeBaseId, + topK: result.topK, + totalResults: result.totalResults, + ...(result.cost ? { cost: result.cost } : {}), + }, + } + if (!result.resultSecretRegistry) { + throw new Error('Internal Knowledge search did not produce a provenance registry') + } + const finalization = finalizeKnowledgeRegistryResponse({ + headers: context.headers, + authType: AuthType.INTERNAL_JWT, + body, + registry: result.resultSecretRegistry, + }) + return { body, ...finalization } +} diff --git a/apps/sim/lib/internal/knowledge/search.ts b/apps/sim/lib/internal/knowledge/search.ts new file mode 100644 index 00000000000..3ac1d3042c3 --- /dev/null +++ b/apps/sim/lib/internal/knowledge/search.ts @@ -0,0 +1,60 @@ +import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' +import { createExecutorPrincipalFromExecutionContext } from '@/lib/internal/principals/executor' +import type { InternalToolOperationContext } from '@/lib/internal/tool-operations/types' +import { KNOWLEDGE_DELEGATION_AUDIENCE } from '@/lib/knowledge/application/authorization' +import { searchKnowledge } from '@/lib/knowledge/application/search' +import type { + ResolvedSecretInputPath, + ResolvedSecretTraceRegistry, +} from '@/executor/utils/resolved-secret-trace-registry' + +export interface SearchKnowledgeAsExecutorInput { + knowledgeBaseIds: string[] + query: string + topK: number + workspaceId: string + context: InternalToolOperationContext + billingAttribution: BillingAttributionSnapshot + resolvedSecretTraceRegistry: ResolvedSecretTraceRegistry + modelInputPaths: readonly ResolvedSecretInputPath[] + signal?: AbortSignal +} + +export async function searchKnowledgeAsExecutor({ + knowledgeBaseIds, + query, + topK, + workspaceId, + context, + billingAttribution, + resolvedSecretTraceRegistry, + modelInputPaths, + signal, +}: SearchKnowledgeAsExecutorInput) { + signal?.throwIfAborted() + const principal = await createExecutorPrincipalFromExecutionContext({ + context, + audience: KNOWLEDGE_DELEGATION_AUDIENCE, + }) + const resultSecretRegistry = resolvedSecretTraceRegistry.forkForInputPaths(modelInputPaths) + if (!resultSecretRegistry.isComplete()) { + throw new Error('Knowledge model input provenance is unavailable') + } + + const result = await searchKnowledge.execute({ + principal, + input: { + workspaceId, + knowledgeBaseIds, + query, + topK, + resolveBillingAttribution: async () => billingAttribution, + resultSecretRegistry, + }, + }) + signal?.throwIfAborted() + if (!result.resultSecretRegistry?.isComplete()) { + throw new Error('Knowledge result secret provenance is unavailable') + } + return { results: result.results, registry: result.resultSecretRegistry } +} diff --git a/apps/sim/lib/internal/latex/errors.ts b/apps/sim/lib/internal/latex/errors.ts new file mode 100644 index 00000000000..78290d91594 --- /dev/null +++ b/apps/sim/lib/internal/latex/errors.ts @@ -0,0 +1,9 @@ +export class LatexOperationError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'LatexOperationError' + } +} diff --git a/apps/sim/lib/internal/latex/execute-tool.test.ts b/apps/sim/lib/internal/latex/execute-tool.test.ts new file mode 100644 index 00000000000..3aeae4c8aaa --- /dev/null +++ b/apps/sim/lib/internal/latex/execute-tool.test.ts @@ -0,0 +1,47 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ compileLatexDocument: vi.fn() })) + +vi.mock('@/lib/internal/latex/operations', () => ({ + compileLatexDocument: mocks.compileLatexDocument, +})) + +import { executeLatexTool } from '@/lib/internal/latex/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +describe('executeLatexTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.compileLatexDocument.mockResolvedValue({ pdfUrl: '/file.pdf' }) + }) + + it('uses trusted execution context instead of tool parameters', async () => { + const controller = new AbortController() + const input = { content: '\\documentclass{article}\\begin{document}x\\end{document}' } + const request: InternalToolOperationCall = { + toolId: 'latex_compile', + input, + headers: new Headers(), + context: { + ...createExecutionContext({ workflowId: 'workflow-1', executionId: 'execution-1' }), + userId: 'user-1', + workspaceId: 'workspace-1', + }, + requestId: 'request-1', + signal: controller.signal, + } + + expect((await executeLatexTool(request)).status).toBe(200) + expect(mocks.compileLatexDocument).toHaveBeenCalledWith(input, { + userId: 'user-1', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + signal: controller.signal, + }) + }) +}) diff --git a/apps/sim/lib/internal/latex/execute-tool.ts b/apps/sim/lib/internal/latex/execute-tool.ts new file mode 100644 index 00000000000..3950a9f6251 --- /dev/null +++ b/apps/sim/lib/internal/latex/execute-tool.ts @@ -0,0 +1,41 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { LatexOperationError } from '@/lib/internal/latex/errors' +import { compileLatexDocument } from '@/lib/internal/latex/operations' +import { latexCompileInputSchema } from '@/lib/internal/latex/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +export const executeLatexTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'latex_compile') { + return Response.json( + { success: false, error: `Unsupported LaTeX tool: ${request.toolId}` }, + { status: 500 } + ) + } + const userId = request.context.userId + if (!userId) return Response.json({ error: 'Unauthorized' }, { status: 401 }) + const parsed = latexCompileInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json({ error: 'Invalid request data' }, { status: 400 }) + } + try { + return Response.json( + await compileLatexDocument(parsed.data, { + userId, + workspaceId: request.context.workspaceId, + workflowId: request.context.workflowId, + executionId: request.context.executionId, + signal: request.signal, + }) + ) + } catch (error) { + request.signal?.throwIfAborted() + const status = isPayloadSizeLimitError(error) + ? 413 + : error instanceof LatexOperationError + ? error.status + : 500 + return Response.json({ error: getErrorMessage(error, 'LaTeX compilation failed') }, { status }) + } +} diff --git a/apps/sim/lib/internal/latex/operations.test.ts b/apps/sim/lib/internal/latex/operations.test.ts new file mode 100644 index 00000000000..bdbad641579 --- /dev/null +++ b/apps/sim/lib/internal/latex/operations.test.ts @@ -0,0 +1,66 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + fetch: vi.fn(), + uploadExecutionFile: vi.fn(), +})) + +vi.mock('@/lib/uploads/contexts/execution', () => ({ + uploadExecutionFile: mocks.uploadExecutionFile, +})) + +vi.mock('@/lib/uploads', () => ({ + StorageService: { uploadFile: vi.fn() }, +})) + +import { compileLatexDocument } from '@/lib/internal/latex/operations' + +describe('compileLatexDocument', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mocks.fetch) + mocks.fetch.mockResolvedValue( + new Response(new Uint8Array([1, 2, 3]), { + headers: { 'content-type': 'application/pdf' }, + }) + ) + mocks.uploadExecutionFile.mockResolvedValue({ id: 'file-1', url: '/file-1' }) + }) + + it('submits once and stores the bounded PDF in execution scope', async () => { + const controller = new AbortController() + const result = await compileLatexDocument( + { + content: '\\documentclass{article}\\begin{document}x\\end{document}', + compiler: 'xelatex', + fileName: '../report.pdf', + }, + { + userId: 'user-1', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + signal: controller.signal, + } + ) + + expect(mocks.fetch).toHaveBeenCalledTimes(1) + expect(mocks.uploadExecutionFile).toHaveBeenCalledWith( + { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }, + expect.any(Buffer), + 'report.pdf', + 'application/pdf', + 'user-1' + ) + expect(result).toEqual( + expect.objectContaining({ pdfUrl: '/file-1', fileName: 'report.pdf', compiler: 'xelatex' }) + ) + }) +}) diff --git a/apps/sim/lib/internal/latex/operations.ts b/apps/sim/lib/internal/latex/operations.ts new file mode 100644 index 00000000000..6baae21f1e3 --- /dev/null +++ b/apps/sim/lib/internal/latex/operations.ts @@ -0,0 +1,149 @@ +import { truncate } from '@sim/utils/string' +import { + readResponseJsonWithLimit, + readResponseToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' +import { getBaseUrl } from '@/lib/core/utils/urls' +import { LatexOperationError } from '@/lib/internal/latex/errors' +import type { LatexCompileInput } from '@/lib/internal/latex/schema' +import { StorageService } from '@/lib/uploads' +import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' + +const LATEX_COMPILE_URL = 'https://latex.ytotech.com/builds/sync' +const MAX_PDF_BYTES = 25 * 1024 * 1024 +const MAX_ERROR_JSON_BYTES = 4 * 1024 * 1024 +const MAX_ERROR_MESSAGE_CHARS = 4000 +const MAX_ERROR_CODE_CHARS = 100 +const COMPILE_TIMEOUT_MS = 50_000 + +export interface LatexOperationContext { + userId: string + workspaceId?: string + workflowId?: string + executionId?: string + signal?: AbortSignal +} + +function buildPdfFileName(fileName: string | undefined): string { + const base = (fileName || 'document').split(/[/\\]/).pop()?.trim() || 'document' + const withoutExtension = base.toLowerCase().endsWith('.pdf') ? base.slice(0, -4) : base + return `${withoutExtension || 'document'}.pdf` +} + +function extractCompilationErrors(logFiles: unknown): string | undefined { + if (typeof logFiles !== 'object' || logFiles === null) return undefined + const snippets: string[] = [] + for (const log of Object.values(logFiles)) { + if (typeof log !== 'string') continue + const lines = log.split('\n') + for (let index = 0; index < lines.length; index++) { + if (lines[index].startsWith('!')) snippets.push(lines.slice(index, index + 3).join('\n')) + } + } + return snippets.length + ? truncate([...new Set(snippets)].join('\n\n'), MAX_ERROR_MESSAGE_CHARS) + : undefined +} + +async function compileError( + response: Response, + signal?: AbortSignal +): Promise { + const body = await readResponseJsonWithLimit(response, { + maxBytes: MAX_ERROR_JSON_BYTES, + label: 'LaTeX compile error response', + signal, + }).catch(() => undefined) + const record = + typeof body === 'object' && body !== null ? (body as Record) : null + const errorCode = + typeof record?.error === 'string' ? truncate(record.error, MAX_ERROR_CODE_CHARS) : undefined + const compilationErrors = extractCompilationErrors(record?.log_files) + const details = compilationErrors ? `:\n${compilationErrors}` : '' + const compilationFailure = + response.status >= 400 && response.status < 500 && Boolean(errorCode || compilationErrors) + return compilationFailure + ? new LatexOperationError( + `LaTeX compilation failed (${errorCode || response.status})${details}`, + 422 + ) + : new LatexOperationError(`LaTeX compile service error: ${response.status}${details}`, 502) +} + +export async function compileLatexDocument( + input: LatexCompileInput, + context: LatexOperationContext +) { + context.signal?.throwIfAborted() + const compiler = input.compiler || 'pdflatex' + const timeoutSignal = AbortSignal.timeout(COMPILE_TIMEOUT_MS) + const signal = context.signal ? AbortSignal.any([context.signal, timeoutSignal]) : timeoutSignal + let response: Response + try { + response = await fetch(LATEX_COMPILE_URL, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + compiler, + resources: [{ main: true, content: input.content }, ...(input.resources ?? [])], + }), + signal, + }) + } catch (error) { + context.signal?.throwIfAborted() + if (timeoutSignal.aborted) { + throw new LatexOperationError('LaTeX compile service timed out', 504) + } + throw error + } + + const contentType = response.headers.get('content-type') || '' + if (!response.ok || !contentType.includes('application/pdf')) { + throw await compileError(response, context.signal) + } + const pdf = await readResponseToBufferWithLimit(response, { + maxBytes: MAX_PDF_BYTES, + label: 'compiled PDF', + signal: context.signal, + }) + if (!pdf.length) { + throw new LatexOperationError('LaTeX compile service returned an empty PDF', 502) + } + context.signal?.throwIfAborted() + const fileName = buildPdfFileName(input.fileName) + if (context.workspaceId && context.workflowId && context.executionId) { + const pdfFile = await uploadExecutionFile( + { + workspaceId: context.workspaceId, + workflowId: context.workflowId, + executionId: context.executionId, + }, + pdf, + fileName, + 'application/pdf', + context.userId + ) + context.signal?.throwIfAborted() + return { + pdfFile, + pdfUrl: pdfFile.url, + fileName, + contentType: 'application/pdf', + compiler, + } + } + + const fileInfo = await StorageService.uploadFile({ + file: pdf, + fileName, + contentType: 'application/pdf', + context: 'copilot', + }) + context.signal?.throwIfAborted() + return { + pdfUrl: `${getBaseUrl()}${fileInfo.path}`, + fileName, + contentType: 'application/pdf', + compiler, + } +} diff --git a/apps/sim/lib/internal/latex/schema.ts b/apps/sim/lib/internal/latex/schema.ts new file mode 100644 index 00000000000..1ea971dec14 --- /dev/null +++ b/apps/sim/lib/internal/latex/schema.ts @@ -0,0 +1,71 @@ +import { z } from 'zod' + +const MAX_LATEX_SOURCE_CHARS = 1_000_000 +const MAX_LATEX_RESOURCES = 25 + +export const latexCompilers = [ + 'pdflatex', + 'xelatex', + 'lualatex', + 'platex', + 'uplatex', + 'context', +] as const + +const latexResourceSchema = z + .object({ + path: z + .string() + .min(1, 'resource path cannot be empty') + .max(512, 'resource path must be at most 512 characters') + .refine( + (path) => !path.startsWith('/') && path.split(/[/\\]/).every((segment) => segment !== '..'), + 'resource path must be relative and must not contain ".." segments' + ), + content: z + .string() + .min(1, 'resource content cannot be empty') + .max(MAX_LATEX_SOURCE_CHARS, 'resource content must be at most 1,000,000 characters') + .optional(), + file: z + .string() + .min(1, 'resource file cannot be empty') + .max(MAX_LATEX_SOURCE_CHARS, 'resource file must be at most 1,000,000 characters of base64') + .optional(), + url: z + .string() + .url('resource url must be a valid URL') + .max(2048, 'resource url must be at most 2048 characters') + .refine( + (url) => url.startsWith('https://') || url.startsWith('http://'), + 'resource url must use http or https' + ) + .optional(), + }) + .superRefine((resource, ctx) => { + const count = [resource.content, resource.file, resource.url].filter( + (value) => value !== undefined + ).length + if (count !== 1) { + ctx.addIssue({ + code: 'custom', + path: ['path'], + message: `resource "${resource.path}" must provide exactly one of content, file, or url`, + }) + } + }) + +export const latexCompileInputSchema = z.object({ + content: z + .string() + .min(1, 'content cannot be empty') + .max(MAX_LATEX_SOURCE_CHARS, 'content must be at most 1,000,000 characters'), + compiler: z.enum(latexCompilers).optional(), + fileName: z.string().max(255, 'fileName must be at most 255 characters').optional(), + resources: z + .array(latexResourceSchema) + .max(MAX_LATEX_RESOURCES, `resources must contain at most ${MAX_LATEX_RESOURCES} entries`) + .optional(), +}) + +export type LatexCompileInput = z.output diff --git a/apps/sim/lib/internal/linq/client.ts b/apps/sim/lib/internal/linq/client.ts new file mode 100644 index 00000000000..52dc3359407 --- /dev/null +++ b/apps/sim/lib/internal/linq/client.ts @@ -0,0 +1,108 @@ +import { isRecordLike } from '@sim/utils/object' +import { + MAX_JSON_API_RESPONSE_BYTES, + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + readResponseJsonWithLimit, + readResponseTextWithLimit, +} from '@/lib/core/utils/stream-limits' +import { LinqOperationError } from '@/lib/internal/linq/errors' +import { extractLinqError, LINQ_API_BASE, linqHeaders } from '@/tools/linq/utils' + +export interface RegisteredLinqAttachment { + attachmentId: string + downloadUrl: string | null + httpMethod: string + requiredHeaders: Record + uploadUrl: string +} + +function stringHeaders(value: unknown, fallback: Record): Record { + if (!isRecordLike(value)) return fallback + const entries = Object.entries(value).filter( + (entry): entry is [string, string] => typeof entry[1] === 'string' + ) + return Object.fromEntries(entries) +} + +export async function registerLinqAttachment( + input: { + apiKey: string + contentType: string + filename: string + sizeBytes: number + }, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const response = await fetch(`${LINQ_API_BASE}/attachments`, { + method: 'POST', + headers: linqHeaders(input.apiKey), + body: JSON.stringify({ + filename: input.filename, + content_type: input.contentType, + size_bytes: input.sizeBytes, + }), + signal, + }) + const data = await readResponseJsonWithLimit(response, { + maxBytes: MAX_JSON_API_RESPONSE_BYTES, + label: 'Linq attachment registration response', + signal, + }).catch(() => null) + if (!response.ok) { + throw new LinqOperationError( + extractLinqError(data, 'Failed to register attachment'), + response.status + ) + } + if (!isRecordLike(data)) { + throw new LinqOperationError('Linq did not return an upload URL or attachment ID', 502) + } + const uploadUrl = typeof data.upload_url === 'string' ? data.upload_url : '' + const attachmentId = typeof data.attachment_id === 'string' ? data.attachment_id : '' + if (!uploadUrl || !attachmentId) { + throw new LinqOperationError('Linq did not return an upload URL or attachment ID', 502) + } + const fallbackHeaders = { + 'Content-Type': input.contentType, + 'Content-Length': String(input.sizeBytes), + } + return { + attachmentId, + downloadUrl: typeof data.download_url === 'string' ? data.download_url : null, + httpMethod: typeof data.http_method === 'string' ? data.http_method : 'PUT', + requiredHeaders: stringHeaders(data.required_headers, fallbackHeaders), + uploadUrl, + } +} + +export async function uploadLinqAttachmentBytes( + registration: RegisteredLinqAttachment, + buffer: Buffer, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const validation = await validateUrlWithDNS(registration.uploadUrl, 'uploadUrl') + signal?.throwIfAborted() + if (!validation.isValid || !validation.resolvedIP) { + throw new LinqOperationError(validation.error || 'Invalid Linq upload URL', 400) + } + const response = await secureFetchWithPinnedIP(registration.uploadUrl, validation.resolvedIP, { + method: registration.httpMethod, + headers: registration.requiredHeaders, + body: new Uint8Array(buffer), + maxResponseBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + signal, + }) + if (response.ok) return + await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Linq presigned upload error response', + signal, + }).catch(() => '') + throw new LinqOperationError(`Failed to upload file bytes to Linq (${response.status})`, 502) +} diff --git a/apps/sim/lib/internal/linq/errors.ts b/apps/sim/lib/internal/linq/errors.ts new file mode 100644 index 00000000000..62f4dabd00b --- /dev/null +++ b/apps/sim/lib/internal/linq/errors.ts @@ -0,0 +1,10 @@ +export class LinqOperationError extends Error { + constructor( + message: string, + readonly status: number, + readonly body: Record = { success: false, error: message } + ) { + super(message) + this.name = 'LinqOperationError' + } +} diff --git a/apps/sim/lib/internal/linq/execute-tool.ts b/apps/sim/lib/internal/linq/execute-tool.ts new file mode 100644 index 00000000000..d3738fab215 --- /dev/null +++ b/apps/sim/lib/internal/linq/execute-tool.ts @@ -0,0 +1,68 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { LinqOperationError } from '@/lib/internal/linq/errors' +import { executeLinqCreateAttachment } from '@/lib/internal/linq/operations' +import { linqCreateAttachmentInputSchema } from '@/lib/internal/linq/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const logger = createLogger('LinqToolExecution') + +function inputSizeError(input: unknown): Response | null { + let serialized: string + try { + serialized = JSON.stringify(input) ?? '' + } catch { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + return Buffer.byteLength(serialized) > DEFAULT_MAX_JSON_BODY_BYTES + ? Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + : null +} + +export const executeLinqTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'linq_create_attachment') { + return Response.json( + { success: false, error: `Unsupported Linq tool: ${request.toolId}` }, + { status: 500 } + ) + } + const userId = request.context.userId + if (!userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + const sizeError = inputSizeError(request.input) + if (sizeError) return sizeError + const parsed = linqCreateAttachmentInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { success: false, error: getValidationErrorMessage(parsed.error, 'Invalid request data') }, + { status: 400 } + ) + } + try { + const result = await executeLinqCreateAttachment(parsed.data, { + headers: request.headers, + requestId: request.requestId, + signal: request.signal, + userId, + }) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof LinqOperationError) { + return Response.json(error.body, { status: error.status }) + } + const message = getErrorMessage(error, 'Unknown error occurred') + logger.error('Linq attachment upload failed', { error: message, requestId: request.requestId }) + return Response.json({ success: false, error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/linq/operations.ts b/apps/sim/lib/internal/linq/operations.ts new file mode 100644 index 00000000000..7344fbb92b3 --- /dev/null +++ b/apps/sim/lib/internal/linq/operations.ts @@ -0,0 +1,134 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { isRecordLike } from '@sim/utils/object' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' +import { registerLinqAttachment, uploadLinqAttachmentBytes } from '@/lib/internal/linq/client' +import { LinqOperationError } from '@/lib/internal/linq/errors' +import type { LinqCreateAttachmentInput } from '@/lib/internal/linq/schema' +import { + isModelSafeWorkspaceFileKey, + MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, +} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/doc-not-ready' +import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +const logger = createLogger('LinqOperations') +const MAX_SIZE_BYTES = 100 * 1024 * 1024 + +export interface LinqOperationContext { + headers: Headers + requestId: string + signal?: AbortSignal + userId: string +} + +async function deniedBody(response: Response): Promise> { + const body: unknown = await response.json().catch(() => null) + return isRecordLike(body) ? body : { success: false, error: 'File not found' } +} + +function fileTooLargeError(sizeBytes: number): LinqOperationError { + return new LinqOperationError( + `File exceeds Linq's 100MB attachment limit (${(sizeBytes / (1024 * 1024)).toFixed(2)}MB)`, + 400 + ) +} + +function validateProvenance(input: LinqCreateAttachmentInput, context: LinqOperationContext): void { + const provenance = validateOpaqueModelInputProvenance({ + headers: context.headers, + payload: input, + isInternalRequest: true, + }) + if (!provenance.success) { + throw new LinqOperationError(provenance.error, provenance.status) + } +} + +export async function executeLinqCreateAttachment( + input: LinqCreateAttachmentInput, + context: LinqOperationContext +) { + context.signal?.throwIfAborted() + validateProvenance(input, context) + let buffer: Buffer + let filename = input.filename ?? '' + let contentType = input.contentType ?? '' + + if (input.file) { + const userFile = processFilesToUserFiles( + [input.file as RawFileInput], + context.requestId, + logger + )[0] + if (!userFile) throw new LinqOperationError('No valid file provided', 400) + const denied = await assertToolFileAccess( + userFile.key, + context.userId, + context.requestId, + logger + ) + context.signal?.throwIfAborted() + if (denied) + throw new LinqOperationError('File not found', denied.status, await deniedBody(denied)) + if (!(await isModelSafeWorkspaceFileKey(userFile.key))) { + throw new LinqOperationError(MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, 400) + } + context.signal?.throwIfAborted() + try { + const downloaded = await downloadServableFileFromStorage( + userFile, + context.requestId, + logger, + { maxBytes: MAX_SIZE_BYTES, signal: context.signal } + ) + buffer = downloaded.buffer + if (!filename) filename = userFile.name + if (!contentType) { + contentType = downloaded.contentType || userFile.type || 'application/octet-stream' + } + } catch (error) { + context.signal?.throwIfAborted() + if (isDocNotReadyError(error)) throw new LinqOperationError(docNotReadyMessage(), 409) + if (isPayloadSizeLimitError(error)) { + throw fileTooLargeError(error.observedBytes ?? userFile.size) + } + throw new LinqOperationError(getErrorMessage(error, 'Unknown error occurred'), 500) + } + } else if (input.fileContent) { + buffer = Buffer.from(input.fileContent, 'base64') + if (!filename) filename = 'file' + if (!contentType) contentType = 'application/octet-stream' + } else { + throw new LinqOperationError('A file is required to upload an attachment', 400) + } + + if (buffer.length === 0) throw new LinqOperationError('File is empty', 400) + if (buffer.length > MAX_SIZE_BYTES) throw fileTooLargeError(buffer.length) + const registration = await registerLinqAttachment( + { + apiKey: input.apiKey, + contentType, + filename, + sizeBytes: buffer.length, + }, + context.signal + ) + await uploadLinqAttachmentBytes(registration, buffer, context.signal) + context.signal?.throwIfAborted() + return { + success: true, + output: { + attachmentId: registration.attachmentId, + downloadUrl: registration.downloadUrl, + filename, + contentType, + sizeBytes: buffer.length, + status: 'complete', + }, + } +} diff --git a/apps/sim/lib/internal/linq/schema.ts b/apps/sim/lib/internal/linq/schema.ts new file mode 100644 index 00000000000..29af68b0fc2 --- /dev/null +++ b/apps/sim/lib/internal/linq/schema.ts @@ -0,0 +1,15 @@ +import { z } from 'zod' +import { resolvedSecretTraceProvenanceSchema } from '@/lib/api/contracts/primitives' +import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' +import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' + +export const linqCreateAttachmentInputSchema = z.object({ + apiKey: z.string().min(1, 'API key is required'), + file: FileInputSchema.optional().nullable(), + fileContent: z.string().optional().nullable(), + filename: z.string().min(1).max(1024).optional().nullable(), + contentType: z.string().min(1).max(255).optional().nullable(), + [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), +}) + +export type LinqCreateAttachmentInput = z.output diff --git a/apps/sim/lib/internal/llm/credentials.ts b/apps/sim/lib/internal/llm/credentials.ts new file mode 100644 index 00000000000..2dbd6407e9a --- /dev/null +++ b/apps/sim/lib/internal/llm/credentials.ts @@ -0,0 +1,41 @@ +import { db } from '@sim/db' +import { account } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { eq } from 'drizzle-orm' +import { + getServiceAccountToken, + refreshTokenIfNeeded, + resolveOAuthAccountId, +} from '@/lib/oauth/credential-service' + +const logger = createLogger('LlmCredentials') + +/** Resolves an already-authorized Vertex credential into a provider access token. */ +export async function resolveVertexAccessToken( + requestId: string, + credentialId: string +): Promise { + logger.info(`[${requestId}] Resolving Vertex AI credential`, { credentialId }) + + const resolved = await resolveOAuthAccountId(credentialId) + if (!resolved) throw new Error(`Vertex AI credential not found: ${credentialId}`) + + if (resolved.credentialType === 'service_account' && resolved.credentialId) { + const accessToken = await getServiceAccountToken(resolved.credentialId, [ + 'https://www.googleapis.com/auth/cloud-platform', + ]) + logger.info(`[${requestId}] Resolved Vertex AI service account credential`) + return accessToken + } + + const credential = await db.query.account.findFirst({ + where: eq(account.id, resolved.accountId), + }) + if (!credential) throw new Error(`Vertex AI credential not found: ${credentialId}`) + + const { accessToken } = await refreshTokenIfNeeded(requestId, credential, resolved.accountId) + if (!accessToken) throw new Error('Failed to get Vertex AI access token') + + logger.info(`[${requestId}] Resolved Vertex AI credential`) + return accessToken +} diff --git a/apps/sim/lib/internal/llm/errors.ts b/apps/sim/lib/internal/llm/errors.ts new file mode 100644 index 00000000000..96a9c5daf4b --- /dev/null +++ b/apps/sim/lib/internal/llm/errors.ts @@ -0,0 +1,9 @@ +export class LlmOperationError extends Error { + constructor( + readonly status: number, + readonly body: { error: string } + ) { + super(body.error) + this.name = 'LlmOperationError' + } +} diff --git a/apps/sim/lib/internal/llm/execute-tool.test.ts b/apps/sim/lib/internal/llm/execute-tool.test.ts new file mode 100644 index 00000000000..f46f4c5e68a --- /dev/null +++ b/apps/sim/lib/internal/llm/execute-tool.test.ts @@ -0,0 +1,91 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const executeOperation = vi.hoisted(() => vi.fn()) + +vi.mock('@/lib/internal/llm/operations', () => ({ + executeLlmProviderOperation: executeOperation, +})) + +import { LlmOperationError } from '@/lib/internal/llm/errors' +import { executeLlmTool } from '@/lib/internal/llm/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +function request(overrides: Partial = {}): InternalToolOperationCall { + return { + toolId: 'llm_chat', + input: { + provider: 'openai', + model: 'gpt-4o', + context: '[{"role":"user","content":"hello"}]', + workspaceId: 'untrusted-workspace', + workflowId: 'untrusted-workflow', + }, + headers: new Headers({ 'x-sim-billing-attribution': 'attribution' }), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + userId: 'user-1', + workspaceId: 'workspace-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeLlmTool', () => { + beforeEach(() => { + vi.clearAllMocks() + executeOperation.mockResolvedValue({ content: 'hello', model: 'gpt-4o' }) + }) + + it('binds provider execution to trusted workflow and workspace scope', async () => { + const controller = new AbortController() + const executionRequest = request({ signal: controller.signal }) + const response = await executeLlmTool(executionRequest) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ content: 'hello', model: 'gpt-4o' }) + expect(executeOperation).toHaveBeenCalledWith( + expect.objectContaining({ + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + stream: false, + }), + expect.objectContaining({ + actorUserId: 'user-1', + headers: executionRequest.headers, + signal: controller.signal, + }) + ) + }) + + it('preserves classified provider errors', async () => { + executeOperation.mockRejectedValueOnce(new LlmOperationError(403, { error: 'Forbidden' })) + + const response = await executeLlmTool(request()) + + expect(response.status).toBe(403) + await expect(response.json()).resolves.toEqual({ error: 'Forbidden' }) + }) + + it('propagates cancellation before and after provider work', async () => { + const before = new AbortController() + before.abort(new DOMException('cancelled', 'AbortError')) + await expect(executeLlmTool(request({ signal: before.signal }))).rejects.toMatchObject({ + name: 'AbortError', + }) + expect(executeOperation).not.toHaveBeenCalled() + + const after = new AbortController() + executeOperation.mockImplementationOnce(async () => { + after.abort(new DOMException('cancelled', 'AbortError')) + return { content: 'unused', model: 'gpt-4o' } + }) + await expect(executeLlmTool(request({ signal: after.signal }))).rejects.toMatchObject({ + name: 'AbortError', + }) + }) +}) diff --git a/apps/sim/lib/internal/llm/execute-tool.ts b/apps/sim/lib/internal/llm/execute-tool.ts new file mode 100644 index 00000000000..4f42bfad539 --- /dev/null +++ b/apps/sim/lib/internal/llm/execute-tool.ts @@ -0,0 +1,73 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { isPlainRecord } from '@sim/utils/object' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { LlmOperationError } from '@/lib/internal/llm/errors' +import { llmProviderOperationInputSchema } from '@/lib/internal/llm/input' +import { executeLlmProviderOperation } from '@/lib/internal/llm/operations' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const logger = createLogger('LlmToolExecution') + +function invalidBodyResponse(): Response { + return Response.json({ error: 'Invalid request body' }, { status: 400 }) +} + +export const executeLlmTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'llm_chat') { + return Response.json( + { success: false, error: `Unsupported LLM tool: ${request.toolId}` }, + { status: 500 } + ) + } + if (!request.context.userId || !request.context.workspaceId) { + return Response.json({ error: 'Unauthorized' }, { status: 401 }) + } + if (!isPlainRecord(request.input)) return invalidBodyResponse() + + let serializedInput: string + try { + serializedInput = JSON.stringify(request.input) + } catch { + return invalidBodyResponse() + } + if (Buffer.byteLength(serializedInput, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + + const parsed = llmProviderOperationInputSchema.safeParse({ + ...request.input, + workspaceId: request.context.workspaceId, + workflowId: request.context.workflowId, + stream: false, + }) + if (!parsed.success) return invalidBodyResponse() + + try { + const result = await executeLlmProviderOperation(parsed.data, { + actorUserId: request.context.userId, + headers: request.headers, + requestId: request.requestId, + signal: request.signal, + }) + request.signal?.throwIfAborted() + if (result instanceof ReadableStream || ('stream' in result && 'execution' in result)) { + throw new Error('LLM chat operation returned an unexpected streaming response') + } + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof LlmOperationError) { + return Response.json(error.body, { status: error.status }) + } + const message = getErrorMessage(error) + logger.error(`[${request.requestId}] LLM chat operation failed`, { error: message }) + return Response.json({ error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/llm/input.ts b/apps/sim/lib/internal/llm/input.ts new file mode 100644 index 00000000000..926db4af854 --- /dev/null +++ b/apps/sim/lib/internal/llm/input.ts @@ -0,0 +1,90 @@ +import { z } from 'zod' +import { resolvedSecretTraceProvenanceSchema } from '@/lib/api/contracts/primitives' +import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' + +const providerToolSchema = z + .object({ + id: z.string(), + name: z.string(), + description: z.string(), + params: z.record(z.string(), z.unknown()), + parameters: z + .object({ + type: z.string(), + properties: z.record(z.string(), z.unknown()), + required: z.array(z.string()), + }) + .passthrough(), + usageControl: z.enum(['auto', 'force', 'none']).optional(), + }) + .passthrough() + +const providerMessageSchema = z + .object({ + role: z.enum(['system', 'user', 'assistant', 'function', 'tool']), + content: z.string().nullable(), + name: z.string().optional(), + function_call: z + .object({ + name: z.string(), + arguments: z.string(), + }) + .optional(), + tool_calls: z + .array( + z.object({ + id: z.string(), + type: z.literal('function'), + function: z.object({ + name: z.string(), + arguments: z.string(), + }), + }) + ) + .optional(), + tool_call_id: z.string().optional(), + }) + .passthrough() + +const providerResponseFormatSchema = z + .object({ + name: z.string(), + schema: z.unknown(), + strict: z.boolean().optional(), + }) + .passthrough() + +export const llmProviderOperationInputSchema = z + .object({ + provider: z.string().min(1), + model: z.string().min(1), + systemPrompt: z.string().optional(), + context: z.string().optional(), + tools: z.array(providerToolSchema).optional(), + temperature: z.number().optional(), + maxTokens: z.number().optional(), + apiKey: z.string().optional(), + azureEndpoint: z.string().optional(), + azureApiVersion: z.string().optional(), + vertexProject: z.string().optional(), + vertexLocation: z.string().optional(), + vertexCredential: z.string().optional(), + bedrockAccessKeyId: z.string().optional(), + bedrockSecretKey: z.string().optional(), + bedrockRegion: z.string().optional(), + responseFormat: providerResponseFormatSchema.optional(), + workflowId: z.string().optional(), + workspaceId: z.string().optional(), + stream: z.boolean().optional(), + messages: z.array(providerMessageSchema).optional(), + environmentVariables: z.record(z.string(), z.string()).optional(), + workflowVariables: z.record(z.string(), z.unknown()).optional(), + blockData: z.record(z.string(), z.unknown()).optional(), + blockNameMapping: z.record(z.string(), z.string()).optional(), + reasoningEffort: z.string().optional(), + verbosity: z.string().optional(), + [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), + }) + .passthrough() + +export type LlmProviderOperationInput = z.input diff --git a/apps/sim/lib/internal/llm/operations.test.ts b/apps/sim/lib/internal/llm/operations.test.ts new file mode 100644 index 00000000000..8bc2f2d57bb --- /dev/null +++ b/apps/sim/lib/internal/llm/operations.test.ts @@ -0,0 +1,173 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + PRIVATE_MODEL_INPUT_PROVENANCE_HEADER, + PRIVATE_MODEL_INPUT_STATE_HEADER, + PROJECTED_MODEL_INPUT_PATHS_V1, +} from '@/lib/execution/model-input-provenance' +import { + RESOLVED_SECRET_PROVENANCE_FIELD, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, +} from '@/lib/execution/private-tool-metadata' + +const mocks = vi.hoisted(() => ({ + assertPermissionsAllowed: vi.fn(), + authorizeCredential: vi.fn(), + checkWorkspaceAccess: vi.fn(), + executeProviderRequest: vi.fn(), + importProvenance: vi.fn(), + isComplete: vi.fn(), + prepareEnvironment: vi.fn(), + requireBillingAttribution: vi.fn(), + resolveVertexAccessToken: vi.fn(), +})) + +vi.mock('@/providers', () => ({ executeProviderRequest: mocks.executeProviderRequest })) +vi.mock('@/lib/auth/credential-access', () => ({ + authorizeCredentialUseForAuth: mocks.authorizeCredential, +})) +vi.mock('@/lib/billing/core/billing-attribution', () => ({ + BILLING_ATTRIBUTION_HEADER: 'x-sim-billing-attribution', + requireBillingAttributionHeader: mocks.requireBillingAttribution, +})) +vi.mock('@/lib/copilot/environment-context', () => ({ + prepareCopilotEnvironmentContext: mocks.prepareEnvironment, +})) +vi.mock('@/lib/internal/llm/credentials', () => ({ + resolveVertexAccessToken: mocks.resolveVertexAccessToken, +})) +vi.mock('@/lib/workspaces/permissions/utils', () => ({ + checkWorkspaceAccess: mocks.checkWorkspaceAccess, +})) +vi.mock('@/executor/utils/resolved-secret-content-projection', () => ({ + projectResolvedSecretModelContent: vi.fn(), +})) +vi.mock('@/ee/access-control/utils/permission-check', () => ({ + assertPermissionsAllowed: mocks.assertPermissionsAllowed, + IntegrationNotAllowedError: class IntegrationNotAllowedError extends Error {}, + ModelNotAllowedError: class ModelNotAllowedError extends Error {}, + ProviderNotAllowedError: class ProviderNotAllowedError extends Error {}, +})) + +import type { LlmOperationError } from '@/lib/internal/llm/errors' +import { executeLlmProviderOperation } from '@/lib/internal/llm/operations' + +const BILLING_ATTRIBUTION = { + actorUserId: 'user-1', + workspaceId: 'workspace-1', + billingEntity: { type: 'user' as const, id: 'user-1' }, +} + +describe('executeLlmProviderOperation', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.checkWorkspaceAccess.mockResolvedValue({ hasAccess: true }) + mocks.requireBillingAttribution.mockReturnValue(BILLING_ATTRIBUTION) + mocks.importProvenance.mockResolvedValue(true) + mocks.isComplete.mockReturnValue(true) + mocks.prepareEnvironment.mockResolvedValue({ + resolvedSecretTraceRegistry: { + importProvenance: mocks.importProvenance, + isComplete: mocks.isComplete, + }, + }) + mocks.executeProviderRequest.mockResolvedValue({ content: 'answer', model: 'gpt-4o' }) + mocks.authorizeCredential.mockResolvedValue({ ok: true }) + mocks.resolveVertexAccessToken.mockResolvedValue('vertex-token') + }) + + it('executes once with billing, provenance, and cancellation bound to provider work', async () => { + const controller = new AbortController() + const provenance = { version: 1, complete: true, entries: [] } + const headers = new Headers({ + 'x-sim-billing-attribution': 'attribution', + [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1, + [PRIVATE_MODEL_INPUT_STATE_HEADER]: PROJECTED_MODEL_INPUT_PATHS_V1, + }) + const result = await executeLlmProviderOperation( + { + provider: 'openai', + model: 'gpt-4o', + context: '[{"role":"user","content":"claim"}]', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + [RESOLVED_SECRET_PROVENANCE_FIELD]: provenance, + }, + { + actorUserId: 'user-1', + headers, + requestId: 'request-1', + signal: controller.signal, + } + ) + + expect(result).toEqual({ content: 'answer', model: 'gpt-4o' }) + expect(mocks.requireBillingAttribution).toHaveBeenCalledWith(headers, { + actorUserId: 'user-1', + workspaceId: 'workspace-1', + }) + expect(mocks.importProvenance).toHaveBeenCalledWith(provenance, { + trusted: true, + origin: 'llmTool.inputProvenance', + }) + expect(mocks.executeProviderRequest).toHaveBeenCalledTimes(1) + expect(mocks.executeProviderRequest).toHaveBeenCalledWith( + 'openai', + expect.objectContaining({ + abortSignal: controller.signal, + billingAttribution: BILLING_ATTRIBUTION, + userId: 'user-1', + }), + expect.objectContaining({ resolvedSecretTraceRegistry: expect.anything() }) + ) + }) + + it('fails before provider work when workspace authorization is denied', async () => { + mocks.checkWorkspaceAccess.mockResolvedValueOnce({ hasAccess: false }) + + await expect( + executeLlmProviderOperation( + { provider: 'openai', model: 'gpt-4o', workspaceId: 'workspace-1' }, + { + actorUserId: 'user-1', + headers: new Headers(), + requestId: 'request-1', + } + ) + ).rejects.toMatchObject({ status: 403, body: { error: 'Forbidden' } }) + expect(mocks.executeProviderRequest).not.toHaveBeenCalled() + }) + + it('authorizes and resolves Vertex credentials before provider work', async () => { + await executeLlmProviderOperation( + { + provider: 'vertex', + model: 'vertex/gemini-2.5-pro', + vertexCredential: 'credential-1', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + }, + { + actorUserId: 'user-1', + headers: new Headers(), + requestId: 'request-1', + } + ) + + expect(mocks.authorizeCredential).toHaveBeenCalledWith( + expect.objectContaining({ userId: 'user-1', authType: 'internal_jwt' }), + expect.objectContaining({ + credentialId: 'credential-1', + workflowId: 'workflow-1', + callerUserId: 'user-1', + }) + ) + expect(mocks.executeProviderRequest).toHaveBeenCalledWith( + 'vertex', + expect.objectContaining({ apiKey: 'vertex-token' }), + expect.anything() + ) + }) +}) diff --git a/apps/sim/lib/internal/llm/operations.ts b/apps/sim/lib/internal/llm/operations.ts new file mode 100644 index 00000000000..fbc2ca61b9f --- /dev/null +++ b/apps/sim/lib/internal/llm/operations.ts @@ -0,0 +1,236 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage, toError } from '@sim/utils/errors' +import { isPlainRecord } from '@sim/utils/object' +import { authorizeCredentialUseForAuth } from '@/lib/auth/credential-access' +import { AuthType } from '@/lib/auth/hybrid' +import { + BILLING_ATTRIBUTION_HEADER, + type BillingAttributionSnapshot, + requireBillingAttributionHeader, +} from '@/lib/billing/core/billing-attribution' +import { prepareCopilotEnvironmentContext } from '@/lib/copilot/environment-context' +import { + inspectModelInputProjectionState, + inspectModelInputProvenanceRequest, +} from '@/lib/execution/model-input-provenance' +import { resolveVertexAccessToken } from '@/lib/internal/llm/credentials' +import { LlmOperationError } from '@/lib/internal/llm/errors' +import type { LlmProviderOperationInput } from '@/lib/internal/llm/input' +import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils' +import { + assertPermissionsAllowed, + IntegrationNotAllowedError, + ModelNotAllowedError, + ProviderNotAllowedError, +} from '@/ee/access-control/utils/permission-check' +import type { StreamingExecution } from '@/executor/types' +import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection' +import { executeProviderRequest } from '@/providers' +import type { ProviderRequest, ProviderResponse } from '@/providers/types' + +const logger = createLogger('LlmOperation') + +export type LlmProviderOperationResult = ProviderResponse | ReadableStream | StreamingExecution + +export interface LlmProviderOperationContext { + actorUserId: string + headers: Headers + requestId: string + signal?: AbortSignal +} + +function fail(status: number, error: string): never { + throw new LlmOperationError(status, { error }) +} + +function authenticatedCaller(context: LlmProviderOperationContext) { + return { + success: true, + userId: context.actorUserId, + authType: AuthType.INTERNAL_JWT, + } as const +} + +async function authorizeVertexCredential( + input: LlmProviderOperationInput, + context: LlmProviderOperationContext +): Promise { + if (input.provider !== 'vertex' || !input.vertexCredential) return + + const access = await authorizeCredentialUseForAuth(authenticatedCaller(context), { + credentialId: input.vertexCredential, + workflowId: input.workflowId || undefined, + callerUserId: context.actorUserId, + }) + if (!access.ok) { + logger.warn(`[${context.requestId}] Vertex credential access denied`, { + error: access.error, + credentialId: input.vertexCredential, + }) + fail(401, access.error || 'Unauthorized') + } +} + +function resolveBillingAttribution( + input: LlmProviderOperationInput, + context: LlmProviderOperationContext +): BillingAttributionSnapshot | undefined { + if (!context.headers.get(BILLING_ATTRIBUTION_HEADER)) return undefined + if (!input.workspaceId) { + fail(400, 'workspaceId is required when billing attribution is supplied') + } + try { + return requireBillingAttributionHeader(context.headers, { + actorUserId: context.actorUserId, + workspaceId: input.workspaceId, + }) + } catch (error) { + fail(400, getErrorMessage(error, 'Invalid billing attribution header')) + } +} + +async function authorizeWorkspace( + input: LlmProviderOperationInput, + context: LlmProviderOperationContext +): Promise { + if (!input.workspaceId) return + const workspaceAccess = await checkWorkspaceAccess(input.workspaceId, context.actorUserId) + if (!workspaceAccess.hasAccess) fail(403, 'Forbidden') + + try { + await assertPermissionsAllowed({ + userId: context.actorUserId, + workspaceId: input.workspaceId, + model: input.model, + }) + } catch (error) { + if ( + error instanceof ProviderNotAllowedError || + error instanceof ModelNotAllowedError || + error instanceof IntegrationNotAllowedError + ) { + fail(403, error.message) + } + throw error + } +} + +async function prepareProviderRequest( + input: LlmProviderOperationInput, + context: LlmProviderOperationContext +): Promise<{ + request: ProviderRequest + runtimeContext: Awaited> +}> { + let apiKey = input.apiKey + try { + await authorizeVertexCredential(input, context) + if (input.provider === 'vertex' && input.vertexCredential) { + apiKey = await resolveVertexAccessToken(context.requestId, input.vertexCredential) + } + } catch (error) { + if (error instanceof LlmOperationError) throw error + logger.error(`[${context.requestId}] Failed to resolve Vertex credential`, { + provider: input.provider, + model: input.model, + error: toError(error).message, + hasVertexCredential: Boolean(input.vertexCredential), + }) + fail(400, getErrorMessage(error, 'Credential error')) + } + + const billingAttribution = resolveBillingAttribution(input, context) + let request: ProviderRequest = { + model: input.model, + systemPrompt: input.systemPrompt, + context: input.context, + tools: input.tools, + temperature: input.temperature, + maxTokens: input.maxTokens, + apiKey, + azureEndpoint: input.azureEndpoint, + azureApiVersion: input.azureApiVersion, + vertexProject: input.vertexProject, + vertexLocation: input.vertexLocation, + bedrockAccessKeyId: input.bedrockAccessKeyId, + bedrockSecretKey: input.bedrockSecretKey, + bedrockRegion: input.bedrockRegion, + responseFormat: input.responseFormat, + workflowId: input.workflowId, + workspaceId: input.workspaceId, + userId: context.actorUserId, + stream: input.stream, + messages: input.messages, + environmentVariables: input.environmentVariables, + workflowVariables: input.workflowVariables, + blockData: input.blockData, + blockNameMapping: input.blockNameMapping, + billingAttribution, + reasoningEffort: input.reasoningEffort, + verbosity: input.verbosity, + abortSignal: context.signal, + } + + const provenanceInspection = inspectModelInputProvenanceRequest(context.headers, input) + const projectionState = inspectModelInputProjectionState(context.headers) + if ( + provenanceInspection.status === 'invalid' || + projectionState === 'invalid' || + (projectionState === 'projected' && provenanceInspection.status !== 'verified') + ) { + fail(400, 'Invalid model input provenance') + } + + const runtimeContext = await prepareCopilotEnvironmentContext( + context.actorUserId, + input.workspaceId + ) + if (provenanceInspection.status === 'verified') { + const provenanceReady = await runtimeContext.resolvedSecretTraceRegistry.importProvenance( + provenanceInspection.value, + { trusted: true, origin: 'llmTool.inputProvenance' } + ) + if (!provenanceReady || !runtimeContext.resolvedSecretTraceRegistry.isComplete()) { + fail(400, 'Model input provenance is unavailable') + } + + if (projectionState === 'unmarked') { + const projection = projectResolvedSecretModelContent( + { systemPrompt: request.systemPrompt, context: request.context }, + runtimeContext.resolvedSecretTraceRegistry + ) + if (!projection.safe || !isPlainRecord(projection.value)) { + fail(400, 'Model input provenance is unavailable') + } + const projectedSystemPrompt = projection.value.systemPrompt + const projectedContext = projection.value.context + if ( + (projectedSystemPrompt !== undefined && typeof projectedSystemPrompt !== 'string') || + (projectedContext !== undefined && typeof projectedContext !== 'string') + ) { + fail(400, 'Invalid model input provenance') + } + request = { + ...request, + systemPrompt: projectedSystemPrompt, + context: projectedContext, + } + } + } + + return { request, runtimeContext } +} + +/** Executes one authorized provider request without transport retries. */ +export async function executeLlmProviderOperation( + input: LlmProviderOperationInput, + context: LlmProviderOperationContext +): Promise { + context.signal?.throwIfAborted() + await authorizeWorkspace(input, context) + const { request, runtimeContext } = await prepareProviderRequest(input, context) + context.signal?.throwIfAborted() + const result = await executeProviderRequest(input.provider, request, runtimeContext) + context.signal?.throwIfAborted() + return result +} diff --git a/apps/sim/lib/internal/logs/execute-tool.test.ts b/apps/sim/lib/internal/logs/execute-tool.test.ts new file mode 100644 index 00000000000..e4daa1dc157 --- /dev/null +++ b/apps/sim/lib/internal/logs/execute-tool.test.ts @@ -0,0 +1,172 @@ +/** + * @vitest-environment node + */ + +import type { WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { ExecutionContext } from '@/executor/types' + +const mocks = vi.hoisted(() => ({ + createPrincipal: vi.fn(), + list: vi.fn(), + get: vi.fn(), + getRun: vi.fn(), + getExecution: vi.fn(), +})) + +vi.mock('@/lib/internal/principals/executor', () => ({ + createExecutorPrincipalFromExecutionContext: mocks.createPrincipal, +})) +vi.mock('@/lib/internal/logs/operations', () => ({ + executeLogsList: mocks.list, + executeLogsGet: mocks.get, + executeLogsGetRunDetails: mocks.getRun, + executeLogsGetExecution: mocks.getExecution, +})) + +import { executeLogsTool } from '@/lib/internal/logs/execute-tool' + +const PRINCIPAL: WorkflowExecutionDelegatedPrincipal = { + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'workspace-canonical', + delegationId: 'delegation-1', + audience: 'sim:logs', + issuedAt: new Date('2026-08-27T00:00:00.000Z'), + expiresAt: new Date('2026-08-27T00:05:00.000Z'), + delegationContext: { kind: 'workflow_execution', workflowId: 'workflow-1' }, +} + +const CONTEXT = { userId: 'user-1', workflowId: 'workflow-1' } as ExecutionContext + +const SUMMARY = { + id: 'log-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + deploymentVersionId: null, + deploymentVersion: null, + deploymentVersionName: null, + executionOrigin: null, + level: 'info', + status: 'success', + duration: '10ms', + trigger: 'manual', + createdAt: '2026-08-27T00:00:00.000Z', + workflow: null, + jobTitle: null, + cost: null, + pauseSummary: { status: null, total: 0, resumed: 0 }, + hasPendingPause: false, +} + +const DETAIL = { + ...SUMMARY, + executionData: {}, + files: null, +} + +const SNAPSHOT = { + executionId: 'execution-1', + workflowId: 'workflow-1', + workflowState: {}, + childWorkflowSnapshots: {}, + executionMetadata: { + trigger: 'manual', + startedAt: '2026-08-27T00:00:00.000Z', + cost: null, + }, +} + +const CASES = [ + { + toolId: 'logs_query', + input: {}, + operation: 'list' as const, + }, + { + toolId: 'logs_query_runs', + input: { limit: 25 }, + operation: 'list' as const, + }, + { + toolId: 'logs_get', + input: { id: 'log-1' }, + operation: 'get' as const, + }, + { + toolId: 'logs_get_run_details', + input: { executionId: 'execution-1' }, + operation: 'getRun' as const, + executionId: 'execution-1', + }, + { + toolId: 'logs_get_execution', + input: { executionId: 'execution-1' }, + operation: 'getExecution' as const, + executionId: 'execution-1', + }, +] + +describe('executeLogsTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.createPrincipal.mockResolvedValue(PRINCIPAL) + mocks.list.mockResolvedValue({ data: [SUMMARY], nextCursor: null }) + mocks.get.mockResolvedValue({ data: DETAIL }) + mocks.getRun.mockResolvedValue({ data: DETAIL }) + mocks.getExecution.mockResolvedValue(SNAPSHOT) + }) + + it.each(CASES)('dispatches $toolId through its canonical contract', async (testCase) => { + const response = await executeLogsTool({ + toolId: testCase.toolId, + input: testCase.input, + headers: new Headers(), + context: CONTEXT, + requestId: 'request-1', + }) + + expect(response.status).toBe(200) + expect(mocks[testCase.operation]).toHaveBeenCalledOnce() + expect(mocks.createPrincipal).toHaveBeenCalledWith({ + context: CONTEXT, + audience: 'sim:logs', + ...(testCase.executionId ? { resourceScope: { executionId: testCase.executionId } } : {}), + }) + }) + + it('authenticates before input validation and preserves response validation', async () => { + mocks.createPrincipal.mockRejectedValueOnce(new Error('Authentication required')) + const unauthenticated = await executeLogsTool({ + toolId: 'logs_get', + input: {}, + headers: new Headers(), + context: CONTEXT, + requestId: 'request-1', + }) + expect(unauthenticated.status).toBe(401) + expect(mocks.get).not.toHaveBeenCalled() + + const invalidInput = await executeLogsTool({ + toolId: 'logs_get', + input: {}, + headers: new Headers(), + context: CONTEXT, + requestId: 'request-1', + }) + expect(invalidInput.status).toBe(400) + expect(await invalidInput.json()).toMatchObject({ error: 'Validation error' }) + + mocks.get.mockResolvedValueOnce({ data: {} }) + const invalidResponse = await executeLogsTool({ + toolId: 'logs_get', + input: { id: 'log-1' }, + headers: new Headers(), + context: CONTEXT, + requestId: 'request-1', + }) + expect(invalidResponse.status).toBe(500) + expect(await invalidResponse.json()).toEqual({ error: 'Failed to fetch log' }) + }) +}) diff --git a/apps/sim/lib/internal/logs/execute-tool.ts b/apps/sim/lib/internal/logs/execute-tool.ts new file mode 100644 index 00000000000..59870040852 --- /dev/null +++ b/apps/sim/lib/internal/logs/execute-tool.ts @@ -0,0 +1,148 @@ +import { createLogger } from '@sim/logger' +import { isPlainRecord } from '@sim/utils/object' +import type { ZodError } from 'zod' +import type { AnyApiRouteContract } from '@/lib/api/contracts' +import { + executionIdParamsSchema, + getExecutionSnapshotContract, + getLogByExecutionIdContract, + getLogDetailContract, + listLogsContract, + listLogsQuerySchema, + logIdParamsSchema, +} from '@/lib/api/contracts/logs' +import { serializeZodIssues } from '@/lib/api/server/validation' +import { InvalidInternalDelegationBindingError } from '@/lib/auth/internal-delegation' +import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' +import { + executeLogsGet, + executeLogsGetExecution, + executeLogsGetRunDetails, + executeLogsList, + type LogsToolOperationContext, +} from '@/lib/internal/logs/operations' +import { createExecutorPrincipalFromExecutionContext } from '@/lib/internal/principals/executor' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { LOGS_DELEGATION_AUDIENCE } from '@/lib/logs/application/authorization' + +const logger = createLogger('LogsInternalOperation') + +const FAILURE_MESSAGES: Record = { + logs_query: 'Failed to list logs', + logs_query_runs: 'Failed to list logs', + logs_get: 'Failed to fetch log', + logs_get_run_details: 'Failed to fetch log', + logs_get_execution: 'Failed to fetch execution data', +} + +function errorResponse(toolId: string, error: unknown): Response { + const classified = asOrchestrationError(error) + if (classified) { + return Response.json( + { error: classified.message }, + { status: statusForOrchestrationError(classified.code) } + ) + } + logger.error(FAILURE_MESSAGES[toolId] ?? 'Logs operation failed', { error }) + return Response.json( + { error: FAILURE_MESSAGES[toolId] ?? 'Logs operation failed' }, + { status: 500 } + ) +} + +async function dispatchLogsTool( + request: Parameters[0], + context: LogsToolOperationContext +): Promise<{ contract: AnyApiRouteContract; body: unknown } | Response> { + const dispatched = async (contract: AnyApiRouteContract, body: Promise) => ({ + contract, + body: await body, + }) + + switch (request.toolId) { + case 'logs_query': + case 'logs_query_runs': { + const parsed = listLogsQuerySchema.safeParse({ + ...(isPlainRecord(request.input) ? request.input : {}), + workspaceId: context.principal.workspaceId, + }) + return parsed.success + ? dispatched(listLogsContract, executeLogsList(parsed.data, context)) + : validationResponse(parsed.error) + } + case 'logs_get': { + const parsed = logIdParamsSchema.safeParse(request.input) + return parsed.success + ? dispatched(getLogDetailContract, executeLogsGet(parsed.data.id, context)) + : validationResponse(parsed.error) + } + case 'logs_get_run_details': { + const parsed = executionIdParamsSchema.safeParse(request.input) + return parsed.success + ? dispatched( + getLogByExecutionIdContract, + executeLogsGetRunDetails(parsed.data.executionId, context) + ) + : validationResponse(parsed.error) + } + case 'logs_get_execution': { + const parsed = executionIdParamsSchema.safeParse(request.input) + return parsed.success + ? dispatched( + getExecutionSnapshotContract, + executeLogsGetExecution(parsed.data.executionId, context) + ) + : validationResponse(parsed.error) + } + default: + return Response.json({ error: `Unsupported Logs tool: ${request.toolId}` }, { status: 500 }) + } +} + +function validationResponse(error: ZodError): Response { + return Response.json( + { error: 'Validation error', details: serializeZodIssues(error) }, + { status: 400 } + ) +} + +export const executeLogsTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!Object.hasOwn(FAILURE_MESSAGES, request.toolId)) { + return Response.json({ error: `Unsupported Logs tool: ${request.toolId}` }, { status: 500 }) + } + + const requestedExecutionId = + (request.toolId === 'logs_get_execution' || request.toolId === 'logs_get_run_details') && + isPlainRecord(request.input) && + typeof request.input.executionId === 'string' + ? request.input.executionId + : undefined + + try { + const principal = await createExecutorPrincipalFromExecutionContext({ + context: request.context, + audience: LOGS_DELEGATION_AUDIENCE, + ...(requestedExecutionId ? { resourceScope: { executionId: requestedExecutionId } } : {}), + }) + request.signal?.throwIfAborted() + const dispatched = await dispatchLogsTool(request, { + principal, + signal: request.signal, + }) + if (dispatched instanceof Response) return dispatched + if (dispatched.contract.response.mode !== 'json') { + throw new Error('Logs tool contract must return JSON') + } + return Response.json(dispatched.contract.response.schema.parse(dispatched.body)) + } catch (error) { + request.signal?.throwIfAborted() + if ( + error instanceof InvalidInternalDelegationBindingError || + (error instanceof Error && error.message === 'Authentication required') + ) { + return Response.json({ error: 'Authentication required' }, { status: 401 }) + } + return errorResponse(request.toolId, error) + } +} diff --git a/apps/sim/lib/internal/logs/operations.test.ts b/apps/sim/lib/internal/logs/operations.test.ts new file mode 100644 index 00000000000..6bb17a75fab --- /dev/null +++ b/apps/sim/lib/internal/logs/operations.test.ts @@ -0,0 +1,102 @@ +/** + * @vitest-environment node + */ + +import type { WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + list: vi.fn(), + detail: vi.fn(), + snapshot: vi.fn(), +})) + +vi.mock('@/lib/logs/application/list-logs', () => ({ + listLogsUseCase: { execute: mocks.list }, +})) +vi.mock('@/lib/logs/application/read-log-detail', () => ({ + readLogDetailUseCase: { execute: mocks.detail }, +})) +vi.mock('@/lib/logs/application/read-execution-snapshot', () => ({ + readExecutionSnapshotUseCase: { execute: mocks.snapshot }, +})) + +import { + executeLogsGet, + executeLogsGetExecution, + executeLogsGetRunDetails, + executeLogsList, + type LogsToolOperationContext, +} from '@/lib/internal/logs/operations' + +const PRINCIPAL: WorkflowExecutionDelegatedPrincipal = { + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'workspace-canonical', + delegationId: 'delegation-1', + audience: 'sim:logs', + issuedAt: new Date('2026-08-27T00:00:00.000Z'), + expiresAt: new Date('2026-08-27T00:05:00.000Z'), + delegationContext: { kind: 'workflow_execution', workflowId: 'workflow-1' }, +} + +function context(): LogsToolOperationContext { + return { principal: PRINCIPAL, signal: undefined } +} + +describe('Logs direct operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.list.mockResolvedValue({ data: [], nextCursor: null }) + mocks.detail.mockResolvedValue({ detail: { id: 'log-1' } }) + mocks.snapshot.mockResolvedValue({ executionId: 'execution-1' }) + }) + + it('uses the canonical delegated workspace for list and detail reads', async () => { + await executeLogsList( + { + workspaceId: 'workspace-forged', + limit: 25, + sortBy: 'date', + sortOrder: 'desc', + }, + context() + ) + await executeLogsGet('log-1', context()) + await executeLogsGetRunDetails('execution-1', context()) + + expect(mocks.list).toHaveBeenCalledWith({ + principal: PRINCIPAL, + input: expect.objectContaining({ + workspaceId: 'workspace-canonical', + limit: 25, + signal: undefined, + }), + }) + expect(mocks.detail).toHaveBeenNthCalledWith(1, { + principal: PRINCIPAL, + input: expect.objectContaining({ + workspaceId: 'workspace-canonical', + lookupColumn: 'id', + lookupValue: 'log-1', + }), + }) + expect(mocks.detail).toHaveBeenNthCalledWith(2, { + principal: PRINCIPAL, + input: expect.objectContaining({ + workspaceId: 'workspace-canonical', + lookupColumn: 'executionId', + lookupValue: 'execution-1', + }), + }) + }) + + it('resolves execution snapshots through the authorized application operation', async () => { + await executeLogsGetExecution('execution-1', context()) + expect(mocks.snapshot).toHaveBeenCalledWith({ + principal: PRINCIPAL, + input: { executionId: 'execution-1', signal: undefined }, + }) + }) +}) diff --git a/apps/sim/lib/internal/logs/operations.ts b/apps/sim/lib/internal/logs/operations.ts new file mode 100644 index 00000000000..f860a7f22b9 --- /dev/null +++ b/apps/sim/lib/internal/logs/operations.ts @@ -0,0 +1,68 @@ +import type { WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' +import type { ContractQuery } from '@/lib/api/contracts' +import type { listLogsContract } from '@/lib/api/contracts/logs' +import { listLogsUseCase } from '@/lib/logs/application/list-logs' +import { readExecutionSnapshotUseCase } from '@/lib/logs/application/read-execution-snapshot' +import { readLogDetailUseCase } from '@/lib/logs/application/read-log-detail' + +export interface LogsToolOperationContext { + principal: WorkflowExecutionDelegatedPrincipal + signal?: AbortSignal +} + +function complete(context: LogsToolOperationContext, value: T): T { + context.signal?.throwIfAborted() + return value +} + +export async function executeLogsList( + query: ContractQuery, + context: LogsToolOperationContext +) { + context.signal?.throwIfAborted() + const result = await listLogsUseCase.execute({ + principal: context.principal, + input: { ...query, workspaceId: context.principal.workspaceId, signal: context.signal }, + }) + return complete(context, result) +} + +export async function executeLogsGet(id: string, context: LogsToolOperationContext) { + const result = await readLogDetailUseCase.execute({ + principal: context.principal, + input: { + workspaceId: context.principal.workspaceId, + lookupColumn: 'id', + lookupValue: id, + signal: context.signal, + }, + }) + return complete(context, { data: result.detail }) +} + +export async function executeLogsGetRunDetails( + executionId: string, + context: LogsToolOperationContext +) { + const result = await readLogDetailUseCase.execute({ + principal: context.principal, + input: { + workspaceId: context.principal.workspaceId, + lookupColumn: 'executionId', + lookupValue: executionId, + signal: context.signal, + }, + }) + return complete(context, { data: result.detail }) +} + +export async function executeLogsGetExecution( + executionId: string, + context: LogsToolOperationContext +) { + const result = await readExecutionSnapshotUseCase.execute({ + principal: context.principal, + input: { executionId, signal: context.signal }, + }) + return complete(context, result) +} diff --git a/apps/sim/lib/internal/mail/attachment-materialization.test.ts b/apps/sim/lib/internal/mail/attachment-materialization.test.ts new file mode 100644 index 00000000000..a26555e66b1 --- /dev/null +++ b/apps/sim/lib/internal/mail/attachment-materialization.test.ts @@ -0,0 +1,77 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + access: vi.fn(), + download: vi.fn(), + process: vi.fn(), +})) + +vi.mock('@/app/api/files/authorization', () => ({ assertToolFileAccess: mocks.access })) +vi.mock('@/lib/uploads/utils/file-utils', () => ({ processFilesToUserFiles: mocks.process })) +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFilesWithinBudget: mocks.download, +})) + +import { + type MailAttachmentMaterializationError, + materializeAuthorizedMailAttachments, +} from '@/lib/internal/mail/attachment-materialization' + +const file = { key: 'workspace/ws-1/a.txt', name: 'a.txt', size: 3, type: 'text/plain' } +const context = { requestId: 'request-1', userId: 'user-1', signal: new AbortController().signal } + +describe('mail attachment materialization', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.process.mockReturnValue([file]) + mocks.access.mockResolvedValue(null) + mocks.download.mockResolvedValue([{ buffer: Buffer.from('abc'), contentType: 'text/plain' }]) + }) + + it('authorizes files and forwards the cumulative budget and signal', async () => { + await expect( + materializeAuthorizedMailAttachments([file], context, { + label: 'Total attachment size', + maxTotalBytes: 25, + }) + ).resolves.toEqual([{ buffer: Buffer.from('abc'), contentType: 'text/plain', name: 'a.txt' }]) + expect(mocks.access).toHaveBeenCalledWith(file.key, 'user-1', 'request-1', expect.anything()) + expect(mocks.download).toHaveBeenCalledWith([file], 'request-1', expect.anything(), { + totalMaxBytes: 25, + label: 'Total attachment size', + signal: context.signal, + }) + }) + + it('rejects declared-size overruns before authorization when requested', async () => { + await expect( + materializeAuthorizedMailAttachments([file], context, { + label: 'Total attachment size', + maxTotalBytes: 2, + preflightDeclaredSize: true, + }) + ).rejects.toMatchObject({ kind: 'size', observedBytes: 3 }) + expect(mocks.access).not.toHaveBeenCalled() + }) + + it('preserves file authorization response bodies', async () => { + mocks.access.mockResolvedValue( + Response.json({ success: false, error: 'Forbidden file' }, { status: 403 }) + ) + await expect( + materializeAuthorizedMailAttachments([file], context, { + label: 'Total attachment size', + maxTotalBytes: 25, + }) + ).rejects.toEqual( + expect.objectContaining>({ + kind: 'access', + status: 403, + body: { success: false, error: 'Forbidden file' }, + }) + ) + }) +}) diff --git a/apps/sim/lib/internal/mail/attachment-materialization.ts b/apps/sim/lib/internal/mail/attachment-materialization.ts new file mode 100644 index 00000000000..563495c2107 --- /dev/null +++ b/apps/sim/lib/internal/mail/attachment-materialization.ts @@ -0,0 +1,130 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { isRecordLike } from '@sim/utils/object' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/doc-not-ready' +import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +const logger = createLogger('MailAttachmentMaterialization') + +export interface MailAttachmentContext { + requestId: string + signal?: AbortSignal + userId: string +} + +export interface MaterializedMailAttachment { + buffer: Buffer + contentType: string + name: string +} + +export type MailAttachmentFailureKind = 'access' | 'download' | 'not-ready' | 'size' + +export class MailAttachmentMaterializationError extends Error { + constructor( + message: string, + readonly kind: MailAttachmentFailureKind, + readonly status: number, + readonly body: Record, + readonly observedBytes?: number + ) { + super(message) + this.name = 'MailAttachmentMaterializationError' + } +} + +interface MaterializeMailAttachmentsOptions { + label: string + maxTotalBytes: number + preflightDeclaredSize?: boolean +} + +function sizeFailure(label: string, observedBytes: number): MailAttachmentMaterializationError { + return new MailAttachmentMaterializationError( + `${label} exceeds its configured limit`, + 'size', + 400, + { success: false, error: `${label} exceeds its configured limit` }, + observedBytes + ) +} + +async function responseBody(response: Response): Promise> { + const body: unknown = await response.json() + return isRecordLike(body) ? body : { success: false, error: 'File not found' } +} + +export async function materializeAuthorizedMailAttachments( + rawAttachments: readonly RawFileInput[], + context: MailAttachmentContext, + options: MaterializeMailAttachmentsOptions +): Promise { + context.signal?.throwIfAborted() + const files = processFilesToUserFiles([...rawAttachments], context.requestId, logger) + if (files.length === 0) return [] + const declaredBytes = files.reduce((total, file) => total + file.size, 0) + if (options.preflightDeclaredSize && declaredBytes > options.maxTotalBytes) { + throw sizeFailure(options.label, declaredBytes) + } + + for (const file of files) { + context.signal?.throwIfAborted() + const denied = await assertToolFileAccess(file.key, context.userId, context.requestId, logger) + context.signal?.throwIfAborted() + if (denied) { + throw new MailAttachmentMaterializationError( + 'File not found', + 'access', + denied.status, + await responseBody(denied) + ) + } + } + + let resolved: Awaited> + try { + resolved = await downloadServableFilesWithinBudget(files, context.requestId, logger, { + totalMaxBytes: options.maxTotalBytes, + label: options.label, + signal: context.signal, + }) + } catch (error) { + context.signal?.throwIfAborted() + if (isDocNotReadyError(error)) { + const message = docNotReadyMessage() + throw new MailAttachmentMaterializationError(message, 'not-ready', 409, { + success: false, + error: message, + }) + } + if (isPayloadSizeLimitError(error)) { + throw sizeFailure(options.label, error.observedBytes ?? declaredBytes) + } + const message = `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}` + throw new MailAttachmentMaterializationError(message, 'download', 500, { + success: false, + error: message, + }) + } + context.signal?.throwIfAborted() + + return files.map((file, index) => { + const materialized = resolved[index] + if (!materialized) { + const message = 'Failed to download attachment: Missing file data' + throw new MailAttachmentMaterializationError(message, 'download', 500, { + success: false, + error: message, + }) + } + return { + buffer: materialized.buffer, + contentType: materialized.contentType || file.type || 'application/octet-stream', + name: file.name, + } + }) +} diff --git a/apps/sim/lib/internal/mail/execute-tools.test.ts b/apps/sim/lib/internal/mail/execute-tools.test.ts new file mode 100644 index 00000000000..d622ac1daa4 --- /dev/null +++ b/apps/sim/lib/internal/mail/execute-tools.test.ts @@ -0,0 +1,102 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ resend: vi.fn(), sendGrid: vi.fn(), smtp: vi.fn() })) +vi.mock('@/lib/internal/resend/operations', () => ({ executeResendSend: mocks.resend })) +vi.mock('@/lib/internal/sendgrid/operations', () => ({ executeSendGridSend: mocks.sendGrid })) +vi.mock('@/lib/internal/smtp/operations', () => ({ executeSmtpSend: mocks.smtp })) + +import { executeResendTool } from '@/lib/internal/resend/execute-tool' +import { executeSendGridTool } from '@/lib/internal/sendgrid/execute-tool' +import { executeSmtpTool } from '@/lib/internal/smtp/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +function request(toolId: string, input: unknown, userId = 'user-1') { + return { + toolId, + input, + headers: new Headers(), + context: { ...createExecutionContext({ workflowId: 'workflow-1' }), userId }, + requestId: 'request-1', + } as InternalToolOperationCall +} + +describe('mail submission handlers', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.resend.mockResolvedValue({ success: true, data: { id: 'resend-1' } }) + mocks.sendGrid.mockResolvedValue({ success: true, output: { success: true } }) + mocks.smtp.mockResolvedValue({ success: true, messageId: 'smtp-1' }) + }) + + it('dispatches each canonical ID to its operation', async () => { + const resend = await executeResendTool( + request('resend_send', { + resendApiKey: 'secret', + fromAddress: 'from@example.com', + to: 'to@example.com', + subject: 'Hello', + body: 'Hello', + }) + ) + const sendGrid = await executeSendGridTool( + request('sendgrid_send_mail', { + apiKey: 'secret', + from: 'from@example.com', + to: 'to@example.com', + }) + ) + const smtp = await executeSmtpTool( + request('smtp_send_mail', { + smtpHost: 'smtp.example.com', + smtpPort: 465, + smtpUsername: 'user', + smtpPassword: 'password', + smtpSecure: 'SSL', + from: 'from@example.com', + to: 'to@example.com', + subject: 'Hello', + body: 'Hello', + }) + ) + + expect([resend.status, sendGrid.status, smtp.status]).toEqual([200, 200, 200]) + expect(mocks.resend).toHaveBeenCalledOnce() + expect(mocks.sendGrid).toHaveBeenCalledOnce() + expect(mocks.smtp).toHaveBeenCalledOnce() + }) + + it.each([ + ['resend_send', executeResendTool], + ['sendgrid_send_mail', executeSendGridTool], + ['smtp_send_mail', executeSmtpTool], + ])('authenticates %s before parsing', async (toolId, execute) => { + const response = await execute(request(toolId, null, '')) + expect(response.status).toBe(401) + expect(mocks.resend).not.toHaveBeenCalled() + expect(mocks.sendGrid).not.toHaveBeenCalled() + expect(mocks.smtp).not.toHaveBeenCalled() + }) + + it('preserves provider-specific validation envelopes', async () => { + const resend = await executeResendTool(request('resend_send', {})) + const sendGrid = await executeSendGridTool(request('sendgrid_send_mail', {})) + const smtp = await executeSmtpTool(request('smtp_send_mail', {})) + await expect(resend.json()).resolves.toMatchObject({ + success: false, + message: expect.any(String), + errors: expect.any(Array), + }) + await expect(sendGrid.json()).resolves.toMatchObject({ + error: 'Validation error', + details: expect.any(Array), + }) + await expect(smtp.json()).resolves.toMatchObject({ + error: 'Validation error', + details: expect.any(Array), + }) + }) +}) diff --git a/apps/sim/lib/internal/mcp/discover-tools.ts b/apps/sim/lib/internal/mcp/discover-tools.ts new file mode 100644 index 00000000000..78050f11e9c --- /dev/null +++ b/apps/sim/lib/internal/mcp/discover-tools.ts @@ -0,0 +1,32 @@ +import { createExecutorPrincipalFromExecutionContext } from '@/lib/internal/principals/executor' +import type { InternalToolOperationContext } from '@/lib/internal/tool-operations/types' +import { MCP_SERVER_DELEGATION_AUDIENCE } from '@/lib/mcp/application/authorization' +import { discoverMcpServerToolsUseCase } from '@/lib/mcp/application/use-cases' + +export interface DiscoverMcpServerToolsAsExecutorInput { + workspaceId: string + context: InternalToolOperationContext + serverId: string + signal?: AbortSignal +} + +export async function discoverMcpServerToolsAsExecutor({ + workspaceId, + context, + serverId, + signal, +}: DiscoverMcpServerToolsAsExecutorInput) { + signal?.throwIfAborted() + const principal = await createExecutorPrincipalFromExecutionContext({ + context, + audience: MCP_SERVER_DELEGATION_AUDIENCE, + }) + + signal?.throwIfAborted() + const result = await discoverMcpServerToolsUseCase.execute({ + principal, + input: { workspaceId, serverId }, + }) + signal?.throwIfAborted() + return result.tools +} diff --git a/apps/sim/lib/internal/mcp/execute-tool.test.ts b/apps/sim/lib/internal/mcp/execute-tool.test.ts new file mode 100644 index 00000000000..c85e58471af --- /dev/null +++ b/apps/sim/lib/internal/mcp/execute-tool.test.ts @@ -0,0 +1,272 @@ +/** + * @vitest-environment node + */ +import type { WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' +import type { InternalToolOperationContext } from '@/lib/internal/tool-operations/types' + +const mocks = vi.hoisted(() => ({ + createPrincipal: vi.fn(), + executeUseCase: vi.fn(), +})) + +vi.mock('@/lib/internal/principals/executor', () => ({ + createExecutorPrincipalFromExecutionContext: mocks.createPrincipal, +})) +vi.mock('@/lib/mcp/application/execute-tool', () => ({ + executeMcpToolUseCase: { execute: mocks.executeUseCase }, + McpToolsNotAllowedError: class McpToolsNotAllowedError extends Error {}, +})) + +import { executeMcpTool } from '@/lib/internal/mcp/execute-tool' + +const PRINCIPAL: WorkflowExecutionDelegatedPrincipal = { + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'workspace-1', + delegationId: 'delegation-1', + audience: 'sim:mcp-servers', + issuedAt: new Date('2026-08-27T00:00:00.000Z'), + expiresAt: new Date('2099-08-27T00:05:00.000Z'), + delegationContext: { kind: 'workflow_execution', workflowId: 'workflow-1' }, +} +const NESTED_HUMAN_PRINCIPAL: WorkflowExecutionDelegatedPrincipal = { + kind: 'delegated', + serviceId: 'executor', + workspaceId: 'workspace-1', + delegationId: 'delegation-nested-human', + audience: 'sim:mcp-servers', + issuedAt: new Date('2026-08-27T00:00:00.000Z'), + expiresAt: new Date('2099-08-27T00:05:00.000Z'), + delegationContext: { + kind: 'workflow_execution', + workflowId: 'workflow-1', + principal: { kind: 'session', userId: 'user-origin', sessionId: 'session-origin' }, + }, +} +const BILLING = { + actorUserId: 'user-1', + workspaceId: 'workspace-1', + organizationId: null, + billedAccountUserId: 'user-1', + billingEntity: { type: 'user', id: 'user-1' }, + billingPeriod: { start: '2026-08-01', end: '2026-09-01' }, + payerSubscription: null, +} as unknown as BillingAttributionSnapshot +const CONTEXT: InternalToolOperationContext = { + userId: 'user-1', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + billingAttribution: BILLING, + callChain: ['workflow-parent'], +} + +describe('executeMcpTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.createPrincipal.mockResolvedValue(PRINCIPAL) + mocks.executeUseCase.mockResolvedValue({ + success: true, + output: { content: [{ type: 'text', text: 'done' }] }, + }) + }) + + it('parses direct block arguments and invokes the authorized use case', async () => { + const response = await executeMcpTool({ + toolId: 'mcp-server-lookup', + input: { arguments: '{"query":"sim"}', _context: { ignored: true } }, + headers: new Headers(), + context: CONTEXT, + requestId: 'request-1', + }) + + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ + success: true, + data: { + success: true, + output: { content: [{ type: 'text', text: 'done' }] }, + }, + }) + expect(mocks.createPrincipal).toHaveBeenCalledWith({ + context: CONTEXT, + audience: 'sim:mcp-servers', + }) + expect(mocks.executeUseCase).toHaveBeenCalledWith({ + principal: PRINCIPAL, + input: expect.objectContaining({ + workspaceId: 'workspace-1', + serverId: 'mcp-server', + toolName: 'lookup', + arguments: { query: 'sim' }, + callChain: ['workflow-parent'], + }), + }) + }) + + it('filters framework parameters for Agent-originated arguments', async () => { + await executeMcpTool({ + toolId: 'mcp-server-lookup', + input: { query: 'sim', serverId: 'untrusted', _context: { workspaceId: 'foreign' } }, + headers: new Headers(), + context: CONTEXT, + requestId: 'request-1', + }) + + expect(mocks.executeUseCase).toHaveBeenCalledWith({ + principal: PRINCIPAL, + input: expect.objectContaining({ arguments: { query: 'sim' } }), + }) + }) + + it('fails closed without trusted workspace or billing context', async () => { + const missingWorkspace = await executeMcpTool({ + toolId: 'mcp-server-lookup', + input: {}, + headers: new Headers(), + context: { ...CONTEXT, workspaceId: undefined }, + requestId: 'request-1', + }) + expect(missingWorkspace.status).toBe(400) + expect(await missingWorkspace.json()).toMatchObject({ + error: 'Missing workspaceId in execution context for MCP tool lookup', + }) + + const missingBilling = await executeMcpTool({ + toolId: 'mcp-server-lookup', + input: {}, + headers: new Headers(), + context: { ...CONTEXT, billingAttribution: undefined }, + requestId: 'request-1', + }) + expect(missingBilling.status).toBe(400) + expect(await missingBilling.json()).toMatchObject({ + error: 'Missing billing attribution in execution context for MCP tool lookup', + }) + expect(mocks.createPrincipal).not.toHaveBeenCalled() + }) + + it('preserves provider tool errors without retrying the operation', async () => { + mocks.executeUseCase.mockResolvedValueOnce({ success: false, error: 'Provider rejected input' }) + + const response = await executeMcpTool({ + toolId: 'mcp-server-lookup', + input: {}, + headers: new Headers(), + context: CONTEXT, + requestId: 'request-1', + }) + + expect(response.status).toBe(400) + expect(await response.json()).toEqual({ success: false, error: 'Provider rejected input' }) + expect(mocks.executeUseCase).toHaveBeenCalledOnce() + }) + + it('throws cancellation before and after submitted work', async () => { + const before = new AbortController() + before.abort(new DOMException('cancelled', 'AbortError')) + await expect( + executeMcpTool({ + toolId: 'mcp-server-lookup', + input: {}, + headers: new Headers(), + context: CONTEXT, + requestId: 'request-1', + signal: before.signal, + }) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mocks.executeUseCase).not.toHaveBeenCalled() + + const after = new AbortController() + mocks.executeUseCase.mockImplementationOnce(async () => { + after.abort(new DOMException('cancelled', 'AbortError')) + return { success: true, output: {} } + }) + await expect( + executeMcpTool({ + toolId: 'mcp-server-lookup', + input: {}, + headers: new Headers(), + context: CONTEXT, + requestId: 'request-1', + signal: after.signal, + }) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mocks.executeUseCase).toHaveBeenCalledOnce() + }) + + it('imports resolved-secret provenance into the isolated tool registry', async () => { + const importCrossingProvenance = vi.fn().mockResolvedValue(true) + const mergeToolCallRegistry = vi.fn() + const fork = { importCrossingProvenance } + const registry = { + forkForToolCall: vi.fn(() => fork), + mergeToolCallRegistry, + } + + const response = await executeMcpTool({ + toolId: 'mcp-server-lookup', + input: {}, + headers: new Headers(), + context: { + ...CONTEXT, + resolvedSecretTraceRegistry: registry as never, + }, + requestId: 'request-1', + }) + + expect(response.status).toBe(200) + expect(importCrossingProvenance).toHaveBeenCalledWith( + expect.objectContaining({ version: 1, complete: true }), + expect.objectContaining({ success: true }), + { trusted: true, origin: 'tool.mcp-server-lookup' } + ) + expect(mergeToolCallRegistry).toHaveBeenCalledWith(fork) + }) + + it('scopes provenance to the trusted nested human without inventing an actor', async () => { + mocks.createPrincipal.mockResolvedValueOnce(NESTED_HUMAN_PRINCIPAL) + mocks.executeUseCase.mockImplementationOnce(async ({ input }) => { + input.onResolvedSecretTraceProvenance?.({ + version: 1, + complete: true, + entries: [], + scope: { userId: 'user-origin', workspaceId: 'workspace-1' }, + }) + return { success: true, output: {} } + }) + const importCrossingProvenance = vi.fn().mockResolvedValue(true) + const fork = { importCrossingProvenance } + const registry = { + forkForToolCall: vi.fn(() => fork), + mergeToolCallRegistry: vi.fn(), + } + + const response = await executeMcpTool({ + toolId: 'mcp-server-lookup', + input: {}, + headers: new Headers(), + context: { + ...CONTEXT, + userId: undefined, + resolvedSecretTraceRegistry: registry as never, + }, + requestId: 'request-1', + }) + + expect(response.status).toBe(200) + expect(mocks.executeUseCase).toHaveBeenCalledWith( + expect.objectContaining({ principal: NESTED_HUMAN_PRINCIPAL }) + ) + expect(importCrossingProvenance).toHaveBeenCalledWith( + expect.objectContaining({ + complete: true, + scope: { userId: 'user-origin', workspaceId: 'workspace-1' }, + }), + expect.objectContaining({ success: true }), + { trusted: true, origin: 'tool.mcp-server-lookup' } + ) + }) +}) diff --git a/apps/sim/lib/internal/mcp/execute-tool.ts b/apps/sim/lib/internal/mcp/execute-tool.ts new file mode 100644 index 00000000000..33ce7d4413f --- /dev/null +++ b/apps/sim/lib/internal/mcp/execute-tool.ts @@ -0,0 +1,239 @@ +import { UnauthorizedError } from '@modelcontextprotocol/sdk/client/auth.js' +import { resolvePrincipalSubject } from '@sim/auth/principal' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { isPlainRecord } from '@sim/utils/object' +import { InvalidInternalDelegationBindingError } from '@/lib/auth/internal-delegation' +import { + capExecutionTimeoutMs, + getAsyncExecutionTimeoutForBillingAttribution, + getRemainingExecutionMs, +} from '@/lib/core/execution-limits' +import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' +import { createExecutorPrincipalFromExecutionContext } from '@/lib/internal/principals/executor' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { MCP_SERVER_DELEGATION_AUDIENCE } from '@/lib/mcp/application/authorization' +import { executeMcpToolUseCase, McpToolsNotAllowedError } from '@/lib/mcp/application/execute-tool' +import { McpOauthRedirectRequired } from '@/lib/mcp/oauth' +import { McpOauthAuthorizationRequiredError } from '@/lib/mcp/types' +import { categorizeError, parseMcpToolId } from '@/lib/mcp/utils' +import { + ResolvedSecretTraceProvenanceAccumulator, + type ResolvedSecretTraceRegistry, +} from '@/executor/utils/resolved-secret-trace-registry' + +const logger = createLogger('McpInternalOperation') + +const MCP_SYSTEM_PARAMETERS = new Set([ + 'serverId', + 'serverUrl', + 'toolName', + 'serverName', + '_context', + 'envVars', + 'workflowVariables', + 'blockData', + 'blockNameMapping', + '_toolSchema', +]) + +function parseArguments(input: unknown): Record | null { + if (!isPlainRecord(input)) return null + if (!Object.hasOwn(input, 'arguments')) { + return Object.fromEntries( + Object.entries(input).filter(([name]) => !MCP_SYSTEM_PARAMETERS.has(name)) + ) + } + + const value = input.arguments + if (typeof value !== 'string') return isPlainRecord(value) ? value : null + try { + const parsed: unknown = JSON.parse(value) + return isPlainRecord(parsed) ? parsed : null + } catch (error) { + logger.warn('Failed to parse MCP arguments JSON', { + errorName: error instanceof Error ? error.name : 'UnknownError', + argumentsLength: value.length, + }) + return {} + } +} + +async function createResponse( + body: Record, + status: number, + provenance: ResolvedSecretTraceProvenanceAccumulator | undefined, + registry: ResolvedSecretTraceRegistry | undefined, + toolId: string +): Promise { + if (!provenance || !registry) return Response.json(body, { status }) + const targetRegistry = registry.forkForToolCall() + const imported = await targetRegistry.importCrossingProvenance( + provenance.exportProvenance(), + body, + { trusted: true, origin: `tool.${toolId}` } + ) + if (!imported) { + return Response.json( + { success: false, error: 'Internal tool response metadata could not be verified' }, + { status: 502, statusText: 'Bad Gateway' } + ) + } + registry.mergeToolCallRegistry(targetRegistry) + return Response.json(body, { status }) +} + +export const executeMcpTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + let serverId: string + let toolName: string + try { + ;({ serverId, toolName } = parseMcpToolId(request.toolId)) + } catch (error) { + return Response.json( + { success: false, error: getErrorMessage(error, 'Invalid MCP tool ID') }, + { status: 400 } + ) + } + + if (!request.context.workspaceId) { + return Response.json( + { + success: false, + error: `Missing workspaceId in execution context for MCP tool ${toolName}`, + }, + { status: 400 } + ) + } + if (!request.context.billingAttribution) { + return Response.json( + { + success: false, + error: `Missing billing attribution in execution context for MCP tool ${toolName}`, + }, + { status: 400 } + ) + } + const args = parseArguments(request.input) + if (!args) + return Response.json({ success: false, error: 'Invalid request format' }, { status: 400 }) + + let provenance: ResolvedSecretTraceProvenanceAccumulator | undefined + try { + const principal = await createExecutorPrincipalFromExecutionContext({ + context: request.context, + audience: MCP_SERVER_DELEGATION_AUDIENCE, + }) + request.signal?.throwIfAborted() + const subject = resolvePrincipalSubject(principal) + provenance = + request.context.resolvedSecretTraceRegistry && subject?.kind === 'sim_user' + ? new ResolvedSecretTraceProvenanceAccumulator({ + userId: subject.userId, + workspaceId: request.context.workspaceId, + }) + : undefined + const policyTimeoutMs = getAsyncExecutionTimeoutForBillingAttribution( + request.context.billingAttribution + ) + const timeoutMs = capExecutionTimeoutMs( + policyTimeoutMs, + getRemainingExecutionMs(request.signal) + ) + const result = await executeMcpToolUseCase.execute({ + principal, + input: { + workspaceId: request.context.workspaceId, + serverId, + toolName, + arguments: args, + callChain: request.context.callChain, + timeoutMs, + signal: request.signal, + onResolvedSecretTraceProvenance: provenance + ? (value) => provenance?.record(value) + : undefined, + }, + }) + request.signal?.throwIfAborted() + const body = result.success + ? { success: true, data: { success: true, output: result.output } } + : { success: false, error: result.error } + return createResponse( + body, + result.success ? 200 : 400, + provenance, + request.context.resolvedSecretTraceRegistry, + request.toolId + ) + } catch (error) { + request.signal?.throwIfAborted() + if ( + error instanceof InvalidInternalDelegationBindingError || + (error instanceof Error && error.message === 'Authentication required') + ) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + if (error instanceof McpToolsNotAllowedError) { + return createResponse( + { success: false, error: error.message }, + 403, + provenance, + request.context.resolvedSecretTraceRegistry, + request.toolId + ) + } + if ( + error instanceof McpOauthAuthorizationRequiredError || + error instanceof McpOauthRedirectRequired || + error instanceof UnauthorizedError + ) { + const oauthServerId = + error instanceof McpOauthAuthorizationRequiredError ? error.serverId : serverId + return createResponse( + { + success: false, + error: 'OAuth re-authorization required', + code: 'reauth_required', + serverId: oauthServerId, + }, + 401, + provenance, + request.context.resolvedSecretTraceRegistry, + request.toolId + ) + } + + const orchestrationError = asOrchestrationError(error) + if (orchestrationError) { + const message = + orchestrationError.code === 'not_found' && + orchestrationError.message !== 'Tool not found on the specified server' + ? 'Resource not found' + : orchestrationError.message + return createResponse( + { success: false, error: message }, + statusForOrchestrationError(orchestrationError.code), + provenance, + request.context.resolvedSecretTraceRegistry, + request.toolId + ) + } + + const categorized = categorizeError(error) + if (categorized.status === 408) provenance?.markIncomplete('mcp-tool-execution-timeout') + logger.error('MCP tool execution failed', { + error: getErrorMessage(error), + requestId: request.requestId, + serverId, + toolName, + }) + return createResponse( + { success: false, error: categorized.message }, + categorized.status, + provenance, + request.context.resolvedSecretTraceRegistry, + request.toolId + ) + } +} diff --git a/apps/sim/lib/internal/memory/execute-tool.test.ts b/apps/sim/lib/internal/memory/execute-tool.test.ts new file mode 100644 index 00000000000..9f5eab78047 --- /dev/null +++ b/apps/sim/lib/internal/memory/execute-tool.test.ts @@ -0,0 +1,224 @@ +/** + * @vitest-environment node + */ + +import type { WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { ExecutionContext } from '@/executor/types' + +const mocks = vi.hoisted(() => ({ + createPrincipal: vi.fn(), + add: vi.fn(), + list: vi.fn(), + get: vi.fn(), + remove: vi.fn(), + createResponse: vi.fn(), +})) + +vi.mock('@/lib/internal/principals/executor', () => ({ + createExecutorPrincipalFromExecutionContext: mocks.createPrincipal, +})) + +vi.mock('@/lib/internal/memory/operations', () => ({ + executeMemoryAdd: mocks.add, + executeMemoryList: mocks.list, + executeMemoryGet: mocks.get, + executeMemoryDelete: mocks.remove, +})) + +vi.mock('@/lib/internal/memory/provenance', () => ({ + MemoryProvenanceError: class MemoryProvenanceError extends Error {}, + createMemoryToolResponse: mocks.createResponse, +})) + +import { executeMemoryTool } from '@/lib/internal/memory/execute-tool' + +const PRINCIPAL: WorkflowExecutionDelegatedPrincipal = { + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'workspace-canonical', + delegationId: 'delegation-1', + audience: 'sim:memory', + issuedAt: new Date('2026-08-27T00:00:00.000Z'), + expiresAt: new Date('2026-08-27T00:05:00.000Z'), + delegationContext: { kind: 'workflow_execution', workflowId: 'workflow-1' }, +} + +const ACTORLESS_DEPLOYED_PRINCIPAL: WorkflowExecutionDelegatedPrincipal = { + kind: 'delegated', + serviceId: 'executor', + workspaceId: 'workspace-canonical', + delegationId: 'delegation-actorless', + audience: 'sim:memory', + issuedAt: new Date('2026-08-27T00:00:00.000Z'), + expiresAt: new Date('2026-08-27T00:05:00.000Z'), + delegationContext: { + kind: 'workflow_execution', + workflowId: 'workflow-1', + executionId: 'execution-1', + principal: { + kind: 'system', + serviceId: 'schedule', + workspaceId: 'workspace-canonical', + workflowId: 'workflow-1', + }, + currentWorkflow: { + workflowId: 'workflow-1', + mode: 'deployment', + deploymentVersionId: 'deployment-1', + }, + }, +} + +const CONTEXT = { userId: 'user-1', workflowId: 'workflow-1' } as ExecutionContext + +const MEMORY = { + conversationId: 'conversation-1', + data: [{ role: 'user', content: 'hello' }], +} + +const CASES = [ + { + toolId: 'memory_add', + input: { + key: 'conversation-1', + data: { role: 'user', content: 'hello' }, + }, + operation: 'add' as const, + }, + { + toolId: 'memory_get_all', + input: {}, + operation: 'list' as const, + }, + { + toolId: 'memory_get', + input: { id: 'conversation-1' }, + operation: 'get' as const, + }, + { + toolId: 'memory_delete', + input: { conversationId: 'conversation-1' }, + operation: 'remove' as const, + }, +] + +describe('executeMemoryTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.createPrincipal.mockResolvedValue(PRINCIPAL) + mocks.add.mockResolvedValue({ body: { success: true, data: MEMORY } }) + mocks.list.mockResolvedValue({ + body: { success: true, data: { memories: [MEMORY] } }, + }) + mocks.get.mockResolvedValue({ body: { success: true, data: MEMORY } }) + mocks.remove.mockResolvedValue({ + body: { + success: true, + data: { message: 'Successfully deleted 1 memories', deletedCount: 1 }, + }, + }) + mocks.createResponse.mockImplementation(async (body) => Response.json(body)) + }) + + it.each(CASES)('dispatches $toolId through its canonical contract', async (testCase) => { + const response = await executeMemoryTool({ + toolId: testCase.toolId, + input: testCase.input, + headers: new Headers(), + context: CONTEXT, + requestId: 'request-1', + }) + + expect(response.status).toBe(200) + expect(mocks[testCase.operation]).toHaveBeenCalledOnce() + expect(mocks.createPrincipal).toHaveBeenCalledWith({ + context: CONTEXT, + audience: 'sim:memory', + }) + }) + + it('authenticates before validating operation input', async () => { + mocks.createPrincipal.mockRejectedValueOnce(new Error('Authentication required')) + const response = await executeMemoryTool({ + toolId: 'memory_add', + input: null, + headers: new Headers(), + context: CONTEXT, + requestId: 'request-1', + }) + + expect(response.status).toBe(401) + expect(await response.json()).toEqual({ + success: false, + error: { message: 'Authentication required' }, + }) + expect(mocks.add).not.toHaveBeenCalled() + }) + + it('preserves actorless deployed authority and uses only post-authorization provenance scope', async () => { + const provenanceScope = { + userId: 'billing-owner', + workspaceId: 'workspace-canonical', + } + const actorlessContext = { + workflowId: 'workflow-1', + workspaceId: 'workspace-canonical', + executionId: 'execution-1', + executorDelegationOrigin: { + workflowId: 'workflow-1', + executionId: 'execution-1', + principal: ACTORLESS_DEPLOYED_PRINCIPAL.delegationContext?.principal, + currentWorkflow: ACTORLESS_DEPLOYED_PRINCIPAL.delegationContext?.currentWorkflow, + }, + } + mocks.createPrincipal.mockResolvedValueOnce(ACTORLESS_DEPLOYED_PRINCIPAL) + mocks.list.mockResolvedValueOnce({ + body: { success: true, data: { memories: [MEMORY] } }, + provenance: [], + provenanceScope, + }) + + const response = await executeMemoryTool({ + toolId: 'memory_get_all', + input: {}, + headers: new Headers(), + context: actorlessContext, + requestId: 'request-1', + }) + + expect(response.status).toBe(200) + expect(mocks.list).toHaveBeenCalledWith( + expect.any(Object), + expect.objectContaining({ principal: ACTORLESS_DEPLOYED_PRINCIPAL }) + ) + expect(mocks.createResponse).toHaveBeenCalledWith(expect.any(Object), [], provenanceScope) + }) + + it('rejects invalid input and invalid operation responses', async () => { + const invalidInput = await executeMemoryTool({ + toolId: 'memory_get', + input: {}, + headers: new Headers(), + context: CONTEXT, + requestId: 'request-1', + }) + expect(invalidInput.status).toBe(400) + expect(await invalidInput.json()).toMatchObject({ error: 'Validation error' }) + + mocks.get.mockResolvedValueOnce({ body: { success: true, data: { conversationId: 42 } } }) + const invalidResponse = await executeMemoryTool({ + toolId: 'memory_get', + input: { id: 'conversation-1' }, + headers: new Headers(), + context: CONTEXT, + requestId: 'request-1', + }) + expect(invalidResponse.status).toBe(500) + expect(await invalidResponse.json()).toEqual({ + success: false, + error: { message: 'Failed to retrieve memory' }, + }) + }) +}) diff --git a/apps/sim/lib/internal/memory/execute-tool.ts b/apps/sim/lib/internal/memory/execute-tool.ts new file mode 100644 index 00000000000..19374b80108 --- /dev/null +++ b/apps/sim/lib/internal/memory/execute-tool.ts @@ -0,0 +1,156 @@ +import { createLogger } from '@sim/logger' +import { isPlainRecord } from '@sim/utils/object' +import type { ZodError } from 'zod' +import type { AnyApiRouteContract } from '@/lib/api/contracts' +import { + createMemoryContract, + deleteMemoryByQueryContract, + getMemoryByIdContract, + listMemoriesContract, + memoryDeleteQuerySchema, + memoryIdParamsSchema, + memoryListQuerySchema, + memoryPostBodySchema, +} from '@/lib/api/contracts/memory' +import { serializeZodIssues } from '@/lib/api/server/validation' +import { InvalidInternalDelegationBindingError } from '@/lib/auth/internal-delegation' +import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' +import { + executeMemoryAdd, + executeMemoryDelete, + executeMemoryGet, + executeMemoryList, + type MemoryToolOperationContext, + type MemoryToolOperationResult, +} from '@/lib/internal/memory/operations' +import { createMemoryToolResponse, MemoryProvenanceError } from '@/lib/internal/memory/provenance' +import { createExecutorPrincipalFromExecutionContext } from '@/lib/internal/principals/executor' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { MEMORY_DELEGATION_AUDIENCE } from '@/lib/memory/application/authorization' + +const logger = createLogger('MemoryInternalOperation') + +const FAILURE_MESSAGES: Record = { + memory_add: 'Failed to create memory', + memory_get: 'Failed to retrieve memory', + memory_get_all: 'Failed to search memories', + memory_delete: 'Failed to delete memories', +} + +function failureResponse(toolId: string, error: unknown): Response { + if (error instanceof MemoryProvenanceError) { + return Response.json({ error: error.message }, { status: 400 }) + } + const classified = asOrchestrationError(error) + if (classified) { + return Response.json( + { success: false, error: { message: classified.message } }, + { status: statusForOrchestrationError(classified.code) } + ) + } + logger.error(FAILURE_MESSAGES[toolId] ?? 'Memory operation failed', { error }) + return Response.json( + { + success: false, + error: { message: FAILURE_MESSAGES[toolId] ?? 'Memory operation failed' }, + }, + { status: 500 } + ) +} + +async function dispatchMemoryTool( + request: Parameters[0], + context: MemoryToolOperationContext +): Promise<{ contract: AnyApiRouteContract; result: MemoryToolOperationResult } | Response> { + const dispatched = async ( + contract: AnyApiRouteContract, + result: Promise + ) => ({ contract, result: await result }) + + switch (request.toolId) { + case 'memory_add': { + const parsed = memoryPostBodySchema.safeParse(request.input) + return parsed.success + ? dispatched(createMemoryContract, executeMemoryAdd(parsed.data, context)) + : validationResponse(parsed.error) + } + case 'memory_get_all': { + const parsed = memoryListQuerySchema.safeParse({ + ...(isPlainRecord(request.input) ? request.input : {}), + workspaceId: context.principal.workspaceId, + }) + return parsed.success + ? dispatched(listMemoriesContract, executeMemoryList(parsed.data, context)) + : validationResponse(parsed.error) + } + case 'memory_get': { + const parsed = memoryIdParamsSchema.safeParse(request.input) + return parsed.success + ? dispatched(getMemoryByIdContract, executeMemoryGet(parsed.data.id, context)) + : validationResponse(parsed.error) + } + case 'memory_delete': { + const parsed = memoryDeleteQuerySchema.safeParse({ + ...(isPlainRecord(request.input) ? request.input : {}), + workspaceId: context.principal.workspaceId, + }) + return parsed.success + ? dispatched(deleteMemoryByQueryContract, executeMemoryDelete(parsed.data, context)) + : validationResponse(parsed.error) + } + default: + return Response.json({ error: `Unsupported Memory tool: ${request.toolId}` }, { status: 500 }) + } +} + +function validationResponse(error: ZodError): Response { + return Response.json( + { error: 'Validation error', details: serializeZodIssues(error) }, + { status: 400 } + ) +} + +export const executeMemoryTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!Object.hasOwn(FAILURE_MESSAGES, request.toolId)) { + return Response.json({ error: `Unsupported Memory tool: ${request.toolId}` }, { status: 500 }) + } + + try { + const principal = await createExecutorPrincipalFromExecutionContext({ + context: request.context, + audience: MEMORY_DELEGATION_AUDIENCE, + }) + request.signal?.throwIfAborted() + const dispatched = await dispatchMemoryTool(request, { + principal, + headers: request.headers, + signal: request.signal, + }) + if (dispatched instanceof Response) return dispatched + if (dispatched.contract.response.mode !== 'json') { + throw new Error('Memory tool contract must return JSON') + } + const body = dispatched.contract.response.schema.parse(dispatched.result.body) as Record< + string, + unknown + > + return createMemoryToolResponse( + body, + dispatched.result.provenance, + dispatched.result.provenanceScope + ) + } catch (error) { + request.signal?.throwIfAborted() + if ( + error instanceof InvalidInternalDelegationBindingError || + (error instanceof Error && error.message === 'Authentication required') + ) { + return Response.json( + { success: false, error: { message: 'Authentication required' } }, + { status: 401 } + ) + } + return failureResponse(request.toolId, error) + } +} diff --git a/apps/sim/lib/internal/memory/operations.test.ts b/apps/sim/lib/internal/memory/operations.test.ts new file mode 100644 index 00000000000..8a202a47365 --- /dev/null +++ b/apps/sim/lib/internal/memory/operations.test.ts @@ -0,0 +1,155 @@ +/** + * @vitest-environment node + */ + +import type { WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + append: vi.fn(), + list: vi.fn(), + read: vi.fn(), + remove: vi.fn(), + requestsProvenance: vi.fn(), + suppliesWriteProvenance: vi.fn(), + readWriteProvenance: vi.fn(), + requireBillingAttribution: vi.fn(), +})) + +vi.mock('@/lib/billing/core/billing-attribution', () => ({ + requireWorkspaceBillingAttributionHeader: mocks.requireBillingAttribution, +})) + +vi.mock('@/lib/memory/application/use-cases', () => ({ + appendMemoryUseCase: { execute: mocks.append }, + listMemoriesUseCase: { execute: mocks.list }, + readMemoryUseCase: { execute: mocks.read }, + deleteMemoryUseCase: { execute: mocks.remove }, +})) + +vi.mock('@/lib/internal/memory/provenance', () => ({ + memoryToolRequestsProvenance: mocks.requestsProvenance, + memoryToolSuppliesWriteProvenance: mocks.suppliesWriteProvenance, + readMemoryWriteProvenance: mocks.readWriteProvenance, +})) + +import { + executeMemoryAdd, + executeMemoryDelete, + executeMemoryGet, + executeMemoryList, + type MemoryToolOperationContext, +} from '@/lib/internal/memory/operations' + +const PRINCIPAL: WorkflowExecutionDelegatedPrincipal = { + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'workspace-canonical', + delegationId: 'delegation-1', + audience: 'sim:memory', + issuedAt: new Date('2026-08-27T00:00:00.000Z'), + expiresAt: new Date('2026-08-27T00:05:00.000Z'), + delegationContext: { kind: 'workflow_execution', workflowId: 'workflow-1' }, +} + +const RECORD = { + id: 'memory-1', + key: 'conversation-1', + data: [{ role: 'user', content: 'hello' }], + secretProvenanceVersion: null, +} + +function context(): MemoryToolOperationContext { + return { principal: PRINCIPAL, headers: new Headers() } +} + +describe('Memory direct operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.requestsProvenance.mockReturnValue(false) + mocks.suppliesWriteProvenance.mockReturnValue(false) + mocks.readWriteProvenance.mockReturnValue(undefined) + mocks.requireBillingAttribution.mockReturnValue({ + billedAccountUserId: 'billing-owner', + workspaceId: 'workspace-canonical', + }) + mocks.append.mockResolvedValue({ record: RECORD }) + mocks.list.mockResolvedValue({ records: [RECORD] }) + mocks.read.mockResolvedValue({ record: RECORD }) + mocks.remove.mockResolvedValue({ deletedCount: 1 }) + }) + + it('binds append authority and provenance to the canonical delegated workspace', async () => { + const writeProvenance = { status: 'exact', entries: [] } + mocks.requestsProvenance.mockReturnValue(true) + mocks.suppliesWriteProvenance.mockReturnValue(true) + mocks.readWriteProvenance.mockReturnValue(writeProvenance) + + await executeMemoryAdd( + { + key: 'conversation-1', + workspaceId: 'workspace-forged', + data: { role: 'user', content: 'hello' }, + }, + context() + ) + + expect(mocks.append).toHaveBeenCalledWith({ + principal: PRINCIPAL, + input: expect.objectContaining({ + workspaceId: 'workspace-canonical', + key: 'conversation-1', + resolveWriteProvenance: expect.any(Function), + resolveBillingAttribution: expect.any(Function), + includePersistedSecretProvenance: true, + }), + }) + const input = mocks.append.mock.calls[0]?.[0].input + const scope = { userId: 'billing-owner', workspaceId: 'workspace-canonical' } + expect(input.resolveWriteProvenance(scope)).toBe(writeProvenance) + expect(mocks.readWriteProvenance).toHaveBeenCalledWith( + expect.any(Headers), + expect.objectContaining({ key: 'conversation-1' }), + scope + ) + await expect(input.resolveBillingAttribution('workspace-canonical')).resolves.toMatchObject({ + billedAccountUserId: 'billing-owner', + }) + expect(mocks.requireBillingAttribution).toHaveBeenCalledWith(expect.any(Headers), { + workspaceId: 'workspace-canonical', + }) + }) + + it('preserves list, read, and delete semantics without trusting workspace parameters', async () => { + await executeMemoryList({ workspaceId: 'workspace-forged', query: null, limit: 50 }, context()) + await executeMemoryGet('conversation-1', context()) + const deleted = await executeMemoryDelete( + { workspaceId: 'workspace-forged', conversationId: 'conversation-1' }, + context() + ) + + expect(mocks.list).toHaveBeenCalledWith({ + principal: PRINCIPAL, + input: expect.objectContaining({ workspaceId: 'workspace-canonical', limit: 50 }), + }) + expect(mocks.read).toHaveBeenCalledWith({ + principal: PRINCIPAL, + input: expect.objectContaining({ + workspaceId: 'workspace-canonical', + key: 'conversation-1', + }), + }) + expect(mocks.remove).toHaveBeenCalledWith({ + principal: PRINCIPAL, + input: expect.objectContaining({ + workspaceId: 'workspace-canonical', + key: 'conversation-1', + }), + }) + expect(deleted.body).toEqual({ + success: true, + data: { message: 'Successfully deleted 1 memories', deletedCount: 1 }, + }) + }) +}) diff --git a/apps/sim/lib/internal/memory/operations.ts b/apps/sim/lib/internal/memory/operations.ts new file mode 100644 index 00000000000..29103641ced --- /dev/null +++ b/apps/sim/lib/internal/memory/operations.ts @@ -0,0 +1,152 @@ +import type { WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' +import type { ContractBody, ContractQuery } from '@/lib/api/contracts' +import type { + createMemoryContract, + deleteMemoryByQueryContract, + listMemoriesContract, +} from '@/lib/api/contracts/memory' +import { requireWorkspaceBillingAttributionHeader } from '@/lib/billing/core/billing-attribution' +import { + memoryToolRequestsProvenance, + memoryToolSuppliesWriteProvenance, + readMemoryWriteProvenance, +} from '@/lib/internal/memory/provenance' +import { + appendMemoryUseCase, + deleteMemoryUseCase, + listMemoriesUseCase, + type MemoryLegacyProvenanceScope, + type MemoryReadProvenance, + readMemoryUseCase, +} from '@/lib/memory/application/use-cases' + +export interface MemoryToolOperationContext { + principal: WorkflowExecutionDelegatedPrincipal + headers: Headers + signal?: AbortSignal +} + +export interface MemoryToolOperationResult { + body: Record + provenance?: MemoryReadProvenance[] + provenanceScope?: MemoryLegacyProvenanceScope +} + +function complete(context: MemoryToolOperationContext, value: T): T { + context.signal?.throwIfAborted() + return value +} + +export async function executeMemoryAdd( + body: ContractBody, + context: MemoryToolOperationContext +): Promise { + const includePersistedSecretProvenance = memoryToolRequestsProvenance(context.headers) + const resolveWriteProvenance = memoryToolSuppliesWriteProvenance(context.headers, body) + ? (scope: MemoryLegacyProvenanceScope) => + readMemoryWriteProvenance(context.headers, body, scope) + : undefined + const result = await appendMemoryUseCase.execute({ + principal: context.principal, + input: { + workspaceId: context.principal.workspaceId, + key: body.key ?? '', + data: body.data, + ...(resolveWriteProvenance ? { resolveWriteProvenance } : {}), + includePersistedSecretProvenance, + resolveBillingAttribution: async (workspaceId) => + requireWorkspaceBillingAttributionHeader(context.headers, { workspaceId }), + signal: context.signal, + }, + }) + return complete(context, { + body: { + success: true, + data: { conversationId: result.record.key, data: result.record.data }, + }, + provenance: result.readProvenance, + provenanceScope: result.provenanceScope, + }) +} + +export async function executeMemoryList( + query: ContractQuery, + context: MemoryToolOperationContext +): Promise { + const result = await listMemoriesUseCase.execute({ + principal: context.principal, + input: { + workspaceId: context.principal.workspaceId, + query: query.query, + limit: query.limit, + includePersistedSecretProvenance: memoryToolRequestsProvenance(context.headers), + resolveBillingAttribution: async (workspaceId) => + requireWorkspaceBillingAttributionHeader(context.headers, { workspaceId }), + signal: context.signal, + }, + }) + return complete(context, { + body: { + success: true, + data: { + memories: result.records.map((record) => ({ + conversationId: record.key, + data: record.data, + })), + }, + }, + provenance: result.readProvenance, + provenanceScope: result.provenanceScope, + }) +} + +export async function executeMemoryGet( + key: string, + context: MemoryToolOperationContext +): Promise { + const result = await readMemoryUseCase.execute({ + principal: context.principal, + input: { + workspaceId: context.principal.workspaceId, + key, + includePersistedSecretProvenance: memoryToolRequestsProvenance(context.headers), + resolveBillingAttribution: async (workspaceId) => + requireWorkspaceBillingAttributionHeader(context.headers, { workspaceId }), + signal: context.signal, + }, + }) + return complete(context, { + body: { + success: true, + data: result.record ? { conversationId: result.record.key, data: result.record.data } : null, + }, + provenance: result.readProvenance, + provenanceScope: result.provenanceScope, + }) +} + +export async function executeMemoryDelete( + query: ContractQuery, + context: MemoryToolOperationContext +): Promise { + const result = await deleteMemoryUseCase.execute({ + principal: context.principal, + input: { + workspaceId: context.principal.workspaceId, + key: query.conversationId ?? '', + signal: context.signal, + }, + }) + return complete(context, { + body: { + success: true, + data: { + message: + result.deletedCount > 0 + ? `Successfully deleted ${result.deletedCount} memories` + : 'No memories found matching the criteria', + deletedCount: result.deletedCount, + }, + }, + }) +} diff --git a/apps/sim/lib/internal/memory/provenance.test.ts b/apps/sim/lib/internal/memory/provenance.test.ts new file mode 100644 index 00000000000..c84611b03d5 --- /dev/null +++ b/apps/sim/lib/internal/memory/provenance.test.ts @@ -0,0 +1,97 @@ +/** + * @vitest-environment node + */ + +import { describe, expect, it } from 'vitest' +import { + PRIVATE_SECRET_PROVENANCE_BUNDLE_V1, + PRIVATE_SECRET_PROVENANCE_FIELD, + PRIVATE_SECRET_PROVENANCE_HEADER, + PRIVATE_TOOL_METADATA_REQUEST_HEADER, + RESOLVED_SECRET_PROVENANCE_FIELD, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, +} from '@/lib/execution/private-tool-metadata' +import { + createMemoryToolResponse, + MemoryProvenanceError, + memoryToolRequestsProvenance, + memoryToolSuppliesWriteProvenance, + readMemoryWriteProvenance, +} from '@/lib/internal/memory/provenance' + +const PROVENANCE_SCOPE = { userId: 'billing-owner', workspaceId: 'workspace-1' } + +function privateWritePayload(workspaceId: string) { + return { + [PRIVATE_SECRET_PROVENANCE_FIELD]: { + version: 1 as const, + complete: true, + selections: [ + { + key: 'data', + provenance: { + version: 1 as const, + complete: true, + entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }], + scope: { userId: 'workflow-owner', workspaceId }, + }, + }, + ], + }, + } +} + +describe('Memory direct provenance', () => { + it('keeps unsupported headerless executor writes on the legacy untracked path', () => { + expect(memoryToolSuppliesWriteProvenance(new Headers(), {})).toBe(false) + expect(readMemoryWriteProvenance(new Headers(), {}, PROVENANCE_SCOPE)).toBeUndefined() + }) + + it('binds authenticated provenance to the canonical workspace and preserves its source owner', () => { + const headers = new Headers({ + [PRIVATE_SECRET_PROVENANCE_HEADER]: PRIVATE_SECRET_PROVENANCE_BUNDLE_V1, + }) + expect(memoryToolSuppliesWriteProvenance(headers, privateWritePayload('workspace-1'))).toBe( + true + ) + + expect( + readMemoryWriteProvenance(headers, privateWritePayload('workspace-1'), PROVENANCE_SCOPE) + ).toEqual({ + status: 'exact', + entries: [ + { + name: 'TOKEN', + encryptedValue: 'encrypted-token', + sourceUserId: 'workflow-owner', + sourceWorkspaceId: 'workspace-1', + }, + ], + }) + expect(() => + readMemoryWriteProvenance(headers, privateWritePayload('workspace-2'), PROVENANCE_SCOPE) + ).toThrow(MemoryProvenanceError) + }) + + it('negotiates and serializes the private response envelope without exposing metadata by default', async () => { + expect(memoryToolRequestsProvenance(new Headers())).toBe(false) + const requestedHeaders = new Headers({ + [PRIVATE_TOOL_METADATA_REQUEST_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1, + }) + expect(memoryToolRequestsProvenance(requestedHeaders)).toBe(true) + + const body = { success: true, data: { memories: [] } } + const ordinary = await createMemoryToolResponse(body, undefined, undefined) + expect(await ordinary.json()).toEqual(body) + + const privateResponse = await createMemoryToolResponse( + body, + [{ data: [], provenance: { status: 'exact', entries: [] } }], + PROVENANCE_SCOPE + ) + expect(await privateResponse.json()).toMatchObject({ + ...body, + [RESOLVED_SECRET_PROVENANCE_FIELD]: { version: 1, complete: true, entries: [] }, + }) + }) +}) diff --git a/apps/sim/lib/internal/memory/provenance.ts b/apps/sim/lib/internal/memory/provenance.ts new file mode 100644 index 00000000000..788186ecef4 --- /dev/null +++ b/apps/sim/lib/internal/memory/provenance.ts @@ -0,0 +1,80 @@ +import { + type DurableSecretProvenance, + durableSecretProvenanceFromPrivateBundle, + importDurableSecretProvenance, +} from '@/lib/execution/durable-secret-provenance' +import { + inspectPrivateSecretProvenanceRequest, + isPrivateSecretProvenanceBundleV1, +} from '@/lib/execution/model-input-provenance' +import { + negotiatePrivateToolMetadataResponse, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, + serializePrivateToolMetadataResponseEnvelope, +} from '@/lib/execution/private-tool-metadata' +import type { + MemoryLegacyProvenanceScope, + MemoryReadProvenance, +} from '@/lib/memory/application/use-cases' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' + +export class MemoryProvenanceError extends Error { + constructor() { + super('Invalid memory secret provenance') + this.name = 'MemoryProvenanceError' + } +} + +export function memoryToolSuppliesWriteProvenance(headers: Headers, payload: unknown): boolean { + return inspectPrivateSecretProvenanceRequest(headers, payload).status !== 'unsupported' +} + +export function readMemoryWriteProvenance( + headers: Headers, + payload: unknown, + scope: MemoryLegacyProvenanceScope +): DurableSecretProvenance | undefined { + const inspection = inspectPrivateSecretProvenanceRequest(headers, payload) + if (inspection.status === 'unsupported') return undefined + if (inspection.status !== 'verified' || !isPrivateSecretProvenanceBundleV1(inspection.value)) { + throw new MemoryProvenanceError() + } + if (!inspection.value.complete) return { status: 'unknown' } + if (inspection.value.selections.length !== 1) throw new MemoryProvenanceError() + + const provenance = durableSecretProvenanceFromPrivateBundle(inspection.value, 'data', scope) + if (!provenance) throw new MemoryProvenanceError() + return provenance +} + +export function memoryToolRequestsProvenance(headers: Headers): boolean { + const negotiation = negotiatePrivateToolMetadataResponse( + headers, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, + true + ) + if (negotiation.status === 'rejected') throw new MemoryProvenanceError() + return negotiation.status !== 'not-requested' +} + +export async function createMemoryToolResponse( + body: Record, + provenance: MemoryReadProvenance[] | undefined, + scope: MemoryLegacyProvenanceScope | undefined +): Promise { + if (provenance === undefined) return Response.json(body) + if (!scope) throw new MemoryProvenanceError() + + const registry = new ResolvedSecretTraceRegistry([], scope) + for (const item of provenance) { + await importDurableSecretProvenance(registry, item.provenance, item.data, 'memory', { + reportUnrecorded: false, + }) + } + const envelope = serializePrivateToolMetadataResponseEnvelope( + body, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, + registry.exportCommittedProvenanceForValue(body) + ) + return Response.json(envelope.body, { headers: envelope.headers }) +} diff --git a/apps/sim/lib/internal/microsoft-dataverse/client.ts b/apps/sim/lib/internal/microsoft-dataverse/client.ts new file mode 100644 index 00000000000..7fc2c067d1d --- /dev/null +++ b/apps/sim/lib/internal/microsoft-dataverse/client.ts @@ -0,0 +1,48 @@ +import { isRecordLike } from '@sim/utils/object' +import { + MAX_JSON_API_RESPONSE_BYTES, + secureFetchWithValidation, +} from '@/lib/core/security/input-validation.server' +import { consumeOrCancelBody } from '@/lib/core/utils/stream-limits' +import { DataverseOperationError } from '@/lib/internal/microsoft-dataverse/errors' + +export async function uploadDataverseFile( + input: { + accessToken: string + fileName: string + uploadUrl: string + }, + buffer: Buffer, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const response = await secureFetchWithValidation( + input.uploadUrl, + { + method: 'PATCH', + headers: { + Authorization: `Bearer ${input.accessToken}`, + 'Content-Type': 'application/octet-stream', + 'OData-MaxVersion': '4.0', + 'OData-Version': '4.0', + 'x-ms-file-name': input.fileName, + }, + body: buffer, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + signal, + stripAuthOnRedirect: true, + }, + 'environmentUrl' + ) + if (response.ok) { + await consumeOrCancelBody(response) + return + } + const data = await response.json().catch(() => null) + const error = isRecordLike(data) && isRecordLike(data.error) ? data.error : null + const message = + error && typeof error.message === 'string' + ? error.message + : `Dataverse API error: ${response.status} ${response.statusText}` + throw new DataverseOperationError(message, response.status) +} diff --git a/apps/sim/lib/internal/microsoft-dataverse/errors.ts b/apps/sim/lib/internal/microsoft-dataverse/errors.ts new file mode 100644 index 00000000000..3ccc0b4463c --- /dev/null +++ b/apps/sim/lib/internal/microsoft-dataverse/errors.ts @@ -0,0 +1,10 @@ +export class DataverseOperationError extends Error { + constructor( + message: string, + readonly status: number, + readonly body: Record = { success: false, error: message } + ) { + super(message) + this.name = 'DataverseOperationError' + } +} diff --git a/apps/sim/lib/internal/microsoft-dataverse/execute-tool.ts b/apps/sim/lib/internal/microsoft-dataverse/execute-tool.ts new file mode 100644 index 00000000000..6807f7d1053 --- /dev/null +++ b/apps/sim/lib/internal/microsoft-dataverse/execute-tool.ts @@ -0,0 +1,67 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { DataverseOperationError } from '@/lib/internal/microsoft-dataverse/errors' +import { executeDataverseUploadFile } from '@/lib/internal/microsoft-dataverse/operations' +import { dataverseUploadFileInputSchema } from '@/lib/internal/microsoft-dataverse/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const logger = createLogger('DataverseToolExecution') + +function inputSizeError(input: unknown): Response | null { + let serialized: string + try { + serialized = JSON.stringify(input) ?? '' + } catch { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + return Buffer.byteLength(serialized) > DEFAULT_MAX_JSON_BODY_BYTES + ? Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + : null +} + +export const executeMicrosoftDataverseTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'microsoft_dataverse_upload_file') { + return Response.json( + { success: false, error: `Unsupported Microsoft Dataverse tool: ${request.toolId}` }, + { status: 500 } + ) + } + const userId = request.context.userId + if (!userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + const sizeError = inputSizeError(request.input) + if (sizeError) return sizeError + const parsed = dataverseUploadFileInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { success: false, error: getValidationErrorMessage(parsed.error, 'Invalid request data') }, + { status: 400 } + ) + } + try { + const result = await executeDataverseUploadFile(parsed.data, { + requestId: request.requestId, + signal: request.signal, + userId, + }) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof DataverseOperationError) { + return Response.json(error.body, { status: error.status }) + } + const message = getErrorMessage(error, 'Internal server error') + logger.error('Dataverse file upload failed', { error: message, requestId: request.requestId }) + return Response.json({ success: false, error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/microsoft-dataverse/operations.ts b/apps/sim/lib/internal/microsoft-dataverse/operations.ts new file mode 100644 index 00000000000..63664c67d76 --- /dev/null +++ b/apps/sim/lib/internal/microsoft-dataverse/operations.ts @@ -0,0 +1,99 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { isRecordLike } from '@sim/utils/object' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { uploadDataverseFile } from '@/lib/internal/microsoft-dataverse/client' +import { DataverseOperationError } from '@/lib/internal/microsoft-dataverse/errors' +import type { DataverseUploadFileInput } from '@/lib/internal/microsoft-dataverse/schema' +import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/doc-not-ready' +import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' +import { getDataverseBaseUrl } from '@/tools/microsoft_dataverse/utils' + +const logger = createLogger('DataverseOperations') +const MAX_UPLOAD_BYTES = 128 * 1024 * 1024 + +export interface DataverseOperationContext { + requestId: string + signal?: AbortSignal + userId: string +} + +async function deniedBody(response: Response): Promise> { + const body: unknown = await response.json().catch(() => null) + return isRecordLike(body) ? body : { success: false, error: 'File not found' } +} + +function uploadTooLargeError(observedBytes: number): DataverseOperationError { + const sizeMB = (observedBytes / (1024 * 1024)).toFixed(2) + return new DataverseOperationError( + `File size (${sizeMB}MB) exceeds Dataverse's 128MB limit for single-request file column uploads. Split the file and use chunked upload instead.`, + 400 + ) +} + +export async function executeDataverseUploadFile( + input: DataverseUploadFileInput, + context: DataverseOperationContext +) { + context.signal?.throwIfAborted() + let buffer: Buffer + if (input.file) { + let userFile + try { + userFile = processSingleFileToUserFile(input.file, context.requestId, logger) + } catch (error) { + throw new DataverseOperationError(getErrorMessage(error, 'Failed to process file'), 400) + } + const denied = await assertToolFileAccess( + userFile.key, + context.userId, + context.requestId, + logger + ) + context.signal?.throwIfAborted() + if (denied) { + throw new DataverseOperationError('File not found', denied.status, await deniedBody(denied)) + } + try { + const downloaded = await downloadServableFileFromStorage( + userFile, + context.requestId, + logger, + { maxBytes: MAX_UPLOAD_BYTES, signal: context.signal } + ) + buffer = downloaded.buffer + } catch (error) { + context.signal?.throwIfAborted() + if (isDocNotReadyError(error)) throw new DataverseOperationError(docNotReadyMessage(), 409) + if (isPayloadSizeLimitError(error)) { + throw uploadTooLargeError(error.observedBytes ?? userFile.size) + } + throw new DataverseOperationError(getErrorMessage(error, 'Failed to download file'), 500) + } + } else if (input.fileContent) { + buffer = Buffer.from(input.fileContent, 'base64') + } else { + throw new DataverseOperationError('Either file or fileContent must be provided', 400) + } + if (buffer.length > MAX_UPLOAD_BYTES) throw uploadTooLargeError(buffer.length) + + const baseUrl = getDataverseBaseUrl(input.environmentUrl) + const uploadUrl = `${baseUrl}/api/data/v9.2/${input.entitySetName.trim()}(${input.recordId.trim()})/${input.fileColumn.trim()}` + await uploadDataverseFile( + { accessToken: input.accessToken, fileName: input.fileName, uploadUrl }, + buffer, + context.signal + ) + context.signal?.throwIfAborted() + return { + success: true, + output: { + recordId: input.recordId, + fileColumn: input.fileColumn, + fileName: input.fileName, + success: true, + }, + } +} diff --git a/apps/sim/lib/internal/microsoft-dataverse/schema.ts b/apps/sim/lib/internal/microsoft-dataverse/schema.ts new file mode 100644 index 00000000000..9589eb859b4 --- /dev/null +++ b/apps/sim/lib/internal/microsoft-dataverse/schema.ts @@ -0,0 +1,15 @@ +import { z } from 'zod' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' + +export const dataverseUploadFileInputSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + environmentUrl: z.string().min(1, 'Environment URL is required'), + entitySetName: z.string().min(1, 'Entity set name is required'), + recordId: z.string().min(1, 'Record ID is required'), + fileColumn: z.string().min(1, 'File column is required'), + fileName: z.string().min(1, 'File name is required'), + file: RawFileInputSchema.optional().nullable(), + fileContent: z.string().optional().nullable(), +}) + +export type DataverseUploadFileInput = z.output diff --git a/apps/sim/lib/internal/microsoft-teams/client.ts b/apps/sim/lib/internal/microsoft-teams/client.ts new file mode 100644 index 00000000000..1ac3399dcb4 --- /dev/null +++ b/apps/sim/lib/internal/microsoft-teams/client.ts @@ -0,0 +1,57 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { isRecordLike } from '@sim/utils/object' +import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' +import { MicrosoftTeamsOperationError } from '@/lib/internal/microsoft-teams/errors' + +const MICROSOFT_GRAPH_BASE_URL = 'https://graph.microsoft.com/v1.0' +const MICROSOFT_GRAPH_RESPONSE_MAX_BYTES = 2 * 1024 * 1024 + +export type MicrosoftTeamsGraphObject = Record + +function asObject(value: unknown): MicrosoftTeamsGraphObject { + return isRecordLike(value) ? value : {} +} + +function errorMessage(data: MicrosoftTeamsGraphObject, fallback: string): string { + const error = asObject(data.error) + return typeof error.message === 'string' && error.message ? error.message : fallback +} + +export class MicrosoftTeamsClient { + constructor(private readonly accessToken: string) {} + + async json( + path: string, + init: RequestInit, + fallbackError: string, + signal?: AbortSignal + ): Promise { + signal?.throwIfAborted() + const response = await fetch(`${MICROSOFT_GRAPH_BASE_URL}${path}`, { + ...init, + headers: { + Authorization: `Bearer ${this.accessToken}`, + ...init.headers, + }, + signal, + }) + const text = await readResponseTextWithLimit(response, { + maxBytes: MICROSOFT_GRAPH_RESPONSE_MAX_BYTES, + label: 'Microsoft Graph response', + signal, + }) + signal?.throwIfAborted() + + let data: MicrosoftTeamsGraphObject + try { + data = text ? asObject(JSON.parse(text)) : {} + } catch (error) { + if (!response.ok) throw new MicrosoftTeamsOperationError(fallbackError, response.status) + throw new Error(getErrorMessage(error, 'Microsoft Graph returned invalid JSON')) + } + if (!response.ok) { + throw new MicrosoftTeamsOperationError(errorMessage(data, fallbackError), response.status) + } + return data + } +} diff --git a/apps/sim/lib/internal/microsoft-teams/errors.ts b/apps/sim/lib/internal/microsoft-teams/errors.ts new file mode 100644 index 00000000000..1e457cb22e6 --- /dev/null +++ b/apps/sim/lib/internal/microsoft-teams/errors.ts @@ -0,0 +1,9 @@ +export class MicrosoftTeamsOperationError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'MicrosoftTeamsOperationError' + } +} diff --git a/apps/sim/lib/internal/microsoft-teams/execute-tool.test.ts b/apps/sim/lib/internal/microsoft-teams/execute-tool.test.ts new file mode 100644 index 00000000000..da09c978423 --- /dev/null +++ b/apps/sim/lib/internal/microsoft-teams/execute-tool.test.ts @@ -0,0 +1,84 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + deleteMicrosoftTeamsChatMessage: vi.fn(), + writeMicrosoftTeamsChannelMessage: vi.fn(), + writeMicrosoftTeamsChatMessage: vi.fn(), +})) + +vi.mock('@/lib/internal/microsoft-teams/operations', () => ({ + deleteMicrosoftTeamsChatMessage: mocks.deleteMicrosoftTeamsChatMessage, + writeMicrosoftTeamsChannelMessage: mocks.writeMicrosoftTeamsChannelMessage, + writeMicrosoftTeamsChatMessage: mocks.writeMicrosoftTeamsChatMessage, +})) + +import { executeMicrosoftTeamsTool } from '@/lib/internal/microsoft-teams/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +describe('executeMicrosoftTeamsTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.deleteMicrosoftTeamsChatMessage.mockResolvedValue({ success: true, output: {} }) + mocks.writeMicrosoftTeamsChannelMessage.mockResolvedValue({ success: true, output: {} }) + mocks.writeMicrosoftTeamsChatMessage.mockResolvedValue({ success: true, output: {} }) + }) + + it('dispatches typed input with cancellation', async () => { + const controller = new AbortController() + const input = { accessToken: 'token', chatId: 'chat-1', messageId: 'message-1' } + const request: InternalToolOperationCall = { + toolId: 'microsoft_teams_delete_chat_message', + input, + headers: new Headers(), + context: createExecutionContext(), + requestId: 'request-1', + signal: controller.signal, + } + + expect((await executeMicrosoftTeamsTool(request)).status).toBe(200) + expect(mocks.deleteMicrosoftTeamsChatMessage).toHaveBeenCalledWith(input, { + signal: controller.signal, + }) + }) + + it.each([ + { + toolId: 'microsoft_teams_write_chat', + input: { accessToken: 'token', chatId: 'chat-1', content: 'hello', files: null }, + operation: mocks.writeMicrosoftTeamsChatMessage, + }, + { + toolId: 'microsoft_teams_write_channel', + input: { + accessToken: 'token', + teamId: 'team-1', + channelId: 'channel-1', + content: 'hello', + files: null, + }, + operation: mocks.writeMicrosoftTeamsChannelMessage, + }, + ])('dispatches $toolId with trusted execution context', async ({ toolId, input, operation }) => { + const controller = new AbortController() + const context = { ...createExecutionContext(), userId: 'user-1' } + const response = await executeMicrosoftTeamsTool({ + toolId, + input, + headers: new Headers(), + context, + requestId: 'request-1', + signal: controller.signal, + }) + + expect(response.status).toBe(200) + expect(operation).toHaveBeenCalledWith(input, { + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + }) + }) +}) diff --git a/apps/sim/lib/internal/microsoft-teams/execute-tool.ts b/apps/sim/lib/internal/microsoft-teams/execute-tool.ts new file mode 100644 index 00000000000..0c371d540b9 --- /dev/null +++ b/apps/sim/lib/internal/microsoft-teams/execute-tool.ts @@ -0,0 +1,87 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { z } from 'zod' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { MicrosoftTeamsOperationError } from '@/lib/internal/microsoft-teams/errors' +import { + deleteMicrosoftTeamsChatMessage, + writeMicrosoftTeamsChannelMessage, + writeMicrosoftTeamsChatMessage, +} from '@/lib/internal/microsoft-teams/operations' +import { + microsoftTeamsWriteChannelInputSchema, + microsoftTeamsWriteChatInputSchema, +} from '@/lib/internal/microsoft-teams/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const deleteInputSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + chatId: z.string().min(1, 'Chat ID is required'), + messageId: z.string().min(1, 'Message ID is required'), +}) + +function inputSizeError(input: unknown): Response | null { + let serialized: string + try { + serialized = JSON.stringify(input) ?? '' + } catch { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + if (Buffer.byteLength(serialized) <= DEFAULT_MAX_JSON_BODY_BYTES) return null + return Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) +} + +export const executeMicrosoftTeamsTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + const sizeError = inputSizeError(request.input) + if (sizeError) return sizeError + try { + const context = { + requestId: request.requestId, + signal: request.signal, + userId: request.context.userId, + } + switch (request.toolId) { + case 'microsoft_teams_write_chat': { + const parsed = microsoftTeamsWriteChatInputSchema.safeParse(request.input) + if (!parsed.success) return validationErrorResponse(parsed.error) + return Response.json(await writeMicrosoftTeamsChatMessage(parsed.data, context)) + } + case 'microsoft_teams_write_channel': { + const parsed = microsoftTeamsWriteChannelInputSchema.safeParse(request.input) + if (!parsed.success) return validationErrorResponse(parsed.error) + return Response.json(await writeMicrosoftTeamsChannelMessage(parsed.data, context)) + } + case 'microsoft_teams_delete_chat_message': { + const parsed = deleteInputSchema.safeParse(request.input) + if (!parsed.success) return validationErrorResponse(parsed.error) + return Response.json( + await deleteMicrosoftTeamsChatMessage(parsed.data, { signal: request.signal }) + ) + } + default: + return Response.json( + { success: false, error: `Unsupported Microsoft Teams tool: ${request.toolId}` }, + { status: 500 } + ) + } + } catch (error) { + request.signal?.throwIfAborted() + return Response.json( + { success: false, error: getErrorMessage(error, 'Unknown error occurred') }, + { status: error instanceof MicrosoftTeamsOperationError ? error.status : 500 } + ) + } +} + +function validationErrorResponse(error: z.ZodError): Response { + return Response.json( + { success: false, error: getValidationErrorMessage(error, 'Invalid request data') }, + { status: 400 } + ) +} diff --git a/apps/sim/lib/internal/microsoft-teams/operations.test.ts b/apps/sim/lib/internal/microsoft-teams/operations.test.ts new file mode 100644 index 00000000000..356c68571f7 --- /dev/null +++ b/apps/sim/lib/internal/microsoft-teams/operations.test.ts @@ -0,0 +1,129 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + assertToolFileAccess: vi.fn(), + downloadServableFileFromStorage: vi.fn(), + fetch: vi.fn(), + processFilesToUserFiles: vi.fn(), +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertToolFileAccess, +})) + +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + processFilesToUserFiles: mocks.processFilesToUserFiles, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mocks.downloadServableFileFromStorage, +})) + +import { + deleteMicrosoftTeamsChatMessage, + writeMicrosoftTeamsChatMessage, +} from '@/lib/internal/microsoft-teams/operations' + +describe('deleteMicrosoftTeamsChatMessage', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mocks.fetch) + mocks.assertToolFileAccess.mockResolvedValue(null) + mocks.processFilesToUserFiles.mockReturnValue([]) + mocks.fetch.mockResolvedValueOnce(Response.json({ id: 'user-1' })).mockResolvedValueOnce( + new Response(null, { + status: 204, + }) + ) + }) + + it('uses the authenticated Graph user and soft-deletes exactly once', async () => { + const controller = new AbortController() + const result = await deleteMicrosoftTeamsChatMessage( + { accessToken: 'token', chatId: ' chat-1 ', messageId: ' message-1 ' }, + { signal: controller.signal } + ) + + expect(mocks.fetch).toHaveBeenCalledTimes(2) + expect(mocks.fetch.mock.calls[1][0]).toContain( + '/users/user-1/chats/chat-1/messages/message-1/softDelete' + ) + expect(mocks.fetch.mock.calls[1][1]).toEqual( + expect.objectContaining({ method: 'POST', signal: controller.signal }) + ) + expect(result.output).toEqual({ + deleted: true, + messageId: 'message-1', + metadata: { messageId: 'message-1', chatId: 'chat-1' }, + }) + }) + + it('sends plain chat content through the same cancellable operation path', async () => { + mocks.fetch.mockReset() + mocks.fetch.mockResolvedValue( + Response.json({ + id: 'message-1', + chatId: 'chat-1', + body: { content: 'hello' }, + createdDateTime: '2026-01-01T00:00:00Z', + webUrl: 'https://teams.example/message-1', + }) + ) + const controller = new AbortController() + const result = await writeMicrosoftTeamsChatMessage( + { accessToken: 'token', chatId: ' chat-1 ', content: 'hello', files: null }, + { requestId: 'request-1', signal: controller.signal, userId: 'user-1' } + ) + + expect(mocks.fetch).toHaveBeenCalledOnce() + expect(mocks.fetch.mock.calls[0][0]).toContain('/chats/chat-1/messages') + expect(mocks.fetch.mock.calls[0][1]).toEqual( + expect.objectContaining({ method: 'POST', signal: controller.signal }) + ) + expect(result.output).toEqual({ + updatedContent: true, + metadata: { + messageId: 'message-1', + chatId: 'chat-1', + content: 'hello', + createdTime: '2026-01-01T00:00:00Z', + url: 'https://teams.example/message-1', + }, + }) + }) + + it('resolves mentions in-process while preserving the enhanced output envelope', async () => { + mocks.fetch.mockReset() + mocks.fetch + .mockResolvedValueOnce( + Response.json({ + value: [{ id: 'member-1', displayName: 'Ada', userIdentityType: 'aadUser' }], + }) + ) + .mockResolvedValueOnce( + Response.json({ + id: 'message-1', + chatId: 'chat-1', + body: { content: 'Ada hello' }, + }) + ) + const result = await writeMicrosoftTeamsChatMessage( + { accessToken: 'token', chatId: 'chat-1', content: 'Ada hello', files: null }, + { requestId: 'request-1', userId: 'user-1' } + ) + const sentBody = JSON.parse(String(mocks.fetch.mock.calls[1][1]?.body)) + + expect(sentBody).toMatchObject({ + body: { contentType: 'html', content: 'Ada hello' }, + mentions: [{ id: 0, mentionText: 'Ada' }], + }) + expect(result.output).toMatchObject({ + updatedContent: true, + metadata: { chatId: 'chat-1', attachmentCount: 0 }, + files: [], + }) + }) +}) diff --git a/apps/sim/lib/internal/microsoft-teams/operations.ts b/apps/sim/lib/internal/microsoft-teams/operations.ts new file mode 100644 index 00000000000..4de7a0cd846 --- /dev/null +++ b/apps/sim/lib/internal/microsoft-teams/operations.ts @@ -0,0 +1,503 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { isRecordLike } from '@sim/utils/object' +import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' +import { + MicrosoftTeamsClient, + type MicrosoftTeamsGraphObject, +} from '@/lib/internal/microsoft-teams/client' +import { MicrosoftTeamsOperationError } from '@/lib/internal/microsoft-teams/errors' +import type { + MicrosoftTeamsWriteChannelInput, + MicrosoftTeamsWriteChatInput, +} from '@/lib/internal/microsoft-teams/schema' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/doc-not-ready' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' +import type { UserFile } from '@/executor/types' + +const MAX_GRAPH_RESPONSE_BYTES = 2 * 1024 * 1024 +const MAX_TEAMS_FILE_SIZE = 4 * 1024 * 1024 +const MENTION_PATTERN = /[^<]+<\/at>/i + +const logger = createLogger('MicrosoftTeamsOperations') + +interface GraphResponse { + id?: string + error?: { message?: string } +} + +export interface MicrosoftTeamsOperationContext { + requestId?: string + signal?: AbortSignal + userId?: string +} + +interface TeamsFileOutput { + name: string + mimeType: string + data: string + size: number +} + +interface TeamsAttachmentRef { + id: string + contentType: 'reference' + contentUrl: string + name: string +} + +interface TeamMember { + id: string + displayName: string + userIdentityType?: string +} + +interface TeamsMention { + id: number + mentionText: string + mentioned: + | { user: { id: string; displayName: string; userIdentityType: string } } + | { application: { displayName: string; id: string; applicationIdentityType: 'bot' } } +} + +interface MentionResult { + mentions: TeamsMention[] + hasMentions: boolean + updatedContent: string +} + +function optionalString(data: MicrosoftTeamsGraphObject, key: string): string | undefined { + return typeof data[key] === 'string' ? data[key] : undefined +} + +function nestedObject(data: MicrosoftTeamsGraphObject, key: string): MicrosoftTeamsGraphObject { + return isRecordLike(data[key]) ? data[key] : {} +} + +function requiredId(value: string, label: string): string { + const id = value.trim() + if (!id) throw new MicrosoftTeamsOperationError(`${label} is required`, 400) + return id +} + +function fileSizeError(file: UserFile, observedBytes = file.size): MicrosoftTeamsOperationError { + const sizeMB = (observedBytes / (1024 * 1024)).toFixed(2) + return new MicrosoftTeamsOperationError( + `File "${file.name}" (${sizeMB}MB) exceeds the 4MB limit for Teams attachments. Use smaller files or upload to SharePoint/OneDrive first.`, + 500 + ) +} + +async function uploadFilesForMessage( + rawFiles: NonNullable, + client: MicrosoftTeamsClient, + context: MicrosoftTeamsOperationContext +): Promise<{ attachments: TeamsAttachmentRef[]; files: TeamsFileOutput[] }> { + if (rawFiles.length === 0) return { attachments: [], files: [] } + if (!context.userId) throw new MicrosoftTeamsOperationError('Authentication required', 401) + const requestId = context.requestId || 'microsoft-teams-operation' + const userFiles = processFilesToUserFiles(rawFiles, requestId, logger) + const attachments: TeamsAttachmentRef[] = [] + const files: TeamsFileOutput[] = [] + let totalBytes = 0 + + for (const file of userFiles) { + context.signal?.throwIfAborted() + if (file.size > MAX_TEAMS_FILE_SIZE) throw fileSizeError(file) + const denied = await assertToolFileAccess(file.key, context.userId, requestId, logger) + context.signal?.throwIfAborted() + if (denied) throw new MicrosoftTeamsOperationError('File not found', denied.status) + + let buffer: Buffer + let contentType: string + try { + const remainingBudget = MAX_BUFFERED_TRANSFER_BYTES - totalBytes + const downloaded = await downloadServableFileFromStorage(file, requestId, logger, { + maxBytes: Math.min(MAX_TEAMS_FILE_SIZE, remainingBudget), + signal: context.signal, + }) + buffer = downloaded.buffer + contentType = downloaded.contentType || file.type || 'application/octet-stream' + } catch (error) { + context.signal?.throwIfAborted() + if (isDocNotReadyError(error)) { + throw new MicrosoftTeamsOperationError(docNotReadyMessage(), 409) + } + if (isPayloadSizeLimitError(error) && totalBytes >= MAX_BUFFERED_TRANSFER_BYTES) { + throw new MicrosoftTeamsOperationError( + 'Teams attachments exceed the upload size limit', + 413 + ) + } + if (isPayloadSizeLimitError(error)) { + throw fileSizeError(file, error.observedBytes ?? file.size) + } + throw error + } + totalBytes += buffer.length + files.push({ + name: file.name, + mimeType: contentType, + data: buffer.toString('base64'), + size: buffer.length, + }) + + let uploaded: MicrosoftTeamsGraphObject + try { + uploaded = await client.json( + `/me/drive/root:/TeamsAttachments/${encodeURIComponent(file.name)}:/content`, + { + method: 'PUT', + headers: { 'Content-Type': contentType }, + body: new Uint8Array(buffer), + }, + 'Unknown error', + context.signal + ) + } catch (error) { + context.signal?.throwIfAborted() + const message = + error instanceof MicrosoftTeamsOperationError ? error.message : 'Unknown error' + throw new MicrosoftTeamsOperationError(`Failed to upload file to Teams: ${message}`, 500) + } + const uploadedId = optionalString(uploaded, 'id') + if (!uploadedId) { + throw new MicrosoftTeamsOperationError('Failed to upload file to Teams: Unknown error', 500) + } + + let details: MicrosoftTeamsGraphObject + try { + details = await client.json( + `/me/drive/items/${encodeURIComponent(uploadedId)}?select=id,name,webDavUrl,eTag,size`, + { method: 'GET' }, + 'Unknown error', + context.signal + ) + } catch (error) { + context.signal?.throwIfAborted() + const message = + error instanceof MicrosoftTeamsOperationError ? error.message : 'Unknown error' + throw new MicrosoftTeamsOperationError(`Failed to get file details: ${message}`, 500) + } + const webDavUrl = optionalString(details, 'webDavUrl') + if (!webDavUrl) { + throw new MicrosoftTeamsOperationError( + `Failed to get file URL for attachment "${file.name}". The file was uploaded but Teams attachment reference could not be created.`, + 500 + ) + } + const detailId = optionalString(details, 'id') || uploadedId + const eTag = optionalString(details, 'eTag') + attachments.push({ + id: eTag?.match(/\{([a-f0-9-]+)\}/i)?.[1] || detailId, + contentType: 'reference', + contentUrl: webDavUrl, + name: file.name, + }) + } + return { attachments, files } +} + +function parseMentionNames(content: string): Array<{ name: string; tag: string; id: number }> { + const parsed: Array<{ name: string; tag: string; id: number }> = [] + const pattern = /([^<]+)<\/at>/gi + let match: RegExpExecArray | null + while ((match = pattern.exec(content)) !== null) { + const name = match[1].trim() + if (name) parsed.push({ name, tag: match[0], id: parsed.length }) + } + return parsed +} + +async function resolveMentions( + content: string, + membersPath: string, + client: MicrosoftTeamsClient, + signal?: AbortSignal +): Promise { + const parsed = parseMentionNames(content) + if (parsed.length === 0) return { mentions: [], hasMentions: false, updatedContent: content } + const data = await client.json(membersPath, { method: 'GET' }, 'Failed to list members', signal) + const rawMembers = Array.isArray(data.value) ? data.value : [] + const members: TeamMember[] = rawMembers.flatMap((value) => { + if (!isRecordLike(value)) return [] + const id = optionalString(value, 'id') + if (!id) return [] + return [ + { + id, + displayName: optionalString(value, 'displayName') || '', + userIdentityType: optionalString(value, 'userIdentityType'), + }, + ] + }) + const mentions: TeamsMention[] = [] + const resolvedTags = new Set() + let updatedContent = content + for (const mention of parsed) { + if (resolvedTags.has(mention.tag)) continue + const normalizedName = mention.name.toLowerCase() + const member = members.find( + (candidate) => candidate.displayName.toLowerCase() === normalizedName + ) + if (!member) continue + mentions.push( + member.userIdentityType === 'bot' + ? { + id: mention.id, + mentionText: mention.name, + mentioned: { + application: { + displayName: member.displayName, + id: member.id, + applicationIdentityType: 'bot', + }, + }, + } + : { + id: mention.id, + mentionText: mention.name, + mentioned: { + user: { + id: member.id, + displayName: member.displayName, + userIdentityType: member.userIdentityType || 'aadUser', + }, + }, + } + ) + resolvedTags.add(mention.tag) + updatedContent = updatedContent.replaceAll( + mention.tag, + `${mention.name}` + ) + } + return { mentions, hasMentions: mentions.length > 0, updatedContent } +} + +async function sendMessage(args: { + accessToken: string + content: string + files: MicrosoftTeamsWriteChatInput['files'] + messagePath: string + membersPath: string + context: MicrosoftTeamsOperationContext + plainMetadata: (data: MicrosoftTeamsGraphObject) => Record + enhancedMetadata: (data: MicrosoftTeamsGraphObject) => Record +}) { + const enhanced = Boolean(args.files?.length || MENTION_PATTERN.test(args.content)) + const client = new MicrosoftTeamsClient(args.accessToken) + if (!enhanced) { + const data = await client.json( + args.messagePath, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ body: { contentType: 'text', content: args.content } }), + }, + 'Failed to send Teams message', + args.context.signal + ) + return { + success: true as const, + output: { updatedContent: true, metadata: args.plainMetadata(data) }, + } + } + + const uploaded = await uploadFilesForMessage(args.files || [], client, args.context) + let messageContent = args.content + let contentType: 'text' | 'html' = 'text' + let mentionResult: MentionResult = { + mentions: [], + hasMentions: false, + updatedContent: args.content, + } + try { + mentionResult = await resolveMentions( + args.content, + args.membersPath, + client, + args.context.signal + ) + } catch (error) { + args.context.signal?.throwIfAborted() + logger.warn('Failed to resolve Teams mentions; continuing without them', { + error: getErrorMessage(error), + requestId: args.context.requestId, + }) + } + if (mentionResult.hasMentions) { + contentType = 'html' + messageContent = mentionResult.updatedContent + } + if (uploaded.attachments.length > 0) { + contentType = 'html' + const tags = uploaded.attachments + .map((attachment) => ``) + .join(' ') + messageContent = `${messageContent}
${tags}` + } + const body: Record = { body: { contentType, content: messageContent } } + if (uploaded.attachments.length > 0) body.attachments = uploaded.attachments + if (mentionResult.mentions.length > 0) body.mentions = mentionResult.mentions + const data = await client.json( + args.messagePath, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }, + 'Failed to send Teams message', + args.context.signal + ) + return { + success: true as const, + output: { + updatedContent: true, + metadata: { + ...args.enhancedMetadata(data), + attachmentCount: uploaded.attachments.length, + }, + files: uploaded.files, + }, + } +} + +export async function writeMicrosoftTeamsChatMessage( + input: MicrosoftTeamsWriteChatInput, + context: MicrosoftTeamsOperationContext +) { + context.signal?.throwIfAborted() + const chatId = requiredId(input.chatId, 'Chat ID') + return sendMessage({ + accessToken: input.accessToken, + content: input.content, + files: input.files, + messagePath: `/chats/${encodeURIComponent(chatId)}/messages`, + membersPath: `/chats/${encodeURIComponent(chatId)}/members`, + context, + plainMetadata: (data) => ({ + messageId: optionalString(data, 'id') || '', + chatId: optionalString(data, 'chatId') || '', + content: optionalString(nestedObject(data, 'body'), 'content') || input.content, + createdTime: optionalString(data, 'createdDateTime') || new Date().toISOString(), + url: optionalString(data, 'webUrl') || '', + }), + enhancedMetadata: (data) => ({ + messageId: optionalString(data, 'id'), + chatId: optionalString(data, 'chatId') || input.chatId, + content: optionalString(nestedObject(data, 'body'), 'content') || input.content, + createdTime: optionalString(data, 'createdDateTime') || new Date().toISOString(), + url: optionalString(data, 'webUrl') || '', + }), + }) +} + +export async function writeMicrosoftTeamsChannelMessage( + input: MicrosoftTeamsWriteChannelInput, + context: MicrosoftTeamsOperationContext +) { + context.signal?.throwIfAborted() + const teamId = requiredId(input.teamId, 'Team ID') + const channelId = requiredId(input.channelId, 'Channel ID') + return sendMessage({ + accessToken: input.accessToken, + content: input.content, + files: input.files, + messagePath: `/teams/${encodeURIComponent(teamId)}/channels/${encodeURIComponent(channelId)}/messages`, + membersPath: `/teams/${encodeURIComponent(teamId)}/channels/${encodeURIComponent(channelId)}/members`, + context, + plainMetadata: (data) => { + const identity = nestedObject(data, 'channelIdentity') + return { + messageId: optionalString(data, 'id') || '', + teamId: optionalString(identity, 'teamId') || '', + channelId: optionalString(identity, 'channelId') || '', + content: optionalString(nestedObject(data, 'body'), 'content') || input.content, + createdTime: optionalString(data, 'createdDateTime') || new Date().toISOString(), + url: optionalString(data, 'webUrl') || '', + } + }, + enhancedMetadata: (data) => { + const identity = nestedObject(data, 'channelIdentity') + return { + messageId: optionalString(data, 'id'), + teamId: optionalString(identity, 'teamId') || input.teamId, + channelId: optionalString(identity, 'channelId') || input.channelId, + content: optionalString(nestedObject(data, 'body'), 'content') || input.content, + createdTime: optionalString(data, 'createdDateTime') || new Date().toISOString(), + url: optionalString(data, 'webUrl') || '', + } + }, + }) +} + +export interface MicrosoftTeamsDeleteChatMessageInput { + accessToken: string + chatId: string + messageId: string +} + +async function readGraphResponse(response: Response, signal?: AbortSignal): Promise { + if (response.status === 204) return {} + return readResponseJsonWithLimit(response, { + maxBytes: MAX_GRAPH_RESPONSE_BYTES, + label: 'Microsoft Graph response', + signal, + }).catch(() => ({})) +} + +export async function deleteMicrosoftTeamsChatMessage( + input: MicrosoftTeamsDeleteChatMessageInput, + context: MicrosoftTeamsOperationContext +) { + context.signal?.throwIfAborted() + const chatId = input.chatId.trim() + const messageId = input.messageId.trim() + if (!chatId || !messageId) { + throw new MicrosoftTeamsOperationError('Chat ID and Message ID are required', 400) + } + + const meResponse = await fetch('https://graph.microsoft.com/v1.0/me', { + headers: { Authorization: `Bearer ${input.accessToken}` }, + signal: context.signal, + }) + const me = await readGraphResponse(meResponse, context.signal) + if (!meResponse.ok || !me.id) { + throw new MicrosoftTeamsOperationError( + me.error?.message || 'Failed to get user information', + meResponse.status + ) + } + + const response = await fetch( + `https://graph.microsoft.com/v1.0/users/${encodeURIComponent(me.id)}/chats/${encodeURIComponent(chatId)}/messages/${encodeURIComponent(messageId)}/softDelete`, + { + method: 'POST', + headers: { + Authorization: `Bearer ${input.accessToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({}), + signal: context.signal, + } + ) + if (!response.ok) { + const error = await readGraphResponse(response, context.signal) + throw new MicrosoftTeamsOperationError( + error.error?.message || 'Failed to delete Teams message', + response.status + ) + } + return { + success: true, + output: { + deleted: true, + messageId, + metadata: { messageId, chatId }, + }, + } +} diff --git a/apps/sim/lib/internal/microsoft-teams/schema.ts b/apps/sim/lib/internal/microsoft-teams/schema.ts new file mode 100644 index 00000000000..4f71d68e5df --- /dev/null +++ b/apps/sim/lib/internal/microsoft-teams/schema.ts @@ -0,0 +1,27 @@ +import { z } from 'zod' +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' + +export const MAX_TEAMS_MESSAGE_FILES = 25 + +const messageFilesSchema = RawFileInputArraySchema.max( + MAX_TEAMS_MESSAGE_FILES, + `At most ${MAX_TEAMS_MESSAGE_FILES} files can be attached to one Teams message` +) + +export const microsoftTeamsWriteChatInputSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + chatId: z.string().min(1, 'Chat ID is required'), + content: z.string().min(1, 'Message content is required'), + files: messageFilesSchema.optional().nullable(), +}) + +export const microsoftTeamsWriteChannelInputSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + teamId: z.string().min(1, 'Team ID is required'), + channelId: z.string().min(1, 'Channel ID is required'), + content: z.string().min(1, 'Message content is required'), + files: messageFilesSchema.optional().nullable(), +}) + +export type MicrosoftTeamsWriteChatInput = z.infer +export type MicrosoftTeamsWriteChannelInput = z.infer diff --git a/apps/sim/lib/internal/microsoft-teams/tool-config.test.ts b/apps/sim/lib/internal/microsoft-teams/tool-config.test.ts new file mode 100644 index 00000000000..19ff9e5f445 --- /dev/null +++ b/apps/sim/lib/internal/microsoft-teams/tool-config.test.ts @@ -0,0 +1,38 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { writeChannelTool } from '@/tools/microsoft_teams/write_channel' +import { writeChatTool } from '@/tools/microsoft_teams/write_chat' + +describe('Microsoft Teams operation configs', () => { + it('preserves resolved variable values without HTTP-shaped metadata', () => { + const chatInput = writeChatTool.operation.input({ + accessToken: '{{MICROSOFT_TEAMS_TOKEN}}', + chatId: '', + content: '', + }) + const channelInput = writeChannelTool.operation.input({ + accessToken: '{{MICROSOFT_TEAMS_TOKEN}}', + teamId: '', + channelId: '', + content: '', + }) + + expect(chatInput).toEqual({ + accessToken: '{{MICROSOFT_TEAMS_TOKEN}}', + chatId: '', + content: '', + files: null, + }) + expect(channelInput).toEqual({ + accessToken: '{{MICROSOFT_TEAMS_TOKEN}}', + teamId: '', + channelId: '', + content: '', + files: null, + }) + expect('request' in writeChatTool).toBe(false) + expect('request' in writeChannelTool).toBe(false) + }) +}) diff --git a/apps/sim/lib/internal/microsoft-word/client.test.ts b/apps/sim/lib/internal/microsoft-word/client.test.ts new file mode 100644 index 00000000000..21b35a15955 --- /dev/null +++ b/apps/sim/lib/internal/microsoft-word/client.test.ts @@ -0,0 +1,192 @@ +/** + * @vitest-environment node + */ +import { inputValidationMock, inputValidationMockFns } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + +import { replaceContentIfUnchanged } from '@/lib/internal/microsoft-word/client' + +const { mockValidateUrlWithDNS, mockSecureFetchWithPinnedIP } = inputValidationMockFns + +const BASE_PATH = 'https://graph.microsoft.com/v1.0/me/drive/items/doc-abc' +const UPLOAD_URL = 'https://sn3302.up.1drv.com/up/session-abc' + +/** Graph's `createUploadSession` response. */ +function sessionResponse() { + const body = { uploadUrl: UPLOAD_URL, expirationDateTime: '2026-01-01T00:00:00Z' } + return { + ok: true, + status: 200, + statusText: '', + headers: new Headers(), + body: null, + text: async () => JSON.stringify(body), + json: async () => body, + arrayBuffer: async () => new ArrayBuffer(0), + } +} + +/** A 202 acknowledging a non-final fragment; carries no driveItem. */ +function fragmentAccepted() { + const body = { nextExpectedRanges: ['1-'] } + return { + ok: true, + status: 202, + statusText: 'Accepted', + headers: new Headers(), + body: null, + text: async () => JSON.stringify(body), + json: async () => body, + arrayBuffer: async () => new ArrayBuffer(0), + } +} + +/** The final fragment's response, carrying the completed driveItem. */ +function completedItem() { + const body = { id: 'doc-abc', name: 'notes.docx', size: 123 } + return { + ok: true, + status: 201, + statusText: 'Created', + headers: new Headers(), + body: null, + text: async () => JSON.stringify(body), + json: async () => body, + arrayBuffer: async () => new ArrayBuffer(0), + } +} + +function preconditionFailed() { + return { + ok: false, + status: 412, + statusText: 'Precondition Failed', + headers: new Headers(), + body: null, + text: async () => '', + json: async () => ({}), + arrayBuffer: async () => new ArrayBuffer(0), + } +} + +/** Parses a `Content-Range: bytes {start}-{end}/{total}` header. */ +function parseRange(header: string): { start: number; end: number; total: number } { + const [range, total] = header.replace('bytes ', '').split('/') + const [start, end] = range.split('-').map(Number) + return { start, end, total: Number(total) } +} + +beforeEach(() => { + /** Reset so an unconsumed one-time result cannot leak into the next test. */ + mockSecureFetchWithPinnedIP.mockReset() + mockValidateUrlWithDNS.mockReset() + mockValidateUrlWithDNS.mockResolvedValue({ + isValid: true, + resolvedIP: '93.184.216.34', + originalHostname: 'graph.microsoft.com', + }) +}) + +describe('replaceContentIfUnchanged', () => { + it('sends a small package as a single fragment', async () => { + mockSecureFetchWithPinnedIP + .mockResolvedValueOnce(sessionResponse()) + .mockResolvedValueOnce(completedItem()) + + await replaceContentIfUnchanged(BASE_PATH, 'token', Buffer.alloc(1024), 'tag-1') + + const puts = mockSecureFetchWithPinnedIP.mock.calls.filter((c) => c[2]?.method === 'PUT') + expect(puts).toHaveLength(1) + expect(parseRange(puts[0][2].headers['Content-Range'])).toEqual({ + start: 0, + end: 1023, + total: 1024, + }) + }) + + it('splits a package larger than one fragment into contiguous ordered ranges', async () => { + /** The package exceeds one upload fragment. */ + const size = 25 * 1024 * 1024 + mockSecureFetchWithPinnedIP + .mockResolvedValueOnce(sessionResponse()) + .mockResolvedValueOnce(fragmentAccepted()) + .mockResolvedValueOnce(fragmentAccepted()) + .mockResolvedValueOnce(completedItem()) + + const item = await replaceContentIfUnchanged(BASE_PATH, 'token', Buffer.alloc(size), 'tag-1') + expect(item.id).toBe('doc-abc') + + const puts = mockSecureFetchWithPinnedIP.mock.calls.filter((c) => c[2]?.method === 'PUT') + expect(puts).toHaveLength(3) + + let expectedStart = 0 + for (const put of puts) { + const { start, end, total } = parseRange(put[2].headers['Content-Range']) + expect(total).toBe(size) + expect(start).toBe(expectedStart) + /** Every non-final fragment must be a multiple of 320 KiB. */ + const length = end - start + 1 + expect(Number(put[2].headers['Content-Length'])).toBe(length) + if (end !== size - 1) expect(length % (320 * 1024)).toBe(0) + expectedStart = end + 1 + } + expect(expectedStart).toBe(size) + }) + + it('never sends the bearer token to the preauthenticated upload URL', async () => { + mockSecureFetchWithPinnedIP + .mockResolvedValueOnce(sessionResponse()) + .mockResolvedValueOnce(completedItem()) + + await replaceContentIfUnchanged(BASE_PATH, 'token', Buffer.alloc(64), 'tag-1') + + const put = mockSecureFetchWithPinnedIP.mock.calls.find((c) => c[2]?.method === 'PUT') + expect(put?.[0]).toBe(UPLOAD_URL) + expect(put?.[2].headers.Authorization).toBeUndefined() + }) + + it('carries the precondition on the session and maps its rejection to a conflict', async () => { + mockSecureFetchWithPinnedIP.mockResolvedValueOnce(preconditionFailed()) + + await expect( + replaceContentIfUnchanged(BASE_PATH, 'token', Buffer.alloc(64), 'tag-1') + ).rejects.toMatchObject({ status: 409 }) + + const session = mockSecureFetchWithPinnedIP.mock.calls[0] + expect(session[0]).toBe(`${BASE_PATH}/createUploadSession`) + expect(session[2].headers['if-match']).toBe('tag-1') + /** The rejected precondition prevents any content upload. */ + expect(mockSecureFetchWithPinnedIP.mock.calls.some((c) => c[2]?.method === 'PUT')).toBe(false) + }) + + it('maps a conflict raised part-way through the fragments to the same error', async () => { + mockSecureFetchWithPinnedIP + .mockResolvedValueOnce(sessionResponse()) + .mockResolvedValueOnce(fragmentAccepted()) + .mockResolvedValueOnce(preconditionFailed()) + + await expect( + replaceContentIfUnchanged(BASE_PATH, 'token', Buffer.alloc(25 * 1024 * 1024), 'tag-1') + ).rejects.toMatchObject({ status: 409 }) + }) + + it('fails loudly when the session response carries no upload URL', async () => { + const body = { expirationDateTime: '2026-01-01T00:00:00Z' } + mockSecureFetchWithPinnedIP.mockResolvedValueOnce({ + ok: true, + status: 200, + statusText: '', + headers: new Headers(), + body: null, + text: async () => JSON.stringify(body), + json: async () => body, + arrayBuffer: async () => new ArrayBuffer(0), + }) + + await expect( + replaceContentIfUnchanged(BASE_PATH, 'token', Buffer.alloc(64), 'tag-1') + ).rejects.toThrow(/did not return an upload URL/) + }) +}) diff --git a/apps/sim/lib/internal/microsoft-word/client.ts b/apps/sim/lib/internal/microsoft-word/client.ts new file mode 100644 index 00000000000..db123397537 --- /dev/null +++ b/apps/sim/lib/internal/microsoft-word/client.ts @@ -0,0 +1,374 @@ +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { DOCX_MIME_TYPE } from '@/lib/microsoft-word/document.server' +import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' +import { parseGraphErrorMessage } from '@/tools/microsoft_excel/utils' +import type { MicrosoftWordDocumentMetadata } from '@/tools/microsoft_word/types' + +/** Microsoft Graph `driveItem` fields the Word routes project. */ +interface GraphDriveItem { + id?: string + name?: string + size?: number + webUrl?: string + createdDateTime?: string + lastModifiedDateTime?: string + /** An eTag for the item's content, unchanged when only metadata changes. */ + cTag?: string + /** An eTag for the whole item, metadata included. */ + eTag?: string + file?: { mimeType?: string } + folder?: Record +} + +/** + * The token that identifies the exact content an edit was based on. + * + * `cTag` is the right one: Graph documents it as "an eTag for the content of the + * item" that does not move when only metadata changes, so a rename will not + * spuriously abort an edit. `eTag` is the fallback for the shapes where Graph + * omits `cTag`. + * + * @see https://learn.microsoft.com/en-us/graph/api/resources/driveitem + */ +export function getContentTag(item: GraphDriveItem): string | undefined { + return item.cTag ?? item.eTag +} + +/** Thrown when Microsoft Graph rejects a request, carrying its HTTP status. */ +export class GraphRequestError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'GraphRequestError' + } +} + +/** + * Issues an IP-pinned request to a Microsoft Graph URL, rejecting the URL when + * DNS resolution points anywhere Sim must not reach. + */ +async function graphFetch( + url: string, + paramName: string, + options: NonNullable[2]> +) { + options.signal?.throwIfAborted() + const validation = await validateUrlWithDNS(url, paramName) + options.signal?.throwIfAborted() + if (!validation.isValid) { + throw new GraphRequestError(validation.error || `Invalid ${paramName}`, 400) + } + return secureFetchWithPinnedIP(url, validation.resolvedIP as string, options) +} + +/** Reads a Graph error body and raises it as a {@link GraphRequestError}. */ +async function raiseGraphError(response: { + status: number + statusText: string + text: () => Promise +}): Promise { + const errorText = await response.text().catch(() => '') + throw new GraphRequestError( + parseGraphErrorMessage(response.status, response.statusText, errorText), + response.status + ) +} + +/** Projects a Graph `driveItem` onto the metadata shape the Word tools return. */ +export function toDocumentMetadata( + item: GraphDriveItem, + fallbackId: string +): MicrosoftWordDocumentMetadata { + return { + documentId: item.id ?? fallbackId, + name: item.name ?? null, + mimeType: item.file?.mimeType ?? null, + webViewLink: item.webUrl ?? null, + size: item.size ?? null, + createdTime: item.createdDateTime ?? null, + modifiedTime: item.lastModifiedDateTime ?? null, + } +} + +/** + * Fetches a drive item's metadata and rejects folders, which have no document + * content and would otherwise fail later with an opaque Graph error. + * + * @see https://learn.microsoft.com/en-us/graph/api/driveitem-get + */ +export async function fetchDocumentItem( + basePath: string, + accessToken: string, + signal?: AbortSignal +): Promise { + const response = await graphFetch(basePath, 'documentUrl', { + headers: { Authorization: `Bearer ${accessToken}` }, + signal, + }) + + if (!response.ok) await raiseGraphError(response) + + const item = (await response.json()) as GraphDriveItem + if (item.folder && !item.file) { + throw new GraphRequestError( + `"${item.name ?? 'The selected item'}" is a folder, not a Word document`, + 400 + ) + } + if (!isWordDocument(item)) { + throw new GraphRequestError( + `"${item.name ?? 'The selected item'}" is not a Word document. Every Microsoft Word operation reads or writes a .docx file.`, + 400 + ) + } + return item +} + +/** + * Whether a drive item is a `.docx` package. + * + * The name suffix is accepted alongside the MIME type because Graph does not + * always populate `file.mimeType`. Getting this wrong is destructive rather than + * merely wrong: without the check, pointing Replace Content at a PDF would + * overwrite it with generated WordprocessingML bytes. + */ +function isWordDocument(item: GraphDriveItem): boolean { + return ( + item.file?.mimeType === DOCX_MIME_TYPE || Boolean(item.name?.toLowerCase().endsWith('.docx')) + ) +} + +/** + * Downloads a drive item's raw content. + * + * @see https://learn.microsoft.com/en-us/graph/api/driveitem-get-content + */ +export async function downloadDocumentContent( + basePath: string, + accessToken: string, + signal?: AbortSignal +): Promise { + const response = await graphFetch(`${basePath}/content`, 'documentContentUrl', { + headers: { Authorization: `Bearer ${accessToken}` }, + maxResponseBytes: MAX_FILE_SIZE, + signal, + /** Graph redirects to a preauthenticated host that must not receive the bearer token. */ + stripAuthOnRedirect: true, + }) + + if (!response.ok) await raiseGraphError(response) + + return Buffer.from(await response.arrayBuffer()) +} + +/** + * Downloads a drive item converted to another format. + * + * @see https://learn.microsoft.com/en-us/graph/api/driveitem-get-content-format + */ +export async function downloadConvertedContent( + basePath: string, + accessToken: string, + format: 'pdf', + signal?: AbortSignal +): Promise { + const response = await graphFetch(`${basePath}/content?format=${format}`, 'documentConvertUrl', { + headers: { Authorization: `Bearer ${accessToken}` }, + maxResponseBytes: MAX_FILE_SIZE, + signal, + stripAuthOnRedirect: true, + }) + + if (!response.ok) await raiseGraphError(response) + + return Buffer.from(await response.arrayBuffer()) +} + +/** Message shown when someone else changed the document mid-edit. */ +const CONFLICT_MESSAGE = + 'The document changed in OneDrive or SharePoint after Sim read it, so the edit was not applied and no other change was overwritten. Run the operation again to edit the current version.' + +/** Raised instead of overwriting a document that changed since it was read. */ +export function documentChangedError(): GraphRequestError { + return new GraphRequestError(CONFLICT_MESSAGE, 409) +} + +/** Message shown when the document carries no version to compare against. */ +const UNVERIFIABLE_MESSAGE = + 'Microsoft Graph did not report a version for this document, so Sim cannot confirm the edit would not overwrite someone else’s change and did not apply it. Use Replace Content if you intend to overwrite the document outright.' + +/** + * Raised when there is no version to compare, rather than writing unguarded. + * + * Graph returns `cTag` for every file and `eTag` for every drive item, so this + * should not be reachable in practice — but a read-modify-write that silently + * degrades to no protection is the failure this whole guard exists to prevent, + * so the missing-version path fails closed instead of proceeding. + */ +function unverifiableDocumentError(): GraphRequestError { + return new GraphRequestError(UNVERIFIABLE_MESSAGE, 409) +} + +/** + * Returns the content tag an edit must be based on, refusing the edit outright + * when the item carries none. + */ +export function requireContentTag(item: GraphDriveItem): string { + const tag = getContentTag(item) + if (!tag) { + throw unverifiableDocumentError() + } + return tag +} + +/** A Graph upload session, used for a precondition-checked content write. */ +interface GraphUploadSession { + uploadUrl?: string +} + +/** + * Replaces a document's content only if it still matches `expectedTag`. + * + * `PUT /items/{id}/content` documents no precondition — its request-headers + * table lists only `Authorization` and `Content-Type` — so a conditional write + * has to go through an upload session, whose `if-match` header Graph documents + * as returning `412 Precondition Failed` on a mismatch. That makes the check + * service-enforced rather than a client-side compare that could itself race. + * + * The residual window is small and stated honestly: the tag is evaluated when + * the session is created and the bytes commit on the following request. Closing + * it completely would need the deferred-commit form, whose conditional commit + * relies on `@microsoft.graph.sourceUrl` — which Microsoft documents as + * unsupported on OneDrive for Business and SharePoint Online, where these + * documents live. + * + * @see https://learn.microsoft.com/en-us/graph/api/driveitem-createuploadsession + * @see https://learn.microsoft.com/en-us/graph/api/resources/driveitem + */ +export async function replaceContentIfUnchanged( + basePath: string, + accessToken: string, + content: Buffer, + expectedTag: string, + signal?: AbortSignal +): Promise { + const sessionResponse = await graphFetch( + `${basePath}/createUploadSession`, + 'documentUploadSessionUrl', + { + method: 'POST', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + 'if-match': expectedTag, + }, + body: '{}', + signal, + } + ) + + if (sessionResponse.status === 412) { + throw documentChangedError() + } + if (!sessionResponse.ok) await raiseGraphError(sessionResponse) + + const { uploadUrl } = (await sessionResponse.json()) as GraphUploadSession + if (!uploadUrl) { + throw new GraphRequestError('Microsoft Graph did not return an upload URL', 502) + } + + return uploadSessionBytes(uploadUrl, content, signal) +} + +/** + * Byte size of each upload fragment. + * + * Graph caps a single upload request below 60 MiB, and requires every fragment + * of a split upload to be a multiple of 320 KiB — 10 MiB is both (327,680 × 32) + * and is inside the 5–10 MiB range Microsoft recommends. Documents are read + * under a 100 MB ceiling, so a single request would not always be enough. + */ +const UPLOAD_FRAGMENT_BYTES = 10 * 1024 * 1024 + +/** + * Sends the package to a session's upload URL, splitting it into sequential + * fragments. Graph answers `202 Accepted` for every fragment but the last, and + * returns the finished driveItem with the one that completes the file. + * + * The URL is preauthenticated and on another host; Graph documents that sending + * `Authorization` here can itself fail the request with a 401, so no bearer + * token is attached. + * + * @see https://learn.microsoft.com/en-us/graph/api/driveitem-createuploadsession + */ +async function uploadSessionBytes( + uploadUrl: string, + content: Buffer, + signal?: AbortSignal +): Promise { + const total = content.length + + for (let start = 0; start < total; start += UPLOAD_FRAGMENT_BYTES) { + const end = Math.min(start + UPLOAD_FRAGMENT_BYTES, total) - 1 + const fragment = content.subarray(start, end + 1) + + const response = await graphFetch(uploadUrl, 'documentUploadUrl', { + method: 'PUT', + headers: { + 'Content-Length': String(fragment.length), + 'Content-Range': `bytes ${start}-${end}/${total}`, + }, + body: fragment, + signal, + }) + + if (response.status === 412 || response.status === 409) { + throw documentChangedError() + } + if (!response.ok) await raiseGraphError(response) + + /** Every fragment but the last is acknowledged with 202 and no item body. */ + if (end === total - 1) { + return (await response.json()) as GraphDriveItem + } + } + + throw new GraphRequestError('Microsoft Graph did not complete the document upload', 502) +} + +/** + * Uploads bytes as a drive item's content and returns the resulting item. + * + * Unconditional by design: this backs creating a new document and the + * deliberate whole-document overwrite. An edit that must not clobber a + * concurrent change uses {@link replaceContentIfUnchanged} instead. + * + * @see https://learn.microsoft.com/en-us/graph/api/driveitem-put-content + */ +export async function uploadDocumentContent( + url: string, + accessToken: string, + content: Buffer, + mimeType: string, + signal?: AbortSignal +): Promise { + const response = await graphFetch(url, 'documentUploadUrl', { + method: 'PUT', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': mimeType, + 'Content-Length': String(content.length), + }, + body: content, + signal, + }) + + if (!response.ok) await raiseGraphError(response) + + return (await response.json()) as GraphDriveItem +} diff --git a/apps/sim/lib/internal/microsoft-word/errors.ts b/apps/sim/lib/internal/microsoft-word/errors.ts new file mode 100644 index 00000000000..22224d9e2d5 --- /dev/null +++ b/apps/sim/lib/internal/microsoft-word/errors.ts @@ -0,0 +1,9 @@ +/** An invalid Microsoft Word operation input. */ +export class MicrosoftWordInputError extends Error { + readonly status = 400 + + constructor(message: string) { + super(message) + this.name = 'MicrosoftWordInputError' + } +} diff --git a/apps/sim/lib/internal/microsoft-word/execute-tool.test.ts b/apps/sim/lib/internal/microsoft-word/execute-tool.test.ts new file mode 100644 index 00000000000..bee8eae27e3 --- /dev/null +++ b/apps/sim/lib/internal/microsoft-word/execute-tool.test.ts @@ -0,0 +1,167 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operationMocks = vi.hoisted(() => ({ + executeMicrosoftWordAppend: vi.fn(), + executeMicrosoftWordCreate: vi.fn(), + executeMicrosoftWordCreateFromTemplate: vi.fn(), + executeMicrosoftWordExportPdf: vi.fn(), + executeMicrosoftWordRead: vi.fn(), + executeMicrosoftWordReplaceText: vi.fn(), + executeMicrosoftWordUpdate: vi.fn(), +})) + +vi.mock('@/lib/internal/microsoft-word/operations', () => operationMocks) + +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { GraphRequestError } from '@/lib/internal/microsoft-word/client' +import { executeMicrosoftWordTool } from '@/lib/internal/microsoft-word/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const READ_INPUT = { accessToken: 'token', documentId: 'document-1' } + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'microsoft_word_read', + input: READ_INPUT, + headers: new Headers(), + context: createExecutionContext({ workflowId: 'workflow-1' }), + requestId: 'request-1', + ...overrides, + } +} + +const TOOL_CASES = [ + [ + 'microsoft_word_append', + { accessToken: 'token', documentId: 'document-1', content: 'Append' }, + operationMocks.executeMicrosoftWordAppend, + ], + [ + 'microsoft_word_create', + { accessToken: 'token', name: 'Document', content: 'Hello' }, + operationMocks.executeMicrosoftWordCreate, + ], + [ + 'microsoft_word_create_from_template', + { accessToken: 'token', templateDocumentId: 'template-1', name: 'Document' }, + operationMocks.executeMicrosoftWordCreateFromTemplate, + ], + [ + 'microsoft_word_export_pdf', + { accessToken: 'token', documentId: 'document-1' }, + operationMocks.executeMicrosoftWordExportPdf, + ], + ['microsoft_word_read', READ_INPUT, operationMocks.executeMicrosoftWordRead], + [ + 'microsoft_word_replace_text', + { accessToken: 'token', documentId: 'document-1', findText: 'old', replaceText: 'new' }, + operationMocks.executeMicrosoftWordReplaceText, + ], + [ + 'microsoft_word_update', + { accessToken: 'token', documentId: 'document-1', content: 'Replacement' }, + operationMocks.executeMicrosoftWordUpdate, + ], +] as const + +describe('executeMicrosoftWordTool', () => { + beforeEach(() => { + vi.clearAllMocks() + for (const operation of Object.values(operationMocks)) { + operation.mockResolvedValue({ success: true, output: { handled: true } }) + } + }) + + it.each(TOOL_CASES)('validates and dispatches %s', async (toolId, input, operation) => { + const controller = new AbortController() + const response = await executeMicrosoftWordTool( + createRequest({ + toolId, + input, + signal: controller.signal, + }) + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ success: true, output: { handled: true } }) + expect(operation).toHaveBeenCalledWith(input, { + requestId: 'request-1', + signal: controller.signal, + }) + }) + + it('returns validation errors before provider work', async () => { + const response = await executeMicrosoftWordTool( + createRequest({ input: { accessToken: '', documentId: '' } }) + ) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(operationMocks.executeMicrosoftWordRead).not.toHaveBeenCalled() + }) + + it('rejects oversized typed inputs before provider work', async () => { + const response = await executeMicrosoftWordTool( + createRequest({ + toolId: 'microsoft_word_update', + input: { + accessToken: 'token', + documentId: 'document-1', + content: 'x'.repeat(DEFAULT_MAX_JSON_BODY_BYTES + 1), + }, + }) + ) + + expect(response.status).toBe(413) + await expect(response.json()).resolves.toMatchObject({ + error: expect.stringMatching(/maximum allowed size/), + }) + expect(operationMocks.executeMicrosoftWordUpdate).not.toHaveBeenCalled() + }) + + it('preserves Microsoft Graph status and the tool error envelope', async () => { + operationMocks.executeMicrosoftWordRead.mockRejectedValue( + new GraphRequestError('Document not found', 404) + ) + + const response = await executeMicrosoftWordTool(createRequest()) + + expect(response.status).toBe(404) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Document not found', + }) + }) + + it('propagates cancellation before provider work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeMicrosoftWordTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(operationMocks.executeMicrosoftWordRead).not.toHaveBeenCalled() + }) + + it('rejects unsupported tool IDs without provider work', async () => { + const response = await executeMicrosoftWordTool( + createRequest({ toolId: 'microsoft_word_unknown' }) + ) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Unsupported Microsoft Word tool: microsoft_word_unknown', + }) + expect(operationMocks.executeMicrosoftWordRead).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/microsoft-word/execute-tool.ts b/apps/sim/lib/internal/microsoft-word/execute-tool.ts new file mode 100644 index 00000000000..9ea1b0ea7cf --- /dev/null +++ b/apps/sim/lib/internal/microsoft-word/execute-tool.ts @@ -0,0 +1,151 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { z } from 'zod' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { GraphRequestError } from '@/lib/internal/microsoft-word/client' +import { MicrosoftWordInputError } from '@/lib/internal/microsoft-word/errors' +import { + executeMicrosoftWordAppend, + executeMicrosoftWordCreate, + executeMicrosoftWordCreateFromTemplate, + executeMicrosoftWordExportPdf, + executeMicrosoftWordRead, + executeMicrosoftWordReplaceText, + executeMicrosoftWordUpdate, + type MicrosoftWordOperationContext, +} from '@/lib/internal/microsoft-word/operations' +import { + microsoftWordAppendInputSchema, + microsoftWordCreateFromTemplateInputSchema, + microsoftWordCreateInputSchema, + microsoftWordExportPdfInputSchema, + microsoftWordReadInputSchema, + microsoftWordReplaceTextInputSchema, + microsoftWordUpdateInputSchema, +} from '@/lib/internal/microsoft-word/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const logger = createLogger('MicrosoftWordToolExecution') + +async function executeOperation( + schema: S, + input: unknown, + execute: (input: z.output, context: MicrosoftWordOperationContext) => Promise, + context: MicrosoftWordOperationContext, + toolId: string +): Promise { + context.signal?.throwIfAborted() + let serializedInput: string + try { + serializedInput = JSON.stringify(input) ?? '' + } catch { + return Response.json({ error: 'Operation input must be valid JSON' }, { status: 400 }) + } + if (Buffer.byteLength(serializedInput, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + error: `Operation input exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + const parsed = schema.safeParse(input) + if (!parsed.success) { + return Response.json( + { error: 'Invalid request data', details: parsed.error.issues }, + { status: 400 } + ) + } + + try { + const result = await execute(parsed.data, context) + context.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + context.signal?.throwIfAborted() + const message = getErrorMessage(error, 'Unknown error occurred') + logger.error('Microsoft Word operation failed', { + error: message, + requestId: context.requestId, + toolId, + }) + + const status = + error instanceof GraphRequestError || error instanceof MicrosoftWordInputError + ? error.status + : isPayloadSizeLimitError(error) + ? 413 + : 500 + return Response.json({ success: false, error: message }, { status }) + } +} + +export const executeMicrosoftWordTool: InternalToolOperationHandler = async (request) => { + const { input, requestId, signal, toolId } = request + const context: MicrosoftWordOperationContext = { requestId, signal } + + switch (toolId) { + case 'microsoft_word_append': + return executeOperation( + microsoftWordAppendInputSchema, + input, + executeMicrosoftWordAppend, + context, + toolId + ) + case 'microsoft_word_create': + return executeOperation( + microsoftWordCreateInputSchema, + input, + executeMicrosoftWordCreate, + context, + toolId + ) + case 'microsoft_word_create_from_template': + return executeOperation( + microsoftWordCreateFromTemplateInputSchema, + input, + executeMicrosoftWordCreateFromTemplate, + context, + toolId + ) + case 'microsoft_word_export_pdf': + return executeOperation( + microsoftWordExportPdfInputSchema, + input, + executeMicrosoftWordExportPdf, + context, + toolId + ) + case 'microsoft_word_read': + return executeOperation( + microsoftWordReadInputSchema, + input, + executeMicrosoftWordRead, + context, + toolId + ) + case 'microsoft_word_replace_text': + return executeOperation( + microsoftWordReplaceTextInputSchema, + input, + executeMicrosoftWordReplaceText, + context, + toolId + ) + case 'microsoft_word_update': + return executeOperation( + microsoftWordUpdateInputSchema, + input, + executeMicrosoftWordUpdate, + context, + toolId + ) + default: + return Response.json( + { success: false, error: `Unsupported Microsoft Word tool: ${toolId}` }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/microsoft-word/operations.test.ts b/apps/sim/lib/internal/microsoft-word/operations.test.ts new file mode 100644 index 00000000000..7fcc9bb4170 --- /dev/null +++ b/apps/sim/lib/internal/microsoft-word/operations.test.ts @@ -0,0 +1,303 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext, inputValidationMock, inputValidationMockFns } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + +import { executeMicrosoftWordTool } from '@/lib/internal/microsoft-word/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' +import { buildDocxFromContent } from '@/lib/microsoft-word/document.server' + +const { mockValidateUrlWithDNS, mockSecureFetchWithPinnedIP } = inputValidationMockFns + +const PINNED_IP = '93.184.216.34' + +const baseBody = { + accessToken: 'token-123', + documentId: 'doc-abc', + content: 'Appended paragraph', +} + +/** A Graph `driveItem` metadata response carrying a content tag. */ +function itemResponse(cTag: string) { + const body = { + id: 'doc-abc', + name: 'notes.docx', + cTag, + file: { mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' }, + } + return { + ok: true, + status: 200, + statusText: '', + headers: new Headers(), + body: null, + text: async () => JSON.stringify(body), + json: async () => body, + arrayBuffer: async () => new ArrayBuffer(0), + } +} + +/** A Graph content response carrying a real `.docx` package. */ +async function docxResponse() { + const buffer = await buildDocxFromContent('Existing paragraph') + return { + ok: true, + status: 200, + statusText: '', + headers: new Headers(), + body: null, + text: async () => '', + json: async () => ({}), + arrayBuffer: async () => + buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength), + } +} + +/** The `createUploadSession` response carrying the preauthenticated upload URL. */ +function uploadSessionResponse(uploadUrl = 'https://sn3302.up.1drv.com/up/session-abc') { + const body = { uploadUrl, expirationDateTime: '2026-01-01T00:00:00Z' } + return { + ok: true, + status: 200, + statusText: '', + headers: new Headers(), + body: null, + text: async () => JSON.stringify(body), + json: async () => body, + arrayBuffer: async () => new ArrayBuffer(0), + } +} + +function preconditionFailedResponse() { + return { + ok: false, + status: 412, + statusText: 'Precondition Failed', + headers: new Headers(), + body: null, + text: async () => '', + json: async () => ({}), + arrayBuffer: async () => new ArrayBuffer(0), + } +} + +beforeEach(() => { + vi.clearAllMocks() + mockValidateUrlWithDNS.mockResolvedValue({ + isValid: true, + resolvedIP: PINNED_IP, + originalHostname: 'graph.microsoft.com', + }) +}) + +function executeTool(toolId: string, input: unknown): Promise { + const request: InternalToolOperationCall = { + toolId, + input, + headers: new Headers(), + context: createExecutionContext({ workflowId: 'workflow-1' }), + requestId: 'request-1', + } + return executeMicrosoftWordTool(request) +} + +function executeAppend(input: typeof baseBody): Promise { + return executeTool('microsoft_word_append', input) +} + +describe('Microsoft Word direct input validation', () => { + it('rejects a whitespace-only document name before provider work', async () => { + const response = await executeTool('microsoft_word_create', { + accessToken: 'token-123', + name: ' ', + content: 'Hello', + }) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + success: false, + error: expect.stringMatching(/Document name is required/), + }) + expect(mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() + }) + + it.each([{ folderId: 'not a valid id' }, { driveId: 'bad/../drive' }])( + 'rejects malformed create paths before provider work', + async (extra) => { + const response = await executeTool('microsoft_word_create', { + accessToken: 'token-123', + name: 'Document', + content: 'Hello', + ...extra, + }) + + expect(response.status).toBe(400) + expect(mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() + } + ) + + it.each([{ '': 'Acme Corp' }, '{" ": "Acme Corp"}', '["a","b"]'])( + 'rejects an invalid template placeholder map before provider work', + async (replacements) => { + const response = await executeTool('microsoft_word_create_from_template', { + accessToken: 'token-123', + templateDocumentId: 'template-abc', + name: 'Acme Agreement', + replacements, + }) + + expect(response.status).toBe(400) + expect(mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() + } + ) +}) + +describe('Microsoft Word append operation', () => { + it('writes through a conditional upload session carrying the content tag', async () => { + mockSecureFetchWithPinnedIP + .mockResolvedValueOnce(itemResponse('tag-1')) + .mockResolvedValueOnce(await docxResponse()) + .mockResolvedValueOnce(uploadSessionResponse()) + .mockResolvedValueOnce(itemResponse('tag-2')) + + const response = await executeAppend(baseBody) + + expect(response.status).toBe(200) + const data = (await response.json()) as { + success: boolean + output: { updatedContent: boolean } + } + expect(data.success).toBe(true) + expect(data.output.updatedContent).toBe(true) + + /** Graph enforces the content precondition when it creates the upload session. */ + const sessionCall = mockSecureFetchWithPinnedIP.mock.calls.find((call) => + String(call[0]).endsWith('/createUploadSession') + ) + expect(sessionCall?.[2]).toMatchObject({ method: 'POST' }) + expect(sessionCall?.[2].headers).toMatchObject({ 'if-match': 'tag-1' }) + + /** The preauthenticated upload URL must not receive the bearer token. */ + const uploadCall = mockSecureFetchWithPinnedIP.mock.calls.at(-1) + expect(uploadCall?.[0]).toBe('https://sn3302.up.1drv.com/up/session-abc') + expect(uploadCall?.[2]).toMatchObject({ method: 'PUT' }) + expect(uploadCall?.[2].headers.Authorization).toBeUndefined() + }) + + it('refuses to overwrite when the service rejects the precondition', async () => { + mockSecureFetchWithPinnedIP + .mockResolvedValueOnce(itemResponse('tag-1')) + .mockResolvedValueOnce(await docxResponse()) + .mockResolvedValueOnce(preconditionFailedResponse()) + + const response = await executeAppend(baseBody) + + expect(response.status).toBe(409) + const data = (await response.json()) as { success: boolean; error: string } + expect(data.success).toBe(false) + expect(data.error).toMatch(/no other change was overwritten/) + + /** A refused upload session prevents any content write. */ + expect(mockSecureFetchWithPinnedIP.mock.calls.some((call) => call[2]?.method === 'PUT')).toBe( + false + ) + }) + + it('maps a malformed document ID to a client error, not a server error', async () => { + const response = await executeAppend({ ...baseBody, documentId: 'bad/../id' }) + + expect(response.status).toBe(400) + const data = (await response.json()) as { success: boolean; error: string } + expect(data.success).toBe(false) + expect(mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() + }) + + it('refuses to write Word bytes over a drive item that is not a .docx', async () => { + const pdf = { + id: 'doc-abc', + name: 'invoice.pdf', + cTag: 'tag-1', + file: { mimeType: 'application/pdf' }, + } + mockSecureFetchWithPinnedIP.mockResolvedValueOnce({ + ok: true, + status: 200, + statusText: '', + headers: new Headers(), + body: null, + text: async () => JSON.stringify(pdf), + json: async () => pdf, + arrayBuffer: async () => new ArrayBuffer(0), + }) + + const response = await executeAppend(baseBody) + + expect(response.status).toBe(400) + const data = (await response.json()) as { error: string } + expect(data.error).toMatch(/not a Word document/) + expect(mockSecureFetchWithPinnedIP.mock.calls.some((call) => call[2]?.method === 'PUT')).toBe( + false + ) + }) + + it('reports a no-op without writing when the content adds no paragraph', async () => { + mockSecureFetchWithPinnedIP + .mockResolvedValueOnce(itemResponse('tag-1')) + .mockResolvedValueOnce(await docxResponse()) + + const response = await executeAppend({ ...baseBody, content: ' \n \n' }) + + expect(response.status).toBe(200) + const data = (await response.json()) as { output: { updatedContent: boolean } } + expect(data.output.updatedContent).toBe(false) + expect(mockSecureFetchWithPinnedIP.mock.calls.some((call) => call[2]?.method === 'PUT')).toBe( + false + ) + }) + + it('refuses to write when Graph reports no version to compare against', async () => { + const untagged = { + id: 'doc-abc', + name: 'notes.docx', + file: { + mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + }, + } + mockSecureFetchWithPinnedIP.mockResolvedValueOnce({ + ok: true, + status: 200, + statusText: '', + headers: new Headers(), + body: null, + text: async () => JSON.stringify(untagged), + json: async () => untagged, + arrayBuffer: async () => new ArrayBuffer(0), + }) + + const response = await executeAppend(baseBody) + + expect(response.status).toBe(409) + const data = (await response.json()) as { error: string } + expect(data.error).toMatch(/did not report a version/) + expect(mockSecureFetchWithPinnedIP.mock.calls.some((call) => call[2]?.method === 'PUT')).toBe( + false + ) + }) + + it('surfaces a conflict raised when the bytes commit', async () => { + mockSecureFetchWithPinnedIP + .mockResolvedValueOnce(itemResponse('tag-1')) + .mockResolvedValueOnce(await docxResponse()) + .mockResolvedValueOnce(uploadSessionResponse()) + .mockResolvedValueOnce(preconditionFailedResponse()) + + const response = await executeAppend(baseBody) + + expect(response.status).toBe(409) + const data = (await response.json()) as { error: string } + expect(data.error).toMatch(/no other change was overwritten/) + }) +}) diff --git a/apps/sim/lib/internal/microsoft-word/operations.ts b/apps/sim/lib/internal/microsoft-word/operations.ts new file mode 100644 index 00000000000..4a3d990bf45 --- /dev/null +++ b/apps/sim/lib/internal/microsoft-word/operations.ts @@ -0,0 +1,336 @@ +import { createLogger } from '@sim/logger' +import { + downloadConvertedContent, + downloadDocumentContent, + fetchDocumentItem, + replaceContentIfUnchanged, + requireContentTag, + toDocumentMetadata, + uploadDocumentContent, +} from '@/lib/internal/microsoft-word/client' +import type { + MicrosoftWordAppendInput, + MicrosoftWordCreateFromTemplateInput, + MicrosoftWordCreateInput, + MicrosoftWordExportPdfInput, + MicrosoftWordReadInput, + MicrosoftWordReplaceTextInput, + MicrosoftWordUpdateInput, +} from '@/lib/internal/microsoft-word/schema' +import { + appendParagraphsToDocx, + buildDocxFromContent, + DOCX_MIME_TYPE, + extractDocxText, + parseReplacements, + replaceTextInDocx, +} from '@/lib/microsoft-word/document.server' +import { + buildCreateUploadUrl, + ensureDocxExtension, + getDocumentBasePath, + getDriveBasePath, + getFolderBasePath, +} from '@/tools/microsoft_word/utils' + +const logger = createLogger('MicrosoftWordOperations') +const PDF_MIME_TYPE = 'application/pdf' + +export interface MicrosoftWordOperationContext { + requestId: string + signal?: AbortSignal +} + +function throwIfAborted(context: MicrosoftWordOperationContext): void { + context.signal?.throwIfAborted() +} + +/** Creates a new Word document without replacing a same-named drive item. */ +export async function executeMicrosoftWordCreate( + input: MicrosoftWordCreateInput, + context: MicrosoftWordOperationContext +) { + throwIfAborted(context) + const fileName = ensureDocxExtension(input.name) + const parentPath = input.folderId?.trim() + ? getFolderBasePath(input.folderId, input.driveId ?? undefined) + : `${getDriveBasePath(input.driveId ?? undefined)}/root` + const uploadUrl = buildCreateUploadUrl(parentPath, fileName) + const documentBuffer = await buildDocxFromContent(input.content ?? '', input.name) + throwIfAborted(context) + const item = await uploadDocumentContent( + uploadUrl, + input.accessToken, + documentBuffer, + DOCX_MIME_TYPE, + context.signal + ) + throwIfAborted(context) + + logger.info('Created Word document', { + requestId: context.requestId, + documentId: item.id, + size: item.size, + }) + return { + success: true as const, + output: { metadata: toDocumentMetadata(item, item.id ?? '') }, + } +} + +/** Reads the text and metadata of an existing Word document. */ +export async function executeMicrosoftWordRead( + input: MicrosoftWordReadInput, + context: MicrosoftWordOperationContext +) { + throwIfAborted(context) + const basePath = getDocumentBasePath(input.documentId, input.driveId ?? undefined) + const item = await fetchDocumentItem(basePath, input.accessToken, context.signal) + const documentBuffer = await downloadDocumentContent(basePath, input.accessToken, context.signal) + throwIfAborted(context) + const content = await extractDocxText(documentBuffer) + throwIfAborted(context) + + logger.info('Read Word document', { + requestId: context.requestId, + documentId: input.documentId, + characterCount: content.length, + }) + return { + success: true as const, + output: { content, metadata: toDocumentMetadata(item, input.documentId) }, + } +} + +/** Deliberately replaces all content in an existing Word document. */ +export async function executeMicrosoftWordUpdate( + input: MicrosoftWordUpdateInput, + context: MicrosoftWordOperationContext +) { + throwIfAborted(context) + const basePath = getDocumentBasePath(input.documentId, input.driveId ?? undefined) + const existing = await fetchDocumentItem(basePath, input.accessToken, context.signal) + throwIfAborted(context) + const documentBuffer = await buildDocxFromContent(input.content, existing.name) + throwIfAborted(context) + const item = await uploadDocumentContent( + `${basePath}/content`, + input.accessToken, + documentBuffer, + DOCX_MIME_TYPE, + context.signal + ) + throwIfAborted(context) + + logger.info('Replaced Word document contents', { + requestId: context.requestId, + documentId: input.documentId, + size: item.size, + }) + return { + success: true as const, + output: { + updatedContent: true, + metadata: toDocumentMetadata(item, input.documentId), + }, + } +} + +/** Appends paragraphs while preserving the rest of the document package. */ +export async function executeMicrosoftWordAppend( + input: MicrosoftWordAppendInput, + context: MicrosoftWordOperationContext +) { + throwIfAborted(context) + const basePath = getDocumentBasePath(input.documentId, input.driveId ?? undefined) + const existingItem = await fetchDocumentItem(basePath, input.accessToken, context.signal) + const contentTag = requireContentTag(existingItem) + const existingBuffer = await downloadDocumentContent(basePath, input.accessToken, context.signal) + throwIfAborted(context) + const { buffer, paragraphsAppended } = await appendParagraphsToDocx(existingBuffer, input.content) + throwIfAborted(context) + + if (paragraphsAppended === 0) { + logger.info('No paragraphs to append; document left untouched', { + requestId: context.requestId, + documentId: input.documentId, + }) + return { + success: true as const, + output: { + updatedContent: false, + metadata: toDocumentMetadata(existingItem, input.documentId), + }, + } + } + + const item = await replaceContentIfUnchanged( + basePath, + input.accessToken, + buffer, + contentTag, + context.signal + ) + throwIfAborted(context) + logger.info('Appended to Word document', { + requestId: context.requestId, + documentId: input.documentId, + paragraphsAppended, + size: item.size, + }) + return { + success: true as const, + output: { + updatedContent: true, + metadata: toDocumentMetadata(item, input.documentId), + }, + } +} + +/** Creates a new document by filling an existing Word template. */ +export async function executeMicrosoftWordCreateFromTemplate( + input: MicrosoftWordCreateFromTemplateInput, + context: MicrosoftWordOperationContext +) { + throwIfAborted(context) + const templatePath = getDocumentBasePath(input.templateDocumentId, input.driveId ?? undefined) + await fetchDocumentItem(templatePath, input.accessToken, context.signal) + const templateBuffer = await downloadDocumentContent( + templatePath, + input.accessToken, + context.signal + ) + throwIfAborted(context) + const pairs = parseReplacements(input.replacements) + const filled = + pairs.length > 0 + ? await replaceTextInDocx(templateBuffer, pairs, input.matchCase ?? false) + : { buffer: templateBuffer, occurrencesChanged: 0 } + throwIfAborted(context) + + const fileName = ensureDocxExtension(input.name) + const parentPath = input.folderId?.trim() + ? getFolderBasePath(input.folderId, input.driveId ?? undefined) + : `${getDriveBasePath(input.driveId ?? undefined)}/root` + const uploadUrl = buildCreateUploadUrl(parentPath, fileName) + const item = await uploadDocumentContent( + uploadUrl, + input.accessToken, + filled.buffer, + DOCX_MIME_TYPE, + context.signal + ) + throwIfAborted(context) + + logger.info('Created Word document from template', { + requestId: context.requestId, + templateDocumentId: input.templateDocumentId, + documentId: item.id, + occurrencesChanged: filled.occurrencesChanged, + }) + return { + success: true as const, + output: { + occurrencesChanged: filled.occurrencesChanged, + metadata: toDocumentMetadata(item, item.id ?? ''), + }, + } +} + +/** Replaces matching text without rewriting an unchanged document. */ +export async function executeMicrosoftWordReplaceText( + input: MicrosoftWordReplaceTextInput, + context: MicrosoftWordOperationContext +) { + throwIfAborted(context) + const basePath = getDocumentBasePath(input.documentId, input.driveId ?? undefined) + const existingItem = await fetchDocumentItem(basePath, input.accessToken, context.signal) + const contentTag = requireContentTag(existingItem) + const existingBuffer = await downloadDocumentContent(basePath, input.accessToken, context.signal) + throwIfAborted(context) + const { buffer, occurrencesChanged } = await replaceTextInDocx( + existingBuffer, + [{ find: input.findText, replace: input.replaceText ?? '' }], + input.matchCase ?? false + ) + throwIfAborted(context) + + if (occurrencesChanged === 0) { + logger.info('No occurrences matched; document left untouched', { + requestId: context.requestId, + documentId: input.documentId, + }) + return { + success: true as const, + output: { + occurrencesChanged: 0, + metadata: toDocumentMetadata(existingItem, input.documentId), + }, + } + } + + const item = await replaceContentIfUnchanged( + basePath, + input.accessToken, + buffer, + contentTag, + context.signal + ) + throwIfAborted(context) + logger.info('Replaced text in Word document', { + requestId: context.requestId, + documentId: input.documentId, + occurrencesChanged, + }) + return { + success: true as const, + output: { + occurrencesChanged, + metadata: toDocumentMetadata(item, input.documentId), + }, + } +} + +function resolvePdfName(override: string | null | undefined, documentName?: string): string { + const explicit = override?.trim() + if (explicit) return explicit.toLowerCase().endsWith('.pdf') ? explicit : `${explicit}.pdf` + + const base = documentName?.trim().replace(/\.docx$/i, '') + return base ? `${base}.pdf` : 'document.pdf' +} + +/** Converts a Word document to a PDF file output through Microsoft Graph. */ +export async function executeMicrosoftWordExportPdf( + input: MicrosoftWordExportPdfInput, + context: MicrosoftWordOperationContext +) { + throwIfAborted(context) + const basePath = getDocumentBasePath(input.documentId, input.driveId ?? undefined) + const item = await fetchDocumentItem(basePath, input.accessToken, context.signal) + const pdfBuffer = await downloadConvertedContent( + basePath, + input.accessToken, + 'pdf', + context.signal + ) + throwIfAborted(context) + const name = resolvePdfName(input.fileName, item.name) + + logger.info('Exported Word document as PDF', { + requestId: context.requestId, + documentId: input.documentId, + name, + size: pdfBuffer.length, + }) + return { + success: true as const, + output: { + file: { + name, + mimeType: PDF_MIME_TYPE, + data: pdfBuffer.toString('base64'), + size: pdfBuffer.length, + }, + }, + } +} diff --git a/apps/sim/lib/internal/microsoft-word/schema.ts b/apps/sim/lib/internal/microsoft-word/schema.ts new file mode 100644 index 00000000000..9a97e991e35 --- /dev/null +++ b/apps/sim/lib/internal/microsoft-word/schema.ts @@ -0,0 +1,107 @@ +import { z } from 'zod' + +const MAX_DOCUMENT_CONTENT_LENGTH = 2_000_000 +const MAX_REPLACEMENTS_LENGTH = 200_000 + +function isPlaceholderMap(value: unknown): boolean { + if (typeof value !== 'object' || value === null || Array.isArray(value)) return false + return Object.keys(value).every((key) => key.trim().length > 0) +} + +function isPlaceholderMapString(value: string): boolean { + if (!value.trim()) return true + try { + return isPlaceholderMap(JSON.parse(value)) + } catch { + return false + } +} + +const wordReplacementsSchema = z + .union([ + z + .string() + .refine( + isPlaceholderMapString, + 'Placeholder values must be a JSON object mapping each non-empty placeholder to its value' + ), + z + .record(z.string(), z.union([z.string(), z.number(), z.boolean(), z.null()])) + .refine(isPlaceholderMap, 'Every placeholder must be a non-empty string'), + ]) + .refine( + (value) => + (typeof value === 'string' ? value.length : JSON.stringify(value).length) <= + MAX_REPLACEMENTS_LENGTH, + 'Placeholder values are too long' + ) + +const accessTokenSchema = z.string().min(1, 'Access token is required') +const documentIdSchema = z.string().min(1, 'Document ID is required') +const driveIdSchema = z.string().optional().nullable() + +export const microsoftWordCreateInputSchema = z.object({ + accessToken: accessTokenSchema, + name: z.string().min(1, 'Document name is required').max(255, 'Document name is too long'), + content: z + .string() + .max(MAX_DOCUMENT_CONTENT_LENGTH, 'Document content is too long') + .optional() + .nullable(), + folderId: z.string().optional().nullable(), + driveId: driveIdSchema, +}) + +export const microsoftWordReadInputSchema = z.object({ + accessToken: accessTokenSchema, + documentId: documentIdSchema, + driveId: driveIdSchema, +}) + +export const microsoftWordUpdateInputSchema = z.object({ + accessToken: accessTokenSchema, + documentId: documentIdSchema, + content: z + .string() + .min(1, 'Document content is required') + .max(MAX_DOCUMENT_CONTENT_LENGTH, 'Document content is too long'), + driveId: driveIdSchema, +}) + +export const microsoftWordAppendInputSchema = microsoftWordUpdateInputSchema + +export const microsoftWordCreateFromTemplateInputSchema = z.object({ + accessToken: accessTokenSchema, + templateDocumentId: z.string().min(1, 'Template document ID is required'), + name: z.string().min(1, 'Document name is required').max(255, 'Document name is too long'), + replacements: wordReplacementsSchema.optional().nullable(), + matchCase: z.boolean().optional().nullable(), + folderId: z.string().optional().nullable(), + driveId: driveIdSchema, +}) + +export const microsoftWordReplaceTextInputSchema = z.object({ + accessToken: accessTokenSchema, + documentId: documentIdSchema, + findText: z.string().min(1, 'Search text is required').max(4000, 'Search text is too long'), + replaceText: z.string().max(20000, 'Replacement text is too long').optional().nullable(), + matchCase: z.boolean().optional().nullable(), + driveId: driveIdSchema, +}) + +export const microsoftWordExportPdfInputSchema = z.object({ + accessToken: accessTokenSchema, + documentId: documentIdSchema, + fileName: z.string().optional().nullable(), + driveId: driveIdSchema, +}) + +export type MicrosoftWordCreateInput = z.output +export type MicrosoftWordReadInput = z.output +export type MicrosoftWordUpdateInput = z.output +export type MicrosoftWordAppendInput = z.output +export type MicrosoftWordCreateFromTemplateInput = z.output< + typeof microsoftWordCreateFromTemplateInputSchema +> +export type MicrosoftWordReplaceTextInput = z.output +export type MicrosoftWordExportPdfInput = z.output diff --git a/apps/sim/lib/internal/mistral/client.ts b/apps/sim/lib/internal/mistral/client.ts new file mode 100644 index 00000000000..593445a037b --- /dev/null +++ b/apps/sim/lib/internal/mistral/client.ts @@ -0,0 +1,50 @@ +import { createLogger } from '@sim/logger' +import { + DEFAULT_MAX_RESPONSE_BYTES, + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { MistralOperationError } from '@/lib/internal/mistral/errors' +import { readBoundedHttpErrorBody } from '@/lib/knowledge/documents/utils' + +const logger = createLogger('MistralClient') +const MISTRAL_ENDPOINT = 'https://api.mistral.ai/v1/ocr' + +export async function submitMistralOcr( + apiKey: string, + body: Record, + signal?: AbortSignal, + maxResponseBytes = DEFAULT_MAX_RESPONSE_BYTES +): Promise { + signal?.throwIfAborted() + const validation = await validateUrlWithDNS(MISTRAL_ENDPOINT, 'Mistral API URL') + signal?.throwIfAborted() + if (!validation.isValid || !validation.resolvedIP) { + throw new MistralOperationError(502, { + success: false, + error: 'Failed to reach Mistral API', + }) + } + + const response = await secureFetchWithPinnedIP(MISTRAL_ENDPOINT, validation.resolvedIP, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify(body), + maxResponseBytes, + signal, + }) + signal?.throwIfAborted() + if (!response.ok) { + const diagnostic = await readBoundedHttpErrorBody(response) + logger.error('Mistral API error', { status: response.status, diagnostic }) + throw new MistralOperationError(response.status, { + success: false, + error: `Mistral API error: ${response.statusText}`, + }) + } + return response.json() +} diff --git a/apps/sim/lib/internal/mistral/errors.ts b/apps/sim/lib/internal/mistral/errors.ts new file mode 100644 index 00000000000..458756cdf7c --- /dev/null +++ b/apps/sim/lib/internal/mistral/errors.ts @@ -0,0 +1,9 @@ +export class MistralOperationError extends Error { + constructor( + readonly status: number, + readonly body: unknown + ) { + super('Mistral operation failed') + this.name = 'MistralOperationError' + } +} diff --git a/apps/sim/lib/internal/mistral/execute-tool.test.ts b/apps/sim/lib/internal/mistral/execute-tool.test.ts new file mode 100644 index 00000000000..bba41a025a9 --- /dev/null +++ b/apps/sim/lib/internal/mistral/execute-tool.test.ts @@ -0,0 +1,43 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operation = vi.hoisted(() => vi.fn()) +vi.mock('@/lib/internal/mistral/operations', () => ({ executeMistralParse: operation })) + +import { executeMistralTool } from '@/lib/internal/mistral/execute-tool' + +describe('executeMistralTool', () => { + beforeEach(() => { + vi.clearAllMocks() + operation.mockResolvedValue({ success: true, output: { pages: [] } }) + }) + + it('dispatches all canonical IDs and propagates cancellation', async () => { + for (const toolId of ['mistral_parser', 'mistral_parser_v2', 'mistral_parser_v3']) { + const response = await executeMistralTool({ + toolId, + input: { apiKey: 'key', filePath: 'https://example.com/file.pdf' }, + headers: new Headers(), + context: { ...createExecutionContext(), userId: 'user-1' }, + requestId: 'request-1', + }) + expect(response.status).toBe(200) + } + + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + await expect( + executeMistralTool({ + toolId: 'mistral_parser', + input: { apiKey: 'key', filePath: 'https://example.com/file.pdf' }, + headers: new Headers(), + context: createExecutionContext(), + requestId: 'request-1', + signal: controller.signal, + }) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) +}) diff --git a/apps/sim/lib/internal/mistral/execute-tool.ts b/apps/sim/lib/internal/mistral/execute-tool.ts new file mode 100644 index 00000000000..88562df2c94 --- /dev/null +++ b/apps/sim/lib/internal/mistral/execute-tool.ts @@ -0,0 +1,83 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { z } from 'zod' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { MistralOperationError } from '@/lib/internal/mistral/errors' +import { + MISTRAL_MAX_OPERATION_INPUT_BYTES, + mistralParseInputSchema, +} from '@/lib/internal/mistral/input' +import { isMistralInputWithinLimit } from '@/lib/internal/mistral/input-size' +import { executeMistralParse } from '@/lib/internal/mistral/operations' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/doc-not-ready' + +const logger = createLogger('MistralToolExecution') + +function validationResponse(error: z.ZodError): Response { + return Response.json( + { + success: false, + error: error.issues[0]?.message || 'Invalid request data', + details: error.issues, + }, + { status: 400 } + ) +} + +export const executeMistralTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!['mistral_parser', 'mistral_parser_v2', 'mistral_parser_v3'].includes(request.toolId)) { + return Response.json( + { success: false, error: `Unsupported Mistral tool: ${request.toolId}` }, + { status: 500 } + ) + } + try { + if (!isMistralInputWithinLimit(request.input, MISTRAL_MAX_OPERATION_INPUT_BYTES)) { + return Response.json( + { + success: false, + error: `Request body exceeds the maximum allowed size of ${MISTRAL_MAX_OPERATION_INPUT_BYTES} bytes`, + }, + { status: 413 } + ) + } + } catch { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + const parsed = mistralParseInputSchema.safeParse(request.input) + if (!parsed.success) return validationResponse(parsed.error) + + try { + const result = await executeMistralParse(parsed.data, { + headers: request.headers, + requestId: request.requestId, + signal: request.signal, + userId: request.context.userId, + }) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof MistralOperationError) { + return Response.json(error.body, { status: error.status }) + } + if (isDocNotReadyError(error)) { + return Response.json({ success: false, error: docNotReadyMessage() }, { status: 409 }) + } + if (isPayloadSizeLimitError(error)) { + return Response.json( + { success: false, error: 'Mistral API response exceeded the safe size limit' }, + { status: 502 } + ) + } + const message = getErrorMessage(error, 'Internal server error') + logger.error('Mistral operation failed', { + error: message, + requestId: request.requestId, + toolId: request.toolId, + }) + return Response.json({ success: false, error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/mistral/input-size.ts b/apps/sim/lib/internal/mistral/input-size.ts new file mode 100644 index 00000000000..e0c37d5d7be --- /dev/null +++ b/apps/sim/lib/internal/mistral/input-size.ts @@ -0,0 +1,39 @@ +function stringBytes(value: string): number { + return Buffer.byteLength(value, 'utf8') + 2 +} + +function countBytes(value: unknown, limit: number, seen: Set): number { + if (value === null) return 4 + if (typeof value === 'string') return stringBytes(value) + if (typeof value === 'boolean') return value ? 4 : 5 + if (typeof value === 'number') return Number.isFinite(value) ? String(value).length : 4 + if (typeof value === 'bigint') throw new TypeError('Do not know how to serialize a BigInt') + if (value instanceof Date) return stringBytes(value.toJSON()) + if (!value || typeof value !== 'object') return 0 + if (seen.has(value)) throw new TypeError('Converting circular structure to JSON') + seen.add(value) + + let bytes = 2 + if (Array.isArray(value)) { + for (let index = 0; index < value.length; index += 1) { + if (index > 0) bytes += 1 + bytes += countBytes(value[index], limit - bytes, seen) + if (bytes > limit) break + } + } else { + let emitted = false + for (const [key, entry] of Object.entries(value)) { + if (entry === undefined || typeof entry === 'function' || typeof entry === 'symbol') continue + if (emitted) bytes += 1 + bytes += stringBytes(key) + 1 + countBytes(entry, limit - bytes, seen) + emitted = true + if (bytes > limit) break + } + } + seen.delete(value) + return bytes +} + +export function isMistralInputWithinLimit(input: unknown, limit: number): boolean { + return countBytes(input, limit, new Set()) <= limit +} diff --git a/apps/sim/lib/internal/mistral/input.ts b/apps/sim/lib/internal/mistral/input.ts new file mode 100644 index 00000000000..c5aefac72c0 --- /dev/null +++ b/apps/sim/lib/internal/mistral/input.ts @@ -0,0 +1,23 @@ +import { z } from 'zod' +import { resolvedSecretTraceProvenanceSchema } from '@/lib/api/contracts/primitives' +import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' +import { MISTRAL_OCR_REQUEST_POLICY } from '@/lib/knowledge/documents/ocr-request-policy' +import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' + +export const mistralParseInputSchema = z.object({ + apiKey: z.string().min(1, 'API key is required'), + filePath: z.string().min(1, 'File path is required').optional(), + fileData: FileInputSchema.optional(), + file: FileInputSchema.optional(), + resultType: z.string().optional(), + pages: z.array(z.number()).max(MISTRAL_OCR_REQUEST_POLICY.maxPages).optional(), + includeImageBase64: z.boolean().optional(), + imageLimit: z.number().optional(), + imageMinSize: z.number().optional(), + [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), +}) + +export type MistralParseInput = z.infer + +export const MISTRAL_MAX_OPERATION_INPUT_BYTES = + Math.ceil(MISTRAL_OCR_REQUEST_POLICY.maxBytes / 3) * 4 + 1024 * 1024 diff --git a/apps/sim/lib/internal/mistral/operations.ts b/apps/sim/lib/internal/mistral/operations.ts new file mode 100644 index 00000000000..4ced4dab583 --- /dev/null +++ b/apps/sim/lib/internal/mistral/operations.ts @@ -0,0 +1,225 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' +import { decodeDataUriWithinLimit } from '@/lib/file-parsers/data-uri' +import { isFileParserError } from '@/lib/file-parsers/errors' +import { submitMistralOcr } from '@/lib/internal/mistral/client' +import { MistralOperationError } from '@/lib/internal/mistral/errors' +import type { MistralParseInput } from '@/lib/internal/mistral/input' +import { MISTRAL_OCR_REQUEST_POLICY } from '@/lib/knowledge/documents/ocr-request-policy' +import { + isModelSafeWorkspaceFileKey, + MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, +} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { + extractStorageKey, + isInternalFileUrl, + processSingleFileToUserFile, +} from '@/lib/uploads/utils/file-utils' +import { + downloadServableFileFromStorage, + resolveInternalFileUrl, + type ServableFile, +} from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +const logger = createLogger('MistralOperations') +const IMAGE_EXTENSIONS = ['.png', '.jpg', '.jpeg', '.gif', '.webp', '.avif'] as const + +export interface MistralOperationContext { + headers: Headers + maxResponseBytes?: number + requestId: string + signal?: AbortSignal + trustedCaller?: 'knowledge-ingestion' + userId?: string +} + +function fileSizeError(): MistralOperationError { + return new MistralOperationError(413, { + success: false, + error: `File exceeds Mistral OCR's ${MISTRAL_OCR_REQUEST_POLICY.maxBytes.toLocaleString()}-byte request limit`, + }) +} + +function inferMimeType(type: string | undefined, name: string | undefined): string { + if (type && type !== 'application/octet-stream') return type + const filename = name?.toLowerCase() ?? '' + if (filename.endsWith('.png')) return 'image/png' + if (filename.endsWith('.jpg') || filename.endsWith('.jpeg')) return 'image/jpeg' + if (filename.endsWith('.gif')) return 'image/gif' + if (filename.endsWith('.webp')) return 'image/webp' + return 'application/pdf' +} + +async function buildInlineDocument( + file: Exclude, + context: MistralOperationContext +): Promise> { + if (!context.userId) { + throw new MistralOperationError(401, { success: false, error: 'Unauthorized' }) + } + let userFile + try { + userFile = processSingleFileToUserFile(file, context.requestId, logger) + } catch (error) { + throw new MistralOperationError(400, { + success: false, + error: getErrorMessage(error, 'Failed to process file'), + }) + } + + let mimeType = inferMimeType(userFile.type, userFile.name) + let base64 = userFile.base64 + if (!base64) { + const denied = await assertToolFileAccess( + userFile.key, + context.userId ?? '', + context.requestId, + logger + ) + context.signal?.throwIfAborted() + if (denied) { + throw new MistralOperationError(404, { success: false, error: 'File not found' }) + } + if (!(await isModelSafeWorkspaceFileKey(userFile.key))) { + throw new MistralOperationError(400, { + success: false, + error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, + }) + } + context.signal?.throwIfAborted() + let servableFile: ServableFile + try { + servableFile = await downloadServableFileFromStorage(userFile, context.requestId, logger, { + maxBytes: MISTRAL_OCR_REQUEST_POLICY.maxBytes, + }) + } catch (error) { + const { isPayloadSizeLimitError } = await import('@/lib/core/utils/stream-limits') + if (isPayloadSizeLimitError(error)) throw fileSizeError() + throw error + } + context.signal?.throwIfAborted() + base64 = servableFile.buffer.toString('base64') + if (servableFile.contentType && servableFile.contentType !== 'application/octet-stream') { + mimeType = servableFile.contentType + } + } + + let inlineBytes: number + try { + inlineBytes = base64.startsWith('data:') + ? decodeDataUriWithinLimit(base64, MISTRAL_OCR_REQUEST_POLICY.maxBytes).buffer.length + : Buffer.byteLength(base64, 'base64') + } catch (error) { + if (isFileParserError(error) && error.code === 'complexity_limit') throw fileSizeError() + throw new MistralOperationError(400, { + success: false, + error: getErrorMessage(error, 'Invalid inline file data'), + }) + } + if (inlineBytes > MISTRAL_OCR_REQUEST_POLICY.maxBytes) throw fileSizeError() + + const payload = base64.startsWith('data:') ? base64 : `data:${mimeType};base64,${base64}` + return mimeType.startsWith('image/') + ? { type: 'image_url', image_url: payload } + : { type: 'document_url', document_url: payload } +} + +async function buildUrlDocument( + filePath: string, + context: MistralOperationContext +): Promise> { + let fileUrl = filePath + if (isInternalFileUrl(filePath)) { + if (!context.userId) { + throw new MistralOperationError(401, { success: false, error: 'Unauthorized' }) + } + const resolution = await resolveInternalFileUrl( + filePath, + context.userId ?? '', + context.requestId, + logger + ) + context.signal?.throwIfAborted() + if (resolution.error) { + throw new MistralOperationError(resolution.error.status, { + success: false, + error: resolution.error.message, + }) + } + fileUrl = resolution.fileUrl || fileUrl + if (!(await isModelSafeWorkspaceFileKey(extractStorageKey(filePath)))) { + throw new MistralOperationError(400, { + success: false, + error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, + }) + } + } else if (filePath.startsWith('/')) { + throw new MistralOperationError(400, { + success: false, + error: 'Invalid file path. Only uploaded files are supported for internal paths.', + }) + } else { + const { validateUrlWithDNS } = await import('@/lib/core/security/input-validation.server') + const validation = await validateUrlWithDNS(fileUrl, 'filePath') + context.signal?.throwIfAborted() + if (!validation.isValid) { + throw new MistralOperationError(400, { success: false, error: validation.error }) + } + } + + const pathname = new URL(fileUrl).pathname.toLowerCase() + return IMAGE_EXTENSIONS.some((extension) => pathname.endsWith(extension)) + ? { type: 'image_url', image_url: fileUrl } + : { type: 'document_url', document_url: fileUrl } +} + +export async function executeMistralParse( + input: MistralParseInput, + context: MistralOperationContext +): Promise<{ success: true; output: unknown }> { + context.signal?.throwIfAborted() + if (!context.userId && context.trustedCaller !== 'knowledge-ingestion') { + throw new MistralOperationError(401, { success: false, error: 'Unauthorized' }) + } + const provenance = validateOpaqueModelInputProvenance({ + headers: context.headers, + payload: input, + isInternalRequest: true, + }) + if (!provenance.success) { + throw new MistralOperationError(provenance.status, { + success: false, + error: provenance.error, + }) + } + + const fileData = input.file || input.fileData + const filePath = typeof fileData === 'string' ? fileData : input.filePath + if (!fileData && (!filePath || filePath.trim() === '')) { + throw new MistralOperationError(400, { success: false, error: 'File input is required' }) + } + + const body: Record = { model: 'mistral-ocr-latest' } + if (fileData && typeof fileData === 'object') { + body.document = await buildInlineDocument(fileData, context) + } else if (filePath) { + body.document = await buildUrlDocument(filePath, context) + } + if (input.pages) body.pages = input.pages + if (input.includeImageBase64 !== undefined) { + body.include_image_base64 = input.includeImageBase64 + } + if (input.imageLimit) body.image_limit = input.imageLimit + if (input.imageMinSize) body.image_min_size = input.imageMinSize + + const output = await submitMistralOcr( + input.apiKey, + body, + context.signal, + context.maxResponseBytes + ) + context.signal?.throwIfAborted() + return { success: true, output } +} diff --git a/apps/sim/lib/internal/mongodb/client.test.ts b/apps/sim/lib/internal/mongodb/client.test.ts new file mode 100644 index 00000000000..8d96f04c3f5 --- /dev/null +++ b/apps/sim/lib/internal/mongodb/client.test.ts @@ -0,0 +1,114 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockClient, mockCreatePinnedLookup, mockMongoClient, mockValidateDatabaseHost } = + vi.hoisted(() => { + const client = { + close: vi.fn().mockResolvedValue(undefined), + connect: vi.fn(), + } + return { + mockClient: client, + mockCreatePinnedLookup: vi.fn(), + mockMongoClient: vi.fn(function MockMongoClient() { + return client + }), + mockValidateDatabaseHost: vi.fn(), + } + }) + +vi.mock('mongodb', () => ({ MongoClient: mockMongoClient })) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + createPinnedLookup: mockCreatePinnedLookup, + validateDatabaseHost: mockValidateDatabaseHost, +})) + +import { createMongodbClient, type MongodbConnectionConfig } from '@/lib/internal/mongodb/client' + +const CONNECTION_CONFIG: MongodbConnectionConfig = { + host: 'db.example.com', + port: 27017, + database: 'application', + username: 'user@example.com', + password: 'p@ss word', + authSource: 'admin', + ssl: 'required', +} + +describe('MongoDB client', () => { + beforeEach(() => { + vi.clearAllMocks() + mockClient.close.mockResolvedValue(undefined) + mockClient.connect.mockResolvedValue(mockClient) + mockValidateDatabaseHost.mockResolvedValue({ + isValid: true, + resolvedIP: '93.184.216.34', + originalHostname: 'db.example.com', + }) + mockCreatePinnedLookup.mockReturnValue('pinned-lookup') + }) + + it('does not construct a client when DNS validation fails', async () => { + mockValidateDatabaseHost.mockResolvedValue({ + isValid: false, + error: 'host resolves to a blocked IP address', + }) + + await expect(createMongodbClient(CONNECTION_CONFIG)).rejects.toThrow( + 'host resolves to a blocked IP address' + ) + expect(mockMongoClient).not.toHaveBeenCalled() + }) + + it('preserves credentials, TLS, auth source, and DNS pinning', async () => { + await createMongodbClient(CONNECTION_CONFIG) + + expect(mockCreatePinnedLookup).toHaveBeenCalledWith('93.184.216.34') + expect(mockMongoClient).toHaveBeenCalledWith( + 'mongodb://user%40example.com:p%40ss%20word@db.example.com:27017/application?authSource=admin&ssl=true', + { + connectTimeoutMS: 10000, + socketTimeoutMS: 10000, + maxPoolSize: 1, + lookup: 'pinned-lookup', + } + ) + expect(mockClient.connect).toHaveBeenCalledOnce() + }) + + it('omits credentials and TLS when they are not configured', async () => { + await createMongodbClient({ + host: 'db.example.com', + port: 27017, + database: 'application', + ssl: 'preferred', + }) + + expect(mockMongoClient).toHaveBeenCalledWith( + 'mongodb://db.example.com:27017/application', + expect.any(Object) + ) + }) + + it('closes a partially connected client when connection fails', async () => { + mockClient.connect.mockRejectedValue(new Error('handshake failed')) + + await expect(createMongodbClient(CONNECTION_CONFIG)).rejects.toThrow('handshake failed') + expect(mockClient.close).toHaveBeenCalledOnce() + }) + + it('cancels connection establishment and propagates the abort reason', async () => { + const controller = new AbortController() + mockClient.connect.mockReturnValue(new Promise(() => undefined)) + + const connection = createMongodbClient(CONNECTION_CONFIG, controller.signal) + await vi.waitFor(() => expect(mockClient.connect).toHaveBeenCalledOnce()) + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect(connection).rejects.toMatchObject({ name: 'AbortError' }) + expect(mockClient.close).toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/mongodb/client.ts b/apps/sim/lib/internal/mongodb/client.ts new file mode 100644 index 00000000000..b6b2098d177 --- /dev/null +++ b/apps/sim/lib/internal/mongodb/client.ts @@ -0,0 +1,74 @@ +import { MongoClient } from 'mongodb' +import { + createPinnedLookup, + validateDatabaseHost, +} from '@/lib/core/security/input-validation.server' + +export interface MongodbConnectionConfig { + host: string + port: number + database: string + username?: string + password?: string + authSource?: string + ssl?: 'disabled' | 'required' | 'preferred' +} + +export async function createMongodbClient( + config: MongodbConnectionConfig, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const hostValidation = await validateDatabaseHost(config.host, 'host') + signal?.throwIfAborted() + + if (!hostValidation.isValid) { + throw new Error(hostValidation.error) + } + + const credentials = + config.username && config.password + ? `${encodeURIComponent(config.username)}:${encodeURIComponent(config.password)}@` + : '' + const queryParams = new URLSearchParams() + + if (config.authSource) { + queryParams.append('authSource', config.authSource) + } + if (config.ssl === 'required') { + queryParams.append('ssl', 'true') + } + + const queryString = queryParams.toString() + const uri = `mongodb://${credentials}${config.host}:${config.port}/${config.database}${queryString ? `?${queryString}` : ''}` + const client = new MongoClient(uri, { + connectTimeoutMS: 10000, + socketTimeoutMS: 10000, + maxPoolSize: 1, + lookup: createPinnedLookup(hostValidation.resolvedIP ?? config.host), + }) + let rejectAbort: ((reason: unknown) => void) | undefined + const abortPromise = signal + ? new Promise((_resolve, reject) => { + rejectAbort = reject + }) + : undefined + const closeOnAbort = () => { + void client.close().catch(() => undefined) + rejectAbort?.(signal?.reason) + } + signal?.addEventListener('abort', closeOnAbort, { once: true }) + + try { + const connectPromise = client.connect() + await (abortPromise ? Promise.race([connectPromise, abortPromise]) : connectPromise) + signal?.throwIfAborted() + return client + } catch (error) { + await client.close().catch(() => undefined) + signal?.throwIfAborted() + throw error + } finally { + signal?.removeEventListener('abort', closeOnAbort) + } +} diff --git a/apps/sim/lib/internal/mongodb/execute-tool.test.ts b/apps/sim/lib/internal/mongodb/execute-tool.test.ts new file mode 100644 index 00000000000..2798bc69a69 --- /dev/null +++ b/apps/sim/lib/internal/mongodb/execute-tool.test.ts @@ -0,0 +1,140 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operationMocks = vi.hoisted(() => { + class MongodbOperationInputError extends Error {} + + return { + MongodbOperationInputError, + executeMongodbAggregation: vi.fn(), + executeMongodbDelete: vi.fn(), + executeMongodbInsert: vi.fn(), + executeMongodbIntrospection: vi.fn(), + executeMongodbQuery: vi.fn(), + executeMongodbUpdate: vi.fn(), + } +}) + +vi.mock('@/lib/internal/mongodb/operations', () => operationMocks) + +import { executeMongodbTool } from '@/lib/internal/mongodb/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const VALID_BODY = { + host: 'db.example.com', + port: 27017, + database: 'application', + username: 'application', + password: 'secret', + authSource: 'admin', + ssl: 'required', + collection: 'users', + query: '{}', + limit: 100, +} as const + +const SUPPORTED_TOOL_IDS = [ + 'mongodb_query', + 'mongodb_execute', + 'mongodb_insert', + 'mongodb_update', + 'mongodb_delete', + 'mongodb_introspect', +] as const + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'mongodb_query', + input: VALID_BODY, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeMongodbTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('validates and executes the matching operation with cancellation', async () => { + const controller = new AbortController() + operationMocks.executeMongodbQuery.mockResolvedValue({ + message: 'Found 1 documents', + documents: [{ id: 1 }], + documentCount: 1, + }) + + const response = await executeMongodbTool(createRequest({ signal: controller.signal })) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + message: 'Found 1 documents', + documents: [{ id: 1 }], + documentCount: 1, + }) + expect(operationMocks.executeMongodbQuery).toHaveBeenCalledWith(VALID_BODY, controller.signal) + }) + + it('returns the canonical contract validation envelope before provider work', async () => { + const response = await executeMongodbTool(createRequest({ input: { host: 'db.example.com' } })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(operationMocks.executeMongodbQuery).not.toHaveBeenCalled() + }) + + it.each(SUPPORTED_TOOL_IDS)('recognizes canonical tool ID %s', async (toolId) => { + const response = await executeMongodbTool(createRequest({ toolId, input: {} })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ error: 'Invalid request data' }) + }) + + it('preserves the route-compatible provider error envelope', async () => { + operationMocks.executeMongodbQuery.mockRejectedValue(new Error('server unavailable')) + + const response = await executeMongodbTool(createRequest()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'MongoDB query failed: server unavailable', + }) + }) + + it('preserves operation input validation as a 400 error', async () => { + operationMocks.executeMongodbQuery.mockRejectedValue( + new operationMocks.MongodbOperationInputError('Filter validation failed: invalid filter') + ) + + const response = await executeMongodbTool(createRequest()) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ + error: 'Filter validation failed: invalid filter', + }) + }) + + it('propagates cancellation without converting it into a provider failure', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeMongodbTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(operationMocks.executeMongodbQuery).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/mongodb/execute-tool.ts b/apps/sim/lib/internal/mongodb/execute-tool.ts new file mode 100644 index 00000000000..252e9eb0b93 --- /dev/null +++ b/apps/sim/lib/internal/mongodb/execute-tool.ts @@ -0,0 +1,112 @@ +import { getErrorMessage } from '@sim/utils/errors' +import type { z } from 'zod' +import { + executeMongodbAggregation, + executeMongodbDelete, + executeMongodbInsert, + executeMongodbIntrospection, + executeMongodbQuery, + executeMongodbUpdate, + MongodbOperationInputError, +} from '@/lib/internal/mongodb/operations' +import { + mongodbDeleteInputSchema, + mongodbExecuteInputSchema, + mongodbInsertInputSchema, + mongodbIntrospectInputSchema, + mongodbQueryInputSchema, + mongodbUpdateInputSchema, +} from '@/lib/internal/mongodb/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +async function executeOperation( + schema: z.ZodType, + input: unknown, + execute: (input: TInput, signal?: AbortSignal) => Promise, + errorMessage: string, + signal?: AbortSignal +): Promise { + const parsed = schema.safeParse(input) + if (!parsed.success) { + return Response.json( + { error: 'Invalid request data', details: parsed.error.issues }, + { status: 400 } + ) + } + + try { + const result = await execute(parsed.data, signal) + signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + signal?.throwIfAborted() + if (error instanceof MongodbOperationInputError) { + return Response.json({ error: error.message }, { status: 400 }) + } + return Response.json( + { error: `${errorMessage}: ${getErrorMessage(error, 'Unknown error occurred')}` }, + { status: 500 } + ) + } +} + +export const executeMongodbTool: InternalToolOperationHandler = async ({ + toolId, + input, + signal, +}) => { + signal?.throwIfAborted() + + switch (toolId) { + case 'mongodb_query': + return executeOperation( + mongodbQueryInputSchema, + input, + executeMongodbQuery, + 'MongoDB query failed', + signal + ) + case 'mongodb_execute': + return executeOperation( + mongodbExecuteInputSchema, + input, + executeMongodbAggregation, + 'MongoDB aggregation failed', + signal + ) + case 'mongodb_insert': + return executeOperation( + mongodbInsertInputSchema, + input, + executeMongodbInsert, + 'MongoDB insert failed', + signal + ) + case 'mongodb_update': + return executeOperation( + mongodbUpdateInputSchema, + input, + executeMongodbUpdate, + 'MongoDB update failed', + signal + ) + case 'mongodb_delete': + return executeOperation( + mongodbDeleteInputSchema, + input, + executeMongodbDelete, + 'MongoDB delete failed', + signal + ) + case 'mongodb_introspect': + return executeOperation( + mongodbIntrospectInputSchema, + input, + executeMongodbIntrospection, + 'MongoDB introspect failed', + signal + ) + default: + return Response.json({ error: `Unsupported MongoDB tool: ${toolId}` }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/mongodb/input-validation.test.ts b/apps/sim/lib/internal/mongodb/input-validation.test.ts new file mode 100644 index 00000000000..902361b657a --- /dev/null +++ b/apps/sim/lib/internal/mongodb/input-validation.test.ts @@ -0,0 +1,42 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + sanitizeMongodbCollectionName, + validateMongodbFilter, + validateMongodbPipeline, +} from '@/lib/internal/mongodb/input-validation' + +describe('MongoDB input validation', () => { + it('preserves filter validation and nested dangerous-operator detection', () => { + expect(validateMongodbFilter('{"status":"active"}')).toEqual({ isValid: true }) + expect(validateMongodbFilter('{"nested":{"$where":"return true"}}')).toEqual({ + isValid: false, + error: 'Filter contains potentially dangerous operators', + }) + expect(validateMongodbFilter('{invalid')).toEqual({ + isValid: false, + error: 'Invalid JSON format in filter', + }) + }) + + it('preserves pipeline shape and dangerous-stage validation', () => { + expect(validateMongodbPipeline('[{"$match":{"status":"active"}}]')).toEqual({ + isValid: true, + }) + expect(validateMongodbPipeline('{"$match":{}}')).toEqual({ + isValid: false, + error: 'Pipeline must be an array', + }) + expect(validateMongodbPipeline('[{"$out":"archive"}]')).toEqual({ + isValid: false, + error: 'Pipeline contains potentially dangerous operators', + }) + }) + + it('preserves collection-name validation', () => { + expect(sanitizeMongodbCollectionName('users_2026')).toBe('users_2026') + expect(() => sanitizeMongodbCollectionName('users.events')).toThrow('Invalid collection name') + }) +}) diff --git a/apps/sim/lib/internal/mongodb/input-validation.ts b/apps/sim/lib/internal/mongodb/input-validation.ts new file mode 100644 index 00000000000..079a8dd9e16 --- /dev/null +++ b/apps/sim/lib/internal/mongodb/input-validation.ts @@ -0,0 +1,86 @@ +function containsDangerousOperator(value: unknown, dangerousOperators: string[]): boolean { + if (typeof value !== 'object' || value === null) return false + + const record = value as Record + for (const key of Object.keys(record)) { + if (dangerousOperators.includes(key)) return true + if ( + typeof record[key] === 'object' && + containsDangerousOperator(record[key], dangerousOperators) + ) { + return true + } + } + return false +} + +export function validateMongodbFilter(filter: string): { isValid: boolean; error?: string } { + try { + const parsed: unknown = JSON.parse(filter) + const dangerousOperators = ['$where', '$regex', '$expr', '$function', '$accumulator', '$let'] + + if (containsDangerousOperator(parsed, dangerousOperators)) { + return { + isValid: false, + error: 'Filter contains potentially dangerous operators', + } + } + + return { isValid: true } + } catch { + return { + isValid: false, + error: 'Invalid JSON format in filter', + } + } +} + +export function validateMongodbPipeline(pipeline: string): { isValid: boolean; error?: string } { + try { + const parsed: unknown = JSON.parse(pipeline) + + if (!Array.isArray(parsed)) { + return { + isValid: false, + error: 'Pipeline must be an array', + } + } + + const dangerousOperators = [ + '$where', + '$function', + '$accumulator', + '$let', + '$merge', + '$out', + '$currentOp', + '$listSessions', + '$listLocalSessions', + ] + + for (const stage of parsed) { + if (containsDangerousOperator(stage, dangerousOperators)) { + return { + isValid: false, + error: 'Pipeline contains potentially dangerous operators', + } + } + } + + return { isValid: true } + } catch { + return { + isValid: false, + error: 'Invalid JSON format in pipeline', + } + } +} + +export function sanitizeMongodbCollectionName(name: string): string { + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + throw new Error( + 'Invalid collection name. Must start with letter or underscore and contain only letters, numbers, and underscores.' + ) + } + return name +} diff --git a/apps/sim/lib/internal/mongodb/introspection.test.ts b/apps/sim/lib/internal/mongodb/introspection.test.ts new file mode 100644 index 00000000000..183f35e0ebd --- /dev/null +++ b/apps/sim/lib/internal/mongodb/introspection.test.ts @@ -0,0 +1,58 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it, vi } from 'vitest' +import { introspectMongodb } from '@/lib/internal/mongodb/introspection' + +describe('MongoDB introspection', () => { + it('preserves collection shaping and passes cancellation to every operation', async () => { + const controller = new AbortController() + const indexes = vi + .fn() + .mockResolvedValue([{ name: 'email_1', key: { email: 1 }, unique: true, sparse: true }]) + const estimatedDocumentCount = vi.fn().mockResolvedValue(42) + const collection = vi.fn(() => ({ indexes, estimatedDocumentCount })) + const listCollections = vi.fn(() => ({ + toArray: vi.fn().mockResolvedValue([{ name: 'users', type: 'collection' }]), + })) + const db = vi.fn(() => ({ collection, listCollections })) + const client = { db } + + await expect( + introspectMongodb(client as never, 'application', controller.signal) + ).resolves.toEqual({ + message: "Found 1 collections in database 'application'", + databases: ['application'], + collections: [ + { + name: 'users', + type: 'collection', + documentCount: 42, + indexes: [{ name: 'email_1', key: { email: 1 }, unique: true, sparse: true }], + }, + ], + }) + expect(listCollections).toHaveBeenCalledWith({}, { signal: controller.signal }) + expect(indexes).toHaveBeenCalledWith({ signal: controller.signal }) + expect(estimatedDocumentCount).toHaveBeenCalledWith({ signal: controller.signal }) + }) + + it('preserves database-list introspection and cancellation', async () => { + const controller = new AbortController() + const listDatabases = vi.fn().mockResolvedValue({ + databases: [{ name: 'application' }, { name: 'analytics' }], + }) + const client = { + db: vi.fn(() => ({ admin: () => ({ listDatabases }) })), + } + + await expect(introspectMongodb(client as never, undefined, controller.signal)).resolves.toEqual( + { + message: 'Found 2 databases', + databases: ['application', 'analytics'], + collections: [], + } + ) + expect(listDatabases).toHaveBeenCalledWith({ signal: controller.signal }) + }) +}) diff --git a/apps/sim/lib/internal/mongodb/introspection.ts b/apps/sim/lib/internal/mongodb/introspection.ts new file mode 100644 index 00000000000..c1209a17b68 --- /dev/null +++ b/apps/sim/lib/internal/mongodb/introspection.ts @@ -0,0 +1,78 @@ +import type { + Abortable, + EstimatedDocumentCountOptions, + IndexDescriptionInfo, + ListDatabasesOptions, + ListIndexesOptions, + MongoClient, +} from 'mongodb' + +export interface MongodbCollectionInfo { + name: string + type: string + documentCount: number + indexes: Array<{ + name: string + key: Record + unique: boolean + sparse?: boolean + }> +} + +export interface MongodbIntrospectionResult { + message: string + databases: string[] + collections: MongodbCollectionInfo[] +} + +export async function introspectMongodb( + client: MongoClient, + database?: string, + signal?: AbortSignal +): Promise { + const databases: string[] = [] + const collections: MongodbCollectionInfo[] = [] + + if (database) { + databases.push(database) + const db = client.db(database) + const collectionList = await db.listCollections({}, { signal }).toArray() + + for (const collectionInfo of collectionList) { + signal?.throwIfAborted() + const collection = db.collection(collectionInfo.name) + const indexOptions: ListIndexesOptions & Abortable = { signal } + const countOptions: EstimatedDocumentCountOptions & Abortable = { signal } + const indexes = await collection.indexes(indexOptions) + const documentCount = await collection.estimatedDocumentCount(countOptions) + + collections.push({ + name: collectionInfo.name, + type: collectionInfo.type || 'collection', + documentCount, + indexes: indexes.map((index: IndexDescriptionInfo) => ({ + name: index.name || '', + key: index.key as Record, + unique: index.unique || false, + sparse: index.sparse, + })), + }) + } + } else { + const options: ListDatabasesOptions & Abortable = { signal } + const databaseList = await client.db().admin().listDatabases(options) + + for (const databaseInfo of databaseList.databases) { + databases.push(databaseInfo.name) + } + } + + signal?.throwIfAborted() + return { + message: database + ? `Found ${collections.length} collections in database '${database}'` + : `Found ${databases.length} databases`, + databases, + collections, + } +} diff --git a/apps/sim/lib/internal/mongodb/operations.test.ts b/apps/sim/lib/internal/mongodb/operations.test.ts new file mode 100644 index 00000000000..5555792495d --- /dev/null +++ b/apps/sim/lib/internal/mongodb/operations.test.ts @@ -0,0 +1,232 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const clientMocks = vi.hoisted(() => ({ + createMongodbClient: vi.fn(), +})) + +const validationMocks = vi.hoisted(() => ({ + sanitizeMongodbCollectionName: vi.fn((name: string) => name), + validateMongodbFilter: vi.fn(), + validateMongodbPipeline: vi.fn(), +})) + +const introspectionMocks = vi.hoisted(() => ({ + introspectMongodb: vi.fn(), +})) + +vi.mock('@/lib/internal/mongodb/client', () => clientMocks) +vi.mock('@/lib/internal/mongodb/input-validation', () => validationMocks) +vi.mock('@/lib/internal/mongodb/introspection', () => introspectionMocks) + +import { + executeMongodbDelete, + executeMongodbInsert, + executeMongodbIntrospection, + executeMongodbQuery, + executeMongodbUpdate, + MongodbOperationInputError, +} from '@/lib/internal/mongodb/operations' + +const CONNECTION = { + host: 'db.example.com', + port: 27017, + database: 'application', + username: 'application', + password: 'secret', + authSource: 'admin', + ssl: 'required', +} as const + +function createClient(collection: Record) { + return { + close: vi.fn().mockResolvedValue(undefined), + db: vi.fn(() => ({ collection: vi.fn(() => collection) })), + } +} + +describe('MongoDB operations', () => { + beforeEach(() => { + vi.clearAllMocks() + validationMocks.validateMongodbFilter.mockReturnValue({ isValid: true }) + validationMocks.validateMongodbPipeline.mockReturnValue({ isValid: true }) + }) + + it('preserves query behavior, cancellation, and cleanup', async () => { + const controller = new AbortController() + const toArray = vi.fn().mockResolvedValue([{ id: 1 }]) + const limit = vi.fn(() => ({ toArray })) + const sort = vi.fn(() => ({ limit })) + const find = vi.fn(() => ({ limit, sort })) + const client = createClient({ find }) + clientMocks.createMongodbClient.mockResolvedValue(client) + + await expect( + executeMongodbQuery( + { + ...CONNECTION, + collection: 'users', + query: '{"active":true}', + sort: '{"createdAt":-1}', + limit: 25, + }, + controller.signal + ) + ).resolves.toEqual({ + message: 'Found 1 documents', + documents: [{ id: 1 }], + documentCount: 1, + }) + expect(find).toHaveBeenCalledWith({ active: true }, { signal: controller.signal }) + expect(sort).toHaveBeenCalledWith({ createdAt: -1 }) + expect(limit).toHaveBeenCalledWith(25) + expect(client.close).toHaveBeenCalledOnce() + }) + + it('returns route-compatible validation errors before opening a connection', () => { + validationMocks.validateMongodbFilter.mockReturnValue({ + isValid: false, + error: 'Filter contains potentially dangerous operators', + }) + + expect(() => + executeMongodbQuery({ + ...CONNECTION, + collection: 'users', + query: '{"$where":"return true"}', + limit: 100, + }) + ).toThrow( + new MongodbOperationInputError( + 'Filter validation failed: Filter contains potentially dangerous operators' + ) + ) + expect(clientMocks.createMongodbClient).not.toHaveBeenCalled() + }) + + it('preserves single and multi-insert response shapes and cancellation', async () => { + const controller = new AbortController() + const insertOne = vi.fn().mockResolvedValue({ insertedId: { toString: () => 'id-1' } }) + const insertMany = vi.fn().mockResolvedValue({ + insertedIds: { + 0: { toString: () => 'id-1' }, + 1: { toString: () => 'id-2' }, + }, + }) + const client = createClient({ insertMany, insertOne }) + clientMocks.createMongodbClient.mockResolvedValue(client) + + await expect( + executeMongodbInsert( + { ...CONNECTION, collection: 'users', documents: [{ name: 'One' }] }, + controller.signal + ) + ).resolves.toEqual({ + message: 'Document inserted successfully', + insertedId: 'id-1', + documentCount: 1, + }) + await expect( + executeMongodbInsert( + { ...CONNECTION, collection: 'users', documents: [{ name: 'One' }, { name: 'Two' }] }, + controller.signal + ) + ).resolves.toEqual({ + message: '2 documents inserted successfully', + insertedIds: ['id-1', 'id-2'], + documentCount: 2, + }) + expect(insertOne).toHaveBeenCalledWith({ name: 'One' }, { signal: controller.signal }) + expect(insertMany).toHaveBeenCalledWith([{ name: 'One' }, { name: 'Two' }], { + signal: controller.signal, + }) + expect(client.close).toHaveBeenCalledTimes(2) + }) + + it('preserves update and delete result envelopes', async () => { + const updateMany = vi.fn().mockResolvedValue({ + matchedCount: 2, + modifiedCount: 2, + upsertedCount: 1, + upsertedId: { toString: () => 'id-3' }, + }) + const deleteMany = vi.fn().mockResolvedValue({ deletedCount: 2 }) + const client = createClient({ deleteMany, updateMany }) + clientMocks.createMongodbClient.mockResolvedValue(client) + + await expect( + executeMongodbUpdate({ + ...CONNECTION, + collection: 'users', + filter: '{"active":false}', + update: '{"$set":{"active":true}}', + multi: true, + upsert: true, + }) + ).resolves.toEqual({ + message: '2 documents updated, 1 documents upserted', + matchedCount: 2, + modifiedCount: 2, + documentCount: 3, + insertedId: 'id-3', + }) + await expect( + executeMongodbDelete({ + ...CONNECTION, + collection: 'users', + filter: '{"active":false}', + multi: true, + }) + ).resolves.toEqual({ message: '2 documents deleted', deletedCount: 2 }) + expect(updateMany).toHaveBeenCalledWith( + { active: false }, + { $set: { active: true } }, + { upsert: true, signal: undefined } + ) + expect(deleteMany).toHaveBeenCalledWith({ active: false }, { signal: undefined }) + expect(client.close).toHaveBeenCalledTimes(2) + }) + + it('normalizes an omitted introspection database to admin and closes the client', async () => { + const controller = new AbortController() + const client = createClient({}) + clientMocks.createMongodbClient.mockResolvedValue(client) + introspectionMocks.introspectMongodb.mockResolvedValue({ + message: 'Found 1 databases', + databases: ['application'], + collections: [], + }) + + await expect( + executeMongodbIntrospection( + { + host: CONNECTION.host, + port: CONNECTION.port, + ssl: CONNECTION.ssl, + }, + controller.signal + ) + ).resolves.toEqual({ + message: 'Found 1 databases', + databases: ['application'], + collections: [], + }) + expect(clientMocks.createMongodbClient).toHaveBeenCalledWith( + { + host: CONNECTION.host, + port: CONNECTION.port, + ssl: CONNECTION.ssl, + database: 'admin', + }, + controller.signal + ) + expect(introspectionMocks.introspectMongodb).toHaveBeenCalledWith( + client, + undefined, + controller.signal + ) + expect(client.close).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/internal/mongodb/operations.ts b/apps/sim/lib/internal/mongodb/operations.ts new file mode 100644 index 00000000000..6807224259d --- /dev/null +++ b/apps/sim/lib/internal/mongodb/operations.ts @@ -0,0 +1,210 @@ +import type { + Abortable, + BulkWriteOptions, + DeleteOptions, + Document, + InsertOneOptions, + MongoClient, + Sort, + UpdateOptions, +} from 'mongodb' +import { createMongodbClient, type MongodbConnectionConfig } from '@/lib/internal/mongodb/client' +import { + sanitizeMongodbCollectionName, + validateMongodbFilter, + validateMongodbPipeline, +} from '@/lib/internal/mongodb/input-validation' +import { introspectMongodb } from '@/lib/internal/mongodb/introspection' +import type { + MongodbDeleteInput, + MongodbExecuteInput, + MongodbInsertInput, + MongodbIntrospectInput, + MongodbQueryInput, + MongodbUpdateInput, +} from '@/lib/internal/mongodb/schema' + +export class MongodbOperationInputError extends Error {} + +async function withMongodbClient( + input: MongodbConnectionConfig, + signal: AbortSignal | undefined, + execute: (client: MongoClient) => Promise +): Promise { + const client = await createMongodbClient(input, signal) + try { + return await execute(client) + } finally { + await client.close() + } +} + +export function executeMongodbQuery(input: MongodbQueryInput, signal?: AbortSignal) { + const collectionName = sanitizeMongodbCollectionName(input.collection) + let filter: Document = {} + + if (input.query?.trim()) { + const validation = validateMongodbFilter(input.query) + if (!validation.isValid) { + throw new MongodbOperationInputError( + `Filter validation failed: ${validation.error ?? 'Invalid filter'}` + ) + } + filter = JSON.parse(input.query) as Document + } + + let sortCriteria: Sort = {} + if (input.sort?.trim()) { + try { + sortCriteria = JSON.parse(input.sort) as Sort + } catch { + throw new MongodbOperationInputError('Invalid JSON format in sort criteria') + } + } + + return withMongodbClient(input, signal, async (client) => { + const collection = client.db(input.database).collection(collectionName) + let cursor = collection.find(filter, { signal }) + + if (Object.keys(sortCriteria).length > 0) { + cursor = cursor.sort(sortCriteria) + } + + cursor = cursor.limit(input.limit || 100) + const documents = await cursor.toArray() + signal?.throwIfAborted() + return { + message: `Found ${documents.length} documents`, + documents, + documentCount: documents.length, + } + }) +} + +export function executeMongodbAggregation(input: MongodbExecuteInput, signal?: AbortSignal) { + const collectionName = sanitizeMongodbCollectionName(input.collection) + const validation = validateMongodbPipeline(input.pipeline) + if (!validation.isValid) { + throw new MongodbOperationInputError( + `Pipeline validation failed: ${validation.error ?? 'Invalid pipeline'}` + ) + } + const pipeline = JSON.parse(input.pipeline) as Document[] + + return withMongodbClient(input, signal, async (client) => { + const documents = await client + .db(input.database) + .collection(collectionName) + .aggregate(pipeline, { signal }) + .toArray() + signal?.throwIfAborted() + return { + message: `Aggregation completed, returned ${documents.length} documents`, + documents, + documentCount: documents.length, + } + }) +} + +export function executeMongodbInsert(input: MongodbInsertInput, signal?: AbortSignal) { + const collectionName = sanitizeMongodbCollectionName(input.collection) + const documents = input.documents as Document[] + + return withMongodbClient(input, signal, async (client) => { + const collection = client.db(input.database).collection(collectionName) + + if (documents.length === 1) { + const options: InsertOneOptions & Abortable = { signal } + const result = await collection.insertOne(documents[0], options) + signal?.throwIfAborted() + return { + message: 'Document inserted successfully', + insertedId: result.insertedId.toString(), + documentCount: 1, + } + } + + const options: BulkWriteOptions & Abortable = { signal } + const result = await collection.insertMany(documents, options) + signal?.throwIfAborted() + const insertedCount = Object.keys(result.insertedIds).length + return { + message: `${insertedCount} documents inserted successfully`, + insertedIds: Object.values(result.insertedIds).map((id) => id.toString()), + documentCount: insertedCount, + } + }) +} + +export function executeMongodbUpdate(input: MongodbUpdateInput, signal?: AbortSignal) { + const collectionName = sanitizeMongodbCollectionName(input.collection) + const validation = validateMongodbFilter(input.filter) + if (!validation.isValid) { + throw new MongodbOperationInputError( + `Filter validation failed: ${validation.error ?? 'Invalid filter'}` + ) + } + + let filter: Document + let update: Document + try { + filter = JSON.parse(input.filter) as Document + update = JSON.parse(input.update) as Document + } catch { + throw new MongodbOperationInputError('Invalid JSON format in filter or update') + } + + return withMongodbClient(input, signal, async (client) => { + const collection = client.db(input.database).collection(collectionName) + const options: UpdateOptions & Abortable = { upsert: input.upsert, signal } + const result = input.multi + ? await collection.updateMany(filter, update, options) + : await collection.updateOne(filter, update, options) + signal?.throwIfAborted() + return { + message: `${result.modifiedCount} documents updated${result.upsertedCount ? `, ${result.upsertedCount} documents upserted` : ''}`, + matchedCount: result.matchedCount, + modifiedCount: result.modifiedCount, + documentCount: result.modifiedCount + (result.upsertedCount || 0), + ...(result.upsertedId && { insertedId: result.upsertedId.toString() }), + } + }) +} + +export function executeMongodbDelete(input: MongodbDeleteInput, signal?: AbortSignal) { + const collectionName = sanitizeMongodbCollectionName(input.collection) + const validation = validateMongodbFilter(input.filter) + if (!validation.isValid) { + throw new MongodbOperationInputError( + `Filter validation failed: ${validation.error ?? 'Invalid filter'}` + ) + } + + let filter: Document + try { + filter = JSON.parse(input.filter) as Document + } catch { + throw new MongodbOperationInputError('Invalid JSON format in filter') + } + + return withMongodbClient(input, signal, async (client) => { + const collection = client.db(input.database).collection(collectionName) + const options: DeleteOptions & Abortable = { signal } + const result = input.multi + ? await collection.deleteMany(filter, options) + : await collection.deleteOne(filter, options) + signal?.throwIfAborted() + return { + message: `${result.deletedCount} documents deleted`, + deletedCount: result.deletedCount, + } + }) +} + +export function executeMongodbIntrospection(input: MongodbIntrospectInput, signal?: AbortSignal) { + return withMongodbClient( + { ...input, database: input.database || 'admin' }, + signal, + async (client) => introspectMongodb(client, input.database, signal) + ) +} diff --git a/apps/sim/lib/internal/mongodb/schema.ts b/apps/sim/lib/internal/mongodb/schema.ts new file mode 100644 index 00000000000..0f7124e6dc1 --- /dev/null +++ b/apps/sim/lib/internal/mongodb/schema.ts @@ -0,0 +1,138 @@ +import { z } from 'zod' + +const sslModeSchema = z.enum(['disabled', 'required', 'preferred']).default('preferred') + +const connectionInputSchema = z.object({ + host: z.string().min(1, 'Host is required'), + port: z.coerce.number().int().positive('Port must be a positive integer'), + database: z.string().min(1, 'Database name is required'), + username: z.string().min(1, 'Username is required').optional(), + password: z.string().min(1, 'Password is required').optional(), + authSource: z.string().optional(), + ssl: sslModeSchema, +}) + +const usernamePasswordPaired = (data: { username?: string; password?: string }) => + Boolean(data.username) === Boolean(data.password) + +const usernamePasswordPairedError = { + message: 'Username and password must be provided together', + path: ['password' as const], +} + +const jsonStringOrObjectSchema = (message: string) => + z + .union([z.string(), z.object({}).passthrough()]) + .transform((value) => + typeof value === 'object' && value !== null ? JSON.stringify(value) : value + ) + .refine((value) => value && value.trim() !== '', { message }) + +const booleanStringSchema = z + .union([z.boolean(), z.string(), z.undefined()]) + .optional() + .transform((value) => { + if (value === 'true' || value === true) return true + if (value === 'false' || value === false) return false + return false + }) + +export const mongodbQueryInputSchema = connectionInputSchema + .extend({ + collection: z.string().min(1, 'Collection name is required'), + query: z + .union([z.string(), z.object({}).passthrough()]) + .optional() + .default('{}') + .transform((value) => { + if (typeof value === 'object' && value !== null) return JSON.stringify(value) + return value || '{}' + }), + limit: z + .union([z.coerce.number().int().positive(), z.literal(''), z.undefined()]) + .optional() + .transform((value) => (value === '' || value === undefined || value === null ? 100 : value)), + sort: z + .union([z.string(), z.object({}).passthrough(), z.null()]) + .optional() + .transform((value) => + typeof value === 'object' && value !== null ? JSON.stringify(value) : value + ), + }) + .refine(usernamePasswordPaired, usernamePasswordPairedError) + +export const mongodbExecuteInputSchema = connectionInputSchema + .extend({ + collection: z.string().min(1, 'Collection name is required'), + pipeline: z + .union([z.string(), z.array(z.object({}).passthrough())]) + .transform((value) => (Array.isArray(value) ? JSON.stringify(value) : value)) + .refine((value) => value && value.trim() !== '', { message: 'Pipeline is required' }), + }) + .refine(usernamePasswordPaired, usernamePasswordPairedError) + +export const mongodbInsertInputSchema = connectionInputSchema + .extend({ + collection: z.string().min(1, 'Collection name is required'), + documents: z + .union([z.array(z.record(z.string(), z.unknown())), z.string()]) + .transform((value) => { + if (typeof value !== 'string') return value + try { + const parsed: unknown = JSON.parse(value) + return Array.isArray(parsed) ? parsed : [parsed] + } catch { + throw new Error('Invalid JSON in documents field') + } + }) + .refine((value) => Array.isArray(value) && value.length > 0, { + message: 'At least one document is required', + }), + }) + .refine(usernamePasswordPaired, usernamePasswordPairedError) + +export const mongodbUpdateInputSchema = connectionInputSchema + .extend({ + collection: z.string().min(1, 'Collection name is required'), + filter: jsonStringOrObjectSchema('Filter is required for MongoDB Update').refine( + (value) => value !== '{}', + { message: 'Filter is required for MongoDB Update' } + ), + update: jsonStringOrObjectSchema('Update is required'), + upsert: booleanStringSchema, + multi: booleanStringSchema, + }) + .refine(usernamePasswordPaired, usernamePasswordPairedError) + +export const mongodbDeleteInputSchema = connectionInputSchema + .extend({ + collection: z.string().min(1, 'Collection name is required'), + filter: jsonStringOrObjectSchema('Filter is required for MongoDB Delete').refine( + (value) => value !== '{}', + { message: 'Filter is required for MongoDB Delete' } + ), + multi: booleanStringSchema, + }) + .refine(usernamePasswordPaired, usernamePasswordPairedError) + +export const mongodbIntrospectInputSchema = z + .object({ + host: z.string().min(1, 'Host is required'), + port: z.coerce.number().int().positive('Port must be a positive integer'), + database: z.string().optional(), + username: z.string().optional(), + password: z.string().optional(), + authSource: z.string().optional(), + ssl: sslModeSchema, + }) + .refine((data) => Boolean(data.username) === Boolean(data.password), { + message: 'Username and password must be provided together', + path: ['password'], + }) + +export type MongodbQueryInput = z.output +export type MongodbExecuteInput = z.output +export type MongodbInsertInput = z.output +export type MongodbUpdateInput = z.output +export type MongodbDeleteInput = z.output +export type MongodbIntrospectInput = z.output diff --git a/apps/sim/lib/internal/mssql/client.ts b/apps/sim/lib/internal/mssql/client.ts new file mode 100644 index 00000000000..5c296cde2a8 --- /dev/null +++ b/apps/sim/lib/internal/mssql/client.ts @@ -0,0 +1,162 @@ +import net from 'node:net' +import sql from 'mssql' +import { validateDatabaseHost } from '@/lib/core/security/input-validation.server' + +export interface MSSQLConnectionConfig { + host: string + port: number + database: string + username: string + password: string + encrypt: 'enabled' | 'disabled' + trustServerCertificate: 'enabled' | 'disabled' + connectionTimeout: number +} + +/** + * Opens a TCP socket to an already-validated IP address. + * + * Tedious calls the `connector` instead of resolving and connecting itself, so + * this is what keeps the connection pinned to the address the SSRF guard + * approved rather than to whatever DNS answers a second time. + * @see https://tediousjs.github.io/tedious/api-connection.html + */ +function connectToPinnedAddress( + address: string, + port: number, + timeoutMs: number, + signal?: AbortSignal +) { + signal?.throwIfAborted() + + return new Promise((resolve, reject) => { + const socket = net.connect({ host: address, port }) + socket.setNoDelay(true) + socket.setTimeout(timeoutMs) + + const cleanup = () => { + signal?.removeEventListener('abort', abort) + socket.removeListener('timeout', timeout) + socket.removeListener('error', fail) + } + const fail = (error: Error) => { + cleanup() + socket.destroy() + reject(error) + } + const timeout = () => fail(new Error(`Connection to ${address}:${port} timed out`)) + const abort = () => { + cleanup() + socket.destroy() + reject(signal?.reason ?? new DOMException('The operation was aborted', 'AbortError')) + } + + socket.once('connect', () => { + cleanup() + socket.setTimeout(0) + resolve(socket) + }) + socket.once('timeout', timeout) + socket.once('error', fail) + signal?.addEventListener('abort', abort, { once: true }) + if (signal?.aborted) abort() + }) +} + +/** + * Opens a single-connection `mssql` pool against the SSRF-validated address. + * + * `options.connector` supplies a socket already connected to the resolved IP, + * which closes the DNS-rebinding window the way the PostgreSQL and MySQL tools + * do. `server` stays the original hostname because tedious derives the TLS + * `servername` from it independently of the connector, so SNI and certificate + * validation are unaffected by the pin. + * + * Named instances are deliberately unsupported: tedious resolves them with a + * UDP SQL Server Browser lookup issued against the hostname *outside* the + * connector, and node-mssql deletes `port` whenever `instanceName` is set, so + * there is no configuration in which a named instance stays pinned. Connect to + * a named instance by giving it a static TCP port instead. + * + * An Azure SQL `Redirect` routing response is unsupported for the same reason + * and fails the same safe way. tedious reconnects on a LOGIN7 routing envchange, + * but a zero-argument connector ignores the redirect target and reconnects to + * the pinned address — which is the behavior we want, since the redirect target + * is chosen by the server and honoring it would be the rebinding the pin exists + * to stop. Reach Azure SQL through a `Proxy`-policy connection. + * + * `requestTimeout` intentionally tracks `connectionTimeout` off the single knob + * the block exposes, which the field labels as covering both. tedious governs + * the whole login handshake (prelogin, TLS, LOGIN7) with `connectTimeout` + * regardless of the connector, so the socket's own timeout is cleared once it is + * connected rather than left to fire during a slow login. + * @see https://tediousjs.github.io/tedious/api-connection.html + * @see https://github.com/tediousjs/node-mssql#general-same-for-all-drivers + */ +export async function createMSSQLConnection( + config: MSSQLConnectionConfig, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const hostValidation = await validateDatabaseHost(config.host, 'host') + signal?.throwIfAborted() + if (!hostValidation.isValid) { + throw new Error(hostValidation.error) + } + + const pinnedAddress = hostValidation.resolvedIP ?? config.host + + const pool = new sql.ConnectionPool({ + server: config.host, + port: config.port, + database: config.database, + user: config.username, + password: config.password, + connectionTimeout: config.connectionTimeout, + requestTimeout: config.connectionTimeout, + pool: { + max: 1, + min: 0, + idleTimeoutMillis: 20000, + }, + options: { + encrypt: config.encrypt === 'enabled', + trustServerCertificate: config.trustServerCertificate === 'enabled', + connector: () => + connectToPinnedAddress(pinnedAddress, config.port, config.connectionTimeout, signal), + }, + }) + + let rejectAbort: ((reason: unknown) => void) | undefined + const aborted = signal + ? new Promise((_, reject) => { + rejectAbort = reject + }) + : undefined + const abortConnection = () => { + void pool.close().catch(() => {}) + rejectAbort?.(signal?.reason ?? new DOMException('The operation was aborted', 'AbortError')) + } + signal?.addEventListener('abort', abortConnection, { once: true }) + + try { + const connect = pool.connect() + await (aborted ? Promise.race([connect, aborted]) : connect) + signal?.throwIfAborted() + } catch (error) { + /** + * Only a pool that was handed back gets closed by the route's `finally`, so + * a pool that failed to connect has to release its own tarn resources here + * or a bad credential retried in a loop leaks one every attempt. `close()` + * on a pool that never connected is a no-op rather than an error, and its + * own failure must not mask the connect error the caller needs to see. + */ + await pool.close().catch(() => {}) + signal?.throwIfAborted() + throw error + } finally { + signal?.removeEventListener('abort', abortConnection) + } + + return pool +} diff --git a/apps/sim/lib/internal/mssql/execute-tool.test.ts b/apps/sim/lib/internal/mssql/execute-tool.test.ts new file mode 100644 index 00000000000..b3a5b542ee8 --- /dev/null +++ b/apps/sim/lib/internal/mssql/execute-tool.test.ts @@ -0,0 +1,139 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operationMocks = vi.hoisted(() => { + class MssqlOperationInputError extends Error {} + + return { + MssqlOperationInputError, + executeMssqlDelete: vi.fn(), + executeMssqlInsert: vi.fn(), + executeMssqlIntrospection: vi.fn(), + executeMssqlQuery: vi.fn(), + executeMssqlStatement: vi.fn(), + executeMssqlUpdate: vi.fn(), + } +}) + +vi.mock('@/lib/internal/mssql/operations', () => operationMocks) + +import { executeMssqlTool } from '@/lib/internal/mssql/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const VALID_BODY = { + host: 'db.example.com', + port: 1433, + database: 'application', + username: 'application', + password: 'secret', + encrypt: 'enabled', + trustServerCertificate: 'disabled', + connectionTimeout: 15000, + query: 'SELECT 1', +} as const + +const SUPPORTED_TOOL_IDS = [ + 'mssql_query', + 'mssql_execute', + 'mssql_insert', + 'mssql_update', + 'mssql_delete', + 'mssql_introspect', +] as const + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'mssql_query', + input: VALID_BODY, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeMssqlTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('validates and executes the matching operation with cancellation', async () => { + const controller = new AbortController() + operationMocks.executeMssqlQuery.mockResolvedValue({ + message: 'Query executed successfully. 1 row(s) returned.', + rows: [{ value: 1 }], + rowCount: 1, + }) + + const response = await executeMssqlTool(createRequest({ signal: controller.signal })) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + message: 'Query executed successfully. 1 row(s) returned.', + rows: [{ value: 1 }], + rowCount: 1, + }) + expect(operationMocks.executeMssqlQuery).toHaveBeenCalledWith(VALID_BODY, controller.signal) + }) + + it('returns the canonical contract validation envelope before database work', async () => { + const response = await executeMssqlTool(createRequest({ input: { host: 'db.example.com' } })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(operationMocks.executeMssqlQuery).not.toHaveBeenCalled() + }) + + it.each(SUPPORTED_TOOL_IDS)('recognizes the canonical tool ID %s', async (toolId) => { + const response = await executeMssqlTool(createRequest({ toolId, input: {} })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ error: 'Invalid request data' }) + }) + + it('preserves the route-compatible provider error envelope', async () => { + operationMocks.executeMssqlQuery.mockRejectedValue(new Error('database unavailable')) + + const response = await executeMssqlTool(createRequest()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'Microsoft SQL Server query failed: database unavailable', + }) + }) + + it('preserves query validation as a 400 error', async () => { + operationMocks.executeMssqlQuery.mockRejectedValue( + new operationMocks.MssqlOperationInputError('Query validation failed: invalid query') + ) + + const response = await executeMssqlTool(createRequest()) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ + error: 'Query validation failed: invalid query', + }) + }) + + it('propagates cancellation without converting it into a database failure', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeMssqlTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(operationMocks.executeMssqlQuery).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/mssql/execute-tool.ts b/apps/sim/lib/internal/mssql/execute-tool.ts new file mode 100644 index 00000000000..2f1fbbd0397 --- /dev/null +++ b/apps/sim/lib/internal/mssql/execute-tool.ts @@ -0,0 +1,111 @@ +import { getErrorMessage } from '@sim/utils/errors' +import type { z } from 'zod' +import { + executeMssqlDelete, + executeMssqlInsert, + executeMssqlIntrospection, + executeMssqlQuery, + executeMssqlStatement, + executeMssqlUpdate, + MssqlOperationInputError, +} from '@/lib/internal/mssql/operations' +import { + mssqlDeleteInputSchema, + mssqlExecuteInputSchema, + mssqlInsertInputSchema, + mssqlIntrospectInputSchema, + mssqlQueryInputSchema, + mssqlUpdateInputSchema, +} from '@/lib/internal/mssql/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +async function executeOperation( + schema: z.ZodType, + input: unknown, + execute: (input: TInput, signal?: AbortSignal) => Promise, + errorMessage: string, + signal?: AbortSignal +): Promise { + const parsed = schema.safeParse(input) + if (!parsed.success) { + return Response.json( + { error: 'Invalid request data', details: parsed.error.issues }, + { status: 400 } + ) + } + + try { + const result = await execute(parsed.data, signal) + signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + signal?.throwIfAborted() + if (error instanceof MssqlOperationInputError) { + return Response.json({ error: error.message }, { status: 400 }) + } + return Response.json( + { error: `${errorMessage}: ${getErrorMessage(error, 'Unknown error occurred')}` }, + { status: 500 } + ) + } +} + +export const executeMssqlTool: InternalToolOperationHandler = async ({ toolId, input, signal }) => { + signal?.throwIfAborted() + + switch (toolId) { + case 'mssql_query': + return executeOperation( + mssqlQueryInputSchema, + input, + executeMssqlQuery, + 'Microsoft SQL Server query failed', + signal + ) + case 'mssql_execute': + return executeOperation( + mssqlExecuteInputSchema, + input, + executeMssqlStatement, + 'Microsoft SQL Server execute failed', + signal + ) + case 'mssql_insert': + return executeOperation( + mssqlInsertInputSchema, + input, + executeMssqlInsert, + 'Microsoft SQL Server insert failed', + signal + ) + case 'mssql_update': + return executeOperation( + mssqlUpdateInputSchema, + input, + executeMssqlUpdate, + 'Microsoft SQL Server update failed', + signal + ) + case 'mssql_delete': + return executeOperation( + mssqlDeleteInputSchema, + input, + executeMssqlDelete, + 'Microsoft SQL Server delete failed', + signal + ) + case 'mssql_introspect': + return executeOperation( + mssqlIntrospectInputSchema, + input, + executeMssqlIntrospection, + 'Microsoft SQL Server introspection failed', + signal + ) + default: + return Response.json( + { error: `Unsupported Microsoft SQL Server tool: ${toolId}` }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/mssql/introspection.ts b/apps/sim/lib/internal/mssql/introspection.ts new file mode 100644 index 00000000000..31832523a3d --- /dev/null +++ b/apps/sim/lib/internal/mssql/introspection.ts @@ -0,0 +1,302 @@ +import type sql from 'mssql' +import { executeMssqlRequest } from '@/lib/internal/mssql/query' + +export interface MSSQLIntrospectionResult { + tables: Array<{ + name: string + schema: string + columns: Array<{ + name: string + type: string + nullable: boolean + default: string | null + isPrimaryKey: boolean + isForeignKey: boolean + references?: { + schema: string + table: string + column: string + } + }> + primaryKey: string[] + foreignKeys: Array<{ + column: string + referencesSchema: string + referencesTable: string + referencesColumn: string + }> + indexes: Array<{ + name: string + columns: string[] + unique: boolean + }> + }> + schemas: string[] +} + +interface SchemaRow { + SCHEMA_NAME: string +} + +interface TableRow { + TABLE_NAME: string + TABLE_SCHEMA: string +} + +interface ColumnRow { + TABLE_NAME: string + COLUMN_NAME: string + DATA_TYPE: string + IS_NULLABLE: string + COLUMN_DEFAULT: string | null +} + +interface KeyColumnRow { + TABLE_NAME: string + COLUMN_NAME: string +} + +interface ForeignKeyRow { + TABLE_NAME: string + COLUMN_NAME: string + REFERENCED_TABLE_SCHEMA: string + REFERENCED_TABLE_NAME: string + REFERENCED_COLUMN_NAME: string +} + +interface IndexRow { + TABLE_NAME: string + INDEX_NAME: string + COLUMN_NAME: string + IS_UNIQUE: boolean | number +} + +/** + * Reads table, column, key, and index metadata for a schema. + * + * Every view read here except `sys.schemas` is metadata-visibility filtered — + * "limited to securables that a user either owns, or on which the user was + * granted some permission" — so a low-privilege login gets a silently partial + * result rather than an error. + * @see https://learn.microsoft.com/en-us/sql/relational-databases/system-information-schema-views/system-information-schema-views-transact-sql + * @see https://learn.microsoft.com/en-us/sql/relational-databases/security/metadata-visibility-configuration + */ +export async function executeIntrospect( + pool: sql.ConnectionPool, + schemaName: string, + signal?: AbortSignal +): Promise { + /** + * `sys.schemas` rather than `INFORMATION_SCHEMA.SCHEMATA` because it needs + * only membership in `public` and carries no metadata-visibility caveat. + * @see https://learn.microsoft.com/en-us/sql/relational-databases/system-catalog-views/schemas-catalog-views-sys-schemas + */ + const schemasResult = await executeMssqlRequest( + pool.request(), + `SELECT s.name AS SCHEMA_NAME + FROM sys.schemas s + WHERE s.name NOT IN ('sys', 'INFORMATION_SCHEMA', 'guest', + 'db_accessadmin', 'db_backupoperator', 'db_datareader', 'db_datawriter', + 'db_ddladmin', 'db_denydatareader', 'db_denydatawriter', 'db_owner', 'db_securityadmin') + ORDER BY s.name`, + signal + ) + const schemas = schemasResult.recordset.map((row: SchemaRow) => row.SCHEMA_NAME) + + const tablesResult = await executeMssqlRequest( + pool.request().input('schema', schemaName), + `SELECT TABLE_NAME, TABLE_SCHEMA + FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = @schema AND TABLE_TYPE = 'BASE TABLE' + ORDER BY TABLE_NAME`, + signal + ) + + const tableRows = tablesResult.recordset as TableRow[] + if (tableRows.length === 0) return { tables: [], schemas } + + /** + * The column, primary key, foreign key, and index reads below are filtered by + * schema and grouped in memory, rather than run once per table. Per-table they + * were four round trips each — a 500-table schema meant ~2,000 sequential + * queries, every one under its own connection timeout. + */ + const columnsResult = await executeMssqlRequest( + pool.request().input('schema', schemaName), + `SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE, IS_NULLABLE, COLUMN_DEFAULT + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = @schema + ORDER BY TABLE_NAME, ORDINAL_POSITION`, + signal + ) + + const pkResult = await executeMssqlRequest( + pool.request().input('schema', schemaName), + `SELECT tc.TABLE_NAME, kcu.COLUMN_NAME + FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc + JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu + ON tc.CONSTRAINT_NAME = kcu.CONSTRAINT_NAME + AND tc.CONSTRAINT_SCHEMA = kcu.CONSTRAINT_SCHEMA + WHERE tc.CONSTRAINT_TYPE = 'PRIMARY KEY' + AND tc.TABLE_SCHEMA = @schema + ORDER BY tc.TABLE_NAME, kcu.ORDINAL_POSITION`, + signal + ) + + const fkResult = await executeMssqlRequest( + pool.request().input('schema', schemaName), + /** + * Resolved through the catalog views rather than + * `INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS`, which reaches the + * referenced side by joining `TABLE_CONSTRAINTS` — a view that returns + * "one row for each table constraint" and so has no row at all when a + * foreign key references a unique *index*, silently dropping the key. + * The catalog views resolve the referenced table and column by ID. + * @see https://learn.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-foreign-key-columns-transact-sql + * @see https://learn.microsoft.com/en-us/sql/relational-databases/system-information-schema-views/table-constraints-transact-sql + */ + `SELECT + pt.name AS TABLE_NAME, + pc.name AS COLUMN_NAME, + rs.name AS REFERENCED_TABLE_SCHEMA, + rt.name AS REFERENCED_TABLE_NAME, + rc.name AS REFERENCED_COLUMN_NAME + FROM sys.foreign_keys fk + JOIN sys.foreign_key_columns fkc ON fkc.constraint_object_id = fk.object_id + JOIN sys.tables pt ON pt.object_id = fk.parent_object_id + JOIN sys.schemas ps ON ps.schema_id = pt.schema_id + JOIN sys.columns pc + ON pc.object_id = fkc.parent_object_id AND pc.column_id = fkc.parent_column_id + JOIN sys.tables rt ON rt.object_id = fkc.referenced_object_id + JOIN sys.schemas rs ON rs.schema_id = rt.schema_id + JOIN sys.columns rc + ON rc.object_id = fkc.referenced_object_id AND rc.column_id = fkc.referenced_column_id + WHERE ps.name = @schema + ORDER BY pt.name, fk.name, fkc.constraint_column_id`, + signal + ) + + const indexResult = await executeMssqlRequest( + pool.request().input('schema', schemaName), + /** + * `key_ordinal > 0` restricts the result to key columns: it is the + * "ordinal (1-based) within set of key-columns", and `0` marks INCLUDEd + * non-key columns, partitioning columns, **and every column of an XML, + * spatial, columnstore, or JSON index**. The partitioning columns are + * why `is_included_column` alone is not enough — those report `0` for it + * too. The index families are the cost of the filter: they contribute no + * key column, so they are absent from the result rather than listed with + * an empty column set. Rowstore keys, which is what a query planner + * reader is after, are reported in full. + * + * `is_hypothetical = 0` drops the statistics-only indexes the Database + * Engine Tuning Advisor leaves behind ("can't be used directly as a data + * access path"), and `is_disabled = 0` drops indexes that exist but are + * not maintained. Reporting either as a live index misleads. + * + * `is_primary_key = 0` keeps the primary key out, since `primaryKey` + * carries it already. A UNIQUE *constraint* is deliberately left in: it + * is a unique index and nothing else in the result reports it. + * @see https://learn.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-index-columns-transact-sql + * @see https://learn.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-indexes-transact-sql + */ + `SELECT t.name AS TABLE_NAME, i.name AS INDEX_NAME, c.name AS COLUMN_NAME, + i.is_unique AS IS_UNIQUE + FROM sys.indexes i + JOIN sys.index_columns ic + ON i.object_id = ic.object_id AND i.index_id = ic.index_id + JOIN sys.columns c + ON ic.object_id = c.object_id AND ic.column_id = c.column_id + JOIN sys.tables t ON i.object_id = t.object_id + JOIN sys.schemas s ON t.schema_id = s.schema_id + WHERE s.name = @schema + AND i.is_primary_key = 0 + AND i.is_hypothetical = 0 + AND i.is_disabled = 0 + AND i.name IS NOT NULL + AND ic.key_ordinal > 0 + ORDER BY t.name, i.name, ic.key_ordinal`, + signal + ) + + /** Groups rows by their `TABLE_NAME`, preserving each group's server order. */ + function groupByTable(rows: TRow[]): Map { + const grouped = new Map() + for (const row of rows) { + const existing = grouped.get(row.TABLE_NAME) + if (existing) existing.push(row) + else grouped.set(row.TABLE_NAME, [row]) + } + return grouped + } + + const columnsByTable = groupByTable(columnsResult.recordset as ColumnRow[]) + const pkByTable = groupByTable(pkResult.recordset as KeyColumnRow[]) + const fkByTable = groupByTable(fkResult.recordset as ForeignKeyRow[]) + const indexRowsByTable = groupByTable(indexResult.recordset as IndexRow[]) + + const tables: MSSQLIntrospectionResult['tables'] = [] + + for (const tableRow of tableRows) { + const tableName = tableRow.TABLE_NAME + const tableSchema = tableRow.TABLE_SCHEMA + + const primaryKeyColumns = (pkByTable.get(tableName) ?? []).map((row) => row.COLUMN_NAME) + + const foreignKeys = (fkByTable.get(tableName) ?? []).map((row) => ({ + column: row.COLUMN_NAME, + referencesSchema: row.REFERENCED_TABLE_SCHEMA, + referencesTable: row.REFERENCED_TABLE_NAME, + referencesColumn: row.REFERENCED_COLUMN_NAME, + })) + + const fkByColumn = new Map() + for (const fk of foreignKeys) { + if (!fkByColumn.has(fk.column)) fkByColumn.set(fk.column, fk) + } + + const indexMap = new Map() + for (const row of indexRowsByTable.get(tableName) ?? []) { + const indexName = row.INDEX_NAME + if (!indexMap.has(indexName)) { + indexMap.set(indexName, { name: indexName, columns: [], unique: Boolean(row.IS_UNIQUE) }) + } + indexMap.get(indexName)!.columns.push(row.COLUMN_NAME) + } + const indexes = Array.from(indexMap.values()) + + const primaryKeySet = new Set(primaryKeyColumns) + + const columns = (columnsByTable.get(tableName) ?? []).map((col) => { + const columnName = col.COLUMN_NAME + const fk = fkByColumn.get(columnName) + + return { + name: columnName, + type: col.DATA_TYPE, + nullable: col.IS_NULLABLE === 'YES', + default: col.COLUMN_DEFAULT ?? null, + isPrimaryKey: primaryKeySet.has(columnName), + isForeignKey: fk !== undefined, + ...(fk && { + references: { + schema: fk.referencesSchema, + table: fk.referencesTable, + column: fk.referencesColumn, + }, + }), + } + }) + + tables.push({ + name: tableName, + schema: tableSchema, + columns, + primaryKey: primaryKeyColumns, + foreignKeys, + indexes, + }) + } + + return { tables, schemas } +} diff --git a/apps/sim/lib/internal/mssql/operations.test.ts b/apps/sim/lib/internal/mssql/operations.test.ts new file mode 100644 index 00000000000..96505ba7295 --- /dev/null +++ b/apps/sim/lib/internal/mssql/operations.test.ts @@ -0,0 +1,144 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const clientMocks = vi.hoisted(() => ({ + createMSSQLConnection: vi.fn(), +})) + +const queryMocks = vi.hoisted(() => ({ + buildDeleteQuery: vi.fn(), + buildInsertQuery: vi.fn(), + buildUpdateQuery: vi.fn(), + executeQuery: vi.fn(), + toRowsResponseBody: vi.fn((result: { rows: unknown[]; rowCount: number }, message: string) => ({ + message, + rows: result.rows, + rowCount: result.rowCount, + })), + validateQuery: vi.fn(), + validateReadOnlyQuery: vi.fn(), +})) + +const introspectionMocks = vi.hoisted(() => ({ + executeIntrospect: vi.fn(), +})) + +vi.mock('@/lib/internal/mssql/client', () => clientMocks) +vi.mock('@/lib/internal/mssql/query', () => queryMocks) +vi.mock('@/lib/internal/mssql/introspection', () => introspectionMocks) + +import { + executeMssqlInsert, + executeMssqlIntrospection, + executeMssqlQuery, + MssqlOperationInputError, +} from '@/lib/internal/mssql/operations' + +const CONNECTION = { + host: 'db.example.com', + port: 1433, + database: 'application', + username: 'application', + password: 'secret', + encrypt: 'enabled', + trustServerCertificate: 'disabled', + connectionTimeout: 15000, +} as const + +describe('Microsoft SQL Server operations', () => { + beforeEach(() => { + vi.clearAllMocks() + queryMocks.validateReadOnlyQuery.mockReturnValue({ isValid: true }) + }) + + it('passes cancellation to the query and closes the pool after success', async () => { + const controller = new AbortController() + const pool = { close: vi.fn().mockResolvedValue(undefined) } + clientMocks.createMSSQLConnection.mockResolvedValue(pool) + queryMocks.executeQuery.mockResolvedValue({ rows: [{ value: 1 }], rowCount: 1 }) + + await expect( + executeMssqlQuery({ ...CONNECTION, query: 'SELECT 1' }, controller.signal) + ).resolves.toEqual({ + message: 'Query executed successfully. 1 row(s) returned.', + rows: [{ value: 1 }], + rowCount: 1, + }) + expect(clientMocks.createMSSQLConnection).toHaveBeenCalledWith( + { ...CONNECTION, query: 'SELECT 1' }, + controller.signal + ) + expect(queryMocks.executeQuery).toHaveBeenCalledWith(pool, 'SELECT 1', [], controller.signal) + expect(pool.close).toHaveBeenCalledOnce() + }) + + it('closes the pool when the database query rejects', async () => { + const pool = { close: vi.fn().mockResolvedValue(undefined) } + clientMocks.createMSSQLConnection.mockResolvedValue(pool) + queryMocks.executeQuery.mockRejectedValue(new Error('database unavailable')) + + await expect(executeMssqlQuery({ ...CONNECTION, query: 'SELECT 1' })).rejects.toThrow( + 'database unavailable' + ) + expect(pool.close).toHaveBeenCalledOnce() + }) + + it('rejects disallowed read-only statements before opening a pool', () => { + queryMocks.validateReadOnlyQuery.mockReturnValue({ + isValid: false, + error: 'The Query operation cannot run DELETE.', + }) + + expect(() => executeMssqlQuery({ ...CONNECTION, query: 'DELETE FROM users' })).toThrow( + new MssqlOperationInputError( + 'Query validation failed: The Query operation cannot run DELETE.' + ) + ) + expect(clientMocks.createMSSQLConnection).not.toHaveBeenCalled() + }) + + it('rejects an invalid insert identifier before opening a pool', () => { + queryMocks.buildInsertQuery.mockImplementation(() => { + throw new Error('Invalid identifier: users-table') + }) + + expect(() => + executeMssqlInsert({ + ...CONNECTION, + table: 'users-table', + data: { value: 1 }, + }) + ).toThrow( + new MssqlOperationInputError( + 'Microsoft SQL Server insert failed: Invalid identifier: users-table' + ) + ) + expect(clientMocks.createMSSQLConnection).not.toHaveBeenCalled() + }) + + it('preserves introspection output, cancellation, and pool cleanup', async () => { + const controller = new AbortController() + const pool = { close: vi.fn().mockResolvedValue(undefined) } + clientMocks.createMSSQLConnection.mockResolvedValue(pool) + introspectionMocks.executeIntrospect.mockResolvedValue({ + tables: [], + schemas: ['dbo'], + }) + + await expect( + executeMssqlIntrospection({ ...CONNECTION, schema: 'dbo' }, controller.signal) + ).resolves.toEqual({ + message: "Schema introspection completed. Found 0 table(s) in schema 'dbo'.", + tables: [], + schemas: ['dbo'], + }) + expect(introspectionMocks.executeIntrospect).toHaveBeenCalledWith( + pool, + 'dbo', + controller.signal + ) + expect(pool.close).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/internal/mssql/operations.ts b/apps/sim/lib/internal/mssql/operations.ts new file mode 100644 index 00000000000..ba2d053db6a --- /dev/null +++ b/apps/sim/lib/internal/mssql/operations.ts @@ -0,0 +1,130 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { createMSSQLConnection, type MSSQLConnectionConfig } from '@/lib/internal/mssql/client' +import { executeIntrospect } from '@/lib/internal/mssql/introspection' +import { + buildDeleteQuery, + buildInsertQuery, + buildUpdateQuery, + executeQuery, + toRowsResponseBody, + validateQuery, + validateReadOnlyQuery, +} from '@/lib/internal/mssql/query' +import type { + MssqlDeleteInput, + MssqlExecuteInput, + MssqlInsertInput, + MssqlIntrospectInput, + MssqlQueryInput, + MssqlUpdateInput, +} from '@/lib/internal/mssql/schema' + +export class MssqlOperationInputError extends Error {} + +async function withMssqlConnection( + input: MSSQLConnectionConfig, + signal: AbortSignal | undefined, + execute: (pool: Awaited>) => Promise +): Promise { + const pool = await createMSSQLConnection(input, signal) + try { + return await execute(pool) + } finally { + await pool.close() + } +} + +function requireValidQuery(query: string, readOnly: boolean): void { + const validation = readOnly ? validateReadOnlyQuery(query) : validateQuery(query) + if (!validation.isValid) { + throw new MssqlOperationInputError( + `Query validation failed: ${validation.error ?? 'Invalid query'}` + ) + } +} + +function buildStatement( + operation: 'insert' | 'update' | 'delete', + build: () => { query: string; values: unknown[] } +): { query: string; values: unknown[] } { + try { + return build() + } catch (error) { + throw new MssqlOperationInputError( + `Microsoft SQL Server ${operation} failed: ${getErrorMessage(error, 'Invalid statement')}` + ) + } +} + +export function executeMssqlQuery(input: MssqlQueryInput, signal?: AbortSignal) { + requireValidQuery(input.query, true) + + return withMssqlConnection(input, signal, async (pool) => { + const result = await executeQuery(pool, input.query, [], signal) + return toRowsResponseBody( + result, + `Query executed successfully. ${result.rowCount} row(s) returned.` + ) + }) +} + +export function executeMssqlStatement(input: MssqlExecuteInput, signal?: AbortSignal) { + requireValidQuery(input.query, false) + + return withMssqlConnection(input, signal, async (pool) => { + const result = await executeQuery(pool, input.query, [], signal) + return toRowsResponseBody( + result, + `SQL executed successfully. ${result.rowCount} row(s) affected.` + ) + }) +} + +export function executeMssqlInsert(input: MssqlInsertInput, signal?: AbortSignal) { + const statement = buildStatement('insert', () => buildInsertQuery(input.table, input.data)) + + return withMssqlConnection(input, signal, async (pool) => { + const result = await executeQuery(pool, statement.query, statement.values, signal) + return toRowsResponseBody( + result, + `Data inserted successfully. ${result.rowCount} row(s) affected.` + ) + }) +} + +export function executeMssqlUpdate(input: MssqlUpdateInput, signal?: AbortSignal) { + const statement = buildStatement('update', () => + buildUpdateQuery(input.table, input.data, input.where) + ) + + return withMssqlConnection(input, signal, async (pool) => { + const result = await executeQuery(pool, statement.query, statement.values, signal) + return toRowsResponseBody( + result, + `Data updated successfully. ${result.rowCount} row(s) affected.` + ) + }) +} + +export function executeMssqlDelete(input: MssqlDeleteInput, signal?: AbortSignal) { + const statement = buildStatement('delete', () => buildDeleteQuery(input.table, input.where)) + + return withMssqlConnection(input, signal, async (pool) => { + const result = await executeQuery(pool, statement.query, statement.values, signal) + return toRowsResponseBody( + result, + `Data deleted successfully. ${result.rowCount} row(s) affected.` + ) + }) +} + +export function executeMssqlIntrospection(input: MssqlIntrospectInput, signal?: AbortSignal) { + return withMssqlConnection(input, signal, async (pool) => { + const result = await executeIntrospect(pool, input.schema, signal) + return { + message: `Schema introspection completed. Found ${result.tables.length} table(s) in schema '${input.schema}'.`, + tables: result.tables, + schemas: result.schemas, + } + }) +} diff --git a/apps/sim/lib/internal/mssql/query.test.ts b/apps/sim/lib/internal/mssql/query.test.ts new file mode 100644 index 00000000000..330c3f64683 --- /dev/null +++ b/apps/sim/lib/internal/mssql/query.test.ts @@ -0,0 +1,748 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockResolveHostAddresses, mockConnectionPool, mockConnect, mockClose } = vi.hoisted(() => { + const connect = vi.fn().mockResolvedValue(undefined) + const close = vi.fn().mockResolvedValue(undefined) + const pool = vi.fn(function ConnectionPool(this: Record) { + this.connect = connect + this.close = close + }) + return { + mockResolveHostAddresses: vi.fn(), + mockConnectionPool: pool, + mockConnect: connect, + mockClose: close, + } +}) + +vi.mock('mssql', () => ({ + default: { ConnectionPool: mockConnectionPool }, + ConnectionPool: mockConnectionPool, +})) + +/** + * Only DNS is stubbed. The SSRF guard and the shared WHERE screens stay real, so + * these tests exercise the same masking behavior production does — which is the + * point, since the bypasses below are a property of that masker. + */ +vi.mock('@sim/security/dns', () => ({ + resolveHostAddresses: mockResolveHostAddresses, + preferIpv4: (addresses: string[]) => addresses[0], +})) + +import { createMSSQLConnection, type MSSQLConnectionConfig } from '@/lib/internal/mssql/client' +import { executeIntrospect } from '@/lib/internal/mssql/introspection' +import { + buildDeleteQuery, + buildInsertQuery, + buildUpdateQuery, + executeMssqlRequest, + executeQuery, + toRowsResponseBody, + validateQuery, + validateReadOnlyQuery, +} from '@/lib/internal/mssql/query' + +function makeConfig(overrides: Partial = {}): MSSQLConnectionConfig { + return { + host: 'db.example.com', + port: 1433, + database: 'app', + username: 'app', + password: 'secret', + encrypt: 'enabled', + trustServerCertificate: 'disabled', + connectionTimeout: 15000, + ...overrides, + } +} + +describe('validateReadOnlyQuery', () => { + it('accepts an ordinary SELECT and a leading CTE', () => { + expect(validateReadOnlyQuery('SELECT TOP (10) * FROM dbo.users').isValid).toBe(true) + expect( + validateReadOnlyQuery('WITH t AS (SELECT id FROM dbo.users) SELECT * FROM t').isValid + ).toBe(true) + }) + + /** T-SQL does not require whitespace after the opening keyword. */ + it.each(['SELECT*FROM dbo.users', 'SELECT(1)', 'WITH(x) AS (SELECT 1) SELECT * FROM x'])( + 'accepts %s, which has no space after the keyword', + (query) => { + expect(validateReadOnlyQuery(query).isValid).toBe(true) + } + ) + + it('still rejects a keyword that merely starts with SELECT', () => { + expect(validateReadOnlyQuery('SELECTX FROM dbo.users').isValid).toBe(false) + }) + + it('accepts a SELECT whose literal contains a doubled quote', () => { + expect(validateReadOnlyQuery("SELECT * FROM dbo.users WHERE name = 'O''Brien'").isValid).toBe( + true + ) + }) + + it.each([ + ['a bare mutation', 'DELETE FROM dbo.users'], + ['a semicolon batch', 'SELECT 1; DROP TABLE dbo.users'], + ['a semicolon-less batch', 'SELECT 1 DELETE FROM dbo.users'], + ['a CTE-led mutation', 'WITH t AS (SELECT id FROM dbo.users) DELETE FROM t'], + ['a comment', 'SELECT 1 -- DELETE FROM dbo.users'], + ['a stored procedure', 'SELECT 1 FROM dbo.t WHERE x = 1 xp_cmdshell'], + ])('rejects %s', (_label, query) => { + expect(validateReadOnlyQuery(query).isValid).toBe(false) + }) + + /** + * The shared masker treats `\` as a literal escape because it was written for + * the MySQL dialect. T-SQL has no backslash escape, so the server closes the + * literal at the quote the masker swallowed and runs the remainder as code — + * with an even quote count, so a parity check alone does not catch it. + */ + it('rejects a backslash-escaped quote that would hide a mutation from the keyword screen', () => { + const smuggled = String.raw`SELECT * FROM dbo.t WHERE a='x\' DELETE FROM dbo.t WHERE b='y'` + + const result = validateReadOnlyQuery(smuggled) + + expect(result.isValid).toBe(false) + expect(result.error).toMatch(/backslash before a quote/) + }) + + it('rejects a quote inside a bracketed identifier', () => { + expect(validateReadOnlyQuery(`SELECT * FROM dbo.t WHERE [a"] = 1 OR 1=1`).isValid).toBe(false) + expect(validateReadOnlyQuery(`SELECT * FROM dbo.t WHERE [a'] = 1 OR 1=1`).isValid).toBe(false) + }) + + it('rejects an unpaired quote', () => { + expect(validateReadOnlyQuery(`SELECT * FROM dbo.t WHERE a = 'x`).isValid).toBe(false) + }) + + /** Semicolon-less batches that change trigger, session, or transaction state. */ + it.each([ + 'SELECT 1 DISABLE TRIGGER dbo.audit_trigger ON dbo.users', + 'SELECT 1 ENABLE TRIGGER dbo.audit_trigger ON dbo.users', + 'SELECT 1 SET IDENTITY_INSERT dbo.t ON', + 'SELECT 1 BEGIN TRAN', + 'SELECT 1 COMMIT', + 'SELECT 1 ROLLBACK', + ])('rejects the state-changing batch %s', (query) => { + expect(validateReadOnlyQuery(query).isValid).toBe(false) + }) + + /** + * `\bupdate\b` cannot match `UPDATETEXT` — there is no word boundary after + * `update` — so each text statement has to be screened in its own right. + */ + it.each([ + "SELECT 1 UPDATETEXT dbo.t.col @ptr 0 NULL 'x'", + "SELECT 1 WRITETEXT dbo.t.col @ptr 'x'", + 'SELECT 1 READTEXT dbo.t.col @ptr 0 16', + ])('rejects the text statement batch %s', (query) => { + expect(validateReadOnlyQuery(query).isValid).toBe(false) + }) + + /** The same statements reached through the WHERE screen, which shares the list. */ + it.each([ + "id = 1 UPDATETEXT dbo.t.col @ptr 0 NULL 'x'", + "id = 1 WRITETEXT dbo.t.col @ptr 'x'", + 'id = 1 READTEXT dbo.t.col @ptr 0 16', + ])('rejects the text statement %s in a WHERE clause', (where) => { + expect(() => buildDeleteQuery('dbo.users', where)).toThrow() + }) + + /** + * The guard against over-screening. `FETCH` is excluded from the keyword list + * because `OFFSET … FETCH NEXT` is the standard paging clause, and the added + * keywords must not catch ordinary identifiers that merely contain them. + */ + it.each([ + 'SELECT * FROM dbo.users ORDER BY id OFFSET 10 ROWS FETCH NEXT 20 ROWS ONLY', + 'WITH p AS (SELECT id FROM dbo.o) SELECT * FROM p ORDER BY id OFFSET 0 ROWS FETCH NEXT 5 ROWS ONLY', + 'SELECT settled, offset_value, begin_date FROM dbo.t', + 'SELECT updatetext_id, writetext_flag, readtext_offset FROM dbo.t', + 'SELECT TOP (100) id, name FROM dbo.users WHERE is_active = 1', + ])('still accepts the legitimate read %s', (query) => { + expect(validateReadOnlyQuery(query).isValid).toBe(true) + }) +}) + +describe('validateQuery (Execute Raw SQL)', () => { + /** T-SQL needs no space after the keyword; `EXEC(@sql)` is ordinary dynamic SQL. */ + it.each([ + 'EXEC(@sql)', + 'EXECUTE(@sql)', + 'EXEC sp_who', + 'EXECUTE dbo.myproc', + 'SELECT(1)', + 'WITH(x) AS (SELECT 1) SELECT * FROM x', + 'DECLARE @x INT', + ])('accepts %s', (query) => { + expect(validateQuery(query).isValid).toBe(true) + }) + + it.each(['SELECTX 1', 'DROP TABLE dbo.t', 'TRUNCATE TABLE dbo.t'])('rejects %s', (query) => { + expect(validateQuery(query).isValid).toBe(false) + }) +}) + +describe('buildUpdateQuery / buildDeleteQuery WHERE screening', () => { + it('builds a parameterized statement for an ordinary condition', () => { + const { query, values } = buildUpdateQuery('dbo.users', { name: 'Jane' }, 'id = 1') + + expect(query).toBe('UPDATE [dbo].[users] SET [name] = @param1 WHERE id = 1') + expect(values).toEqual(['Jane']) + }) + + /** + * Same masker desynchronisation as above, reached through the mutation path: + * an even quote count, no semicolon, and the tautology invisible to every + * screen that runs over masked text. + */ + it('rejects a backslash-escaped quote that would hide a tautology', () => { + const smuggled = String.raw`id = 'a\' OR 1=1 OR 2>1 AND b = 'x'` + + expect(() => buildDeleteQuery('dbo.users', smuggled)).toThrow(/backslash before a quote/) + expect(() => buildUpdateQuery('dbo.users', { a: 1 }, smuggled)).toThrow( + /backslash before a quote/ + ) + }) + + it('rejects a quote hidden inside a bracketed identifier', () => { + expect(() => buildDeleteQuery('dbo.users', `[a"] = 1 OR 1=1`)).toThrow(/bracketed identifier/) + }) + + /** + * The shared guard sees `OR 1` but not a parenthesised or negated constant. + * These are the specific forms it documents as undetected; the class as a + * whole is not lexically decidable, so this narrows rather than closes it. + */ + it.each([ + 'id = 1 OR (1)', + 'id = 1 OR ((1))', + 'id = 1 OR NOT 0', + 'id = 1 OR NOT (0)', + 'id = 1 OR (TRUE)', + ])('rejects the constant tautology %s', (where) => { + expect(() => buildDeleteQuery('dbo.users', where)).toThrow() + }) + + it.each([ + 'id = 1 OR (priority = 2)', + 'id = 1 OR (1 = priority)', + "status = 'open' OR (retries < 3)", + ])('still accepts the real disjunct %s', (where) => { + expect(() => buildDeleteQuery('dbo.users', where)).not.toThrow() + }) + + it.each([ + ['a semicolon-less batch', "id = 1 DBCC SHRINKDATABASE('app')"], + ['an appended SELECT', 'id = 1 SELECT secret FROM dbo.credentials'], + ['a catalog probe', 'id = 1 AND EXISTS (sys.objects)'], + ['a stored procedure', 'id = 1 AND xp_cmdshell'], + ])('rejects %s', (_label, where) => { + expect(() => buildDeleteQuery('dbo.users', where)).toThrow() + }) +}) + +describe('identifier handling', () => { + it('bracket-quotes every part of a qualified name and binds values', () => { + const { query, values } = buildInsertQuery('dbo.users', { name: 'Jane', age: 30 }) + + expect(query).toBe('INSERT INTO [dbo].[users] ([name], [age]) VALUES (@param1, @param2)') + expect(values).toEqual(['Jane', 30]) + }) + + it('rejects an identifier that is not a plain word', () => { + expect(() => buildInsertQuery('users; DROP TABLE x', { a: 1 })).toThrow(/Invalid identifier/) + expect(() => buildInsertQuery('users', { 'a b': 1 })).toThrow(/Invalid identifier/) + }) + + it('cannot be escaped by pre-closing a bracket', () => { + expect(() => buildInsertQuery('users] DROP TABLE x --[', { a: 1 })).toThrow( + /Invalid identifier/ + ) + }) +}) + +describe('executeQuery parameter binding', () => { + function makePool(recordset: unknown[] = [], rowsAffected: number[] = [0]) { + const input = vi.fn() + const query = vi.fn().mockResolvedValue({ recordset, rowsAffected }) + return { + pool: { request: () => ({ input, query }) } as never, + input, + query, + } + } + + it('binds every value positionally, never interpolating it', async () => { + const { pool, input, query } = makePool() + + await executeQuery(pool, 'INSERT INTO [dbo].[t] ([a]) VALUES (@param1)', ["'; DROP TABLE t --"]) + + expect(query).toHaveBeenCalledWith('INSERT INTO [dbo].[t] ([a]) VALUES (@param1)') + expect(input).toHaveBeenCalledWith('param1', "'; DROP TABLE t --") + }) + + /** + * node-mssql infers NVarChar for an unrecognised object and tedious then + * rejects it with a bare `Invalid string.`, so a nested JSON value has to be + * serialized before it reaches the driver. + */ + it('serializes nested objects and arrays, passing scalars and Dates through', async () => { + const { pool, input } = makePool() + const when = new Date('2020-01-01T00:00:00Z') + + await executeQuery(pool, 'INSERT INTO [dbo].[t] VALUES (@param1, @param2, @param3, @param4)', [ + { nested: true }, + ['a', 'b'], + when, + 42, + ]) + + expect(input).toHaveBeenNthCalledWith(1, 'param1', '{"nested":true}') + expect(input).toHaveBeenNthCalledWith(2, 'param2', '["a","b"]') + expect(input).toHaveBeenNthCalledWith(3, 'param3', when) + expect(input).toHaveBeenNthCalledWith(4, 'param4', 42) + }) + + it('reports affected rows when the statement returns no recordset', async () => { + const { pool } = makePool([], [3]) + + await expect( + executeQuery(pool, 'DELETE FROM [dbo].[t] WHERE id = @param1', [1]) + ).resolves.toEqual({ rows: [], rowCount: 3 }) + }) + + it('cancels an in-flight driver request when the signal aborts', async () => { + const controller = new AbortController() + let rejectQuery: ((error: Error) => void) | undefined + const request = { + cancel: vi.fn(() => rejectQuery?.(new Error('Canceled.'))), + query: vi.fn( + () => + new Promise((_, reject) => { + rejectQuery = reject + }) + ), + } + + const pending = executeMssqlRequest(request as never, 'WAITFOR DELAY', controller.signal) + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect(pending).rejects.toMatchObject({ name: 'AbortError' }) + expect(request.cancel).toHaveBeenCalledOnce() + }) +}) + +describe('createMSSQLConnection DNS pinning', () => { + beforeEach(() => { + vi.clearAllMocks() + mockConnect.mockResolvedValue(undefined) + mockClose.mockResolvedValue(undefined) + mockResolveHostAddresses.mockResolvedValue({ + addresses: ['93.184.216.34'], + preferred: '93.184.216.34', + }) + }) + + it('never opens a connection when the host cannot be resolved (no SSRF window)', async () => { + mockResolveHostAddresses.mockRejectedValue(new Error('ENOTFOUND')) + + await expect( + createMSSQLConnection(makeConfig({ host: 'rebind.attacker.example' })) + ).rejects.toThrow(/could not be resolved/) + expect(mockConnectionPool).not.toHaveBeenCalled() + }) + + it('keeps the hostname as `server` so TLS SNI and certificate validation still apply', async () => { + await createMSSQLConnection(makeConfig({ host: 'rebind.attacker.example' })) + + expect(mockResolveHostAddresses).toHaveBeenCalledWith('rebind.attacker.example') + const config = mockConnectionPool.mock.calls[0][0] + expect(config.server).toBe('rebind.attacker.example') + }) + + it('routes the socket through a connector bound to the validated IP, not the hostname', async () => { + await createMSSQLConnection(makeConfig({ host: 'rebind.attacker.example' })) + + const config = mockConnectionPool.mock.calls[0][0] + expect(typeof config.options.connector).toBe('function') + expect(config.options.instanceName).toBeUndefined() + }) + + /** + * Only a pool that is handed back reaches the route's `finally`, so a pool + * whose connect rejected has to release itself or a bad credential retried in + * a loop leaks one per attempt. + */ + it('closes the pool and surfaces the original error when connect fails', async () => { + mockConnect.mockRejectedValue(new Error('Login failed for user')) + + await expect(createMSSQLConnection(makeConfig())).rejects.toThrow('Login failed for user') + expect(mockClose).toHaveBeenCalledTimes(1) + }) + + it('does not let a close failure mask the connect error', async () => { + mockConnect.mockRejectedValue(new Error('Login failed for user')) + mockClose.mockRejectedValue(new Error('close blew up')) + + await expect(createMSSQLConnection(makeConfig())).rejects.toThrow('Login failed for user') + }) + + it('leaves the pool open on success so the route controls its lifetime', async () => { + await createMSSQLConnection(makeConfig()) + + expect(mockClose).not.toHaveBeenCalled() + }) + + it('maps the string toggles onto driver booleans without coercing "disabled" to true', async () => { + await createMSSQLConnection( + makeConfig({ encrypt: 'disabled', trustServerCertificate: 'enabled' }) + ) + + const config = mockConnectionPool.mock.calls[0][0] + expect(config.options.encrypt).toBe(false) + expect(config.options.trustServerCertificate).toBe(true) + }) + + it('aborts a pending pool connection and closes its resources', async () => { + const controller = new AbortController() + mockConnect.mockReturnValue(new Promise(() => {})) + + const pending = createMSSQLConnection(makeConfig(), controller.signal) + await vi.waitFor(() => expect(mockConnectionPool).toHaveBeenCalledOnce()) + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect(pending).rejects.toMatchObject({ name: 'AbortError' }) + expect(mockClose).toHaveBeenCalled() + }) +}) + +describe('read-only screens cover the rest of the session and transaction family', () => { + /** + * Each is a valid semicolon-less second statement, and the file's stated rule + * is that a second statement is rejected structurally rather than by what it + * happens to do. + */ + it.each([ + ['SAVE TRANSACTION', 'SELECT 1 SAVE TRANSACTION sp1'], + ['SAVE TRAN', 'SELECT 1 SAVE TRAN sp1'], + ['OPEN SYMMETRIC KEY', 'SELECT 1 OPEN SYMMETRIC KEY k DECRYPTION BY CERTIFICATE c'], + ['OPEN MASTER KEY', "SELECT 1 OPEN MASTER KEY DECRYPTION BY PASSWORD = 'p'"], + ['CLOSE ALL SYMMETRIC KEYS', 'SELECT 1 CLOSE ALL SYMMETRIC KEYS'], + ['CLOSE MASTER KEY', 'SELECT 1 CLOSE MASTER KEY'], + ['DEALLOCATE', 'SELECT 1 DEALLOCATE cur'], + ['ADD SIGNATURE', 'SELECT 1 ADD SIGNATURE TO dbo.p BY CERTIFICATE c'], + ['RAISERROR WITH LOG', "SELECT 1 RAISERROR ('boom', 16, 1) WITH LOG"], + ])('rejects %s in the Query operation', (_label, query) => { + expect(validateReadOnlyQuery(query).isValid).toBe(false) + }) + + it.each([ + ['SAVE TRANSACTION', 'id = 1 SAVE TRANSACTION sp1'], + ['OPEN SYMMETRIC KEY', 'id = 1 OPEN SYMMETRIC KEY k DECRYPTION BY CERTIFICATE c'], + ['CLOSE ALL SYMMETRIC KEYS', 'id = 1 CLOSE ALL SYMMETRIC KEYS'], + ['DEALLOCATE', 'id = 1 DEALLOCATE cur'], + ['ADD SIGNATURE', 'id = 1 ADD SIGNATURE TO dbo.p BY CERTIFICATE c'], + ['RAISERROR WITH LOG', "id = 1 RAISERROR ('boom', 16, 1) WITH LOG"], + ])('rejects %s in an update or delete WHERE clause', (_label, where) => { + expect(() => buildUpdateQuery('t', { a: 1 }, where)).toThrow() + expect(() => buildDeleteQuery('t', where)).toThrow() + }) + + /** + * The over-screening guard. `open`, `close`, `save`, and `add` are ordinary + * column names (a price table has all four), so a bare-word screen would make + * the plain SELECTs this operation exists to run un-runnable. + */ + it('still accepts ordinary identifiers that start with a screened phrase word', () => { + const allowed = [ + 'SELECT open, close, high, low FROM dbo.prices', + 'SELECT close FROM dbo.prices WHERE open > 10', + 'SELECT save_id, add_on, open_date, close_date FROM dbo.orders', + 'SELECT o.open, o.close FROM dbo.ohlc o ORDER BY o.open DESC', + ] + + for (const query of allowed) { + expect(validateReadOnlyQuery(query)).toEqual({ isValid: true }) + } + + expect(() => buildUpdateQuery('prices', { close: 2 }, 'open > 10')).not.toThrow() + expect(() => buildDeleteQuery('prices', 'close < 1 AND open_date > 0')).not.toThrow() + }) +}) + +describe('read-only screens cover RENAME and the Service Broker statement family', () => { + /** + * `RENAME` is documented T-SQL DDL for Azure Synapse dedicated SQL pools and + * Analytics Platform System, both reachable over TDS with the connection + * fields this block exposes — so a schema change was passing an operation + * advertised as read-only. + */ + it.each([ + ['RENAME OBJECT', 'SELECT 1 RENAME OBJECT dbo.Customer TO Customer1'], + ['RENAME OBJECT COLUMN', 'SELECT 1 RENAME OBJECT dbo.t COLUMN c1 TO c2'], + ['RENAME DATABASE', 'SELECT 1 RENAME DATABASE db1 TO db2'], + ['RECEIVE', 'SELECT 1 RECEIVE TOP(1) * FROM dbo.MyQueue'], + ['END CONVERSATION', "SELECT 1 END CONVERSATION '00000000-0000-0000-0000-000000000000'"], + [ + 'MOVE CONVERSATION', + "SELECT 1 MOVE CONVERSATION '00000000-0000-0000-0000-000000000000' TO '00000000-0000-0000-0000-000000000001'", + ], + ['GET CONVERSATION GROUP', 'SELECT 1 GET CONVERSATION GROUP @g FROM dbo.MyQueue'], + [ + 'SEND ON CONVERSATION', + "SELECT 1 SEND ON CONVERSATION '00000000-0000-0000-0000-000000000000' MESSAGE TYPE [t] ('x')", + ], + ])('rejects %s in the Query operation', (_label, query) => { + expect(validateReadOnlyQuery(query).isValid).toBe(false) + }) + + it.each([ + ['RENAME OBJECT', 'id = 1 RENAME OBJECT dbo.t TO t2'], + ['RECEIVE', 'id = 1 RECEIVE TOP(1) * FROM dbo.MyQueue'], + ['END CONVERSATION', "id = 1 END CONVERSATION '00000000-0000-0000-0000-000000000000'"], + ['GET CONVERSATION GROUP', 'id = 1 GET CONVERSATION GROUP @g FROM dbo.MyQueue'], + ])('rejects %s in an update or delete WHERE clause', (_label, where) => { + expect(() => buildUpdateQuery('t', { a: 1 }, where)).toThrow() + expect(() => buildDeleteQuery('t', where)).toThrow() + }) + + /** + * The over-screening guard. `END` closes every `CASE`, and `rename`/`receive` + * are the stems of ordinary column names, so neither addition may cost the + * plain SELECTs this operation exists to run. + */ + it('still accepts CASE … END and ordinary identifiers built on the new words', () => { + const allowed = [ + "SELECT CASE WHEN status = 1 THEN 'on' ELSE 'off' END FROM dbo.jobs", + "SELECT CASE WHEN a = 1 THEN 'x' END AS conversation_state FROM dbo.t", + 'SELECT renamed_at, rename_log, received_at, receive_queue FROM dbo.audit', + 'SELECT conversation_id, get_flag, move_order, send_at, end_date FROM dbo.t', + ] + + for (const query of allowed) { + expect(validateReadOnlyQuery(query)).toEqual({ isValid: true }) + } + + expect(() => buildUpdateQuery('audit', { a: 1 }, 'renamed_at > 0')).not.toThrow() + expect(() => buildDeleteQuery('audit', 'received_at > 0 AND conversation_id = 3')).not.toThrow() + }) +}) + +describe('executeQuery result caps', () => { + function makeCapPool(recordset: unknown[]) { + return { + request: () => ({ + input: vi.fn(), + query: vi.fn().mockResolvedValue({ recordset, rowsAffected: [0] }), + }), + } as never + } + + it('caps the recordset at the row ceiling and says so', async () => { + const result = await executeQuery( + makeCapPool(Array.from({ length: 10_001 }, (_, i) => ({ i }))), + 'SELECT 1' + ) + + expect(result.rows).toHaveLength(10_000) + expect(result.rowCount).toBe(10_000) + expect(result.truncated).toBe(true) + expect(result.truncationReason).toMatch(/OFFSET/) + }) + + it('caps on bytes even when the row count is small', async () => { + // 20 rows of ~1MB each: well under the row ceiling, well over the byte one. + const fat = Array.from({ length: 20 }, () => ({ blob: 'x'.repeat(1024 * 1024) })) + const result = await executeQuery(makeCapPool(fat), 'SELECT 1') + + expect(result.rows.length).toBeLessThan(20) + expect(result.truncated).toBe(true) + }) + + it('leaves an ordinary result untouched', async () => { + const rows = [{ id: 1 }, { id: 2 }] + const result = await executeQuery(makeCapPool(rows), 'SELECT 1') + + expect(result.rows).toEqual(rows) + expect(result.truncated).toBeUndefined() + expect(result.truncationReason).toBeUndefined() + }) + + it('never serializes past the byte ceiling', async () => { + const fat = Array.from({ length: 20 }, () => ({ blob: 'x'.repeat(1024 * 1024) })) + const result = await executeQuery(makeCapPool(fat), 'SELECT 1') + + expect(JSON.stringify(result.rows).length).toBeLessThanOrEqual(10 * 1024 * 1024) + }) + + it('drops a lone row that is larger than the byte ceiling rather than admitting it', async () => { + const oversized = [{ blob: 'x'.repeat(11 * 1024 * 1024) }] + const result = await executeQuery(makeCapPool(oversized), 'SELECT 1') + + expect(result.rows).toEqual([]) + expect(result.truncated).toBe(true) + expect(result.truncationReason).toMatch(/exceeds the 10 MB response ceiling/) + }) + + /** + * `String.length` counts UTF-16 code units and the response is emitted as + * UTF-8, so a CJK recordset costs three bytes for every unit the old + * accounting charged one for. Measured with `length` these rows fit; measured + * as the bytes that actually go on the wire they are ~3x over. + */ + it('bounds a multibyte recordset by UTF-8 bytes, not UTF-16 code units', async () => { + const cjk = Array.from({ length: 20 }, () => ({ blob: '世'.repeat(1024 * 1024) })) + const result = await executeQuery(makeCapPool(cjk), 'SELECT 1') + + expect(Buffer.byteLength(JSON.stringify(result.rows), 'utf8')).toBeLessThanOrEqual( + 10 * 1024 * 1024 + ) + expect(result.rows.length).toBeGreaterThan(0) + expect(result.truncated).toBe(true) + }) + + /** Emoji are 4 UTF-8 bytes across 2 surrogate code units — a 2:1 undercount. */ + it('bounds an astral-plane recordset by UTF-8 bytes', async () => { + const emoji = Array.from({ length: 20 }, () => ({ blob: '😀'.repeat(1024 * 1024) })) + const result = await executeQuery(makeCapPool(emoji), 'SELECT 1') + + expect(Buffer.byteLength(JSON.stringify(result.rows), 'utf8')).toBeLessThanOrEqual( + 10 * 1024 * 1024 + ) + expect(result.truncated).toBe(true) + }) + + /** + * Rows sized to divide the ceiling exactly, so an accounting that ignores the + * array's commas and the fields around it lands precisely on the limit and the + * body it emits is over by the punctuation and the envelope. + */ + it('keeps the emitted body inside the ceiling once array and envelope overhead is counted', async () => { + const rowPayload = 'x'.repeat(2048 - '{"blob":""}'.length) + const packed = Array.from({ length: 6000 }, () => ({ blob: rowPayload })) + const result = await executeQuery(makeCapPool(packed), 'SELECT 1') + + const body = toRowsResponseBody(result, 'Query executed successfully. rows returned.') + + expect(result.truncated).toBe(true) + expect(Buffer.byteLength(JSON.stringify(body), 'utf8')).toBeLessThanOrEqual(10 * 1024 * 1024) + }) +}) + +describe('toRowsResponseBody truncation disclosure', () => { + it('discloses a truncated result in both the message and machine-readable fields', () => { + const body = toRowsResponseBody( + { + rows: [{ id: 1 }], + rowCount: 1, + truncated: true, + truncationReason: 'Result truncated to 1 row(s): page with OFFSET ... FETCH NEXT.', + }, + 'Query executed successfully. 1 row(s) returned.' + ) + + expect(body.truncated).toBe(true) + expect(body.truncationReason).toMatch(/OFFSET/) + expect(body.message).toBe( + 'Query executed successfully. 1 row(s) returned. Result truncated to 1 row(s): page with OFFSET ... FETCH NEXT.' + ) + }) + + it('leaves a complete result free of truncation fields', () => { + const body = toRowsResponseBody( + { rows: [{ id: 1 }], rowCount: 1 }, + 'Query executed successfully. 1 row(s) returned.' + ) + + expect(body.message).toBe('Query executed successfully. 1 row(s) returned.') + expect(body).not.toHaveProperty('truncated') + expect(body).not.toHaveProperty('truncationReason') + }) +}) + +describe('executeIntrospect issues a fixed number of queries', () => { + const schemas = [{ SCHEMA_NAME: 'dbo' }] + const introspectTables = Array.from({ length: 50 }, (_, i) => ({ + TABLE_NAME: `t${i}`, + TABLE_SCHEMA: 'dbo', + })) + const introspectColumns = introspectTables.flatMap((t) => [ + { + TABLE_NAME: t.TABLE_NAME, + COLUMN_NAME: 'id', + DATA_TYPE: 'int', + IS_NULLABLE: 'NO', + COLUMN_DEFAULT: null, + }, + { + TABLE_NAME: t.TABLE_NAME, + COLUMN_NAME: 'owner_id', + DATA_TYPE: 'int', + IS_NULLABLE: 'YES', + COLUMN_DEFAULT: null, + }, + ]) + const introspectPks = introspectTables.map((t) => ({ + TABLE_NAME: t.TABLE_NAME, + COLUMN_NAME: 'id', + })) + const introspectFks = introspectTables.map((t) => ({ + TABLE_NAME: t.TABLE_NAME, + COLUMN_NAME: 'owner_id', + REFERENCED_TABLE_SCHEMA: 'dbo', + REFERENCED_TABLE_NAME: 'owners', + REFERENCED_COLUMN_NAME: 'id', + })) + const introspectIndexes = introspectTables.map((t) => ({ + TABLE_NAME: t.TABLE_NAME, + INDEX_NAME: `ix_${t.TABLE_NAME}_owner`, + COLUMN_NAME: 'owner_id', + IS_UNIQUE: 0, + })) + + function makeIntrospectPool() { + const query = vi.fn(async (text: string) => { + if (text.includes('FROM sys.schemas s')) return { recordset: schemas } + if (text.includes('INFORMATION_SCHEMA.TABLES')) return { recordset: introspectTables } + if (text.includes('INFORMATION_SCHEMA.COLUMNS')) return { recordset: introspectColumns } + if (text.includes('PRIMARY KEY')) return { recordset: introspectPks } + if (text.includes('sys.foreign_keys')) return { recordset: introspectFks } + if (text.includes('sys.index_columns')) return { recordset: introspectIndexes } + throw new Error(`unexpected query: ${text}`) + }) + return { pool: { request: () => ({ input: vi.fn().mockReturnThis(), query }) } as never, query } + } + + it('does not scale its round trips with the table count', async () => { + // Previously 4 queries per table plus 2: 50 tables meant 202 sequential + // round trips, each under its own request timeout. + const { pool, query } = makeIntrospectPool() + + const result = await executeIntrospect(pool, 'dbo') + + expect(result.tables).toHaveLength(50) + expect(query.mock.calls.length).toBeLessThanOrEqual(6) + }) + + it('still attributes columns, keys, and indexes to the right table', async () => { + const { pool } = makeIntrospectPool() + + const result = await executeIntrospect(pool, 'dbo') + const table = result.tables.find((t) => t.name === 't7')! + + expect(table.schema).toBe('dbo') + expect(table.columns.map((c) => c.name)).toEqual(['id', 'owner_id']) + expect(table.primaryKey).toEqual(['id']) + expect(table.columns[0].isPrimaryKey).toBe(true) + expect(table.columns[1].isForeignKey).toBe(true) + expect(table.columns[1].references).toEqual({ schema: 'dbo', table: 'owners', column: 'id' }) + expect(table.indexes).toEqual([{ name: 'ix_t7_owner', columns: ['owner_id'], unique: false }]) + }) +}) diff --git a/apps/sim/lib/internal/mssql/query.ts b/apps/sim/lib/internal/mssql/query.ts new file mode 100644 index 00000000000..6e0fb10f6d1 --- /dev/null +++ b/apps/sim/lib/internal/mssql/query.ts @@ -0,0 +1,640 @@ +import type sql from 'mssql' +import { + maskSqlStringLiterals, + validateSqlWhereClause, +} from '@/lib/core/security/input-validation.server' + +export interface MSSQLQueryResult { + rows: unknown[] + rowCount: number + /** Set when the recordset hit a row or byte ceiling and rows were dropped. */ + truncated?: boolean + /** Human-readable explanation of the ceiling that was hit. */ + truncationReason?: string +} + +/** + * Prepares a JSON-sourced value for `request.input()`. + * + * With no explicit type, node-mssql infers one from the value — and its object + * branch only recognises `String`, `Number`, `Boolean`, `Date`, `Buffer`, and + * `Table`. Anything else (a nested object, an array) is inferred as `NVarChar` + * while staying an object, and tedious's `NVarChar.validate` then throws a bare + * `Invalid string.` with nothing naming the column. Since `data` arrives as + * arbitrary JSON, serializing those to JSON text is both the only sound binding + * and what a caller writing into an `nvarchar`/JSON column means. + * @see https://github.com/tediousjs/node-mssql#data-types + */ +function toBindableValue(value: unknown): unknown { + if (value === null || value === undefined) return value + if (typeof value !== 'object') return value + if (value instanceof Date || Buffer.isBuffer(value)) return value + return JSON.stringify(value) +} + +/** + * Ceilings on what a single statement may materialize into the response. + * + * The driver buffers the whole recordset before `request.query` resolves, and + * the route then serializes it into a JSON body, so an unbounded `SELECT` over a + * large table is held in memory twice. A caller who wants more pages it with + * `OFFSET ... FETCH NEXT`. The byte ceiling exists because row count alone does + * not bound size — 1,000 rows of `nvarchar(max)` is not a small result. + */ +const MSSQL_MAX_RESULT_ROWS = 10_000 +const MSSQL_MAX_RESULT_BYTES = 10 * 1024 * 1024 + +/** + * Bytes held back from {@link MSSQL_MAX_RESULT_BYTES} for the part of the + * response body that is not a row. + * + * {@link toRowsResponseBody} wraps `rows` in `message`, `rowCount`, and — when + * the recordset was capped — `truncated` and `truncationReason`, none of which + * the per-row accounting can see. Those are a few hundred bytes at their + * longest (the truncation prose is the bulk of it), so the reserve is set an + * order of magnitude above the worst case and costs 0.04% of the ceiling. The + * alternative, serializing the assembled body to check it, would re-serialize + * the whole recordset a second time for no useful precision. + */ +const MSSQL_RESPONSE_ENVELOPE_BYTES = 4096 + +/** What the serialized `rows` array itself may occupy. */ +const MSSQL_MAX_ROWS_BYTES = MSSQL_MAX_RESULT_BYTES - MSSQL_RESPONSE_ENVELOPE_BYTES + +/** + * Truncates a recordset to the row and byte ceilings. + * + * Measures each row with `JSON.stringify` because that is what the route will do + * anyway, so the number bounds the response the caller actually receives rather + * than an in-memory estimate that does not correspond to it. Each row is + * serialized exactly once and its cost accumulated, rather than re-serializing + * the growing array per row, which would be quadratic on a large recordset. + * + * The size is `Buffer.byteLength(..., 'utf8')`, not `String.length`. `length` + * counts UTF-16 code units while `NextResponse.json` emits UTF-8, and every + * character above U+007F costs more bytes than code units — worst case 3:1, for + * the U+0800–U+FFFF range that holds CJK, so a recordset of Chinese text passed + * a 10 MB `length` budget while serializing to nearly 30 MB. (Astral characters + * such as emoji are only 2:1: 4 bytes across 2 surrogate code units.) + * + * The array's own punctuation is counted too — one byte per row covers the + * opening `[` for the first row and the separating `,` for each one after it, + * with the leading byte standing in for the closing `]` — and + * {@link MSSQL_RESPONSE_ENVELOPE_BYTES} covers the fields around it. Without + * both, a result packed exactly to the ceiling still emitted a body over it. + * + * A row is admitted only when it still fits, so a single row larger than the + * byte ceiling is dropped rather than admitted as a lone exception — otherwise + * `SELECT` of one `nvarchar(max)` value would serialize an unbounded body and + * the ceiling would bound everything except the case it exists for. The drop is + * disclosed through {@link MSSQLQueryResult.truncationReason}, so an empty + * recordset is never mistaken for an empty table. + */ +function capRecordset(rows: unknown[]): { rows: unknown[]; truncated: boolean } { + if (rows.length === 0) return { rows, truncated: false } + + const capped: unknown[] = [] + /** The closing `]`; each row below pays for its own `[` or `,`. */ + let bytes = 1 + + for (const row of rows) { + if (capped.length >= MSSQL_MAX_RESULT_ROWS) break + const serialized = JSON.stringify(row) + /** + * `JSON.stringify` answers `undefined` for a value it cannot represent, but + * an array element in that position serializes as the four bytes of `null`. + */ + const rowBytes = serialized === undefined ? 4 : Buffer.byteLength(serialized, 'utf8') + if (bytes + rowBytes + 1 > MSSQL_MAX_ROWS_BYTES) break + bytes += rowBytes + 1 + capped.push(row) + } + + return { rows: capped, truncated: capped.length < rows.length } +} + +/** + * Runs a statement with positional values bound as `@param1`, `@param2`, … . + * + * `recordset` holds the first result set and `rowsAffected` holds one count per + * statement, so a SELECT reports its row count and a DML statement reports the + * summed affected rows. + * @see https://github.com/tediousjs/node-mssql#request + */ +export async function executeQuery( + pool: sql.ConnectionPool, + query: string, + values: unknown[] = [], + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const request = pool.request() + values.forEach((value, index) => { + request.input(`param${index + 1}`, toBindableValue(value)) + }) + + const result = await executeMssqlRequest(request, query, signal) + const { rows, truncated } = capRecordset(result.recordset ?? []) + const affected = (result.rowsAffected ?? []).reduce( + (total: number, count: number) => total + count, + 0 + ) + + return { + rows, + rowCount: rows.length > 0 ? rows.length : affected, + ...(truncated && { + truncated: true, + truncationReason: + rows.length === 0 + ? `No rows returned: the first row alone exceeds the ${MSSQL_MAX_RESULT_BYTES / (1024 * 1024)} MB response ceiling. Select fewer columns, or slice large values with SUBSTRING.` + : `Result truncated to ${rows.length} row(s): a single statement returns at most ${MSSQL_MAX_RESULT_ROWS} rows or ${MSSQL_MAX_RESULT_BYTES / (1024 * 1024)} MB. Page with OFFSET ... FETCH NEXT to read the rest.`, + }), + } +} + +/** Executes one driver request and maps an abort into node-mssql cancellation. */ +export async function executeMssqlRequest( + request: sql.Request, + query: string, + signal?: AbortSignal +): Promise> { + signal?.throwIfAborted() + + const cancelRequest = () => request.cancel() + signal?.addEventListener('abort', cancelRequest, { once: true }) + + try { + if (signal?.aborted) { + cancelRequest() + signal.throwIfAborted() + } + const result = await request.query(query) + signal?.throwIfAborted() + return result + } catch (error) { + signal?.throwIfAborted() + throw error + } finally { + signal?.removeEventListener('abort', cancelRequest) + } +} + +/** + * Builds the success body every statement route returns. + * + * A truncated recordset is disclosed twice on purpose: folded into `message`, so + * an agent that reads only the status line still learns rows were dropped, and + * as `truncated`/`truncationReason`, so a caller can branch on it without + * parsing prose. Without this the route reported a capped result as a complete + * one and paging looked unnecessary. + */ +export function toRowsResponseBody(result: MSSQLQueryResult, message: string) { + return { + message: result.truncationReason ? `${message} ${result.truncationReason}` : message, + rows: result.rows, + rowCount: result.rowCount, + ...(result.truncated && { + truncated: true, + truncationReason: result.truncationReason, + }), + } +} + +/** + * Every T-SQL keyword that introduces a statement with an effect — DML, DDL, + * permissions, and the administrative commands (`DBCC`, `BACKUP`, `RESTORE`, + * `SHUTDOWN`, `KILL`, …) that are easy to forget precisely because they are not + * DML. One list serves both the read-only query screen and the WHERE screen so + * the two cannot drift apart; a gap in either is a gap in both, which is how + * `DBCC` slipped past a per-site list. + * + * `DISABLE`/`ENABLE` are here because `SELECT 1 DISABLE TRIGGER dbo.audit ON + * dbo.users` is a valid semicolon-less batch that turns auditing off, and + * `SET`/`BEGIN`/`COMMIT`/`ROLLBACK` because session and transaction state are + * changed the same way (`SET IDENTITY_INSERT`, `SET ANSI_NULLS`). The rest of + * that family — `SAVE TRANSACTION`, the symmetric/master key statements, + * `ADD SIGNATURE`, and `RAISERROR ... WITH LOG` — opens with a word that is also + * an ordinary identifier, so it is screened as a two-token phrase in + * {@link MSSQL_STATEMENT_PHRASES} instead. + * + * The text statements `UPDATETEXT`, `WRITETEXT`, and `READTEXT` are listed in + * their own right rather than left to `update`: there is no word boundary after + * `update` in `UPDATETEXT`, so `\bupdate\b` never matches it and + * `SELECT 1 UPDATETEXT dbo.t.col @ptr 0 NULL 'x'` would otherwise pass every + * screen on the advertised read-only path. `READTEXT` reads rather than writes, + * but it introduces a second statement in exactly the same semicolon-less way, + * which is what this list exists to reject. + * + * `RENAME` is documented T-SQL DDL — it applies to Azure Synapse Analytics + * dedicated SQL pools and Analytics Platform System, both of which speak TDS on + * port 1433 and are reachable with exactly the connection fields this block + * exposes. `SELECT 1 RENAME OBJECT dbo.Customer TO Customer1` is a valid + * semicolon-less batch that changes schema through an operation advertised as + * read-only, and `RENAME DATABASE` and `RENAME OBJECT … COLUMN … TO …` reach it + * the same way. + * + * `RECEIVE` is the Service Broker read that *removes* the messages it returns, + * so it is a write in everything but name. Its siblings — `END`/`MOVE`/`GET` + * `CONVERSATION` and `SEND ON CONVERSATION` — open with words that are ordinary + * identifiers (`END` closes every `CASE`), so they are screened as phrases in + * {@link MSSQL_STATEMENT_PHRASES} instead. + * + * `FETCH` is deliberately **absent**: `OFFSET … FETCH NEXT` is the standard + * T-SQL paging clause, so screening it would reject the ordinary paged SELECT + * this operation exists to run. Word boundaries keep the additions off ordinary + * identifiers — `settled`, `offset_value`, and `begin_date` all match nothing. + * @see https://learn.microsoft.com/en-us/sql/t-sql/statements/statements + */ +const MSSQL_STATEMENT_KEYWORDS = + /\b(?:insert|update|updatetext|writetext|readtext|delete|merge|drop|create|alter|truncate|rename|receive|disable|enable|set|begin|commit|rollback|grant|revoke|deny|exec|execute|backup|restore|shutdown|reconfigure|dbcc|kill|checkpoint|use|bulk|revert|setuser|openrowset|opendatasource|openquery|openxml|waitfor|into|deallocate)\b/i + +/** + * The remaining session, transaction, cursor, and key-management statements, + * every one of which is a valid semicolon-less second statement the single-word + * list above cannot carry. + * + * Each is matched as a **two-token** phrase rather than a bare word, because the + * leading words are ordinary identifiers: `open` and `close` are columns in any + * price table, `save` and `add` are common verbs, and `END` closes every `CASE`. + * Screening those bare would reject the plain SELECTs this operation exists to + * run. `DEALLOCATE` is the one exception and lives in the word list above — it + * has no ordinary-identifier reading. + * + * Most of these write neither table data nor schema, which is why they were + * missed; they are screened because the file's stated rule is that a second + * statement is rejected structurally, not by what it happens to do. + * `RAISERROR ... WITH LOG` writes to the error log and the Windows application + * log, so it is not inert. The Service Broker conversation statements are not + * inert either: `END CONVERSATION ... WITH CLEANUP` drops every message in a + * conversation, `MOVE CONVERSATION` reassigns it, and `SEND ON CONVERSATION` + * enqueues a message — and the handles they need are enumerable through this + * same path, because the catalog screen applies only to WHERE clauses. + * @see https://learn.microsoft.com/en-us/sql/t-sql/statements/end-conversation-transact-sql + * @see https://learn.microsoft.com/en-us/sql/t-sql/statements/statements + */ +const MSSQL_STATEMENT_PHRASES: readonly RegExp[] = [ + /\bsave\s+tran(?:saction)?\b/i, + /\bopen\s+(?:symmetric|master)\s+key\b/i, + /\bclose\s+(?:all\s+symmetric\s+keys|master\s+key|symmetric\s+key)\b/i, + /\badd\s+signature\b/i, + /\braiserror[\s\S]*?\bwith\s+log\b/i, + /\b(?:end|move|get)\s+conversation\b/i, + /\bsend\s+on\s+conversation\b/i, +] + +/** Matches the first screened statement phrase, or `null`. */ +function matchStatementPhrase(masked: string): string | null { + for (const pattern of MSSQL_STATEMENT_PHRASES) { + const match = pattern.exec(masked) + if (match) return match[0] + } + return null +} + +/** Extended, OLE-automation, and system stored procedures, called with or without `EXEC`. */ +const MSSQL_PROCEDURE_PATTERN = /\b(?:xp_|sp_)\w+/i + +/** + * Rejects a second statement in a batch. + * + * T-SQL treats the semicolon as optional, so this catches only the explicit + * form; the keyword screen above is what catches the semicolon-less one. Run on + * literal-masked text so a semicolon inside a quoted value is not a statement. + */ +const MSSQL_STACKED_STATEMENT = /;\s*\S/ + +/** + * Rejects SQL comments in the read-only path. + * + * A block comment placed inside a keyword splits it as far as a lexical scan is + * concerned, so no amount of keyword coverage helps if the server rejoins the + * halves into one token. Rather than model how the server's tokenizer treats an + * interior comment — which cannot be settled without a live instance — the + * read-only path refuses comments outright. A SELECT submitted through this + * operation has no need for one, and Execute Raw SQL still accepts them. + * + * {@link maskSqlStringLiterals} deliberately leaves comment markers intact, so + * this still fires after masking while `'-- not a comment'` inside a literal + * does not trip it. + */ +const MSSQL_COMMENT = /--|\/\*|\*\// + +/** + * A quote character {@link maskSqlStringLiterals} treats as opening a literal. + * Backtick is included because the masker honours it even though T-SQL does not. + */ +const MSSQL_MASKER_QUOTES = ["'", '"', '`'] as const + +/** A backslash directly before one of {@link MSSQL_MASKER_QUOTES}. */ +const MSSQL_BACKSLASH_ESCAPED_QUOTE = /\\['"`]/ + +/** Any quote character inside a bracket-quoted identifier. */ +const MSSQL_QUOTE_IN_BRACKETS = /\[[^\]]*['"`]/ + +/** + * Rejects input whose quoting would make literal masking unreliable. + * + * Every screen below runs over {@link maskSqlStringLiterals} output, so anything + * the masker hides is invisible to all of them — and the masker is written for + * the ANSI/MySQL dialect, not T-SQL. Three ways to desynchronise it, each of + * which lets real SQL hide inside what the masker believes is a string: + * + * 1. **An unpaired quote.** The masker runs to the end of the input looking for + * a partner, masking everything on the way. + * 2. **A quote inside a bracketed identifier.** Brackets are SQL Server's other + * quoting form and the masker does not track them, so `[a"] = 1 OR 1=1` + * reads as one unterminated literal. + * 3. **A backslash before a quote.** The masker treats `\` as an escape and + * swallows the quote that follows — but **T-SQL has no backslash escape**, so + * the server closes the literal there and executes the rest as code. This is + * the dangerous one, because it survives an even quote count: + * `a='x\' DELETE FROM dbo.t WHERE b='y'` holds four quotes and masks to + * `a= y`, hiding the DELETE from the keyword screen entirely. + * + * All three are rejected rather than modelled. T-SQL escapes a quote by doubling + * it and has no use for a backslash escape or a backtick, so nothing correct is + * turned away except a value ending in a literal backslash (`'C:\'`) — which the + * Execute Raw SQL operation still accepts. Failing closed here is what lets + * every screen below trust that what the mask left visible is all the code there + * is. + * @see https://learn.microsoft.com/en-us/sql/relational-databases/databases/database-identifiers + * @see https://learn.microsoft.com/en-us/sql/t-sql/data-types/constants-transact-sql + */ +function hasUnreliableQuoting(value: string): boolean { + for (const quote of MSSQL_MASKER_QUOTES) { + let count = 0 + for (const char of value) { + if (char === quote) count++ + } + if (count % 2 !== 0) return true + } + + return MSSQL_BACKSLASH_ESCAPED_QUOTE.test(value) || MSSQL_QUOTE_IN_BRACKETS.test(value) +} + +/** + * Restricts the Query operation to statements that only read. + * + * The block label, the tool description, and the docs all present this + * operation as SELECT-only, so it must not double as a second path to DML — + * `mssql_execute` is the operation that accepts mutations. Without this an + * agent choosing `mssql_query` because "it is only a SELECT" could delete rows. + * + * Four screens, deliberately layered so no single one has to be exhaustive: the + * statement must open with `SELECT` (or a leading `WITH`, since a CTE is the + * normal way to write a non-trivial SELECT); it may not contain a comment, which + * would otherwise let a keyword be split in half; it may not contain a second + * statement after a semicolon, which rejects `SELECT 1; ` structurally + * rather than by naming the anything; and it may not mention a + * statement-introducing keyword or a stored procedure, which covers both the + * semicolon-less batch and the `WITH x AS (...) DELETE FROM x` form that a + * leading-token check alone would miss. + * + * All screening runs over literal-masked text, so ordinary prose in a WHERE + * clause does not trip it. The screens are lexical rather than a parser, so a + * query using a keyword as a bare identifier is rejected too — it fails closed, + * and the Execute Raw SQL operation is the escape hatch. + */ +export function validateReadOnlyQuery(query: string): { isValid: boolean; error?: string } { + const trimmedQuery = query.trim() + + /** + * `\b` rather than `\s`: T-SQL does not require whitespace after the keyword, + * so `SELECT*FROM dbo.users` and `SELECT(1)` are valid reads that a + * whitespace-anchored check would push to Execute Raw SQL for no reason. The + * boundary still refuses `SELECTX`, and it cannot loosen the screen overall — + * the keyword and batch checks below run on the whole statement regardless of + * how it opens. + */ + if (!/^(?:select|with)\b/i.test(trimmedQuery)) { + return { + isValid: false, + error: + 'The Query operation only accepts SELECT statements, optionally led by a WITH clause. Use the Execute Raw SQL operation to run anything else.', + } + } + + if (hasUnreliableQuoting(trimmedQuery)) { + return { + isValid: false, + error: + 'The Query operation could not read this statement reliably: it has an unpaired quote, a quote inside a bracketed identifier, or a backslash before a quote. T-SQL escapes a quote by doubling it.', + } + } + + const masked = maskSqlStringLiterals(trimmedQuery) + + if (MSSQL_COMMENT.test(masked)) { + return { + isValid: false, + error: + 'The Query operation does not accept SQL comments, because a comment can split a keyword. Use the Execute Raw SQL operation if the statement needs one.', + } + } + + if (MSSQL_STACKED_STATEMENT.test(masked)) { + return { + isValid: false, + error: + 'The Query operation runs a single SELECT statement. Use the Execute Raw SQL operation to run a batch.', + } + } + + const disallowed = + MSSQL_STATEMENT_KEYWORDS.exec(masked)?.[0] ?? + MSSQL_PROCEDURE_PATTERN.exec(masked)?.[0] ?? + matchStatementPhrase(masked) + if (disallowed) { + return { + isValid: false, + error: `The Query operation cannot run ${disallowed.toUpperCase()}. Use the Execute Raw SQL operation for statements that modify data, schema, or server state.`, + } + } + + return { isValid: true } +} + +/** + * Restricts Execute Raw SQL to a statement kind the operation advertises. + * + * The anchor is `\b` rather than `\s`, matching the read-only screen, because + * T-SQL does not require whitespace after the opening keyword: `EXEC(@sql)` is + * the ordinary way to run dynamic SQL and `SELECT(1)` is a valid read, both of + * which a whitespace anchor refused on the one operation meant to accept them. + * `EXECUTE x` still matches — the alternation backtracks from `exec` to + * `execute` when the boundary fails — and `SELECTX` is still refused. + * + * This is a statement-kind check, not a security screen. Execute Raw SQL is the + * deliberate escape hatch: the caller supplies their own credentials, so the + * boundary is what those credentials may do, not what this pattern matches. + */ +export function validateQuery(query: string): { isValid: boolean; error?: string } { + const trimmedQuery = query.trim() + + const allowedStatements = /^(select|insert|update|delete|with|merge|exec|execute|declare)\b/i + if (!allowedStatements.test(trimmedQuery)) { + return { + isValid: false, + error: + 'Only SELECT, INSERT, UPDATE, DELETE, WITH, MERGE, EXEC, EXECUTE, and DECLARE statements are allowed', + } + } + + return { isValid: true } +} + +export function buildInsertQuery(table: string, data: Record) { + const sanitizedTable = sanitizeIdentifier(table) + const columns = Object.keys(data) + const values = Object.values(data) + const placeholders = columns.map((_, index) => `@param${index + 1}`).join(', ') + + const query = `INSERT INTO ${sanitizedTable} (${columns.map(sanitizeIdentifier).join(', ')}) VALUES (${placeholders})` + + return { query, values } +} + +export function buildUpdateQuery(table: string, data: Record, where: string) { + validateWhereClause(where) + + const sanitizedTable = sanitizeIdentifier(table) + const columns = Object.keys(data) + const values = Object.values(data) + + const setClause = columns + .map((col, index) => `${sanitizeIdentifier(col)} = @param${index + 1}`) + .join(', ') + const query = `UPDATE ${sanitizedTable} SET ${setClause} WHERE ${where}` + + return { query, values } +} + +export function buildDeleteQuery(table: string, where: string) { + validateWhereClause(where) + + const sanitizedTable = sanitizeIdentifier(table) + const query = `DELETE FROM ${sanitizedTable} WHERE ${where}` + + return { query, values: [] as unknown[] } +} + +/** + * Rejects `SELECT` inside an update or delete WHERE clause. + * + * `SELECT` cannot live in the shared {@link MSSQL_STATEMENT_KEYWORDS} list, since + * the read-only query screen exists to *permit* it — but appended to a WHERE it + * is an exfiltration channel: `id = 1 SELECT secret FROM dbo.credentials` runs + * as a second statement and the route hands back its recordset as though it were + * the mutation's result. + * + * The cost is that a subquery condition (`id IN (SELECT ...)`) is refused here. + * That is a deliberate trade — the WHERE text is interpolated rather than bound, + * so the screen cannot tell a subquery from an appended statement, and the + * Execute Raw SQL operation covers a subquery-driven update. + */ +const MSSQL_WHERE_SELECT = /\bselect\b/i + +/** + * SQL Server catalog surfaces a WHERE clause has no business reading: the + * information schema, the `sys.*` catalog views, and the legacy compatibility + * views still reachable as `master..sysobjects`. + * @see https://learn.microsoft.com/en-us/sql/relational-databases/system-catalog-views/catalog-views-transact-sql + */ +/** + * Constant tautologies the shared guard's `OR ` rule cannot see because + * a parenthesis or a `NOT` sits between the operator and the constant — + * `OR (1)`, `OR ((1))`, `OR NOT 0`, `OR NOT (FALSE)`. + * + * Both patterns require the constant to be the *whole* parenthesised term, so a + * real disjunct is untouched: `OR (1 = priority)` does not match, because a + * closing paren does not follow the digit. + * + * This narrows the gap; it does not close it, and it is not meant to. An + * always-true expression cannot be recognised lexically in general — + * `OR 2 > 1`, `OR LEN(x) >= 0`, and `OR id IS NOT NULL` all survive any pattern + * list — which is why {@link validateWhereClause} is documented as + * defense-in-depth rather than a boundary. + */ +const MSSQL_WHERE_CONSTANT_TAUTOLOGY: readonly RegExp[] = [ + /\bor\s+(?:not\s+)*\(+\s*(?:\d+(?:\.\d+)?|true|false)\s*\)+/i, + /\bor\s+not\s+(?:\d+(?:\.\d+)?|true|false)\b/i, +] + +const MSSQL_CATALOG_PATTERNS: readonly RegExp[] = [ + /information_schema/i, + /\bsys\./i, + /\.\.\s*sys\w*/i, + /\bsys(?:objects|columns|databases|users|indexes|comments)\b/i, +] + +/** + * Rejects WHERE clauses containing injection or always-true tautology patterns + * so a user-supplied condition cannot broaden an update or delete to every row. + * + * Delegates the shared checks to {@link validateSqlWhereClause} — which masks + * string literals before scanning, so prose inside a quoted value cannot trip a + * structural pattern — then adds the two things it cannot know about. + * + * The first is the one that does not generalize: **T-SQL does not require a + * statement terminator**, so `id = 1 DBCC SHRINKDATABASE('db')` is a valid + * two-statement batch and every semicolon-anchored stacked-query check, the + * shared guard's included, reads straight past it. That is why the screen is + * {@link MSSQL_STATEMENT_KEYWORDS} — the same list the read-only query screen + * uses, so a keyword can never be covered in one place and missed in the other. + * Word boundaries keep ordinary column names (`updated_at`, `deleted_at`, + * `created_by`) matching nothing; a column whose name *is* a bare keyword has + * to be reached through the Execute Raw SQL operation. The second is the + * catalog surface above, plus the `xp_`/`sp_` procedures. + * + * As the shared guard's own documentation states, this is defense-in-depth + * rather than a security boundary: the caller supplies their own database + * credentials and can run equivalent SQL through the Execute Raw SQL operation. + * It stops the easy ways an injected condition escalates, nothing more. + * @throws {Error} If the WHERE clause matches any screened pattern + */ +function validateWhereClause(where: string): void { + if (hasUnreliableQuoting(where)) { + throw new Error( + 'WHERE clause has an unpaired quote, a quote inside a bracketed identifier, or a backslash before a quote. T-SQL escapes a quote by doubling it.' + ) + } + + const shared = validateSqlWhereClause(where, 'WHERE clause') + if (!shared.isValid) { + throw new Error(shared.error) + } + + const masked = maskSqlStringLiterals(where) + if ( + MSSQL_STATEMENT_KEYWORDS.test(masked) || + matchStatementPhrase(masked) !== null || + MSSQL_PROCEDURE_PATTERN.test(masked) || + MSSQL_WHERE_SELECT.test(masked) || + MSSQL_WHERE_CONSTANT_TAUTOLOGY.some((pattern) => pattern.test(masked)) || + MSSQL_CATALOG_PATTERNS.some((pattern) => pattern.test(masked)) + ) { + throw new Error('WHERE clause contains potentially dangerous operation') + } +} + +export function sanitizeIdentifier(identifier: string): string { + if (identifier.includes('.')) { + const parts = identifier.split('.') + return parts.map((part) => sanitizeSingleIdentifier(part)).join('.') + } + + return sanitizeSingleIdentifier(identifier) +} + +function sanitizeSingleIdentifier(identifier: string): string { + const cleaned = identifier.replace(/[[\]]/g, '') + + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(cleaned)) { + throw new Error( + `Invalid identifier: ${identifier}. Identifiers must start with a letter or underscore and contain only letters, numbers, and underscores.` + ) + } + + return `[${cleaned}]` +} diff --git a/apps/sim/lib/internal/mssql/schema.ts b/apps/sim/lib/internal/mssql/schema.ts new file mode 100644 index 00000000000..ee10ba4a5ad --- /dev/null +++ b/apps/sim/lib/internal/mssql/schema.ts @@ -0,0 +1,84 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { z } from 'zod' + +export const mssqlToggleSchema = z.enum(['enabled', 'disabled']) + +const nonEmptyRecordSchema = (message: string) => + z.record(z.string(), z.unknown()).refine((value) => Object.keys(value).length > 0, { message }) + +const jsonObjectStringSchema = (message: string, includeReceivedValue = false) => + z + .string() + .min(1) + .transform((value) => { + try { + const parsed = JSON.parse(value) + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw new Error('Data must be a JSON object') + } + return parsed + } catch (error) { + if (!includeReceivedValue) throw new Error(message) + throw new Error( + `${message}: ${getErrorMessage(error, 'Unknown error')}. Received: ${value.substring(0, 100)}...` + ) + } + }) + +const insertDataSchema = z.union([ + nonEmptyRecordSchema('Data object cannot be empty'), + jsonObjectStringSchema('Invalid JSON format in data field', true), +]) +const updateDataSchema = z.union([ + nonEmptyRecordSchema('Data object cannot be empty'), + jsonObjectStringSchema('Invalid JSON format in data field'), +]) + +export const mssqlConnectionInputSchema = z.object({ + host: z.string().min(1, 'Host is required'), + port: z.coerce + .number() + .int() + .min(1, 'Port must be between 1 and 65535') + .max(65535, 'Port must be between 1 and 65535') + .default(1433), + database: z.string().min(1, 'Database name is required'), + username: z.string().min(1, 'Username is required'), + password: z.string().min(1, 'Password is required'), + encrypt: mssqlToggleSchema.default('enabled'), + trustServerCertificate: mssqlToggleSchema.default('disabled'), + connectionTimeout: z.coerce + .number() + .int() + .min(1000, 'connectionTimeout must be at least 1000 ms') + .max(120000, 'connectionTimeout must be at most 120000 ms') + .default(15000), +}) + +export const mssqlQueryInputSchema = mssqlConnectionInputSchema.extend({ + query: z.string().min(1, 'Query is required'), +}) +export const mssqlExecuteInputSchema = mssqlQueryInputSchema +export const mssqlInsertInputSchema = mssqlConnectionInputSchema.extend({ + table: z.string().min(1, 'Table name is required'), + data: insertDataSchema, +}) +export const mssqlUpdateInputSchema = mssqlConnectionInputSchema.extend({ + table: z.string().min(1, 'Table name is required'), + data: updateDataSchema, + where: z.string().min(1, 'WHERE clause is required'), +}) +export const mssqlDeleteInputSchema = mssqlConnectionInputSchema.extend({ + table: z.string().min(1, 'Table name is required'), + where: z.string().min(1, 'WHERE clause is required'), +}) +export const mssqlIntrospectInputSchema = mssqlConnectionInputSchema.extend({ + schema: z.string().min(1, 'Schema name cannot be empty').default('dbo'), +}) + +export type MssqlQueryInput = z.output +export type MssqlExecuteInput = z.output +export type MssqlInsertInput = z.output +export type MssqlUpdateInput = z.output +export type MssqlDeleteInput = z.output +export type MssqlIntrospectInput = z.output diff --git a/apps/sim/lib/internal/mysql/client.test.ts b/apps/sim/lib/internal/mysql/client.test.ts new file mode 100644 index 00000000000..3ae9faff36e --- /dev/null +++ b/apps/sim/lib/internal/mysql/client.test.ts @@ -0,0 +1,117 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockCreateConnection, mockNetConnect, mockValidateDatabaseHost } = vi.hoisted(() => ({ + mockCreateConnection: vi.fn(), + mockNetConnect: vi.fn(), + mockValidateDatabaseHost: vi.fn(), +})) + +vi.mock('node:net', () => ({ + default: { connect: mockNetConnect }, +})) + +vi.mock('mysql2/promise', () => ({ + default: { createConnection: mockCreateConnection }, +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + validateDatabaseHost: mockValidateDatabaseHost, +})) + +import { + createMysqlConnection, + executeMysqlCommand, + type MysqlConnectionConfig, +} from '@/lib/internal/mysql/client' + +const CONNECTION_CONFIG: MysqlConnectionConfig = { + host: 'db.example.com', + port: 3306, + database: 'application', + username: 'application', + password: 'secret', + ssl: 'required', +} + +describe('MySQL client', () => { + beforeEach(() => { + vi.clearAllMocks() + mockValidateDatabaseHost.mockResolvedValue({ + isValid: true, + resolvedIP: '93.184.216.34', + originalHostname: 'db.example.com', + }) + mockCreateConnection.mockResolvedValue({ end: vi.fn(), destroy: vi.fn() }) + mockNetConnect.mockReturnValue({ setNoDelay: vi.fn(), destroy: vi.fn() }) + }) + + it('does not create a connection when DNS validation fails', async () => { + mockValidateDatabaseHost.mockResolvedValue({ + isValid: false, + error: 'host resolves to a blocked IP address', + }) + + await expect(createMysqlConnection(CONNECTION_CONFIG)).rejects.toThrow( + 'host resolves to a blocked IP address' + ) + expect(mockCreateConnection).not.toHaveBeenCalled() + }) + + it.each([ + ['disabled', undefined], + ['required', { rejectUnauthorized: true }], + ['preferred', { rejectUnauthorized: false }], + ] as const)('pins the validated IP and preserves ssl=%s', async (ssl, expectedSsl) => { + await createMysqlConnection({ ...CONNECTION_CONFIG, ssl }) + + const options = mockCreateConnection.mock.calls[0][0] + expect(options.host).toBe('db.example.com') + expect(options.ssl).toEqual(expectedSsl) + const socket = options.stream() + expect(mockNetConnect).toHaveBeenCalledWith({ + host: '93.184.216.34', + port: 3306, + timeout: 10000, + }) + expect(socket.setNoDelay).toHaveBeenCalledWith(true) + }) + + it('does no DNS or connection work when already cancelled', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect(createMysqlConnection(CONNECTION_CONFIG, controller.signal)).rejects.toMatchObject( + { name: 'AbortError' } + ) + expect(mockValidateDatabaseHost).not.toHaveBeenCalled() + expect(mockCreateConnection).not.toHaveBeenCalled() + }) + + it('destroys the connection when an in-flight command is cancelled', async () => { + const controller = new AbortController() + let rejectCommand: (reason: Error) => void = () => undefined + const connection = { + execute: vi.fn( + () => + new Promise((_resolve, reject) => { + rejectCommand = reject + }) + ), + destroy: vi.fn(() => rejectCommand(new Error('connection closed'))), + } + + const execution = executeMysqlCommand( + connection as never, + 'SELECT SLEEP(10)', + undefined, + controller.signal + ) + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect(execution).rejects.toMatchObject({ name: 'AbortError' }) + expect(connection.destroy).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/internal/mysql/client.ts b/apps/sim/lib/internal/mysql/client.ts new file mode 100644 index 00000000000..6c72f55816e --- /dev/null +++ b/apps/sim/lib/internal/mysql/client.ts @@ -0,0 +1,82 @@ +import net from 'node:net' +import mysql from 'mysql2/promise' +import { validateDatabaseHost } from '@/lib/core/security/input-validation.server' + +export interface MysqlConnectionConfig { + host: string + port: number + database: string + username: string + password: string + ssl: 'disabled' | 'required' | 'preferred' +} + +export async function createMysqlConnection( + config: MysqlConnectionConfig, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const hostValidation = await validateDatabaseHost(config.host, 'host') + signal?.throwIfAborted() + + if (!hostValidation.isValid) { + throw new Error(hostValidation.error) + } + + const resolvedIp = hostValidation.resolvedIP ?? config.host + let socket: net.Socket | undefined + const destroySocket = () => socket?.destroy() + signal?.addEventListener('abort', destroySocket, { once: true }) + + const connectionConfig: mysql.ConnectionOptions = { + host: config.host, + port: config.port, + database: config.database, + user: config.username, + password: config.password, + stream: () => { + socket = net.connect({ host: resolvedIp, port: config.port, timeout: 10000 }) + socket.setNoDelay(true) + return socket + }, + } + + if (config.ssl === 'required') { + connectionConfig.ssl = { rejectUnauthorized: true } + } else if (config.ssl === 'preferred') { + connectionConfig.ssl = { rejectUnauthorized: false } + } + + try { + const connection = await mysql.createConnection(connectionConfig) + signal?.throwIfAborted() + return connection + } catch (error) { + signal?.throwIfAborted() + throw error + } finally { + signal?.removeEventListener('abort', destroySocket) + } +} + +export async function executeMysqlCommand( + connection: mysql.Connection, + query: string, + values?: unknown[], + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const destroyConnection = () => connection.destroy() + signal?.addEventListener('abort', destroyConnection, { once: true }) + + try { + const [result] = await connection.execute(query, values) + signal?.throwIfAborted() + return result + } catch (error) { + signal?.throwIfAborted() + throw error + } finally { + signal?.removeEventListener('abort', destroyConnection) + } +} diff --git a/apps/sim/lib/internal/mysql/execute-tool.test.ts b/apps/sim/lib/internal/mysql/execute-tool.test.ts new file mode 100644 index 00000000000..9a5cde418ed --- /dev/null +++ b/apps/sim/lib/internal/mysql/execute-tool.test.ts @@ -0,0 +1,137 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operationMocks = vi.hoisted(() => { + class MysqlOperationInputError extends Error {} + + return { + MysqlOperationInputError, + executeMysqlDelete: vi.fn(), + executeMysqlInsert: vi.fn(), + executeMysqlIntrospection: vi.fn(), + executeMysqlQuery: vi.fn(), + executeMysqlStatement: vi.fn(), + executeMysqlUpdate: vi.fn(), + } +}) + +vi.mock('@/lib/internal/mysql/operations', () => operationMocks) + +import { executeMysqlTool } from '@/lib/internal/mysql/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const VALID_BODY = { + host: 'db.example.com', + port: 3306, + database: 'application', + username: 'application', + password: 'secret', + ssl: 'required', + query: 'SELECT 1', +} as const + +const SUPPORTED_TOOL_IDS = [ + 'mysql_query', + 'mysql_execute', + 'mysql_insert', + 'mysql_update', + 'mysql_delete', + 'mysql_introspect', +] as const + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'mysql_query', + input: VALID_BODY, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeMysqlTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('validates and executes the matching operation with cancellation', async () => { + const controller = new AbortController() + operationMocks.executeMysqlQuery.mockResolvedValue({ + message: 'Query executed successfully. 1 row(s) returned.', + rows: [{ value: 1 }], + rowCount: 1, + }) + + const response = await executeMysqlTool(createRequest({ signal: controller.signal })) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + message: 'Query executed successfully. 1 row(s) returned.', + rows: [{ value: 1 }], + rowCount: 1, + }) + expect(operationMocks.executeMysqlQuery).toHaveBeenCalledWith(VALID_BODY, controller.signal) + }) + + it('returns the canonical contract validation envelope before database work', async () => { + const response = await executeMysqlTool(createRequest({ input: { host: 'db.example.com' } })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(operationMocks.executeMysqlQuery).not.toHaveBeenCalled() + }) + + it.each(SUPPORTED_TOOL_IDS)('recognizes the canonical tool ID %s', async (toolId) => { + const response = await executeMysqlTool(createRequest({ toolId, input: {} })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ error: 'Invalid request data' }) + }) + + it('preserves the route-compatible provider error envelope', async () => { + operationMocks.executeMysqlQuery.mockRejectedValue(new Error('database unavailable')) + + const response = await executeMysqlTool(createRequest()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'MySQL query failed: database unavailable', + }) + }) + + it('preserves query validation as a 400 error', async () => { + operationMocks.executeMysqlQuery.mockRejectedValue( + new operationMocks.MysqlOperationInputError('Query validation failed: invalid query') + ) + + const response = await executeMysqlTool(createRequest()) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ + error: 'Query validation failed: invalid query', + }) + }) + + it('propagates cancellation without converting it into a database failure', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeMysqlTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(operationMocks.executeMysqlQuery).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/mysql/execute-tool.ts b/apps/sim/lib/internal/mysql/execute-tool.ts new file mode 100644 index 00000000000..dd454d7847a --- /dev/null +++ b/apps/sim/lib/internal/mysql/execute-tool.ts @@ -0,0 +1,108 @@ +import { getErrorMessage } from '@sim/utils/errors' +import type { z } from 'zod' +import { + executeMysqlDelete, + executeMysqlInsert, + executeMysqlIntrospection, + executeMysqlQuery, + executeMysqlStatement, + executeMysqlUpdate, + MysqlOperationInputError, +} from '@/lib/internal/mysql/operations' +import { + mysqlDeleteInputSchema, + mysqlExecuteInputSchema, + mysqlInsertInputSchema, + mysqlIntrospectInputSchema, + mysqlQueryInputSchema, + mysqlUpdateInputSchema, +} from '@/lib/internal/mysql/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +async function executeOperation( + schema: z.ZodType, + input: unknown, + execute: (input: TInput, signal?: AbortSignal) => Promise, + errorMessage: string, + signal?: AbortSignal +): Promise { + const parsed = schema.safeParse(input) + if (!parsed.success) { + return Response.json( + { error: 'Invalid request data', details: parsed.error.issues }, + { status: 400 } + ) + } + + try { + const result = await execute(parsed.data, signal) + signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + signal?.throwIfAborted() + if (error instanceof MysqlOperationInputError) { + return Response.json({ error: error.message }, { status: 400 }) + } + return Response.json( + { error: `${errorMessage}: ${getErrorMessage(error, 'Unknown error occurred')}` }, + { status: 500 } + ) + } +} + +export const executeMysqlTool: InternalToolOperationHandler = async ({ toolId, input, signal }) => { + signal?.throwIfAborted() + + switch (toolId) { + case 'mysql_query': + return executeOperation( + mysqlQueryInputSchema, + input, + executeMysqlQuery, + 'MySQL query failed', + signal + ) + case 'mysql_execute': + return executeOperation( + mysqlExecuteInputSchema, + input, + executeMysqlStatement, + 'MySQL execute failed', + signal + ) + case 'mysql_insert': + return executeOperation( + mysqlInsertInputSchema, + input, + executeMysqlInsert, + 'MySQL insert failed', + signal + ) + case 'mysql_update': + return executeOperation( + mysqlUpdateInputSchema, + input, + executeMysqlUpdate, + 'MySQL update failed', + signal + ) + case 'mysql_delete': + return executeOperation( + mysqlDeleteInputSchema, + input, + executeMysqlDelete, + 'MySQL delete failed', + signal + ) + case 'mysql_introspect': + return executeOperation( + mysqlIntrospectInputSchema, + input, + executeMysqlIntrospection, + 'MySQL introspection failed', + signal + ) + default: + return Response.json({ error: `Unsupported MySQL tool: ${toolId}` }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/mysql/operations.test.ts b/apps/sim/lib/internal/mysql/operations.test.ts new file mode 100644 index 00000000000..20d53bd5de0 --- /dev/null +++ b/apps/sim/lib/internal/mysql/operations.test.ts @@ -0,0 +1,117 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const clientMocks = vi.hoisted(() => ({ + createMysqlConnection: vi.fn(), +})) + +const queryMocks = vi.hoisted(() => ({ + buildMysqlDeleteQuery: vi.fn(), + buildMysqlInsertQuery: vi.fn(), + buildMysqlUpdateQuery: vi.fn(), + introspectMysqlDatabase: vi.fn(), + queryMysql: vi.fn(), + validateMysqlQuery: vi.fn(), +})) + +vi.mock('@/lib/internal/mysql/client', () => clientMocks) +vi.mock('@/lib/internal/mysql/queries', () => queryMocks) + +import { + executeMysqlIntrospection, + executeMysqlQuery, + MysqlOperationInputError, +} from '@/lib/internal/mysql/operations' + +const CONNECTION = { + host: 'db.example.com', + port: 3306, + database: 'application', + username: 'application', + password: 'secret', + ssl: 'required', +} as const + +describe('MySQL operations', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('passes cancellation to the driver and closes the connection after success', async () => { + const controller = new AbortController() + const connection = { end: vi.fn().mockResolvedValue(undefined) } + clientMocks.createMysqlConnection.mockResolvedValue(connection) + queryMocks.validateMysqlQuery.mockReturnValue({ isValid: true }) + queryMocks.queryMysql.mockResolvedValue({ rows: [{ value: 1 }], rowCount: 1 }) + + await expect( + executeMysqlQuery({ ...CONNECTION, query: 'SELECT 1' }, controller.signal) + ).resolves.toEqual({ + message: 'Query executed successfully. 1 row(s) returned.', + rows: [{ value: 1 }], + rowCount: 1, + }) + expect(clientMocks.createMysqlConnection).toHaveBeenCalledWith( + { ...CONNECTION, query: 'SELECT 1' }, + controller.signal + ) + expect(queryMocks.queryMysql).toHaveBeenCalledWith( + connection, + 'SELECT 1', + undefined, + controller.signal + ) + expect(connection.end).toHaveBeenCalledOnce() + }) + + it('closes the connection when a database query rejects', async () => { + const connection = { end: vi.fn().mockResolvedValue(undefined) } + clientMocks.createMysqlConnection.mockResolvedValue(connection) + queryMocks.validateMysqlQuery.mockReturnValue({ isValid: true }) + queryMocks.queryMysql.mockRejectedValue(new Error('database unavailable')) + + await expect(executeMysqlQuery({ ...CONNECTION, query: 'SELECT 1' })).rejects.toThrow( + 'database unavailable' + ) + expect(connection.end).toHaveBeenCalledOnce() + }) + + it('rejects disallowed statements before opening a connection', () => { + queryMocks.validateMysqlQuery.mockReturnValue({ + isValid: false, + error: + 'Only SELECT, INSERT, UPDATE, DELETE, WITH, SHOW, DESCRIBE, and EXPLAIN statements are allowed', + }) + + expect(() => executeMysqlQuery({ ...CONNECTION, query: 'DROP TABLE users' })).toThrow( + new MysqlOperationInputError( + 'Query validation failed: Only SELECT, INSERT, UPDATE, DELETE, WITH, SHOW, DESCRIBE, and EXPLAIN statements are allowed' + ) + ) + expect(clientMocks.createMysqlConnection).not.toHaveBeenCalled() + }) + + it('preserves introspection output and cancellation', async () => { + const controller = new AbortController() + const connection = { end: vi.fn().mockResolvedValue(undefined) } + clientMocks.createMysqlConnection.mockResolvedValue(connection) + queryMocks.introspectMysqlDatabase.mockResolvedValue({ + tables: [], + databases: ['application'], + }) + + await expect(executeMysqlIntrospection(CONNECTION, controller.signal)).resolves.toEqual({ + message: "Schema introspection completed. Found 0 table(s) in database 'application'.", + tables: [], + databases: ['application'], + }) + expect(queryMocks.introspectMysqlDatabase).toHaveBeenCalledWith( + connection, + 'application', + controller.signal + ) + expect(connection.end).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/internal/mysql/operations.ts b/apps/sim/lib/internal/mysql/operations.ts new file mode 100644 index 00000000000..48f215d33c7 --- /dev/null +++ b/apps/sim/lib/internal/mysql/operations.ts @@ -0,0 +1,114 @@ +import { createMysqlConnection, type MysqlConnectionConfig } from '@/lib/internal/mysql/client' +import { + buildMysqlDeleteQuery, + buildMysqlInsertQuery, + buildMysqlUpdateQuery, + introspectMysqlDatabase, + queryMysql, + validateMysqlQuery, +} from '@/lib/internal/mysql/queries' +import type { + MysqlDeleteInput, + MysqlExecuteInput, + MysqlInsertInput, + MysqlIntrospectInput, + MysqlQueryInput, + MysqlUpdateInput, +} from '@/lib/internal/mysql/schema' + +export class MysqlOperationInputError extends Error {} + +async function withMysqlConnection( + input: MysqlConnectionConfig, + signal: AbortSignal | undefined, + execute: (connection: Awaited>) => Promise +): Promise { + const connection = await createMysqlConnection(input, signal) + try { + return await execute(connection) + } finally { + await connection.end() + } +} + +function validateOperationQuery(query: string): void { + const validation = validateMysqlQuery(query) + if (!validation.isValid) { + throw new MysqlOperationInputError( + `Query validation failed: ${validation.error ?? 'Invalid query'}` + ) + } +} + +export function executeMysqlQuery(input: MysqlQueryInput, signal?: AbortSignal) { + validateOperationQuery(input.query) + + return withMysqlConnection(input, signal, async (connection) => { + const result = await queryMysql(connection, input.query, undefined, signal) + return { + message: `Query executed successfully. ${result.rowCount} row(s) returned.`, + rows: result.rows, + rowCount: result.rowCount, + } + }) +} + +export function executeMysqlStatement(input: MysqlExecuteInput, signal?: AbortSignal) { + validateOperationQuery(input.query) + + return withMysqlConnection(input, signal, async (connection) => { + const result = await queryMysql(connection, input.query, undefined, signal) + return { + message: `SQL executed successfully. ${result.rowCount} row(s) affected.`, + rows: result.rows, + rowCount: result.rowCount, + } + }) +} + +export function executeMysqlInsert(input: MysqlInsertInput, signal?: AbortSignal) { + return withMysqlConnection(input, signal, async (connection) => { + const { query, values } = buildMysqlInsertQuery(input.table, input.data) + const result = await queryMysql(connection, query, values, signal) + return { + message: `Data inserted successfully. ${result.rowCount} row(s) affected.`, + rows: result.rows, + rowCount: result.rowCount, + } + }) +} + +export function executeMysqlUpdate(input: MysqlUpdateInput, signal?: AbortSignal) { + return withMysqlConnection(input, signal, async (connection) => { + const { query, values } = buildMysqlUpdateQuery(input.table, input.data, input.where) + const result = await queryMysql(connection, query, values, signal) + return { + message: `Data updated successfully. ${result.rowCount} row(s) affected.`, + rows: result.rows, + rowCount: result.rowCount, + } + }) +} + +export function executeMysqlDelete(input: MysqlDeleteInput, signal?: AbortSignal) { + return withMysqlConnection(input, signal, async (connection) => { + const { query, values } = buildMysqlDeleteQuery(input.table, input.where) + const result = await queryMysql(connection, query, values, signal) + return { + message: `Data deleted successfully. ${result.rowCount} row(s) affected.`, + rows: result.rows, + rowCount: result.rowCount, + } + }) +} + +export function executeMysqlIntrospection(input: MysqlIntrospectInput, signal?: AbortSignal) { + return withMysqlConnection(input, signal, async (connection) => { + const result = await introspectMysqlDatabase(connection, input.database, signal) + return { + message: `Schema introspection completed. Found ${result.tables.length} table(s) in database '${input.database}'.`, + tables: result.tables, + databases: result.databases, + } + }) +} diff --git a/apps/sim/lib/internal/mysql/queries.test.ts b/apps/sim/lib/internal/mysql/queries.test.ts new file mode 100644 index 00000000000..a8f6f28fe56 --- /dev/null +++ b/apps/sim/lib/internal/mysql/queries.test.ts @@ -0,0 +1,136 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockExecuteMysqlCommand } = vi.hoisted(() => ({ + mockExecuteMysqlCommand: vi.fn(), +})) + +vi.mock('@/lib/internal/mysql/client', () => ({ + executeMysqlCommand: mockExecuteMysqlCommand, +})) + +import { + buildMysqlDeleteQuery, + buildMysqlInsertQuery, + buildMysqlUpdateQuery, + introspectMysqlDatabase, + queryMysql, + sanitizeMysqlIdentifier, + validateMysqlQuery, +} from '@/lib/internal/mysql/queries' + +const connection = { execute: vi.fn(), destroy: vi.fn() } + +describe('MySQL queries', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('preserves row and affected-row result semantics', async () => { + mockExecuteMysqlCommand.mockResolvedValueOnce([{ id: 1 }]).mockResolvedValueOnce({ + affectedRows: 3, + }) + + await expect(queryMysql(connection as never, 'SELECT 1')).resolves.toEqual({ + rows: [{ id: 1 }], + rowCount: 1, + }) + await expect(queryMysql(connection as never, 'UPDATE users SET active = 1')).resolves.toEqual({ + rows: [], + rowCount: 3, + }) + }) + + it('preserves parameterized INSERT, UPDATE, and DELETE construction', () => { + expect( + buildMysqlInsertQuery('application.users', { + email: 'person@example.com', + active: true, + }) + ).toEqual({ + query: 'INSERT INTO `application`.`users` (`email`, `active`) VALUES (?, ?)', + values: ['person@example.com', true], + }) + expect(buildMysqlUpdateQuery('users', { active: false }, 'id = 42')).toEqual({ + query: 'UPDATE `users` SET `active` = ? WHERE id = 42', + values: [false], + }) + expect(buildMysqlDeleteQuery('users', 'id = 42')).toEqual({ + query: 'DELETE FROM `users` WHERE id = 42', + values: [], + }) + }) + + it('preserves identifier, WHERE, and statement validation', () => { + expect(sanitizeMysqlIdentifier('application.users')).toBe('`application`.`users`') + expect(() => sanitizeMysqlIdentifier('users; DROP TABLE users')).toThrow('Invalid identifier') + expect(() => buildMysqlDeleteQuery('users', "id = 42 OR 'x'='x'")).toThrow( + 'WHERE clause contains potentially dangerous operation' + ) + expect(validateMysqlQuery('DESCRIBE users')).toEqual({ isValid: true }) + expect(validateMysqlQuery('DROP TABLE users')).toEqual({ + isValid: false, + error: + 'Only SELECT, INSERT, UPDATE, DELETE, WITH, SHOW, DESCRIBE, and EXPLAIN statements are allowed', + }) + }) + + it('preserves introspection shaping and passes cancellation to every query', async () => { + const controller = new AbortController() + mockExecuteMysqlCommand + .mockResolvedValueOnce([{ SCHEMA_NAME: 'application' }]) + .mockResolvedValueOnce([{ TABLE_NAME: 'users' }]) + .mockResolvedValueOnce([ + { + COLUMN_NAME: 'role', + DATA_TYPE: 'enum', + COLUMN_TYPE: "enum('admin','member')", + IS_NULLABLE: 'NO', + COLUMN_DEFAULT: 'member', + EXTRA: 'auto_increment', + }, + ]) + .mockResolvedValueOnce([{ COLUMN_NAME: 'role' }]) + .mockResolvedValueOnce([ + { + COLUMN_NAME: 'role', + REFERENCED_TABLE_NAME: 'roles', + REFERENCED_COLUMN_NAME: 'name', + }, + ]) + .mockResolvedValueOnce([{ INDEX_NAME: 'users_role_idx', COLUMN_NAME: 'role', NON_UNIQUE: 0 }]) + + await expect( + introspectMysqlDatabase(connection as never, 'application', controller.signal) + ).resolves.toEqual({ + databases: ['application'], + tables: [ + { + name: 'users', + database: 'application', + columns: [ + { + name: 'role', + type: "enum('admin','member')", + nullable: false, + default: 'member', + isPrimaryKey: true, + isForeignKey: true, + autoIncrement: true, + references: { table: 'roles', column: 'name' }, + }, + ], + primaryKey: ['role'], + foreignKeys: [{ column: 'role', referencesTable: 'roles', referencesColumn: 'name' }], + indexes: [{ name: 'users_role_idx', columns: ['role'], unique: true }], + }, + ], + }) + expect(mockExecuteMysqlCommand).toHaveBeenCalledTimes(6) + for (const call of mockExecuteMysqlCommand.mock.calls) { + expect(call[3]).toBe(controller.signal) + } + }) +}) diff --git a/apps/sim/lib/internal/mysql/queries.ts b/apps/sim/lib/internal/mysql/queries.ts new file mode 100644 index 00000000000..d72a1c80a4e --- /dev/null +++ b/apps/sim/lib/internal/mysql/queries.ts @@ -0,0 +1,311 @@ +import type mysql from 'mysql2/promise' +import { executeMysqlCommand } from '@/lib/internal/mysql/client' + +export interface MysqlRowsResult { + rows: unknown[] + rowCount: number +} + +export interface MysqlIntrospectionResult { + tables: Array<{ + name: string + database: string + columns: Array<{ + name: string + type: string + nullable: boolean + default: string | null + isPrimaryKey: boolean + isForeignKey: boolean + autoIncrement: boolean + references?: { + table: string + column: string + } + }> + primaryKey: string[] + foreignKeys: Array<{ + column: string + referencesTable: string + referencesColumn: string + }> + indexes: Array<{ + name: string + columns: string[] + unique: boolean + }> + }> + databases: string[] +} + +interface DatabaseRow extends mysql.RowDataPacket { + SCHEMA_NAME: string +} + +interface TableRow extends mysql.RowDataPacket { + TABLE_NAME: string +} + +interface ColumnRow extends mysql.RowDataPacket { + COLUMN_NAME: string + DATA_TYPE: string + COLUMN_TYPE: string + IS_NULLABLE: string + COLUMN_DEFAULT: string | null + EXTRA?: string +} + +interface PrimaryKeyRow extends mysql.RowDataPacket { + COLUMN_NAME: string +} + +interface ForeignKeyRow extends mysql.RowDataPacket { + COLUMN_NAME: string + REFERENCED_TABLE_NAME: string + REFERENCED_COLUMN_NAME: string +} + +interface IndexRow extends mysql.RowDataPacket { + INDEX_NAME: string + COLUMN_NAME: string + NON_UNIQUE: number +} + +export async function queryMysql( + connection: mysql.Connection, + query: string, + values?: unknown[], + signal?: AbortSignal +): Promise { + const result = await executeMysqlCommand(connection, query, values, signal) + + if (Array.isArray(result)) { + return { + rows: result, + rowCount: result.length, + } + } + + return { + rows: [], + rowCount: (result as mysql.ResultSetHeader).affectedRows || 0, + } +} + +export function validateMysqlQuery(query: string): { isValid: boolean; error?: string } { + const trimmedQuery = query.trim().toLowerCase() + const allowedStatements = /^(select|insert|update|delete|with|show|describe|explain)\s+/i + + if (!allowedStatements.test(trimmedQuery)) { + return { + isValid: false, + error: + 'Only SELECT, INSERT, UPDATE, DELETE, WITH, SHOW, DESCRIBE, and EXPLAIN statements are allowed', + } + } + + return { isValid: true } +} + +export function buildMysqlInsertQuery(table: string, data: Record) { + const sanitizedTable = sanitizeMysqlIdentifier(table) + const columns = Object.keys(data) + const values = Object.values(data) + const placeholders = columns.map(() => '?').join(', ') + const query = `INSERT INTO ${sanitizedTable} (${columns.map(sanitizeMysqlIdentifier).join(', ')}) VALUES (${placeholders})` + + return { query, values } +} + +export function buildMysqlUpdateQuery(table: string, data: Record, where: string) { + validateWhereClause(where) + + const sanitizedTable = sanitizeMysqlIdentifier(table) + const columns = Object.keys(data) + const values = Object.values(data) + const setClause = columns.map((column) => `${sanitizeMysqlIdentifier(column)} = ?`).join(', ') + const query = `UPDATE ${sanitizedTable} SET ${setClause} WHERE ${where}` + + return { query, values } +} + +export function buildMysqlDeleteQuery(table: string, where: string) { + validateWhereClause(where) + + const sanitizedTable = sanitizeMysqlIdentifier(table) + const query = `DELETE FROM ${sanitizedTable} WHERE ${where}` + + return { query, values: [] } +} + +export function sanitizeMysqlIdentifier(identifier: string): string { + if (identifier.includes('.')) { + return identifier + .split('.') + .map((part) => sanitizeSingleIdentifier(part)) + .join('.') + } + + return sanitizeSingleIdentifier(identifier) +} + +export async function introspectMysqlDatabase( + connection: mysql.Connection, + databaseName: string, + signal?: AbortSignal +): Promise { + const databasesRows = await executeMysqlCommand( + connection, + `SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA + WHERE SCHEMA_NAME NOT IN ('mysql', 'information_schema', 'performance_schema', 'sys') + ORDER BY SCHEMA_NAME`, + undefined, + signal + ) + const databases = databasesRows.map((row) => row.SCHEMA_NAME) + + const tablesRows = await executeMysqlCommand( + connection, + `SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = 'BASE TABLE' + ORDER BY TABLE_NAME`, + [databaseName], + signal + ) + const tables: MysqlIntrospectionResult['tables'] = [] + + for (const tableRow of tablesRows) { + signal?.throwIfAborted() + const tableName = tableRow.TABLE_NAME + const queryValues = [databaseName, tableName] + + const columnsRows = await executeMysqlCommand( + connection, + `SELECT COLUMN_NAME, DATA_TYPE, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, EXTRA + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? + ORDER BY ORDINAL_POSITION`, + queryValues, + signal + ) + + const primaryKeyRows = await executeMysqlCommand( + connection, + `SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE + WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND CONSTRAINT_NAME = 'PRIMARY' + ORDER BY ORDINAL_POSITION`, + queryValues, + signal + ) + const primaryKey = primaryKeyRows.map((row) => row.COLUMN_NAME) + + const foreignKeyRows = await executeMysqlCommand( + connection, + `SELECT kcu.COLUMN_NAME, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME + FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu + WHERE kcu.TABLE_SCHEMA = ? AND kcu.TABLE_NAME = ? AND kcu.REFERENCED_TABLE_NAME IS NOT NULL`, + queryValues, + signal + ) + const foreignKeys = foreignKeyRows.map((row) => ({ + column: row.COLUMN_NAME, + referencesTable: row.REFERENCED_TABLE_NAME, + referencesColumn: row.REFERENCED_COLUMN_NAME, + })) + const foreignKeyColumns = new Set(foreignKeys.map((foreignKey) => foreignKey.column)) + + const indexRows = await executeMysqlCommand( + connection, + `SELECT INDEX_NAME, COLUMN_NAME, SEQ_IN_INDEX, NON_UNIQUE + FROM INFORMATION_SCHEMA.STATISTICS + WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND INDEX_NAME != 'PRIMARY' + ORDER BY INDEX_NAME, SEQ_IN_INDEX`, + queryValues, + signal + ) + const indexesByName = new Map() + for (const row of indexRows) { + const index = indexesByName.get(row.INDEX_NAME) ?? { + name: row.INDEX_NAME, + columns: [], + unique: row.NON_UNIQUE === 0, + } + index.columns.push(row.COLUMN_NAME) + indexesByName.set(row.INDEX_NAME, index) + } + + const columns = columnsRows.map((column) => { + const foreignKey = foreignKeys.find((candidate) => candidate.column === column.COLUMN_NAME) + return { + name: column.COLUMN_NAME, + type: column.COLUMN_TYPE || column.DATA_TYPE, + nullable: column.IS_NULLABLE === 'YES', + default: column.COLUMN_DEFAULT, + isPrimaryKey: primaryKey.includes(column.COLUMN_NAME), + isForeignKey: foreignKeyColumns.has(column.COLUMN_NAME), + autoIncrement: column.EXTRA?.toLowerCase().includes('auto_increment') || false, + ...(foreignKey && { + references: { + table: foreignKey.referencesTable, + column: foreignKey.referencesColumn, + }, + }), + } + }) + + tables.push({ + name: tableName, + database: databaseName, + columns, + primaryKey, + foreignKeys, + indexes: Array.from(indexesByName.values()), + }) + } + + return { tables, databases } +} + +function sanitizeSingleIdentifier(identifier: string): string { + const cleaned = identifier.replace(/`/g, '') + + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(cleaned)) { + throw new Error( + `Invalid identifier: ${identifier}. Identifiers must start with a letter or underscore and contain only letters, numbers, and underscores.` + ) + } + + return `\`${cleaned}\`` +} + +function validateWhereClause(where: string): void { + const dangerousPatterns = [ + /;\s*(drop|delete|insert|update|create|alter|grant|revoke)/i, + /union\s+(all\s+)?select/i, + /into\s+outfile/i, + /into\s+dumpfile/i, + /load_file\s*\(/i, + /--/, + /\/\*/, + /\*\//, + /\bor\s+(['"]?)(\w+)\1\s*=\s*\1\2\1/i, + /\bor\s+true\b/i, + /\bor\s+false\b/i, + /\band\s+(['"]?)(\w+)\1\s*=\s*\1\2\1/i, + /\band\s+true\b/i, + /\band\s+false\b/i, + /\bsleep\s*\(/i, + /\bbenchmark\s*\(/i, + /\bwaitfor\s+delay/i, + /;\s*\w+/, + /information_schema/i, + /mysql\./i, + /\bxp_cmdshell/i, + ] + + for (const pattern of dangerousPatterns) { + if (pattern.test(where)) { + throw new Error('WHERE clause contains potentially dangerous operation') + } + } +} diff --git a/apps/sim/lib/internal/mysql/schema.ts b/apps/sim/lib/internal/mysql/schema.ts new file mode 100644 index 00000000000..3b205709e33 --- /dev/null +++ b/apps/sim/lib/internal/mysql/schema.ts @@ -0,0 +1,71 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { z } from 'zod' + +const sslModeSchema = z.enum(['disabled', 'required', 'preferred']).default('preferred') + +const nonEmptyRecordSchema = (message: string) => + z.record(z.string(), z.unknown()).refine((value) => Object.keys(value).length > 0, { message }) + +const jsonObjectStringSchema = (message: string, includeReceivedValue = false) => + z + .string() + .min(1) + .transform((value) => { + try { + const parsed = JSON.parse(value) + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw new Error('Data must be a JSON object') + } + return parsed + } catch (error) { + if (!includeReceivedValue) throw new Error(message) + throw new Error( + `${message}: ${getErrorMessage(error, 'Unknown error')}. Received: ${value.substring(0, 100)}...` + ) + } + }) + +const connectionInputSchema = z.object({ + host: z.string().min(1, 'Host is required'), + port: z.coerce.number().int().positive('Port must be a positive integer'), + database: z.string().min(1, 'Database name is required'), + username: z.string().min(1, 'Username is required'), + password: z.string().min(1, 'Password is required'), + ssl: sslModeSchema, +}) + +const queryInputSchema = connectionInputSchema.extend({ + query: z.string().min(1, 'Query is required'), +}) +const insertDataSchema = z.union([ + nonEmptyRecordSchema('Data object cannot be empty'), + jsonObjectStringSchema('Invalid JSON format in data field', true), +]) +const updateDataSchema = z.union([ + nonEmptyRecordSchema('Data object cannot be empty'), + jsonObjectStringSchema('Invalid JSON format in data field'), +]) + +export const mysqlQueryInputSchema = queryInputSchema +export const mysqlExecuteInputSchema = queryInputSchema +export const mysqlInsertInputSchema = connectionInputSchema.extend({ + table: z.string().min(1, 'Table name is required'), + data: insertDataSchema, +}) +export const mysqlUpdateInputSchema = connectionInputSchema.extend({ + table: z.string().min(1, 'Table name is required'), + data: updateDataSchema, + where: z.string().min(1, 'WHERE clause is required'), +}) +export const mysqlDeleteInputSchema = connectionInputSchema.extend({ + table: z.string().min(1, 'Table name is required'), + where: z.string().min(1, 'WHERE clause is required'), +}) +export const mysqlIntrospectInputSchema = connectionInputSchema + +export type MysqlQueryInput = z.output +export type MysqlExecuteInput = z.output +export type MysqlInsertInput = z.output +export type MysqlUpdateInput = z.output +export type MysqlDeleteInput = z.output +export type MysqlIntrospectInput = z.output diff --git a/apps/sim/lib/internal/neo4j/client.ts b/apps/sim/lib/internal/neo4j/client.ts new file mode 100644 index 00000000000..89ae5de8e40 --- /dev/null +++ b/apps/sim/lib/internal/neo4j/client.ts @@ -0,0 +1,51 @@ +import neo4j, { type Driver } from 'neo4j-driver' +import { validateDatabaseHost } from '@/lib/core/security/input-validation.server' +import type { Neo4jConnectionConfig } from '@/tools/neo4j/types' + +function isAuraHost(host: string): boolean { + return host === 'databases.neo4j.io' || host.endsWith('.databases.neo4j.io') +} + +export async function createNeo4jDriver( + config: Neo4jConnectionConfig, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const hostValidation = await validateDatabaseHost(config.host, 'host') + signal?.throwIfAborted() + if (!hostValidation.isValid) throw new Error(hostValidation.error) + + const aura = isAuraHost(config.host) + const protocol = aura ? 'neo4j+s' : config.encryption === 'enabled' ? 'bolt+s' : 'bolt' + const usePinnedIp = !protocol.endsWith('+s') + const resolvedHost = hostValidation.resolvedIP ?? config.host + const uriHost = usePinnedIp + ? resolvedHost.includes(':') + ? `[${resolvedHost}]` + : resolvedHost + : config.host + const uri = `${protocol}://${uriHost}:${config.port}` + const driverConfig: Exclude[2], undefined> = { + maxConnectionPoolSize: 1, + connectionTimeout: 10_000, + } + if (!protocol.endsWith('+s')) { + driverConfig.encrypted = config.encryption === 'enabled' ? 'ENCRYPTION_ON' : 'ENCRYPTION_OFF' + } + + const driver = neo4j.driver(uri, neo4j.auth.basic(config.username, config.password), driverConfig) + const abort = () => { + void driver.close() + } + signal?.addEventListener('abort', abort, { once: true }) + try { + await driver.verifyConnectivity() + signal?.throwIfAborted() + return driver + } catch (error) { + await driver.close().catch(() => undefined) + throw error + } finally { + signal?.removeEventListener('abort', abort) + } +} diff --git a/apps/sim/lib/internal/neo4j/execute-tool.test.ts b/apps/sim/lib/internal/neo4j/execute-tool.test.ts new file mode 100644 index 00000000000..537464ceef4 --- /dev/null +++ b/apps/sim/lib/internal/neo4j/execute-tool.test.ts @@ -0,0 +1,107 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + create: vi.fn(), + delete: vi.fn(), + execute: vi.fn(), + introspect: vi.fn(), + merge: vi.fn(), + query: vi.fn(), + update: vi.fn(), + InputError: class Neo4jOperationInputError extends Error {}, +})) + +vi.mock('@/lib/internal/neo4j/operations', () => ({ + executeNeo4jCreate: mocks.create, + executeNeo4jDelete: mocks.delete, + executeNeo4jStatement: mocks.execute, + executeNeo4jIntrospection: mocks.introspect, + executeNeo4jMerge: mocks.merge, + executeNeo4jQuery: mocks.query, + executeNeo4jUpdate: mocks.update, + Neo4jOperationInputError: mocks.InputError, +})) + +import { executeNeo4jTool } from '@/lib/internal/neo4j/execute-tool' + +const BASE_BODY = { + host: 'neo4j.example.com', + port: 7687, + database: 'neo4j', + username: 'neo4j', + password: 'password', + encryption: 'enabled', + cypherQuery: 'MATCH (n) RETURN n', + parameters: {}, +} + +function request(toolId: string, signal?: AbortSignal) { + return { + toolId, + input: toolId === 'neo4j_introspect' ? { ...BASE_BODY, cypherQuery: undefined } : BASE_BODY, + headers: new Headers(), + context: createExecutionContext(), + requestId: 'request-1', + signal, + } +} + +describe('executeNeo4jTool', () => { + beforeEach(() => { + vi.clearAllMocks() + for (const operation of [ + mocks.create, + mocks.delete, + mocks.execute, + mocks.introspect, + mocks.merge, + mocks.query, + mocks.update, + ]) { + operation.mockResolvedValue({ message: 'ok' }) + } + }) + + it.each([ + ['neo4j_query', mocks.query], + ['neo4j_execute', mocks.execute], + ['neo4j_create', mocks.create], + ['neo4j_update', mocks.update], + ['neo4j_merge', mocks.merge], + ['neo4j_delete', mocks.delete], + ['neo4j_introspect', mocks.introspect], + ])('dispatches %s directly', async (toolId, operation) => { + const response = await executeNeo4jTool(request(toolId)) + + expect(response.status).toBe(200) + expect(operation).toHaveBeenCalledOnce() + await expect(response.json()).resolves.toEqual({ message: 'ok' }) + }) + + it('preserves query-validation errors as 400 responses', async () => { + mocks.query.mockRejectedValueOnce(new mocks.InputError('Query cannot be empty')) + + const response = await executeNeo4jTool(request('neo4j_query')) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ + error: 'Query validation failed: Query cannot be empty', + }) + }) + + it('propagates cancellation without converting it to a provider error', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect(executeNeo4jTool(request('neo4j_query', controller.signal))).rejects.toMatchObject( + { + name: 'AbortError', + } + ) + expect(mocks.query).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/neo4j/execute-tool.ts b/apps/sim/lib/internal/neo4j/execute-tool.ts new file mode 100644 index 00000000000..f542d648bb3 --- /dev/null +++ b/apps/sim/lib/internal/neo4j/execute-tool.ts @@ -0,0 +1,110 @@ +import { getErrorMessage } from '@sim/utils/errors' +import type { AnyApiRouteContract, ContractBody } from '@/lib/api/contracts' +import { + neo4jCreateContract, + neo4jDeleteContract, + neo4jExecuteContract, + neo4jIntrospectContract, + neo4jMergeContract, + neo4jQueryContract, + neo4jUpdateContract, +} from '@/lib/api/contracts/tools/databases/neo4j' +import { + executeNeo4jCreate, + executeNeo4jDelete, + executeNeo4jIntrospection, + executeNeo4jMerge, + executeNeo4jQuery, + executeNeo4jStatement, + executeNeo4jUpdate, + Neo4jOperationInputError, +} from '@/lib/internal/neo4j/operations' +import { parseInternalToolInput } from '@/lib/internal/tool-operations/parse-input' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +async function executeOperation( + contract: C, + input: unknown, + execute: (input: ContractBody, signal?: AbortSignal) => Promise, + errorPrefix: string, + signal?: AbortSignal +): Promise { + const parsed = parseInternalToolInput(contract, input) + if (!parsed.success) return parsed.response + try { + return Response.json(await execute(parsed.data, signal)) + } catch (error) { + signal?.throwIfAborted() + if (error instanceof Neo4jOperationInputError) { + return Response.json({ error: `Query validation failed: ${error.message}` }, { status: 400 }) + } + return Response.json( + { error: `${errorPrefix}: ${getErrorMessage(error, 'Unknown error occurred')}` }, + { status: 500 } + ) + } +} + +export const executeNeo4jTool: InternalToolOperationHandler = async ({ toolId, input, signal }) => { + signal?.throwIfAborted() + switch (toolId) { + case 'neo4j_query': + return executeOperation( + neo4jQueryContract, + input, + executeNeo4jQuery, + 'Neo4j query failed', + signal + ) + case 'neo4j_execute': + return executeOperation( + neo4jExecuteContract, + input, + executeNeo4jStatement, + 'Neo4j execute failed', + signal + ) + case 'neo4j_create': + return executeOperation( + neo4jCreateContract, + input, + executeNeo4jCreate, + 'Neo4j create failed', + signal + ) + case 'neo4j_update': + return executeOperation( + neo4jUpdateContract, + input, + executeNeo4jUpdate, + 'Neo4j update failed', + signal + ) + case 'neo4j_merge': + return executeOperation( + neo4jMergeContract, + input, + executeNeo4jMerge, + 'Neo4j merge failed', + signal + ) + case 'neo4j_delete': + return executeOperation( + neo4jDeleteContract, + input, + executeNeo4jDelete, + 'Neo4j delete failed', + signal + ) + case 'neo4j_introspect': + return executeOperation( + neo4jIntrospectContract, + input, + executeNeo4jIntrospection, + 'Neo4j introspection failed', + signal + ) + default: + return Response.json({ error: `Unsupported Neo4j tool: ${toolId}` }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/neo4j/operations.test.ts b/apps/sim/lib/internal/neo4j/operations.test.ts new file mode 100644 index 00000000000..03a66938099 --- /dev/null +++ b/apps/sim/lib/internal/neo4j/operations.test.ts @@ -0,0 +1,90 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + createDriver: vi.fn(), + sessionClose: vi.fn(), + driverClose: vi.fn(), + run: vi.fn(), +})) + +vi.mock('@/lib/internal/neo4j/client', () => ({ + createNeo4jDriver: mocks.createDriver, +})) + +import { executeNeo4jQuery, Neo4jOperationInputError } from '@/lib/internal/neo4j/operations' + +const INPUT = { + host: 'neo4j.example.com', + port: 7687, + database: 'neo4j', + username: 'neo4j', + password: 'password', + encryption: 'enabled' as const, + cypherQuery: 'MATCH (n) RETURN n', + parameters: {}, +} + +function queryResult() { + const updates = { + nodesCreated: 0, + nodesDeleted: 0, + relationshipsCreated: 0, + relationshipsDeleted: 0, + propertiesSet: 0, + labelsAdded: 0, + labelsRemoved: 0, + indexesAdded: 0, + indexesRemoved: 0, + constraintsAdded: 0, + constraintsRemoved: 0, + } + return { + records: [{ keys: ['name'], get: vi.fn().mockReturnValue('Ada') }], + summary: { + resultAvailableAfter: { toNumber: () => 2 }, + resultConsumedAfter: { toNumber: () => 3 }, + counters: { updates: () => updates }, + }, + } +} + +describe('Neo4j operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.run.mockResolvedValue(queryResult()) + mocks.sessionClose.mockResolvedValue(undefined) + mocks.driverClose.mockResolvedValue(undefined) + mocks.createDriver.mockResolvedValue({ + session: vi.fn().mockReturnValue({ run: mocks.run, close: mocks.sessionClose }), + close: mocks.driverClose, + }) + }) + + it('projects records and always closes the session and driver', async () => { + await expect(executeNeo4jQuery(INPUT)).resolves.toMatchObject({ + message: 'Found 1 records', + records: [{ name: 'Ada' }], + recordCount: 1, + }) + expect(mocks.sessionClose).toHaveBeenCalledOnce() + expect(mocks.driverClose).toHaveBeenCalledOnce() + }) + + it('closes resources when the query fails', async () => { + mocks.run.mockRejectedValueOnce(new Error('provider failed')) + + await expect(executeNeo4jQuery(INPUT)).rejects.toThrow('provider failed') + expect(mocks.sessionClose).toHaveBeenCalledOnce() + expect(mocks.driverClose).toHaveBeenCalledOnce() + }) + + it('rejects invalid Cypher before opening a driver', async () => { + await expect(executeNeo4jQuery({ ...INPUT, cypherQuery: ' ' })).rejects.toBeInstanceOf( + Neo4jOperationInputError + ) + expect(mocks.createDriver).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/neo4j/operations.ts b/apps/sim/lib/internal/neo4j/operations.ts new file mode 100644 index 00000000000..476b1151072 --- /dev/null +++ b/apps/sim/lib/internal/neo4j/operations.ts @@ -0,0 +1,277 @@ +import { createLogger } from '@sim/logger' +import type { QueryResult, Session } from 'neo4j-driver' +import type { + Neo4jCreateRequest, + Neo4jDeleteRequest, + Neo4jExecuteRequest, + Neo4jIntrospectRequest, + Neo4jMergeRequest, + Neo4jQueryRequest, + Neo4jUpdateRequest, +} from '@/lib/api/contracts/tools/databases/neo4j' +import { createNeo4jDriver } from '@/lib/internal/neo4j/client' +import { convertNeo4jValue } from '@/lib/internal/neo4j/values' +import type { Neo4jNodeSchema, Neo4jRelationshipSchema } from '@/tools/neo4j/types' + +const logger = createLogger('Neo4jOperations') + +export class Neo4jOperationInputError extends Error {} + +type StatementInput = + | Neo4jQueryRequest + | Neo4jExecuteRequest + | Neo4jCreateRequest + | Neo4jUpdateRequest + | Neo4jMergeRequest + | Neo4jDeleteRequest + +type StatementKind = 'query' | 'execute' | 'create' | 'update' | 'merge' | 'delete' + +function validateCypherQuery(query: string): void { + if (!query || typeof query !== 'string') { + throw new Neo4jOperationInputError('Query must be a non-empty string') + } + if (!query.trim()) throw new Neo4jOperationInputError('Query cannot be empty') +} + +function bindSessionAbort(session: Session, signal?: AbortSignal): () => void { + if (!signal) return () => undefined + const abort = () => { + void session.close() + } + signal.addEventListener('abort', abort, { once: true }) + return () => signal.removeEventListener('abort', abort) +} + +function projectRecords(result: QueryResult): Array> { + return result.records.map((record) => { + const projected: Record = {} + for (const key of record.keys) { + if (typeof key === 'string') projected[key] = convertNeo4jValue(record.get(key)) + } + return projected + }) +} + +function projectSummary(result: QueryResult) { + const updates = result.summary.counters.updates() + return { + resultAvailableAfter: result.summary.resultAvailableAfter.toNumber(), + resultConsumedAfter: result.summary.resultConsumedAfter.toNumber(), + counters: { + nodesCreated: updates.nodesCreated, + nodesDeleted: updates.nodesDeleted, + relationshipsCreated: updates.relationshipsCreated, + relationshipsDeleted: updates.relationshipsDeleted, + propertiesSet: updates.propertiesSet, + labelsAdded: updates.labelsAdded, + labelsRemoved: updates.labelsRemoved, + indexesAdded: updates.indexesAdded, + indexesRemoved: updates.indexesRemoved, + constraintsAdded: updates.constraintsAdded, + constraintsRemoved: updates.constraintsRemoved, + }, + } +} + +function statementResponse(kind: StatementKind, result: QueryResult) { + const records = projectRecords(result) + const summary = projectSummary(result) + switch (kind) { + case 'query': + return { + message: `Found ${records.length} records`, + records, + recordCount: records.length, + summary, + } + case 'execute': + return { + message: `Query executed successfully, returned ${records.length} records`, + records, + recordCount: records.length, + summary, + } + case 'create': + return { + message: `Created ${summary.counters.nodesCreated} nodes and ${summary.counters.relationshipsCreated} relationships`, + records, + recordCount: records.length, + summary, + } + case 'update': + return { + message: `Updated ${summary.counters.propertiesSet} properties`, + records, + recordCount: records.length, + summary, + } + case 'merge': + return { + message: `Merge completed: ${summary.counters.nodesCreated} nodes created, ${summary.counters.relationshipsCreated} relationships created`, + records, + recordCount: records.length, + summary, + } + case 'delete': + return { + message: `Deleted ${summary.counters.nodesDeleted} nodes and ${summary.counters.relationshipsDeleted} relationships`, + summary, + } + } +} + +async function executeStatement(input: StatementInput, kind: StatementKind, signal?: AbortSignal) { + signal?.throwIfAborted() + validateCypherQuery(input.cypherQuery) + const driver = await createNeo4jDriver({ ...input, port: Number(input.port) }, signal) + const session = driver.session({ database: input.database }) + const unbindAbort = bindSessionAbort(session, signal) + try { + const result = await session.run(input.cypherQuery, input.parameters ?? {}) + signal?.throwIfAborted() + return statementResponse(kind, result) + } finally { + unbindAbort() + await session.close().catch(() => undefined) + await driver.close().catch(() => undefined) + } +} + +export const executeNeo4jQuery = (input: Neo4jQueryRequest, signal?: AbortSignal) => + executeStatement(input, 'query', signal) +export const executeNeo4jStatement = (input: Neo4jExecuteRequest, signal?: AbortSignal) => + executeStatement(input, 'execute', signal) +export const executeNeo4jCreate = (input: Neo4jCreateRequest, signal?: AbortSignal) => + executeStatement(input, 'create', signal) +export const executeNeo4jUpdate = (input: Neo4jUpdateRequest, signal?: AbortSignal) => + executeStatement(input, 'update', signal) +export const executeNeo4jMerge = (input: Neo4jMergeRequest, signal?: AbortSignal) => + executeStatement(input, 'merge', signal) +export const executeNeo4jDelete = (input: Neo4jDeleteRequest, signal?: AbortSignal) => + executeStatement(input, 'delete', signal) + +function stringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((entry): entry is string => typeof entry === 'string') + : [] +} + +export async function executeNeo4jIntrospection( + input: Neo4jIntrospectRequest, + signal?: AbortSignal +) { + signal?.throwIfAborted() + const driver = await createNeo4jDriver({ ...input, port: Number(input.port) }, signal) + const session = driver.session({ database: input.database }) + const unbindAbort = bindSessionAbort(session, signal) + try { + const labelsResult = await session.run( + 'CALL db.labels() YIELD label RETURN label ORDER BY label' + ) + signal?.throwIfAborted() + const labels = labelsResult.records.map((record) => String(record.get('label'))) + const relationshipTypesResult = await session.run( + 'CALL db.relationshipTypes() YIELD relationshipType RETURN relationshipType ORDER BY relationshipType' + ) + signal?.throwIfAborted() + const relationshipTypes = relationshipTypesResult.records.map((record) => + String(record.get('relationshipType')) + ) + + const nodeSchemas: Neo4jNodeSchema[] = [] + try { + const result = await session.run( + 'CALL db.schema.nodeTypeProperties() YIELD nodeLabels, propertyName, propertyTypes RETURN nodeLabels, propertyName, propertyTypes' + ) + const byLabel = new Map>() + for (const record of result.records) { + const label = stringArray(record.get('nodeLabels')).join(':') + const properties = byLabel.get(label) ?? [] + properties.push({ + name: String(record.get('propertyName')), + types: stringArray(record.get('propertyTypes')), + }) + byLabel.set(label, properties) + } + for (const [label, properties] of byLabel) nodeSchemas.push({ label, properties }) + } catch (error) { + signal?.throwIfAborted() + logger.warn('Could not fetch Neo4j node properties', { error }) + } + + const relationshipSchemas: Neo4jRelationshipSchema[] = [] + try { + const result = await session.run( + 'CALL db.schema.relTypeProperties() YIELD relationshipType, propertyName, propertyTypes RETURN relationshipType, propertyName, propertyTypes' + ) + const byType = new Map>() + for (const record of result.records) { + const type = String(record.get('relationshipType')) + const properties = byType.get(type) ?? [] + const propertyName = record.get('propertyName') + if (typeof propertyName === 'string') { + properties.push({ name: propertyName, types: stringArray(record.get('propertyTypes')) }) + } + byType.set(type, properties) + } + for (const [type, properties] of byType) relationshipSchemas.push({ type, properties }) + } catch (error) { + signal?.throwIfAborted() + logger.warn('Could not fetch Neo4j relationship properties', { error }) + } + + const constraints: Array<{ + name: string + type: string + entityType: string + properties: string[] + }> = [] + try { + const result = await session.run('SHOW CONSTRAINTS') + for (const record of result.records) { + constraints.push({ + name: String(record.get('name')), + type: String(record.get('type')), + entityType: String(record.get('entityType')), + properties: stringArray(record.get('properties')), + }) + } + } catch (error) { + signal?.throwIfAborted() + logger.warn('Could not fetch Neo4j constraints', { error }) + } + + const indexes: Array<{ name: string; type: string; entityType: string; properties: string[] }> = + [] + try { + const result = await session.run('SHOW INDEXES') + for (const record of result.records) { + indexes.push({ + name: String(record.get('name')), + type: String(record.get('type')), + entityType: String(record.get('entityType')), + properties: stringArray(record.get('properties')), + }) + } + } catch (error) { + signal?.throwIfAborted() + logger.warn('Could not fetch Neo4j indexes', { error }) + } + signal?.throwIfAborted() + + return { + message: `Database introspection completed: found ${labels.length} labels, ${relationshipTypes.length} relationship types, ${nodeSchemas.length} node schemas, ${relationshipSchemas.length} relationship schemas, ${constraints.length} constraints, ${indexes.length} indexes`, + labels, + relationshipTypes, + nodeSchemas, + relationshipSchemas, + constraints, + indexes, + } + } finally { + unbindAbort() + await session.close().catch(() => undefined) + await driver.close().catch(() => undefined) + } +} diff --git a/apps/sim/lib/internal/neo4j/values.ts b/apps/sim/lib/internal/neo4j/values.ts new file mode 100644 index 00000000000..88e71cd63c9 --- /dev/null +++ b/apps/sim/lib/internal/neo4j/values.ts @@ -0,0 +1,68 @@ +import neo4j from 'neo4j-driver' + +interface NumberLike { + toNumber(): number +} + +function isNumberLike(value: unknown): value is NumberLike { + return ( + typeof value === 'object' && + value !== null && + 'toNumber' in value && + typeof value.toNumber === 'function' + ) +} + +function asRecord(value: object): Record { + return value as Record +} + +export function convertNeo4jValue(value: unknown): unknown { + if (value === null || value === undefined) return value + if (neo4j.isInt(value) || isNumberLike(value)) return value.toNumber() + if (Array.isArray(value)) return value.map(convertNeo4jValue) + if (typeof value !== 'object') return value + + const record = asRecord(value) + if (Array.isArray(record.labels) && record.properties && 'identity' in record) { + return { + identity: isNumberLike(record.identity) ? record.identity.toNumber() : record.identity, + labels: record.labels, + properties: convertNeo4jValue(record.properties), + } + } + if ( + typeof record.type === 'string' && + record.properties && + 'identity' in record && + 'start' in record && + 'end' in record + ) { + return { + identity: isNumberLike(record.identity) ? record.identity.toNumber() : record.identity, + start: isNumberLike(record.start) ? record.start.toNumber() : record.start, + end: isNumberLike(record.end) ? record.end.toNumber() : record.end, + type: record.type, + properties: convertNeo4jValue(record.properties), + } + } + if ('start' in record && 'end' in record && Array.isArray(record.segments)) { + return { + start: convertNeo4jValue(record.start), + end: convertNeo4jValue(record.end), + segments: record.segments.map((segment) => { + const fields = typeof segment === 'object' && segment !== null ? asRecord(segment) : {} + return { + start: convertNeo4jValue(fields.start), + relationship: convertNeo4jValue(fields.relationship), + end: convertNeo4jValue(fields.end), + } + }), + length: record.length, + } + } + + return Object.fromEntries( + Object.entries(record).map(([key, entry]) => [key, convertNeo4jValue(entry)]) + ) +} diff --git a/apps/sim/lib/internal/onedrive/errors.ts b/apps/sim/lib/internal/onedrive/errors.ts new file mode 100644 index 00000000000..f9e3b0a9525 --- /dev/null +++ b/apps/sim/lib/internal/onedrive/errors.ts @@ -0,0 +1,10 @@ +export class OneDriveOperationError extends Error { + constructor( + message: string, + readonly status: number, + readonly body: Record = { success: false, error: message } + ) { + super(message) + this.name = 'OneDriveOperationError' + } +} diff --git a/apps/sim/lib/internal/onedrive/execute-tool.test.ts b/apps/sim/lib/internal/onedrive/execute-tool.test.ts new file mode 100644 index 00000000000..ae324b007b6 --- /dev/null +++ b/apps/sim/lib/internal/onedrive/execute-tool.test.ts @@ -0,0 +1,72 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + downloadOneDriveFile: vi.fn(), + uploadOneDriveFile: vi.fn(), +})) + +vi.mock('@/lib/internal/onedrive/operations', () => ({ + downloadOneDriveFile: mocks.downloadOneDriveFile, + uploadOneDriveFile: mocks.uploadOneDriveFile, +})) + +import { executeOneDriveTool } from '@/lib/internal/onedrive/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +describe('executeOneDriveTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.downloadOneDriveFile.mockResolvedValue({ success: true, output: {} }) + mocks.uploadOneDriveFile.mockResolvedValue({ success: true, output: {} }) + }) + + it('dispatches typed input with cancellation', async () => { + const controller = new AbortController() + const request: InternalToolOperationCall = { + toolId: 'onedrive_download', + input: { accessToken: 'token', fileId: 'file-1' }, + headers: new Headers(), + context: createExecutionContext(), + requestId: 'request-1', + signal: controller.signal, + } + + expect((await executeOneDriveTool(request)).status).toBe(200) + expect(mocks.downloadOneDriveFile).toHaveBeenCalledWith( + { accessToken: 'token', fileId: 'file-1', fileName: undefined }, + { signal: controller.signal } + ) + }) + + it('dispatches uploads with trusted file scope and cancellation', async () => { + const controller = new AbortController() + const input = { + accessToken: 'token', + fileName: 'notes', + content: 'hello', + file: null, + folderId: null, + mimeType: null, + values: null, + } + const response = await executeOneDriveTool({ + toolId: 'onedrive_upload', + input, + headers: new Headers(), + context: { ...createExecutionContext(), userId: 'user-1' }, + requestId: 'request-1', + signal: controller.signal, + }) + + expect(response.status).toBe(200) + expect(mocks.uploadOneDriveFile).toHaveBeenCalledWith(input, { + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + }) + }) +}) diff --git a/apps/sim/lib/internal/onedrive/execute-tool.ts b/apps/sim/lib/internal/onedrive/execute-tool.ts new file mode 100644 index 00000000000..a4791fd31d5 --- /dev/null +++ b/apps/sim/lib/internal/onedrive/execute-tool.ts @@ -0,0 +1,87 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { z } from 'zod' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { OneDriveOperationError } from '@/lib/internal/onedrive/errors' +import { downloadOneDriveFile, uploadOneDriveFile } from '@/lib/internal/onedrive/operations' +import { oneDriveUploadInputSchema } from '@/lib/internal/onedrive/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const downloadInputSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + fileId: z.string().min(1, 'File ID is required'), + fileName: z.string().optional().nullable(), +}) + +function inputSizeError(input: unknown): Response | null { + let serialized: string + try { + serialized = JSON.stringify(input) ?? '' + } catch { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + if (Buffer.byteLength(serialized) <= DEFAULT_MAX_JSON_BODY_BYTES) return null + return Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) +} + +export const executeOneDriveTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + const sizeError = inputSizeError(request.input) + if (sizeError) return sizeError + try { + switch (request.toolId) { + case 'onedrive_download': { + const parsed = downloadInputSchema.safeParse(request.input) + if (!parsed.success) return validationErrorResponse(parsed.error) + return Response.json( + await downloadOneDriveFile( + { ...parsed.data, fileName: parsed.data.fileName ?? undefined }, + { signal: request.signal } + ) + ) + } + case 'onedrive_upload': { + const parsed = oneDriveUploadInputSchema.safeParse(request.input) + if (!parsed.success) return validationErrorResponse(parsed.error) + return Response.json( + await uploadOneDriveFile(parsed.data, { + requestId: request.requestId, + signal: request.signal, + userId: request.context.userId, + }) + ) + } + default: + return Response.json( + { success: false, error: `Unsupported OneDrive tool: ${request.toolId}` }, + { status: 500 } + ) + } + } catch (error) { + request.signal?.throwIfAborted() + const status = isPayloadSizeLimitError(error) + ? 413 + : error instanceof OneDriveOperationError + ? error.status + : 500 + return Response.json( + error instanceof OneDriveOperationError + ? error.body + : { success: false, error: getErrorMessage(error, 'Unknown error occurred') }, + { status } + ) + } +} + +function validationErrorResponse(error: z.ZodError): Response { + return Response.json( + { success: false, error: getValidationErrorMessage(error, 'Invalid request data') }, + { status: 400 } + ) +} diff --git a/apps/sim/lib/internal/onedrive/operations.test.ts b/apps/sim/lib/internal/onedrive/operations.test.ts new file mode 100644 index 00000000000..975cd038361 --- /dev/null +++ b/apps/sim/lib/internal/onedrive/operations.test.ts @@ -0,0 +1,233 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + assertToolFileAccess: vi.fn(), + downloadServableFileFromStorage: vi.fn(), + processSingleFileToUserFile: vi.fn(), + secureFetchWithPinnedIP: vi.fn(), + secureFetchWithValidation: vi.fn(), + validateMicrosoftGraphId: vi.fn(), + validateUrlWithDNS: vi.fn(), +})) + +vi.mock('@/lib/core/security/input-validation', () => ({ + validateMicrosoftGraphId: mocks.validateMicrosoftGraphId, +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + secureFetchWithPinnedIP: mocks.secureFetchWithPinnedIP, + secureFetchWithValidation: mocks.secureFetchWithValidation, + validateUrlWithDNS: mocks.validateUrlWithDNS, +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertToolFileAccess, +})) + +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + getExtensionFromMimeType: vi.fn(() => 'bin'), + processSingleFileToUserFile: mocks.processSingleFileToUserFile, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mocks.downloadServableFileFromStorage, +})) + +import { downloadOneDriveFile, uploadOneDriveFile } from '@/lib/internal/onedrive/operations' + +describe('downloadOneDriveFile', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.assertToolFileAccess.mockResolvedValue(null) + mocks.validateMicrosoftGraphId.mockReturnValue({ isValid: true }) + mocks.validateUrlWithDNS.mockResolvedValue({ isValid: true, resolvedIP: '203.0.113.1' }) + mocks.secureFetchWithPinnedIP + .mockResolvedValueOnce( + Response.json({ id: 'file-1', name: 'report.pdf', file: { mimeType: 'application/pdf' } }) + ) + .mockResolvedValueOnce(new Response(new Uint8Array([1, 2, 3]))) + }) + + it('pins metadata and content requests and returns a bounded file envelope', async () => { + const controller = new AbortController() + const result = await downloadOneDriveFile( + { accessToken: 'token', fileId: 'folder/file-1' }, + { signal: controller.signal } + ) + + expect(mocks.secureFetchWithPinnedIP).toHaveBeenCalledTimes(2) + expect(mocks.secureFetchWithPinnedIP.mock.calls[0][0]).toContain('folder%2Ffile-1') + expect(mocks.secureFetchWithPinnedIP.mock.calls[1][2]).toEqual( + expect.objectContaining({ signal: controller.signal }) + ) + expect(result.output.file).toEqual({ + name: 'report.pdf', + mimeType: 'application/pdf', + data: 'AQID', + size: 3, + }) + }) + + it('uploads plain content without an HTTP route hop and preserves text-file behavior', async () => { + mocks.secureFetchWithValidation.mockResolvedValue( + Response.json({ + id: 'file-1', + name: 'notes.txt', + size: 5, + webUrl: 'https://onedrive.example/file-1', + createdDateTime: 'created', + lastModifiedDateTime: 'modified', + file: { mimeType: 'text/plain' }, + }) + ) + const controller = new AbortController() + const result = await uploadOneDriveFile( + { + accessToken: 'token', + fileName: 'notes.md', + content: 'hello', + file: null, + folderId: null, + mimeType: null, + values: null, + conflictBehavior: null, + }, + { requestId: 'request-1', signal: controller.signal, userId: 'user-1' } + ) + + expect(mocks.secureFetchWithValidation).toHaveBeenCalledWith( + expect.stringContaining('/notes.txt:/content'), + expect.objectContaining({ body: 'hello', method: 'PUT', signal: controller.signal }), + 'uploadUrl' + ) + expect(result.output.file).toMatchObject({ + id: 'file-1', + name: 'notes.txt', + mimeType: 'text/plain', + }) + }) + + it('authorizes and bounds a stored file before uploading it', async () => { + const storedFile = { + key: 'workspace/file.bin', + name: 'file.bin', + size: 3, + type: 'application/octet-stream', + } + mocks.processSingleFileToUserFile.mockReturnValue(storedFile) + mocks.downloadServableFileFromStorage.mockResolvedValue({ + buffer: Buffer.from([1, 2, 3]), + contentType: 'application/octet-stream', + }) + mocks.secureFetchWithValidation.mockResolvedValue( + Response.json({ + id: 'file-1', + name: 'file.bin', + size: 3, + webUrl: 'https://onedrive.example/file-1', + createdDateTime: 'created', + lastModifiedDateTime: 'modified', + file: { mimeType: 'application/octet-stream' }, + }) + ) + const controller = new AbortController() + await uploadOneDriveFile( + { + accessToken: 'token', + fileName: 'file.bin', + file: storedFile, + content: null, + folderId: null, + mimeType: null, + values: null, + conflictBehavior: null, + }, + { requestId: 'request-1', signal: controller.signal, userId: 'user-1' } + ) + + expect(mocks.assertToolFileAccess).toHaveBeenCalledWith( + storedFile.key, + 'user-1', + 'request-1', + expect.anything() + ) + expect(mocks.downloadServableFileFromStorage).toHaveBeenCalledWith( + storedFile, + 'request-1', + expect.anything(), + expect.objectContaining({ maxBytes: 250 * 1024 * 1024, signal: controller.signal }) + ) + expect(mocks.assertToolFileAccess).toHaveBeenCalledBefore(mocks.downloadServableFileFromStorage) + }) + + it('creates a workbook and writes bounded Excel values in the same operation', async () => { + mocks.secureFetchWithValidation + .mockResolvedValueOnce( + Response.json({ + id: 'file-1', + name: 'report.xlsx', + size: 100, + webUrl: 'https://onedrive.example/file-1', + createdDateTime: 'created', + lastModifiedDateTime: 'modified', + file: { + mimeType: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + }, + }) + ) + .mockResolvedValueOnce(Response.json({ id: 'session-1' })) + .mockResolvedValueOnce(Response.json({ value: [{ name: 'Sheet1' }] })) + .mockResolvedValueOnce( + Response.json({ + address: 'Sheet1!A1:B2', + values: [ + [1, 2], + [3, 4], + ], + }) + ) + .mockResolvedValueOnce(new Response(null, { status: 204 })) + + const result = await uploadOneDriveFile( + { + accessToken: 'token', + fileName: 'report.xlsx', + file: null, + content: null, + folderId: null, + mimeType: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + values: [ + [1, 2], + [3, 4], + ], + conflictBehavior: null, + }, + { requestId: 'request-1', userId: 'user-1' } + ) + + expect(mocks.secureFetchWithValidation).toHaveBeenCalledTimes(5) + const writeCall = mocks.secureFetchWithValidation.mock.calls[3] + expect(writeCall[0]).toContain("range(address='A1%3AB2')") + expect(writeCall[1]).toEqual( + expect.objectContaining({ + method: 'PATCH', + body: JSON.stringify({ + values: [ + [1, 2], + [3, 4], + ], + }), + }) + ) + expect(result.output.excelWriteResult).toEqual({ + success: true, + updatedRange: 'Sheet1!A1:B2', + updatedRows: 2, + updatedColumns: 2, + updatedCells: 4, + }) + }) +}) diff --git a/apps/sim/lib/internal/onedrive/operations.ts b/apps/sim/lib/internal/onedrive/operations.ts new file mode 100644 index 00000000000..0eaab6fb5f7 --- /dev/null +++ b/apps/sim/lib/internal/onedrive/operations.ts @@ -0,0 +1,511 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { isRecordLike } from '@sim/utils/object' +import * as XLSX from 'xlsx' +import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation' +import { + type SecureFetchResponse, + secureFetchWithPinnedIP, + secureFetchWithValidation, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { + isPayloadSizeLimitError, + readResponseJsonWithLimit, + readResponseTextWithLimit, + readResponseToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' +import { OneDriveOperationError } from '@/lib/internal/onedrive/errors' +import type { OneDriveUploadInput } from '@/lib/internal/onedrive/schema' +import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/doc-not-ready' +import { + getExtensionFromMimeType, + processSingleFileToUserFile, +} from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' +import { assertToolFileAccess } from '@/app/api/files/authorization' +import type { OneDriveDownloadResponse, OneDriveToolParams } from '@/tools/onedrive/types' +import { normalizeExcelValues } from '@/tools/onedrive/utils' + +const MAX_GRAPH_JSON_BYTES = 2 * 1024 * 1024 +const MAX_SIMPLE_UPLOAD_BYTES = 250 * 1024 * 1024 +const MAX_EXCEL_CELLS = 1_000_000 +const EXCEL_MIME_TYPE = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + +const logger = createLogger('OneDriveOperations') + +interface GraphApiError { + error?: { message?: string } +} + +interface DriveItemMetadata { + id?: string + name?: string + folder?: Record + file?: { mimeType?: string } +} + +type OneDriveDownloadInput = Pick & { + fileId: string +} + +export interface OneDriveOperationContext { + requestId?: string + signal?: AbortSignal + userId?: string +} + +interface OneDriveFileData extends Record { + id: string + name: string + size: number + webUrl: string + createdDateTime: string + lastModifiedDateTime: string + file?: { mimeType?: string } + parentReference?: { id: string; path: string } + '@microsoft.graph.downloadUrl'?: string +} + +interface ExcelWriteResult { + success: boolean + updatedRange?: string + updatedRows?: number + updatedColumns?: number + updatedCells?: number + error?: string + details?: string +} + +function oneDriveFileData(value: unknown): OneDriveFileData { + if (!isRecordLike(value)) throw new Error('Microsoft Graph returned invalid file metadata') + return value as OneDriveFileData +} + +function fileTooLargeError(observedBytes: number): OneDriveOperationError { + const sizeMB = (observedBytes / (1024 * 1024)).toFixed(2) + return new OneDriveOperationError( + `File size (${sizeMB}MB) exceeds OneDrive's limit of 250MB for simple uploads. Use chunked upload for larger files.`, + 400 + ) +} + +async function readGraphJson( + response: SecureFetchResponse, + signal?: AbortSignal +): Promise { + return readResponseJsonWithLimit(response, { + maxBytes: MAX_GRAPH_JSON_BYTES, + label: 'Microsoft Graph response', + signal, + }) +} + +async function graphRequest( + url: string, + init: Parameters[1], + label: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + return secureFetchWithValidation( + url, + { ...init, maxResponseBytes: MAX_GRAPH_JSON_BYTES, signal }, + label + ) +} + +function uploadedFileOutput(fileData: OneDriveFileData, mimeType: string) { + return { + id: fileData.id, + name: fileData.name, + mimeType: fileData.file?.mimeType || mimeType, + webViewLink: fileData.webUrl, + webContentLink: fileData['@microsoft.graph.downloadUrl'], + size: fileData.size, + createdTime: fileData.createdDateTime, + modifiedTime: fileData.lastModifiedDateTime, + parentReference: fileData.parentReference, + } +} + +function excelColumnName(index: number): string { + let current = index + let name = '' + while (current > 0) { + const remainder = (current - 1) % 26 + name = String.fromCharCode(65 + remainder) + name + current = Math.floor((current - 1) / 26) + } + return name +} + +function rectangularExcelValues(values: ReturnType): unknown[][] { + if (!values?.length) return [] + let rows: unknown[][] + if (Array.isArray(values[0])) { + rows = values as unknown[][] + } else { + const worksheet = XLSX.utils.json_to_sheet(values) + rows = XLSX.utils.sheet_to_json(worksheet, { header: 1, defval: '' }) + } + let columns = 0 + for (const row of rows) columns = Math.max(columns, row.length) + if (rows.length * columns > MAX_EXCEL_CELLS) { + throw new Error(`Excel values exceed the ${MAX_EXCEL_CELLS.toLocaleString()}-cell limit`) + } + return rows.map((row) => + row.length === columns + ? row + : [...row, ...Array.from({ length: columns - row.length }, () => '')] + ) +} + +async function writeExcelValues( + fileId: string, + accessToken: string, + values: ReturnType, + context: OneDriveOperationContext +): Promise { + if (!values?.length) return undefined + let sessionId: string | undefined + try { + const sessionResponse = await graphRequest( + `https://graph.microsoft.com/v1.0/me/drive/items/${encodeURIComponent(fileId)}/workbook/createSession`, + { + method: 'POST', + headers: { Authorization: `Bearer ${accessToken}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ persistChanges: true }), + }, + 'sessionUrl', + context.signal + ) + if (sessionResponse.ok) { + const session = await readGraphJson(sessionResponse, context.signal) + if (isRecordLike(session) && typeof session.id === 'string') sessionId = session.id + } else { + await sessionResponse.body?.cancel() + } + + let sheetName = 'Sheet1' + try { + const listResponse = await graphRequest( + `https://graph.microsoft.com/v1.0/me/drive/items/${encodeURIComponent(fileId)}/workbook/worksheets?$select=name&$orderby=position&$top=1`, + { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + ...(sessionId ? { 'workbook-session-id': sessionId } : {}), + }, + }, + 'listUrl', + context.signal + ) + if (listResponse.ok) { + const listed = await readGraphJson(listResponse, context.signal) + const first = isRecordLike(listed) && Array.isArray(listed.value) ? listed.value[0] : null + if (isRecordLike(first) && typeof first.name === 'string' && first.name) { + sheetName = first.name + } + } else { + await readResponseTextWithLimit(listResponse, { + maxBytes: MAX_GRAPH_JSON_BYTES, + label: 'Microsoft Graph worksheet error', + signal: context.signal, + }) + } + } catch (error) { + context.signal?.throwIfAborted() + logger.warn('Failed to list OneDrive workbook worksheets; using Sheet1', { + error: getErrorMessage(error), + requestId: context.requestId, + }) + } + + const processedValues = rectangularExcelValues(values) + const rowCount = processedValues.length + const columnCount = processedValues[0]?.length || 0 + const range = `A1:${columnCount > 0 ? excelColumnName(columnCount) : 'A'}${rowCount || 1}` + const rangeUrl = new URL( + `https://graph.microsoft.com/v1.0/me/drive/items/${encodeURIComponent(fileId)}/workbook/worksheets('${encodeURIComponent(sheetName)}')/range(address='${encodeURIComponent(range)}')` + ) + const writeResponse = await graphRequest( + rangeUrl.toString(), + { + method: 'PATCH', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + ...(sessionId ? { 'workbook-session-id': sessionId } : {}), + }, + body: JSON.stringify({ values: processedValues }), + }, + 'excelWriteUrl', + context.signal + ) + if (!writeResponse.ok) { + const details = await readResponseTextWithLimit(writeResponse, { + maxBytes: MAX_GRAPH_JSON_BYTES, + label: 'Microsoft Graph Excel error', + signal: context.signal, + }) + return { + success: false, + error: `Excel write failed: ${writeResponse.statusText || 'unknown'}`, + details, + } + } + const written = await readGraphJson(writeResponse, context.signal) + const data = isRecordLike(written) ? written : {} + const returnedValues = Array.isArray(data.values) ? data.values : [] + const firstRow = Array.isArray(returnedValues[0]) ? returnedValues[0] : [] + return { + success: true, + updatedRange: + typeof data.address === 'string' + ? data.address + : typeof data.addressLocal === 'string' + ? data.addressLocal + : undefined, + updatedRows: returnedValues.length, + updatedColumns: firstRow.length, + updatedCells: returnedValues.length * firstRow.length, + } + } catch (error) { + context.signal?.throwIfAborted() + return { success: false, error: getErrorMessage(error, 'Unknown error during Excel write') } + } finally { + if (sessionId && !context.signal?.aborted) { + try { + const response = await graphRequest( + `https://graph.microsoft.com/v1.0/me/drive/items/${encodeURIComponent(fileId)}/workbook/closeSession`, + { + method: 'POST', + headers: { + Authorization: `Bearer ${accessToken}`, + 'workbook-session-id': sessionId, + }, + }, + 'closeSessionUrl', + context.signal + ) + await response.body?.cancel() + } catch (error) { + context.signal?.throwIfAborted() + logger.warn('Failed to close OneDrive workbook session', { + error: getErrorMessage(error), + requestId: context.requestId, + }) + } + } + } +} + +export async function uploadOneDriveFile( + input: OneDriveUploadInput, + context: OneDriveOperationContext +) { + context.signal?.throwIfAborted() + const excelValues = normalizeExcelValues(input.values) + const isExcelCreation = input.mimeType === EXCEL_MIME_TYPE && !input.file + const isStoredFileMode = Boolean(input.file || isExcelCreation) + let fileBuffer: Buffer | string + let mimeType: string + let fileName = input.fileName + + if (!isStoredFileMode) { + fileBuffer = input.content || '' + mimeType = 'text/plain' + if (!fileName.endsWith('.txt')) fileName = `${fileName.replace(/\.[^.]*$/, '')}.txt` + } else if (isExcelCreation) { + const workbook = XLSX.utils.book_new() + XLSX.utils.book_append_sheet(workbook, XLSX.utils.aoa_to_sheet([[]]), 'Sheet1') + fileBuffer = Buffer.from(XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' })) + mimeType = EXCEL_MIME_TYPE + } else { + if (!input.file) throw new OneDriveOperationError('No file provided', 400) + if (!context.userId) throw new OneDriveOperationError('Authentication required', 401) + const requestId = context.requestId || 'onedrive-operation' + let userFile + try { + userFile = processSingleFileToUserFile(input.file, requestId, logger) + } catch (error) { + throw new OneDriveOperationError(getErrorMessage(error, 'Failed to process file'), 400) + } + const denied = await assertToolFileAccess(userFile.key, context.userId, requestId, logger) + context.signal?.throwIfAborted() + if (denied) throw new OneDriveOperationError('File not found', denied.status) + try { + const downloaded = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_SIMPLE_UPLOAD_BYTES, + signal: context.signal, + }) + fileBuffer = downloaded.buffer + mimeType = downloaded.contentType || userFile.type || 'application/octet-stream' + } catch (error) { + context.signal?.throwIfAborted() + if (isDocNotReadyError(error)) throw new OneDriveOperationError(docNotReadyMessage(), 409) + if (isPayloadSizeLimitError(error)) { + throw fileTooLargeError(error.observedBytes ?? userFile.size) + } + throw new OneDriveOperationError( + `Failed to download file: ${getErrorMessage(error, 'Unknown error')}`, + 500 + ) + } + } + + if (Buffer.byteLength(fileBuffer) > MAX_SIMPLE_UPLOAD_BYTES) { + throw fileTooLargeError(Buffer.byteLength(fileBuffer)) + } + const hasExtension = fileName.includes('.') && fileName.lastIndexOf('.') > 0 + if (!hasExtension) { + const extension = getExtensionFromMimeType(mimeType) + if (extension) fileName = `${fileName}.${extension}` + } else if (isExcelCreation && !fileName.endsWith('.xlsx')) { + fileName = `${fileName.replace(/\.[^.]*$/, '')}.xlsx` + } + + const folderId = input.folderId?.trim() + if (isStoredFileMode && folderId) { + const validation = validateMicrosoftGraphId(folderId, 'folderId') + if (!validation.isValid) { + throw new OneDriveOperationError(validation.error || 'Invalid folderId', 400) + } + } + const encodedName = encodeURIComponent(fileName) + let uploadUrl = folderId + ? `https://graph.microsoft.com/v1.0/me/drive/items/${encodeURIComponent(folderId)}:/${encodedName}:/content` + : `https://graph.microsoft.com/v1.0/me/drive/root:/${encodedName}:/content` + if (isStoredFileMode && input.conflictBehavior) { + uploadUrl += `?@microsoft.graph.conflictBehavior=${input.conflictBehavior}` + } + const uploadResponse = await graphRequest( + uploadUrl, + { + method: 'PUT', + headers: { Authorization: `Bearer ${input.accessToken}`, 'Content-Type': mimeType }, + body: fileBuffer, + }, + 'uploadUrl', + context.signal + ) + if (!uploadResponse.ok) { + const details = await readResponseTextWithLimit(uploadResponse, { + maxBytes: MAX_GRAPH_JSON_BYTES, + label: 'OneDrive upload error', + signal: context.signal, + }) + throw new OneDriveOperationError( + `OneDrive upload failed: ${uploadResponse.statusText}`, + uploadResponse.status, + { + success: false, + error: `OneDrive upload failed: ${uploadResponse.statusText}`, + details, + } + ) + } + const fileData = oneDriveFileData(await readGraphJson(uploadResponse, context.signal)) + const excelWriteResult = isExcelCreation + ? await writeExcelValues(fileData.id, input.accessToken, excelValues, context) + : undefined + return { + success: true as const, + output: { + file: uploadedFileOutput(fileData, mimeType), + ...(excelWriteResult ? { excelWriteResult } : {}), + }, + } +} + +async function fetchGraph( + url: string, + label: string, + accessToken: string, + maxResponseBytes: number, + signal?: AbortSignal +) { + const validation = await validateUrlWithDNS(url, label) + signal?.throwIfAborted() + if (!validation.isValid || !validation.resolvedIP) { + throw new OneDriveOperationError(validation.error || `Invalid ${label}`, 400) + } + return secureFetchWithPinnedIP(url, validation.resolvedIP, { + headers: { Authorization: `Bearer ${accessToken}` }, + maxResponseBytes, + signal, + }) +} + +async function graphError(response: SecureFetchResponse, fallback: string, signal?: AbortSignal) { + const error: GraphApiError = await readResponseJsonWithLimit(response, { + maxBytes: MAX_GRAPH_JSON_BYTES, + label: 'OneDrive error response', + signal, + }).catch((): GraphApiError => ({})) + return error.error?.message || fallback +} + +export async function downloadOneDriveFile( + input: OneDriveDownloadInput, + context: OneDriveOperationContext +): Promise { + context.signal?.throwIfAborted() + const fileId = encodeURIComponent(input.fileId) + const metadataResponse = await fetchGraph( + `https://graph.microsoft.com/v1.0/me/drive/items/${fileId}`, + 'metadataUrl', + input.accessToken, + MAX_GRAPH_JSON_BYTES, + context.signal + ) + if (!metadataResponse.ok) { + throw new OneDriveOperationError( + await graphError(metadataResponse, 'Failed to get file metadata', context.signal), + 400 + ) + } + const metadata = await readResponseJsonWithLimit(metadataResponse, { + maxBytes: MAX_GRAPH_JSON_BYTES, + label: 'OneDrive metadata response', + signal: context.signal, + }) + if (metadata.folder && !metadata.file) { + throw new OneDriveOperationError( + `Cannot download folder "${metadata.name}". Please select a file instead.`, + 400 + ) + } + + const downloadResponse = await fetchGraph( + `https://graph.microsoft.com/v1.0/me/drive/items/${fileId}/content`, + 'downloadUrl', + input.accessToken, + MAX_FILE_SIZE, + context.signal + ) + if (!downloadResponse.ok) { + throw new OneDriveOperationError( + await graphError(downloadResponse, 'Failed to download file', context.signal), + 400 + ) + } + const buffer = await readResponseToBufferWithLimit(downloadResponse, { + maxBytes: MAX_FILE_SIZE, + label: 'OneDrive file download', + signal: context.signal, + }) + return { + success: true, + output: { + file: { + name: input.fileName || metadata.name || 'download', + mimeType: metadata.file?.mimeType || 'application/octet-stream', + data: buffer.toString('base64'), + size: buffer.length, + }, + }, + } +} diff --git a/apps/sim/lib/internal/onedrive/schema.ts b/apps/sim/lib/internal/onedrive/schema.ts new file mode 100644 index 00000000000..3b0aff49041 --- /dev/null +++ b/apps/sim/lib/internal/onedrive/schema.ts @@ -0,0 +1,21 @@ +import { z } from 'zod' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' + +const excelCellSchema = z.union([z.string(), z.number(), z.boolean(), z.null()]) +const excelRowSchema = z.array(excelCellSchema) + +export const oneDriveUploadInputSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + fileName: z.string().min(1, 'File name is required'), + file: RawFileInputSchema.optional().nullable(), + content: z.string().optional().nullable(), + folderId: z.string().optional().nullable(), + mimeType: z.string().optional().nullable(), + values: z + .union([z.string(), z.array(excelRowSchema), z.array(z.record(z.string(), excelCellSchema))]) + .optional() + .nullable(), + conflictBehavior: z.enum(['fail', 'replace', 'rename']).optional().nullable(), +}) + +export type OneDriveUploadInput = z.infer diff --git a/apps/sim/lib/internal/onedrive/tool-config.test.ts b/apps/sim/lib/internal/onedrive/tool-config.test.ts new file mode 100644 index 00000000000..185b8cfee31 --- /dev/null +++ b/apps/sim/lib/internal/onedrive/tool-config.test.ts @@ -0,0 +1,21 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { uploadTool } from '@/tools/onedrive/upload' + +describe('OneDrive upload operation config', () => { + it('passes resolved text and structured values through without coercion', () => { + const values = [[''], ['{{WORKFLOW_VALUE}}']] + const input = uploadTool.operation.input({ + accessToken: '{{ONEDRIVE_TOKEN}}', + fileName: '', + content: '', + values, + }) as { content: unknown; values: unknown } + + expect(input.content).toBe('') + expect(input.values).toBe(values) + expect('request' in uploadTool).toBe(false) + }) +}) diff --git a/apps/sim/lib/internal/onepassword/client.test.ts b/apps/sim/lib/internal/onepassword/client.test.ts new file mode 100644 index 00000000000..1dda28e2e38 --- /dev/null +++ b/apps/sim/lib/internal/onepassword/client.test.ts @@ -0,0 +1,176 @@ +/** + * @vitest-environment node + */ +import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockDnsLookup, mockSecureFetch } = vi.hoisted(() => ({ + mockDnsLookup: vi.fn(), + mockSecureFetch: vi.fn(), +})) + +vi.mock('dns/promises', () => ({ + default: { lookup: mockDnsLookup }, +})) +vi.mock('@/lib/core/security/input-validation.server', () => ({ + MAX_JSON_API_RESPONSE_BYTES: 10 * 1024 * 1024, + secureFetchWithPinnedIP: mockSecureFetch, +})) + +import { connectRequest, validateConnectServerUrl } from '@/lib/internal/onepassword/client' + +afterAll(resetEnvFlagsMock) + +describe('validateConnectServerUrl', () => { + beforeEach(() => { + vi.clearAllMocks() + setEnvFlags({ isHosted: false }) + }) + + it('rejects a non-URL string', async () => { + await expect(validateConnectServerUrl('not a url')).rejects.toThrow('is not a valid URL') + }) + + describe('hosted deployment', () => { + beforeEach(() => { + setEnvFlags({ isHosted: true }) + }) + + it.each([ + ['loopback', 'http://127.0.0.1:8080'], + ['RFC1918 10.x', 'http://10.0.0.5'], + ['RFC1918 192.168.x', 'http://192.168.1.1:8443'], + ['RFC1918 172.16.x', 'http://172.16.0.9'], + ['link-local metadata', 'http://169.254.169.254'], + ['IPv4-mapped IPv6 private', 'http://[::ffff:10.0.0.1]'], + ['IPv6 loopback', 'http://[::1]'], + ])('blocks %s', async (_label, url) => { + await expect(validateConnectServerUrl(url)).rejects.toThrow( + 'cannot point to a private or reserved IP address' + ) + }) + + it('allows a public IP literal', async () => { + await expect(validateConnectServerUrl('https://8.8.8.8')).resolves.toBe('8.8.8.8') + }) + + it('blocks a hostname that resolves to a private IP', async () => { + mockDnsLookup.mockResolvedValue([{ address: '10.1.2.3', family: 4 }]) + await expect(validateConnectServerUrl('https://connect.internal')).rejects.toThrow( + 'cannot point to a private or reserved IP address' + ) + }) + + it('allows a hostname that resolves to a public IP', async () => { + mockDnsLookup.mockResolvedValue([{ address: '93.184.216.34', family: 4 }]) + await expect(validateConnectServerUrl('https://connect.example.com')).resolves.toBe( + '93.184.216.34' + ) + }) + + it('prefers the IPv4 address for a dual-stack host (avoids unreachable IPv6 pin)', async () => { + mockDnsLookup.mockResolvedValue([ + { address: '2606:4700::6810:85e5', family: 6 }, + { address: '93.184.216.34', family: 4 }, + ]) + await expect(validateConnectServerUrl('https://connect.example.com')).resolves.toBe( + '93.184.216.34' + ) + }) + + it('pins the sole IPv6 address for an IPv6-only host', async () => { + mockDnsLookup.mockResolvedValue([{ address: '2606:4700::6810:85e5', family: 6 }]) + await expect(validateConnectServerUrl('https://connect.example.com')).resolves.toBe( + '2606:4700::6810:85e5' + ) + }) + }) + + describe('self-hosted deployment', () => { + beforeEach(() => { + setEnvFlags({ isHosted: false }) + }) + + it.each([ + ['loopback', 'http://127.0.0.1:8080', '127.0.0.1'], + ['RFC1918 10.x', 'http://10.0.0.5', '10.0.0.5'], + ['RFC1918 192.168.x', 'http://192.168.1.1:8443', '192.168.1.1'], + ])('allows %s (private Connect server)', async (_label, url, expected) => { + await expect(validateConnectServerUrl(url)).resolves.toBe(expected) + }) + + it('still blocks link-local metadata', async () => { + await expect(validateConnectServerUrl('http://169.254.169.254')).rejects.toThrow( + 'cannot point to a link-local address' + ) + }) + + it('still blocks IPv6 link-local', async () => { + await expect(validateConnectServerUrl('http://[fe80::1]')).rejects.toThrow( + 'cannot point to a link-local address' + ) + }) + + it('allows a hostname that resolves to a private IP', async () => { + mockDnsLookup.mockResolvedValue([{ address: '10.1.2.3', family: 4 }]) + await expect(validateConnectServerUrl('https://connect.internal')).resolves.toBe('10.1.2.3') + }) + }) + + it('rejects when DNS resolution fails', async () => { + mockDnsLookup.mockRejectedValue(new Error('ENOTFOUND')) + await expect(validateConnectServerUrl('https://nope.invalid')).rejects.toThrow( + 'could not be resolved' + ) + }) +}) + +describe('connectRequest', () => { + beforeEach(() => { + vi.clearAllMocks() + setEnvFlags({ isHosted: false }) + mockSecureFetch.mockResolvedValue({ ok: true, status: 200 }) + }) + + it('pins the resolved server and applies the JSON byte cap and cancellation', async () => { + const controller = new AbortController() + + await connectRequest({ + serverUrl: 'https://8.8.8.8', + apiKey: 'not-a-real-connect-token', + path: '/v1/vaults', + method: 'POST', + body: { title: 'Example' }, + signal: controller.signal, + }) + + expect(mockSecureFetch).toHaveBeenCalledWith('https://8.8.8.8/v1/vaults', '8.8.8.8', { + method: 'POST', + headers: { + Authorization: 'Bearer not-a-real-connect-token', + 'Content-Type': 'application/json', + }, + body: '{"title":"Example"}', + allowHttp: true, + maxResponseBytes: 10 * 1024 * 1024, + signal: controller.signal, + }) + }) + + it('does no DNS or provider work after cancellation', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + connectRequest({ + serverUrl: 'https://connect.example.com', + apiKey: 'not-a-real-connect-token', + path: '/v1/vaults', + method: 'GET', + signal: controller.signal, + }) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mockDnsLookup).not.toHaveBeenCalled() + expect(mockSecureFetch).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/onepassword/client.ts b/apps/sim/lib/internal/onepassword/client.ts new file mode 100644 index 00000000000..8ac448c09a8 --- /dev/null +++ b/apps/sim/lib/internal/onepassword/client.ts @@ -0,0 +1,609 @@ +import type { + FileAttributes, + Item, + ItemCategory, + ItemField, + ItemFieldType, + ItemFile, + ItemOverview, + ItemSection, + VaultOverview, + Website, +} from '@1password/sdk' +import { createLogger } from '@sim/logger' +import { resolveHostAddresses } from '@sim/security/dns' +import { isPrivateIp, unwrapIpv6Brackets } from '@sim/security/ssrf' +import { toError } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import * as ipaddr from 'ipaddr.js' +import { isHosted } from '@/lib/core/config/env-flags' +import { + MAX_JSON_API_RESPONSE_BYTES, + type SecureFetchResponse, + secureFetchWithPinnedIP, +} from '@/lib/core/security/input-validation.server' + +/** Connect-format field type strings returned by normalization. */ +type ConnectFieldType = + | 'STRING' + | 'CONCEALED' + | 'EMAIL' + | 'URL' + | 'OTP' + | 'PHONE' + | 'DATE' + | 'MONTH_YEAR' + | 'MENU' + | 'ADDRESS' + | 'REFERENCE' + | 'SSHKEY' + | 'CREDIT_CARD_NUMBER' + | 'CREDIT_CARD_TYPE' + +/** Connect-format category strings returned by normalization. */ +type ConnectCategory = + | 'LOGIN' + | 'PASSWORD' + | 'API_CREDENTIAL' + | 'SECURE_NOTE' + | 'SERVER' + | 'DATABASE' + | 'CREDIT_CARD' + | 'IDENTITY' + | 'SSH_KEY' + | 'DOCUMENT' + | 'SOFTWARE_LICENSE' + | 'EMAIL_ACCOUNT' + | 'MEMBERSHIP' + | 'PASSPORT' + | 'REWARD_PROGRAM' + | 'DRIVER_LICENSE' + | 'BANK_ACCOUNT' + | 'MEDICAL_RECORD' + | 'OUTDOOR_LICENSE' + | 'WIRELESS_ROUTER' + | 'SOCIAL_SECURITY_NUMBER' + | 'CUSTOM' + +/** Normalized vault shape matching the Connect API response. */ +export interface NormalizedVault { + id: string + name: string + description: null + attributeVersion: number + contentVersion: number + items: number + type: string + createdAt: string | null + updatedAt: string | null +} + +/** Normalized item overview shape matching the Connect API response. */ +export interface NormalizedItemOverview { + id: string + title: string + vault: { id: string } + category: ConnectCategory + urls: Array<{ href: string; label: string | null; primary: boolean }> + favorite: boolean + tags: string[] + version: number + state: string | null + createdAt: string | null + updatedAt: string | null + lastEditedBy: null +} + +/** Normalized field shape matching the Connect API response. */ +interface NormalizedField { + id: string + label: string + type: ConnectFieldType + purpose: string + value: string | null + section: { id: string } | null + generate: boolean + recipe: null + entropy: null +} + +/** Normalized attached-file metadata shape matching the Connect API response. */ +export interface NormalizedItemFile { + id: string + name: string + size: number + section: { id: string } | null +} + +/** Normalized full item shape matching the Connect API response. */ +export interface NormalizedItem extends NormalizedItemOverview { + fields: NormalizedField[] + sections: Array<{ id: string; label: string }> + files: NormalizedItemFile[] +} + +/** + * SDK field type string values → Connect field type mapping. + * Uses string literals instead of enum imports to avoid loading the WASM module at build time. + */ +const SDK_TO_CONNECT_FIELD_TYPE: Record = { + Text: 'STRING', + Concealed: 'CONCEALED', + Email: 'EMAIL', + Url: 'URL', + Totp: 'OTP', + Phone: 'PHONE', + Date: 'DATE', + MonthYear: 'MONTH_YEAR', + Menu: 'MENU', + Address: 'ADDRESS', + Reference: 'REFERENCE', + SshKey: 'SSHKEY', + CreditCardNumber: 'CREDIT_CARD_NUMBER', + CreditCardType: 'CREDIT_CARD_TYPE', +} + +/** SDK category string values → Connect category mapping. */ +const SDK_TO_CONNECT_CATEGORY: Record = { + Login: 'LOGIN', + Password: 'PASSWORD', + ApiCredentials: 'API_CREDENTIAL', + SecureNote: 'SECURE_NOTE', + Server: 'SERVER', + Database: 'DATABASE', + CreditCard: 'CREDIT_CARD', + Identity: 'IDENTITY', + SshKey: 'SSH_KEY', + Document: 'DOCUMENT', + SoftwareLicense: 'SOFTWARE_LICENSE', + Email: 'EMAIL_ACCOUNT', + Membership: 'MEMBERSHIP', + Passport: 'PASSPORT', + Rewards: 'REWARD_PROGRAM', + DriverLicense: 'DRIVER_LICENSE', + BankAccount: 'BANK_ACCOUNT', + MedicalRecord: 'MEDICAL_RECORD', + OutdoorLicense: 'OUTDOOR_LICENSE', + Router: 'WIRELESS_ROUTER', + SocialSecurityNumber: 'SOCIAL_SECURITY_NUMBER', + CryptoWallet: 'CUSTOM', + Person: 'CUSTOM', + Unsupported: 'CUSTOM', +} + +/** Connect category → SDK category string mapping. */ +const CONNECT_TO_SDK_CATEGORY: Record = { + LOGIN: 'Login', + PASSWORD: 'Password', + API_CREDENTIAL: 'ApiCredentials', + SECURE_NOTE: 'SecureNote', + SERVER: 'Server', + DATABASE: 'Database', + CREDIT_CARD: 'CreditCard', + IDENTITY: 'Identity', + SSH_KEY: 'SshKey', + DOCUMENT: 'Document', + SOFTWARE_LICENSE: 'SoftwareLicense', + EMAIL_ACCOUNT: 'Email', + MEMBERSHIP: 'Membership', + PASSPORT: 'Passport', + REWARD_PROGRAM: 'Rewards', + DRIVER_LICENSE: 'DriverLicense', + BANK_ACCOUNT: 'BankAccount', + MEDICAL_RECORD: 'MedicalRecord', + OUTDOOR_LICENSE: 'OutdoorLicense', + WIRELESS_ROUTER: 'Router', + SOCIAL_SECURITY_NUMBER: 'SocialSecurityNumber', +} + +/** Connect field type → SDK field type string mapping. */ +const CONNECT_TO_SDK_FIELD_TYPE: Record = { + STRING: 'Text', + CONCEALED: 'Concealed', + EMAIL: 'Email', + URL: 'Url', + OTP: 'Totp', + TOTP: 'Totp', + PHONE: 'Phone', + DATE: 'Date', + MONTH_YEAR: 'MonthYear', + MENU: 'Menu', + ADDRESS: 'Address', + REFERENCE: 'Reference', + SSHKEY: 'SshKey', + CREDIT_CARD_NUMBER: 'CreditCardNumber', + CREDIT_CARD_TYPE: 'CreditCardType', +} + +export type ConnectionMode = 'service_account' | 'connect' + +export interface CredentialParams { + connectionMode?: ConnectionMode | null + serviceAccountToken?: string | null + serverUrl?: string | null + apiKey?: string | null +} + +export type ResolvedCredentials = + | { mode: 'service_account'; serviceAccountToken: string } + | { mode: 'connect'; serverUrl: string; apiKey: string } + +/** Determine which backend to use based on provided credentials. */ +export function resolveCredentials(params: CredentialParams): ResolvedCredentials { + const mode = params.connectionMode ?? (params.serviceAccountToken ? 'service_account' : 'connect') + + if (mode === 'service_account') { + if (!params.serviceAccountToken) { + throw new Error('Service Account token is required for Service Account mode') + } + return { mode, serviceAccountToken: params.serviceAccountToken } + } + + if (!params.serverUrl || !params.apiKey) { + throw new Error('Server URL and Connect token are required for Connect Server mode') + } + return { mode, serverUrl: params.serverUrl, apiKey: params.apiKey } +} + +/** + * Create a 1Password SDK client from a service account token. + * Uses dynamic import to avoid loading the WASM module at build time. + */ +export async function createOnePasswordClient(serviceAccountToken: string, signal?: AbortSignal) { + signal?.throwIfAborted() + const { createClient } = await import('@1password/sdk') + signal?.throwIfAborted() + const client = await createClient({ + auth: serviceAccountToken, + integrationName: 'Sim Studio', + integrationVersion: '1.0.0', + }) + signal?.throwIfAborted() + return client +} + +const connectLogger = createLogger('OnePasswordConnect') + +/** + * Enforces the SSRF policy for a resolved Connect server IP. + * + * On the hosted service, all private and reserved IPs are blocked — a tenant has + * no legitimate reason to point Connect at the platform's internal network. On + * self-hosted deployments only link-local (cloud metadata) is blocked, since the + * operator controls both the workflows and the network and Connect servers + * legitimately live on private (RFC1918) addresses. + * + * @throws Error if the IP is not permitted under the active policy. + */ +function assertConnectIpAllowed(ip: string, hostname: string): void { + if (isHosted) { + if (isPrivateIp(ip)) { + connectLogger.warn('1Password Connect server URL resolves to a private or reserved IP', { + hostname, + resolvedIP: ip, + }) + throw new Error('1Password server URL cannot point to a private or reserved IP address') + } + return + } + + if (ipaddr.isValid(ip) && ipaddr.process(ip).range() === 'linkLocal') { + connectLogger.warn('1Password Connect server URL resolves to a link-local IP', { + hostname, + resolvedIP: ip, + }) + throw new Error('1Password server URL cannot point to a link-local address') + } +} + +/** + * Validates a Connect server URL against the SSRF policy and returns the resolved + * IP for DNS pinning to prevent TOCTOU rebinding. See {@link assertConnectIpAllowed} + * for the hosted vs. self-hosted policy. + * @throws Error if the URL is invalid, fails the IP policy, or DNS fails. + */ +export async function validateConnectServerUrl( + serverUrl: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + let hostname: string + try { + hostname = new URL(serverUrl).hostname + } catch { + throw new Error('1Password server URL is not a valid URL') + } + + const clean = unwrapIpv6Brackets(hostname) + + if (ipaddr.isValid(clean)) { + assertConnectIpAllowed(clean, clean) + return clean + } + + let addresses: string[] + let address: string + try { + const resolved = await resolveHostAddresses(clean) + signal?.throwIfAborted() + addresses = resolved.addresses + address = resolved.preferred + } catch (error) { + signal?.throwIfAborted() + connectLogger.warn('DNS lookup failed for 1Password Connect server URL', { + hostname: clean, + error: toError(error).message, + }) + throw new Error('1Password server URL hostname could not be resolved') + } + + for (const candidate of addresses) { + assertConnectIpAllowed(candidate, clean) + } + return address +} + +/** + * Proxy a request to the 1Password Connect Server. + * + * The Connect server is self-hosted at a user-supplied `serverUrl`, so the response body + * is always capped. JSON endpoints use {@link MAX_JSON_API_RESPONSE_BYTES}; callers + * downloading file content pass a larger `maxResponseBytes` explicitly. + */ +export async function connectRequest(options: { + serverUrl: string + apiKey: string + path: string + method: string + body?: unknown + query?: string + maxResponseBytes?: number + signal?: AbortSignal +}): Promise { + options.signal?.throwIfAborted() + const resolvedIP = await validateConnectServerUrl(options.serverUrl, options.signal) + + const base = options.serverUrl.replace(/\/$/, '') + const queryStr = options.query ? `?${options.query}` : '' + const url = `${base}${options.path}${queryStr}` + + const headers: Record = { + Authorization: `Bearer ${options.apiKey}`, + } + + if (options.body) { + headers['Content-Type'] = 'application/json' + } + + return secureFetchWithPinnedIP(url, resolvedIP, { + method: options.method, + headers, + body: options.body ? JSON.stringify(options.body) : undefined, + allowHttp: true, + maxResponseBytes: options.maxResponseBytes ?? MAX_JSON_API_RESPONSE_BYTES, + signal: options.signal, + }) +} + +/** Normalize an SDK VaultOverview to match Connect API vault shape. */ +export function normalizeSdkVault(vault: VaultOverview): NormalizedVault { + return { + id: vault.id, + name: vault.title, + description: null, + attributeVersion: 0, + contentVersion: 0, + items: 0, + type: 'USER_CREATED', + createdAt: + vault.createdAt instanceof Date ? vault.createdAt.toISOString() : (vault.createdAt ?? null), + updatedAt: + vault.updatedAt instanceof Date ? vault.updatedAt.toISOString() : (vault.updatedAt ?? null), + } +} + +/** Normalize an SDK ItemOverview to match Connect API item summary shape. */ +export function normalizeSdkItemOverview(item: ItemOverview): NormalizedItemOverview { + return { + id: item.id, + title: item.title, + vault: { id: item.vaultId }, + category: SDK_TO_CONNECT_CATEGORY[item.category] ?? 'CUSTOM', + urls: (item.websites ?? []).map((w: Website) => ({ + href: w.url, + label: w.label ?? null, + primary: false, + })), + favorite: false, + tags: item.tags ?? [], + version: 0, + state: item.state === 'archived' ? 'ARCHIVED' : null, + createdAt: + item.createdAt instanceof Date ? item.createdAt.toISOString() : (item.createdAt ?? null), + updatedAt: + item.updatedAt instanceof Date ? item.updatedAt.toISOString() : (item.updatedAt ?? null), + lastEditedBy: null, + } +} + +/** Normalize a full SDK Item to match Connect API FullItem shape. */ +export function normalizeSdkItem(item: Item): NormalizedItem { + return { + id: item.id, + title: item.title, + vault: { id: item.vaultId }, + category: SDK_TO_CONNECT_CATEGORY[item.category] ?? 'CUSTOM', + urls: (item.websites ?? []).map((w: Website) => ({ + href: w.url, + label: w.label ?? null, + primary: false, + })), + favorite: false, + tags: item.tags ?? [], + version: item.version ?? 0, + state: null, + fields: (item.fields ?? []).map((field: ItemField) => ({ + id: field.id, + label: field.title, + type: SDK_TO_CONNECT_FIELD_TYPE[field.fieldType] ?? 'STRING', + purpose: '', + value: field.value ?? null, + section: field.sectionId ? { id: field.sectionId } : null, + generate: false, + recipe: null, + entropy: null, + })), + sections: (item.sections ?? []).map((section: ItemSection) => ({ + id: section.id, + label: section.title, + })), + files: [ + ...(item.files ?? []).map((file: ItemFile) => ({ + id: file.attributes.id, + name: file.attributes.name, + size: file.attributes.size, + section: file.sectionId ? { id: file.sectionId } : null, + })), + ...(item.document + ? [ + { + id: item.document.id, + name: item.document.name, + size: item.document.size, + section: null, + }, + ] + : []), + ], + createdAt: + item.createdAt instanceof Date ? item.createdAt.toISOString() : (item.createdAt ?? null), + updatedAt: + item.updatedAt instanceof Date ? item.updatedAt.toISOString() : (item.updatedAt ?? null), + lastEditedBy: null, + } +} + +/** + * Find an attached file's SDK {@link FileAttributes} on an item by file ID. + * Checks both the `files` array and the single `document` attribute that + * Document-category items carry instead of a `files` entry. + */ +export function findItemFileAttributes(item: Item, fileId: string): FileAttributes | undefined { + if (item.document?.id === fileId) return item.document + return item.files?.find((file) => file.attributes.id === fileId)?.attributes +} + +/** + * Convert a Connect-shaped item (the vocabulary `normalizeSdkItem` produces and + * this integration's tools document — `label`/`type`/`section: {id}`) back into + * an SDK-compatible {@link Item} for `client.items.put()`. Falls back to `existing` + * for any array the caller didn't provide, so partial input (e.g. Replace Item's + * optional fields) is preserved. + * + * Service Account mode must always convert through this function before calling + * `put()` — never apply a Connect-shaped JSON Patch directly onto a raw SDK + * {@link Item}, since SDK field/category vocabulary differs from Connect's + * (`title` vs `label`, `fieldType` vs `type`, `sectionId` vs `section.id`, SDK + * category enum strings vs Connect's SCREAMING_SNAKE_CASE) and silently no-ops or + * corrupts the write otherwise. + */ +function objectValue(value: unknown): Record { + return value !== null && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : {} +} + +function optionalString(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined +} + +export function connectItemToSdkItem(connectItem: Record, existing: Item): Item { + const existingFieldsById = new Map((existing.fields ?? []).map((f) => [f.id, f])) + const existingSectionsById = new Map((existing.sections ?? []).map((s) => [s.id, s])) + + const fields = Array.isArray(connectItem.fields) + ? connectItem.fields.map((value) => { + const field = objectValue(value) + const section = objectValue(field.section) + const id = optionalString(field.id) + return { + /** Preserve SDK-only metadata on fields that already existed. */ + ...(id ? existingFieldsById.get(id) : undefined), + id: id || generateId().slice(0, 8), + title: optionalString(field.label) || optionalString(field.title) || '', + fieldType: toSdkFieldType(optionalString(field.type) || 'STRING'), + value: optionalString(field.value) || '', + sectionId: optionalString(section.id) ?? optionalString(field.sectionId), + } + }) + : existing.fields + + const sections = Array.isArray(connectItem.sections) + ? connectItem.sections.map((value) => { + const section = objectValue(value) + const id = optionalString(section.id) + return { + ...(id ? existingSectionsById.get(id) : undefined), + id: id || '', + title: optionalString(section.label) || optionalString(section.title) || '', + } + }) + : existing.sections + + const websitesValue = connectItem.urls ?? connectItem.websites + const websites = Array.isArray(websitesValue) + ? websitesValue.map((value) => { + const website = objectValue(value) + return { + url: optionalString(website.href) || optionalString(website.url) || '', + label: optionalString(website.label) || '', + autofillBehavior: 'AnywhereOnWebsite' as const, + } + }) + : existing.websites + const category = optionalString(connectItem.category) + + return { + ...existing, + id: existing.id, + vaultId: existing.vaultId, + title: optionalString(connectItem.title) || existing.title, + category: category ? toSdkCategory(category) : existing.category, + fields, + sections, + notes: optionalString(connectItem.notes) ?? existing.notes, + tags: Array.isArray(connectItem.tags) ? (connectItem.tags as string[]) : existing.tags, + websites, + } as Item +} + +/** + * Best-effort SCIM `eq` filter matcher for Service Account mode, which has no + * server-side filtering (unlike Connect, whose `filter` query param is forwarded + * verbatim and evaluated by the Connect server). Recognizes `attribute eq "value"` + * (quotes optional) as an exact, case-insensitive match against the named attribute + * — `id` compares against the id, anything else (name/title/etc.) against the + * display value; anything that doesn't parse as `eq` falls back to a + * case-insensitive substring match against both so the field remains useful for + * free-text search. + */ +export function matchesFilter(value: string, id: string, filter: string): boolean { + const eqMatch = filter.match(/^\s*(\S+)\s+eq\s+"?([^"]*)"?\s*$/i) + if (eqMatch) { + const [, attribute, needle] = eqMatch + const target = attribute.toLowerCase() === 'id' ? id : value + return target.toLowerCase() === needle.toLowerCase() + } + const needle = filter.toLowerCase() + return value.toLowerCase().includes(needle) || id.toLowerCase().includes(needle) +} + +/** Convert a Connect-style category string to the SDK category string. */ +export function toSdkCategory(category: string): `${ItemCategory}` { + return CONNECT_TO_SDK_CATEGORY[category] ?? 'Login' +} + +/** Convert a Connect-style field type string to the SDK field type string. */ +export function toSdkFieldType(type: string): `${ItemFieldType}` { + return CONNECT_TO_SDK_FIELD_TYPE[type] ?? 'Text' +} diff --git a/apps/sim/lib/internal/onepassword/errors.ts b/apps/sim/lib/internal/onepassword/errors.ts new file mode 100644 index 00000000000..c799d7e1568 --- /dev/null +++ b/apps/sim/lib/internal/onepassword/errors.ts @@ -0,0 +1,9 @@ +export class OnePasswordOperationError extends Error { + constructor( + readonly status: number, + readonly body: Record + ) { + super(typeof body.error === 'string' ? body.error : '1Password operation failed') + this.name = 'OnePasswordOperationError' + } +} diff --git a/apps/sim/lib/internal/onepassword/execute-tool.test.ts b/apps/sim/lib/internal/onepassword/execute-tool.test.ts new file mode 100644 index 00000000000..0b78f8f9949 --- /dev/null +++ b/apps/sim/lib/internal/onepassword/execute-tool.test.ts @@ -0,0 +1,186 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operationMocks = vi.hoisted(() => ({ + executeOnePasswordCreateItem: vi.fn(), + executeOnePasswordDeleteItem: vi.fn(), + executeOnePasswordGetItem: vi.fn(), + executeOnePasswordGetItemFile: vi.fn(), + executeOnePasswordGetVault: vi.fn(), + executeOnePasswordListItems: vi.fn(), + executeOnePasswordListVaults: vi.fn(), + executeOnePasswordReplaceItem: vi.fn(), + executeOnePasswordResolveSecret: vi.fn(), + executeOnePasswordUpdateItem: vi.fn(), +})) + +vi.mock('@/lib/internal/onepassword/operations', () => operationMocks) + +import { OnePasswordOperationError } from '@/lib/internal/onepassword/errors' +import { executeOnePasswordTool } from '@/lib/internal/onepassword/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const CREDENTIALS = { + connectionMode: 'service_account', + serviceAccountToken: 'not-a-real-service-account-token', +} + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'onepassword_list_vaults', + input: CREDENTIALS, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +const TOOL_CASES = [ + ['onepassword_list_vaults', CREDENTIALS, operationMocks.executeOnePasswordListVaults], + [ + 'onepassword_get_vault', + { ...CREDENTIALS, vaultId: 'vault-1' }, + operationMocks.executeOnePasswordGetVault, + ], + [ + 'onepassword_list_items', + { ...CREDENTIALS, vaultId: 'vault-1' }, + operationMocks.executeOnePasswordListItems, + ], + [ + 'onepassword_get_item', + { ...CREDENTIALS, vaultId: 'vault-1', itemId: 'item-1' }, + operationMocks.executeOnePasswordGetItem, + ], + [ + 'onepassword_create_item', + { ...CREDENTIALS, vaultId: 'vault-1', category: 'LOGIN' }, + operationMocks.executeOnePasswordCreateItem, + ], + [ + 'onepassword_update_item', + { + ...CREDENTIALS, + vaultId: 'vault-1', + itemId: 'item-1', + operations: '[{"op":"replace","path":"/title","value":"Updated"}]', + }, + operationMocks.executeOnePasswordUpdateItem, + ], + [ + 'onepassword_replace_item', + { + ...CREDENTIALS, + vaultId: 'vault-1', + itemId: 'item-1', + item: '{"title":"Replacement"}', + }, + operationMocks.executeOnePasswordReplaceItem, + ], + [ + 'onepassword_delete_item', + { ...CREDENTIALS, vaultId: 'vault-1', itemId: 'item-1' }, + operationMocks.executeOnePasswordDeleteItem, + ], + [ + 'onepassword_resolve_secret', + { ...CREDENTIALS, secretReference: 'op://vault/item/password' }, + operationMocks.executeOnePasswordResolveSecret, + ], + [ + 'onepassword_get_item_file', + { ...CREDENTIALS, vaultId: 'vault-1', itemId: 'item-1', fileId: 'file-1' }, + operationMocks.executeOnePasswordGetItemFile, + ], +] as const + +describe('executeOnePasswordTool', () => { + beforeEach(() => { + vi.clearAllMocks() + for (const operation of Object.values(operationMocks)) { + operation.mockResolvedValue({ handled: true }) + } + }) + + it.each(TOOL_CASES)('validates and dispatches %s', async (toolId, input, operation) => { + const controller = new AbortController() + const response = await executeOnePasswordTool( + createRequest({ toolId, input, signal: controller.signal }) + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ handled: true }) + expect(operation).toHaveBeenCalledWith(expect.objectContaining(input), { + signal: controller.signal, + }) + }) + + it('preserves canonical validation envelopes for semantic input', async () => { + const invalidInput = await executeOnePasswordTool(createRequest({ input: '{' })) + expect(invalidInput.status).toBe(400) + await expect(invalidInput.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + + const invalidBody = await executeOnePasswordTool( + createRequest({ + toolId: 'onepassword_get_item', + input: { ...CREDENTIALS, vaultId: '', itemId: '' }, + }) + ) + expect(invalidBody.status).toBe(400) + await expect(invalidBody.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(operationMocks.executeOnePasswordGetItem).not.toHaveBeenCalled() + }) + + it('preserves provider statuses and operation-specific generic failures', async () => { + operationMocks.executeOnePasswordListVaults.mockRejectedValueOnce( + new OnePasswordOperationError(429, { error: 'rate limited' }) + ) + const provider = await executeOnePasswordTool(createRequest()) + expect(provider.status).toBe(429) + await expect(provider.json()).resolves.toEqual({ error: 'rate limited' }) + + operationMocks.executeOnePasswordListVaults.mockRejectedValueOnce( + new Error('network unavailable') + ) + const generic = await executeOnePasswordTool(createRequest()) + expect(generic.status).toBe(500) + await expect(generic.json()).resolves.toEqual({ + error: 'Failed to list vaults: network unavailable', + }) + }) + + it('propagates cancellation before provider work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeOnePasswordTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(operationMocks.executeOnePasswordListVaults).not.toHaveBeenCalled() + }) + + it('returns a deterministic error for unsupported IDs', async () => { + const response = await executeOnePasswordTool(createRequest({ toolId: 'onepassword_unknown' })) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'Unsupported 1Password tool: onepassword_unknown', + }) + }) +}) diff --git a/apps/sim/lib/internal/onepassword/execute-tool.ts b/apps/sim/lib/internal/onepassword/execute-tool.ts new file mode 100644 index 00000000000..47aec72789c --- /dev/null +++ b/apps/sim/lib/internal/onepassword/execute-tool.ts @@ -0,0 +1,145 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { AnyApiRouteContract, ContractBody } from '@/lib/api/contracts' +import { + onePasswordCreateItemContract, + onePasswordDeleteItemContract, + onePasswordGetItemContract, + onePasswordGetItemFileContract, + onePasswordGetVaultContract, + onePasswordListItemsContract, + onePasswordListVaultsContract, + onePasswordReplaceItemContract, + onePasswordResolveSecretContract, + onePasswordUpdateItemContract, +} from '@/lib/api/contracts/tools/onepassword' +import { OnePasswordOperationError } from '@/lib/internal/onepassword/errors' +import { + executeOnePasswordCreateItem, + executeOnePasswordDeleteItem, + executeOnePasswordGetItem, + executeOnePasswordGetItemFile, + executeOnePasswordGetVault, + executeOnePasswordListItems, + executeOnePasswordListVaults, + executeOnePasswordReplaceItem, + executeOnePasswordResolveSecret, + executeOnePasswordUpdateItem, + type OnePasswordOperationContext, +} from '@/lib/internal/onepassword/operations' +import { parseInternalToolInput } from '@/lib/internal/tool-operations/parse-input' +import type { + InternalToolOperationCall, + InternalToolOperationHandler, +} from '@/lib/internal/tool-operations/types' + +const logger = createLogger('OnePasswordToolExecution') + +async function executeOperation( + contract: C, + request: InternalToolOperationCall, + operation: (input: ContractBody, context: OnePasswordOperationContext) => Promise, + failureMessage: string +): Promise { + request.signal?.throwIfAborted() + const parsed = parseInternalToolInput(contract, request.input) + if (!parsed.success) return parsed.response + + try { + const result = await operation(parsed.data, { signal: request.signal }) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof OnePasswordOperationError) { + return Response.json(error.body, { status: error.status }) + } + const message = getErrorMessage(error, 'Unknown error') + logger.error('1Password operation failed', { + error: message, + requestId: request.requestId, + toolId: request.toolId, + }) + return Response.json({ error: `${failureMessage}: ${message}` }, { status: 500 }) + } +} + +export const executeOnePasswordTool: InternalToolOperationHandler = async (request) => { + switch (request.toolId) { + case 'onepassword_list_vaults': + return executeOperation( + onePasswordListVaultsContract, + request, + executeOnePasswordListVaults, + 'Failed to list vaults' + ) + case 'onepassword_get_vault': + return executeOperation( + onePasswordGetVaultContract, + request, + executeOnePasswordGetVault, + 'Failed to get vault' + ) + case 'onepassword_list_items': + return executeOperation( + onePasswordListItemsContract, + request, + executeOnePasswordListItems, + 'Failed to list items' + ) + case 'onepassword_get_item': + return executeOperation( + onePasswordGetItemContract, + request, + executeOnePasswordGetItem, + 'Failed to get item' + ) + case 'onepassword_create_item': + return executeOperation( + onePasswordCreateItemContract, + request, + executeOnePasswordCreateItem, + 'Failed to create item' + ) + case 'onepassword_update_item': + return executeOperation( + onePasswordUpdateItemContract, + request, + executeOnePasswordUpdateItem, + 'Failed to update item' + ) + case 'onepassword_replace_item': + return executeOperation( + onePasswordReplaceItemContract, + request, + executeOnePasswordReplaceItem, + 'Failed to replace item' + ) + case 'onepassword_delete_item': + return executeOperation( + onePasswordDeleteItemContract, + request, + executeOnePasswordDeleteItem, + 'Failed to delete item' + ) + case 'onepassword_resolve_secret': + return executeOperation( + onePasswordResolveSecretContract, + request, + executeOnePasswordResolveSecret, + 'Failed to resolve secret' + ) + case 'onepassword_get_item_file': + return executeOperation( + onePasswordGetItemFileContract, + request, + executeOnePasswordGetItemFile, + 'Failed to get item file' + ) + default: + return Response.json( + { error: `Unsupported 1Password tool: ${request.toolId}` }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/onepassword/json-patch.test.ts b/apps/sim/lib/internal/onepassword/json-patch.test.ts new file mode 100644 index 00000000000..0d84bdeee18 --- /dev/null +++ b/apps/sim/lib/internal/onepassword/json-patch.test.ts @@ -0,0 +1,36 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { applyOnePasswordPatch } from '@/lib/internal/onepassword/json-patch' + +describe('applyOnePasswordPatch', () => { + it('resolves array segments by 1Password ID before numeric index', () => { + const item: Record = { + fields: [ + { id: '1', value: 'by-id' }, + { id: 'other', value: 'by-index' }, + ], + } + + applyOnePasswordPatch(item, { + op: 'replace', + path: '/fields/1/value', + value: 'updated-by-id', + }) + + expect(item.fields).toEqual([ + { id: '1', value: 'updated-by-id' }, + { id: 'other', value: 'by-index' }, + ]) + }) + + it('supports append and remove operations', () => { + const item: Record = { tags: ['one'] } + + applyOnePasswordPatch(item, { op: 'add', path: '/tags/-', value: 'two' }) + applyOnePasswordPatch(item, { op: 'remove', path: '/tags/0' }) + + expect(item.tags).toEqual(['two']) + }) +}) diff --git a/apps/sim/lib/internal/onepassword/json-patch.ts b/apps/sim/lib/internal/onepassword/json-patch.ts new file mode 100644 index 00000000000..dbf8ac02d12 --- /dev/null +++ b/apps/sim/lib/internal/onepassword/json-patch.ts @@ -0,0 +1,78 @@ +export interface JsonPatchOperation { + op: 'add' | 'remove' | 'replace' + path: string + value?: unknown +} + +function recordValue(value: unknown): Record | undefined { + return value !== null && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : undefined +} + +function arrayIndexForSegment(target: unknown[], segment: string): number { + const byId = target.findIndex((element) => recordValue(element)?.id === segment) + if (byId !== -1) return byId + const index = Number(segment) + return Number.isInteger(index) && index >= 0 && index < target.length ? index : -1 +} + +function arrayElementForSegment(target: unknown[], segment: string): unknown { + const index = arrayIndexForSegment(target, segment) + return index === -1 ? undefined : target[index] +} + +/** Applies the Connect API's ID-aware RFC6902 path semantics. */ +export function applyOnePasswordPatch( + item: Record, + operation: JsonPatchOperation +): void { + const segments = operation.path.split('/').filter(Boolean) + const rootKey = segments[0] + if (!rootKey) return + + if (segments.length === 1) { + if (operation.op === 'replace' || operation.op === 'add') { + item[rootKey] = operation.value + } else { + delete item[rootKey] + } + return + } + + let target: unknown = item + for (let index = 0; index < segments.length - 1; index++) { + const segment = segments[index] + if (!segment) return + if (Array.isArray(target)) { + target = arrayElementForSegment(target, segment) + } else { + target = recordValue(target)?.[segment] + } + if (target === undefined || target === null) return + } + + const lastSegment = segments.at(-1) + if (!lastSegment) return + + if (operation.op === 'replace' || operation.op === 'add') { + if (Array.isArray(target) && lastSegment === '-') { + target.push(operation.value) + } else if (Array.isArray(target)) { + const index = arrayIndexForSegment(target, lastSegment) + if (index !== -1) target[index] = operation.value + } else { + const record = recordValue(target) + if (record) record[lastSegment] = operation.value + } + return + } + + if (Array.isArray(target)) { + const index = arrayIndexForSegment(target, lastSegment) + if (index !== -1) target.splice(index, 1) + } else { + const record = recordValue(target) + if (record) delete record[lastSegment] + } +} diff --git a/apps/sim/lib/internal/onepassword/operations.test.ts b/apps/sim/lib/internal/onepassword/operations.test.ts new file mode 100644 index 00000000000..76d7518f514 --- /dev/null +++ b/apps/sim/lib/internal/onepassword/operations.test.ts @@ -0,0 +1,258 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const clientMocks = vi.hoisted(() => ({ + connectItemToSdkItem: vi.fn(), + connectRequest: vi.fn(), + createOnePasswordClient: vi.fn(), + findItemFileAttributes: vi.fn(), + matchesFilter: vi.fn(), + normalizeSdkItem: vi.fn(), + normalizeSdkItemOverview: vi.fn(), + normalizeSdkVault: vi.fn(), + resolveCredentials: vi.fn(), + toSdkCategory: vi.fn(), + toSdkFieldType: vi.fn(), +})) + +vi.mock('@/lib/internal/onepassword/client', () => clientMocks) +vi.mock('@/lib/uploads/utils/validation', () => ({ MAX_FILE_SIZE: 5 })) + +import type { OnePasswordOperationError } from '@/lib/internal/onepassword/errors' +import { + executeOnePasswordGetItemFile, + executeOnePasswordListVaults, + executeOnePasswordResolveSecret, + executeOnePasswordUpdateItem, +} from '@/lib/internal/onepassword/operations' + +const SERVICE_CREDENTIALS = { + connectionMode: 'service_account' as const, + serviceAccountToken: 'not-a-real-service-account-token', +} + +const CONNECT_CREDENTIALS = { + connectionMode: 'connect' as const, + serverUrl: 'https://connect.example.com', + apiKey: 'not-a-real-connect-token', +} + +function response(options: { + status?: number + json?: unknown + bytes?: Uint8Array + contentType?: string +}) { + const status = options.status ?? 200 + const bytes = options.bytes ?? new Uint8Array() + return { + ok: status >= 200 && status < 300, + status, + statusText: '', + headers: { + get: (name: string) => + name.toLowerCase() === 'content-type' ? (options.contentType ?? null) : null, + }, + body: null, + json: async () => options.json ?? {}, + text: async () => '', + arrayBuffer: async () => + bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength), + } +} + +describe('1Password operations', () => { + beforeEach(() => { + vi.clearAllMocks() + clientMocks.resolveCredentials.mockImplementation((input: { connectionMode?: string }) => + input.connectionMode === 'connect' + ? { + mode: 'connect', + serverUrl: CONNECT_CREDENTIALS.serverUrl, + apiKey: CONNECT_CREDENTIALS.apiKey, + } + : { + mode: 'service_account', + serviceAccountToken: SERVICE_CREDENTIALS.serviceAccountToken, + } + ) + clientMocks.normalizeSdkVault.mockImplementation((vault) => vault) + clientMocks.normalizeSdkItem.mockImplementation((item) => item) + clientMocks.connectItemToSdkItem.mockImplementation((item) => item) + }) + + it('preserves Connect provider statuses and forwards cancellation', async () => { + const controller = new AbortController() + clientMocks.connectRequest.mockResolvedValue( + response({ status: 429, json: { message: 'rate limited' } }) + ) + + await expect( + executeOnePasswordListVaults(CONNECT_CREDENTIALS, { signal: controller.signal }) + ).rejects.toMatchObject>({ + status: 429, + body: { error: 'rate limited' }, + }) + expect(clientMocks.connectRequest).toHaveBeenCalledWith({ + serverUrl: CONNECT_CREDENTIALS.serverUrl, + apiKey: CONNECT_CREDENTIALS.apiKey, + path: '/v1/vaults', + method: 'GET', + query: undefined, + signal: controller.signal, + }) + }) + + it('preserves ID-aware JSON Patch semantics before an SDK update', async () => { + const existing = { + id: 'item-1', + title: 'Login', + fields: [{ id: 'password', value: 'old' }], + } + const put = vi.fn().mockImplementation(async (item) => item) + clientMocks.createOnePasswordClient.mockResolvedValue({ + items: { + get: vi.fn().mockResolvedValue(existing), + put, + }, + }) + + await executeOnePasswordUpdateItem( + { + ...SERVICE_CREDENTIALS, + vaultId: 'vault-1', + itemId: 'item-1', + operations: '[{"op":"replace","path":"/fields/password/value","value":"new"}]', + }, + {} + ) + + expect(clientMocks.connectItemToSdkItem).toHaveBeenCalledWith( + expect.objectContaining({ + fields: [{ id: 'password', value: 'new' }], + }), + existing + ) + expect(put).toHaveBeenCalledOnce() + }) + + it('bounds SDK file reads before and after materialization', async () => { + const read = vi.fn().mockResolvedValue(new Uint8Array(6)) + clientMocks.createOnePasswordClient.mockResolvedValue({ + items: { + get: vi.fn().mockResolvedValue({ id: 'item-1' }), + files: { read }, + }, + }) + clientMocks.findItemFileAttributes.mockReturnValue({ + id: 'file-1', + name: 'secret.bin', + size: 5, + }) + + await expect( + executeOnePasswordGetItemFile( + { + ...SERVICE_CREDENTIALS, + vaultId: 'vault-1', + itemId: 'item-1', + fileId: 'file-1', + }, + {} + ) + ).rejects.toThrow('1Password item file exceeds maximum size of 5 bytes') + + clientMocks.findItemFileAttributes.mockReturnValueOnce({ + id: 'file-1', + name: 'secret.bin', + size: 6, + }) + await expect( + executeOnePasswordGetItemFile( + { + ...SERVICE_CREDENTIALS, + vaultId: 'vault-1', + itemId: 'item-1', + fileId: 'file-1', + }, + {} + ) + ).rejects.toThrow('1Password item file exceeds maximum size of 5 bytes') + expect(read).toHaveBeenCalledOnce() + }) + + it('keeps Connect file content bounded and preserves the file envelope', async () => { + const controller = new AbortController() + clientMocks.connectRequest + .mockResolvedValueOnce(response({ json: { name: 'secret.txt', size: 5 } })) + .mockResolvedValueOnce( + response({ bytes: new TextEncoder().encode('hello'), contentType: 'text/plain' }) + ) + + const result = await executeOnePasswordGetItemFile( + { + ...CONNECT_CREDENTIALS, + vaultId: 'vault-1', + itemId: 'item-1', + fileId: 'file-1', + }, + { signal: controller.signal } + ) + + expect(result).toEqual({ + file: { + name: 'secret.txt', + mimeType: 'text/plain', + data: Buffer.from('hello').toString('base64'), + size: 5, + }, + }) + expect(clientMocks.connectRequest.mock.calls[1]?.[0]).toMatchObject({ + maxResponseBytes: 5, + signal: controller.signal, + }) + }) + + it('preserves the private secret value and rejects Connect mode', async () => { + const resolve = vi.fn().mockResolvedValue('resolved-secret') + clientMocks.createOnePasswordClient.mockResolvedValue({ secrets: { resolve } }) + + await expect( + executeOnePasswordResolveSecret( + { ...SERVICE_CREDENTIALS, secretReference: 'op://vault/item/password' }, + {} + ) + ).resolves.toEqual({ + value: 'resolved-secret', + reference: 'op://vault/item/password', + }) + + await expect( + executeOnePasswordResolveSecret( + { ...CONNECT_CREDENTIALS, secretReference: 'op://vault/item/password' }, + {} + ) + ).rejects.toMatchObject>({ + status: 400, + body: { error: 'Resolve Secret is only available in Service Account mode' }, + }) + }) + + it('propagates cancellation after an SDK call returns', async () => { + const controller = new AbortController() + clientMocks.createOnePasswordClient.mockResolvedValue({ + vaults: { + list: vi.fn().mockImplementation(async () => { + controller.abort(new DOMException('cancelled', 'AbortError')) + return [] + }), + }, + }) + + await expect( + executeOnePasswordListVaults(SERVICE_CREDENTIALS, { signal: controller.signal }) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) +}) diff --git a/apps/sim/lib/internal/onepassword/operations.ts b/apps/sim/lib/internal/onepassword/operations.ts new file mode 100644 index 00000000000..27721610f4b --- /dev/null +++ b/apps/sim/lib/internal/onepassword/operations.ts @@ -0,0 +1,445 @@ +import type { ItemCreateParams } from '@1password/sdk' +import { generateId } from '@sim/utils/id' +import type { ContractBody } from '@/lib/api/contracts' +import type { + onePasswordCreateItemContract, + onePasswordDeleteItemContract, + onePasswordGetItemContract, + onePasswordGetItemFileContract, + onePasswordGetVaultContract, + onePasswordListItemsContract, + onePasswordListVaultsContract, + onePasswordReplaceItemContract, + onePasswordResolveSecretContract, + onePasswordUpdateItemContract, +} from '@/lib/api/contracts/tools/onepassword' +import { assertKnownSizeWithinLimit } from '@/lib/core/utils/stream-limits' +import { + connectItemToSdkItem, + connectRequest, + createOnePasswordClient, + findItemFileAttributes, + matchesFilter, + normalizeSdkItem, + normalizeSdkItemOverview, + normalizeSdkVault, + resolveCredentials, + toSdkCategory, + toSdkFieldType, +} from '@/lib/internal/onepassword/client' +import { OnePasswordOperationError } from '@/lib/internal/onepassword/errors' +import { + applyOnePasswordPatch, + type JsonPatchOperation, +} from '@/lib/internal/onepassword/json-patch' +import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' + +export interface OnePasswordOperationContext { + signal?: AbortSignal +} + +type ListVaultsInput = ContractBody +type GetVaultInput = ContractBody +type ListItemsInput = ContractBody +type GetItemInput = ContractBody +type CreateItemInput = ContractBody +type UpdateItemInput = ContractBody +type ReplaceItemInput = ContractBody +type DeleteItemInput = ContractBody +type ResolveSecretInput = ContractBody +type GetItemFileInput = ContractBody + +function asRecord(value: unknown): Record { + return value !== null && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : {} +} + +function providerMessage(data: unknown, fallback: string): string { + const message = asRecord(data).message + return typeof message === 'string' && message ? message : fallback +} + +async function runSdk(signal: AbortSignal | undefined, operation: () => Promise): Promise { + signal?.throwIfAborted() + const result = await operation() + signal?.throwIfAborted() + return result +} + +export async function executeOnePasswordListVaults( + input: ListVaultsInput, + context: OnePasswordOperationContext +): Promise { + const credentials = resolveCredentials(input) + if (credentials.mode === 'service_account') { + const client = await createOnePasswordClient(credentials.serviceAccountToken, context.signal) + const vaults = await runSdk(context.signal, () => client.vaults.list()) + const normalized = vaults.map(normalizeSdkVault) + const filter = input.filter + if (!filter) return normalized + return normalized.filter((vault) => matchesFilter(vault.name ?? '', vault.id ?? '', filter)) + } + + const response = await connectRequest({ + serverUrl: credentials.serverUrl, + apiKey: credentials.apiKey, + path: '/v1/vaults', + method: 'GET', + query: input.filter ? `filter=${encodeURIComponent(input.filter)}` : undefined, + signal: context.signal, + }) + const data = await response.json() + context.signal?.throwIfAborted() + if (!response.ok) { + throw new OnePasswordOperationError(response.status, { + error: providerMessage(data, 'Failed to list vaults'), + }) + } + return data +} + +export async function executeOnePasswordGetVault( + input: GetVaultInput, + context: OnePasswordOperationContext +): Promise { + const credentials = resolveCredentials(input) + if (credentials.mode === 'service_account') { + const client = await createOnePasswordClient(credentials.serviceAccountToken, context.signal) + const vaults = await runSdk(context.signal, () => client.vaults.list()) + const vault = vaults.find((candidate) => candidate.id === input.vaultId) + if (!vault) { + throw new OnePasswordOperationError(404, { error: 'Vault not found' }) + } + return normalizeSdkVault(vault) + } + + const response = await connectRequest({ + serverUrl: credentials.serverUrl, + apiKey: credentials.apiKey, + path: `/v1/vaults/${input.vaultId}`, + method: 'GET', + signal: context.signal, + }) + const data = await response.json() + context.signal?.throwIfAborted() + if (!response.ok) { + throw new OnePasswordOperationError(response.status, { + error: providerMessage(data, 'Failed to get vault'), + }) + } + return data +} + +export async function executeOnePasswordListItems( + input: ListItemsInput, + context: OnePasswordOperationContext +): Promise { + const credentials = resolveCredentials(input) + if (credentials.mode === 'service_account') { + const client = await createOnePasswordClient(credentials.serviceAccountToken, context.signal) + const items = await runSdk(context.signal, () => client.items.list(input.vaultId)) + const normalized = items.map(normalizeSdkItemOverview) + const filter = input.filter + if (!filter) return normalized + return normalized.filter((item) => matchesFilter(item.title ?? '', item.id ?? '', filter)) + } + + const response = await connectRequest({ + serverUrl: credentials.serverUrl, + apiKey: credentials.apiKey, + path: `/v1/vaults/${input.vaultId}/items`, + method: 'GET', + query: input.filter ? `filter=${encodeURIComponent(input.filter)}` : undefined, + signal: context.signal, + }) + const data = await response.json() + context.signal?.throwIfAborted() + if (!response.ok) { + throw new OnePasswordOperationError(response.status, { + error: providerMessage(data, 'Failed to list items'), + }) + } + return data +} + +export async function executeOnePasswordGetItem( + input: GetItemInput, + context: OnePasswordOperationContext +): Promise { + const credentials = resolveCredentials(input) + if (credentials.mode === 'service_account') { + const client = await createOnePasswordClient(credentials.serviceAccountToken, context.signal) + const item = await runSdk(context.signal, () => client.items.get(input.vaultId, input.itemId)) + return normalizeSdkItem(item) + } + + const response = await connectRequest({ + serverUrl: credentials.serverUrl, + apiKey: credentials.apiKey, + path: `/v1/vaults/${input.vaultId}/items/${input.itemId}`, + method: 'GET', + signal: context.signal, + }) + const data = await response.json() + context.signal?.throwIfAborted() + if (!response.ok) { + throw new OnePasswordOperationError(response.status, { + error: providerMessage(data, 'Failed to get item'), + }) + } + return data +} + +export async function executeOnePasswordCreateItem( + input: CreateItemInput, + context: OnePasswordOperationContext +): Promise { + const credentials = resolveCredentials(input) + if (credentials.mode === 'service_account') { + const client = await createOnePasswordClient(credentials.serviceAccountToken, context.signal) + const tags = input.tags + ? input.tags + .split(',') + .map((tag) => tag.trim()) + .filter(Boolean) + : undefined + const fields = input.fields + ? (JSON.parse(input.fields) as Array>).map((field) => { + const section = asRecord(field.section) + return { + id: (field.id as string) || generateId().slice(0, 8), + title: (field.label as string) || (field.title as string) || '', + fieldType: toSdkFieldType((field.type as string) || 'STRING'), + value: (field.value as string) || '', + sectionId: + (section.id as string | undefined) ?? (field.sectionId as string | undefined), + } + }) + : undefined + const item = await runSdk(context.signal, () => + client.items.create({ + vaultId: input.vaultId, + category: toSdkCategory(input.category), + title: input.title || '', + tags, + fields, + } as ItemCreateParams) + ) + return normalizeSdkItem(item) + } + + const body: Record = { + vault: { id: input.vaultId }, + category: input.category, + } + if (input.title) body.title = input.title + if (input.tags) body.tags = input.tags.split(',').map((tag) => tag.trim()) + if (input.fields) body.fields = JSON.parse(input.fields) + const response = await connectRequest({ + serverUrl: credentials.serverUrl, + apiKey: credentials.apiKey, + path: `/v1/vaults/${input.vaultId}/items`, + method: 'POST', + body, + signal: context.signal, + }) + const data = await response.json() + context.signal?.throwIfAborted() + if (!response.ok) { + throw new OnePasswordOperationError(response.status, { + error: providerMessage(data, 'Failed to create item'), + }) + } + return data +} + +export async function executeOnePasswordUpdateItem( + input: UpdateItemInput, + context: OnePasswordOperationContext +): Promise { + const credentials = resolveCredentials(input) + const operations = JSON.parse(input.operations) as JsonPatchOperation[] + if (credentials.mode === 'service_account') { + const client = await createOnePasswordClient(credentials.serviceAccountToken, context.signal) + const existing = await runSdk(context.signal, () => + client.items.get(input.vaultId, input.itemId) + ) + const connectItem: Record = { ...normalizeSdkItem(existing) } + for (const operation of operations) applyOnePasswordPatch(connectItem, operation) + const result = await runSdk(context.signal, () => + client.items.put(connectItemToSdkItem(connectItem, existing)) + ) + return normalizeSdkItem(result) + } + + const response = await connectRequest({ + serverUrl: credentials.serverUrl, + apiKey: credentials.apiKey, + path: `/v1/vaults/${input.vaultId}/items/${input.itemId}`, + method: 'PATCH', + body: operations, + signal: context.signal, + }) + const data = await response.json() + context.signal?.throwIfAborted() + if (!response.ok) { + throw new OnePasswordOperationError(response.status, { + error: providerMessage(data, 'Failed to update item'), + }) + } + return data +} + +export async function executeOnePasswordReplaceItem( + input: ReplaceItemInput, + context: OnePasswordOperationContext +): Promise { + const credentials = resolveCredentials(input) + const itemData = JSON.parse(input.item) as Record + if (credentials.mode === 'service_account') { + const client = await createOnePasswordClient(credentials.serviceAccountToken, context.signal) + const existing = await runSdk(context.signal, () => + client.items.get(input.vaultId, input.itemId) + ) + const result = await runSdk(context.signal, () => + client.items.put(connectItemToSdkItem(itemData, existing)) + ) + return normalizeSdkItem(result) + } + + const response = await connectRequest({ + serverUrl: credentials.serverUrl, + apiKey: credentials.apiKey, + path: `/v1/vaults/${input.vaultId}/items/${input.itemId}`, + method: 'PUT', + body: itemData, + signal: context.signal, + }) + const data = await response.json() + context.signal?.throwIfAborted() + if (!response.ok) { + throw new OnePasswordOperationError(response.status, { + error: providerMessage(data, 'Failed to replace item'), + }) + } + return data +} + +export async function executeOnePasswordDeleteItem( + input: DeleteItemInput, + context: OnePasswordOperationContext +): Promise<{ success: true }> { + const credentials = resolveCredentials(input) + if (credentials.mode === 'service_account') { + const client = await createOnePasswordClient(credentials.serviceAccountToken, context.signal) + await runSdk(context.signal, () => client.items.delete(input.vaultId, input.itemId)) + return { success: true } + } + + const response = await connectRequest({ + serverUrl: credentials.serverUrl, + apiKey: credentials.apiKey, + path: `/v1/vaults/${input.vaultId}/items/${input.itemId}`, + method: 'DELETE', + signal: context.signal, + }) + context.signal?.throwIfAborted() + if (!response.ok) { + const data = await response.json().catch(() => ({})) + context.signal?.throwIfAborted() + throw new OnePasswordOperationError(response.status, { + error: providerMessage(data, 'Failed to delete item'), + }) + } + return { success: true } +} + +export async function executeOnePasswordResolveSecret( + input: ResolveSecretInput, + context: OnePasswordOperationContext +): Promise<{ value: string; reference: string }> { + const credentials = resolveCredentials(input) + if (credentials.mode !== 'service_account') { + throw new OnePasswordOperationError(400, { + error: 'Resolve Secret is only available in Service Account mode', + }) + } + const client = await createOnePasswordClient(credentials.serviceAccountToken, context.signal) + const value = await runSdk(context.signal, () => client.secrets.resolve(input.secretReference)) + return { value, reference: input.secretReference } +} + +export async function executeOnePasswordGetItemFile( + input: GetItemFileInput, + context: OnePasswordOperationContext +): Promise<{ + file: { name: string; mimeType: string; data: string; size: number } +}> { + const credentials = resolveCredentials(input) + if (credentials.mode === 'service_account') { + const client = await createOnePasswordClient(credentials.serviceAccountToken, context.signal) + const item = await runSdk(context.signal, () => client.items.get(input.vaultId, input.itemId)) + const attributes = findItemFileAttributes(item, input.fileId) + if (!attributes) { + throw new OnePasswordOperationError(404, { error: 'File not found on item' }) + } + assertKnownSizeWithinLimit(attributes.size, MAX_FILE_SIZE, '1Password item file') + const content = await runSdk(context.signal, () => + client.items.files.read(input.vaultId, input.itemId, attributes) + ) + assertKnownSizeWithinLimit(content.byteLength, MAX_FILE_SIZE, '1Password item file') + const buffer = Buffer.from(content.buffer, content.byteOffset, content.byteLength) + return { + file: { + name: attributes.name, + mimeType: 'application/octet-stream', + data: buffer.toString('base64'), + size: attributes.size, + }, + } + } + + const metadataResponse = await connectRequest({ + serverUrl: credentials.serverUrl, + apiKey: credentials.apiKey, + path: `/v1/vaults/${input.vaultId}/items/${input.itemId}/files/${input.fileId}`, + method: 'GET', + signal: context.signal, + }) + if (!metadataResponse.ok) { + const data = await metadataResponse.json().catch(() => ({})) + context.signal?.throwIfAborted() + throw new OnePasswordOperationError(metadataResponse.status, { + error: providerMessage(data, 'Failed to get file metadata'), + }) + } + const metadata = asRecord(await metadataResponse.json()) + context.signal?.throwIfAborted() + + const contentResponse = await connectRequest({ + serverUrl: credentials.serverUrl, + apiKey: credentials.apiKey, + path: `/v1/vaults/${input.vaultId}/items/${input.itemId}/files/${input.fileId}/content`, + method: 'GET', + maxResponseBytes: MAX_FILE_SIZE, + signal: context.signal, + }) + if (!contentResponse.ok) { + const data = await contentResponse.json().catch(() => ({})) + context.signal?.throwIfAborted() + throw new OnePasswordOperationError(contentResponse.status, { + error: providerMessage(data, 'Failed to download file content'), + }) + } + const buffer = Buffer.from(await contentResponse.arrayBuffer()) + context.signal?.throwIfAborted() + return { + file: { + name: typeof metadata.name === 'string' ? metadata.name : 'attachment', + mimeType: contentResponse.headers.get('content-type') || 'application/octet-stream', + data: buffer.toString('base64'), + size: typeof metadata.size === 'number' ? metadata.size : buffer.length, + }, + } +} diff --git a/apps/sim/lib/internal/outlook/client.test.ts b/apps/sim/lib/internal/outlook/client.test.ts new file mode 100644 index 00000000000..1954a636768 --- /dev/null +++ b/apps/sim/lib/internal/outlook/client.test.ts @@ -0,0 +1,104 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { OutlookClient } from '@/lib/internal/outlook/client' +import { OutlookOperationError } from '@/lib/internal/outlook/errors' + +describe('OutlookClient', () => { + const fetchMock = vi.fn() + + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', fetchMock) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('sends OAuth credentials, provider input, and cancellation to Microsoft Graph', async () => { + fetchMock.mockResolvedValue( + Response.json({ id: 'copied-1', parentFolderId: 'folder-1' }, { status: 200 }) + ) + const controller = new AbortController() + const client = new OutlookClient('access-token') + + const result = await client.json( + '/me/messages/message-1/copy', + { method: 'POST', body: JSON.stringify({ destinationId: 'folder-1' }) }, + 'Failed to copy email', + controller.signal + ) + + expect(result).toEqual({ id: 'copied-1', parentFolderId: 'folder-1' }) + expect(fetchMock).toHaveBeenCalledWith( + 'https://graph.microsoft.com/v1.0/me/messages/message-1/copy', + expect.objectContaining({ + method: 'POST', + signal: controller.signal, + headers: expect.objectContaining({ + Authorization: 'Bearer access-token', + 'Content-Type': 'application/json', + }), + }) + ) + }) + + it('preserves Graph status and message errors', async () => { + fetchMock.mockResolvedValue( + Response.json({ error: { message: 'Message not found' } }, { status: 404 }) + ) + const client = new OutlookClient('access-token') + + await expect( + client.json('/me/messages/missing', { method: 'GET' }, 'Failed to read email') + ).rejects.toEqual(new OutlookOperationError('Message not found', 404)) + }) + + it('uses operation fallback errors for malformed Graph error bodies', async () => { + fetchMock.mockResolvedValue(new Response('bad gateway', { status: 502 })) + const client = new OutlookClient('access-token') + + await expect( + client.empty('/me/sendMail', { method: 'POST' }, 'Failed to send email') + ).rejects.toEqual(new OutlookOperationError('Failed to send email', 502)) + }) + + it('caps Graph JSON responses before materializing oversized bodies', async () => { + fetchMock.mockResolvedValue( + new Response('{}', { + status: 200, + headers: { 'content-length': String(10 * 1024 * 1024 + 1) }, + }) + ) + const client = new OutlookClient('access-token') + + await expect( + client.json('/me/messages/message-1', { method: 'GET' }, 'Failed to read email') + ).rejects.toEqual( + new PayloadSizeLimitError({ + label: 'Microsoft Graph response', + maxBytes: 10 * 1024 * 1024, + observedBytes: 10 * 1024 * 1024 + 1, + }) + ) + }) + + it('stops before provider work when already cancelled', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + const client = new OutlookClient('access-token') + + await expect( + client.json( + '/me/messages/message-1', + { method: 'GET' }, + 'Failed to read email', + controller.signal + ) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(fetchMock).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/outlook/client.ts b/apps/sim/lib/internal/outlook/client.ts new file mode 100644 index 00000000000..7c4e70035bb --- /dev/null +++ b/apps/sim/lib/internal/outlook/client.ts @@ -0,0 +1,101 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' +import { OutlookOperationError } from '@/lib/internal/outlook/errors' + +const MICROSOFT_GRAPH_BASE_URL = 'https://graph.microsoft.com/v1.0' +const MICROSOFT_GRAPH_RESPONSE_MAX_BYTES = 10 * 1024 * 1024 + +export type OutlookJsonObject = Record + +export function asObject(value: unknown): OutlookJsonObject { + return value && typeof value === 'object' && !Array.isArray(value) + ? (value as OutlookJsonObject) + : {} +} + +function parseJson(text: string): OutlookJsonObject { + if (!text) return {} + return asObject(JSON.parse(text)) +} + +function graphErrorMessage(data: OutlookJsonObject, fallback: string): string { + const error = asObject(data.error) + return typeof error.message === 'string' && error.message ? error.message : fallback +} + +export class OutlookClient { + constructor(private readonly accessToken: string) {} + + private url(path: string): string { + return `${MICROSOFT_GRAPH_BASE_URL}${path}` + } + + async json( + path: string, + init: RequestInit, + fallbackError: string, + signal?: AbortSignal + ): Promise { + signal?.throwIfAborted() + const response = await fetch(this.url(path), { + ...init, + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.accessToken}`, + ...init.headers, + }, + signal, + }) + const text = await readResponseTextWithLimit(response, { + maxBytes: MICROSOFT_GRAPH_RESPONSE_MAX_BYTES, + label: 'Microsoft Graph response', + signal, + }) + signal?.throwIfAborted() + + let data: OutlookJsonObject + try { + data = parseJson(text) + } catch (error) { + if (!response.ok) throw new OutlookOperationError(fallbackError, response.status) + throw new Error(getErrorMessage(error, 'Microsoft Graph returned invalid JSON')) + } + if (!response.ok) { + throw new OutlookOperationError(graphErrorMessage(data, fallbackError), response.status) + } + return data + } + + async empty( + path: string, + init: RequestInit, + fallbackError: string, + signal?: AbortSignal + ): Promise { + signal?.throwIfAborted() + const response = await fetch(this.url(path), { + ...init, + headers: { + Authorization: `Bearer ${this.accessToken}`, + ...init.headers, + }, + signal, + }) + if (!response.ok) { + const text = await readResponseTextWithLimit(response, { + maxBytes: MICROSOFT_GRAPH_RESPONSE_MAX_BYTES, + label: 'Microsoft Graph error response', + signal, + }) + let data: OutlookJsonObject = {} + try { + data = parseJson(text) + } catch { + data = {} + } + throw new OutlookOperationError(graphErrorMessage(data, fallbackError), response.status) + } + await response.body?.cancel() + signal?.throwIfAborted() + } +} diff --git a/apps/sim/lib/internal/outlook/errors.ts b/apps/sim/lib/internal/outlook/errors.ts new file mode 100644 index 00000000000..3122aeece66 --- /dev/null +++ b/apps/sim/lib/internal/outlook/errors.ts @@ -0,0 +1,10 @@ +export class OutlookOperationError extends Error { + constructor( + message: string, + readonly status: number, + readonly body: { success: false; error: string } = { success: false, error: message } + ) { + super(message) + this.name = 'OutlookOperationError' + } +} diff --git a/apps/sim/lib/internal/outlook/execute-tool.test.ts b/apps/sim/lib/internal/outlook/execute-tool.test.ts new file mode 100644 index 00000000000..63a032b41b4 --- /dev/null +++ b/apps/sim/lib/internal/outlook/execute-tool.test.ts @@ -0,0 +1,171 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operationMocks = vi.hoisted(() => ({ + executeOutlookCopy: vi.fn(), + executeOutlookDelete: vi.fn(), + executeOutlookDraft: vi.fn(), + executeOutlookMarkRead: vi.fn(), + executeOutlookMarkUnread: vi.fn(), + executeOutlookMove: vi.fn(), + executeOutlookSend: vi.fn(), +})) + +vi.mock('@/lib/internal/outlook/operations', () => operationMocks) + +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { OutlookOperationError } from '@/lib/internal/outlook/errors' +import { executeOutlookTool } from '@/lib/internal/outlook/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const MESSAGE_BODY = { accessToken: 'access-token', messageId: 'message-1' } +const COPY_MOVE_BODY = { ...MESSAGE_BODY, destinationId: 'folder-1' } +const MAIL_BODY = { + accessToken: 'access-token', + to: 'recipient@example.com', + subject: 'Hello', + body: 'Message body', +} + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'outlook_copy', + input: COPY_MOVE_BODY, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +const TOOL_CASES = [ + ['outlook_copy', COPY_MOVE_BODY, operationMocks.executeOutlookCopy, 'provider'], + ['outlook_delete', MESSAGE_BODY, operationMocks.executeOutlookDelete, 'provider'], + ['outlook_draft', MAIL_BODY, operationMocks.executeOutlookDraft, 'mail'], + ['outlook_mark_read', MESSAGE_BODY, operationMocks.executeOutlookMarkRead, 'provider'], + ['outlook_mark_unread', MESSAGE_BODY, operationMocks.executeOutlookMarkUnread, 'provider'], + ['outlook_move', COPY_MOVE_BODY, operationMocks.executeOutlookMove, 'provider'], + ['outlook_send', MAIL_BODY, operationMocks.executeOutlookSend, 'mail'], +] as const + +describe('executeOutlookTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it.each(TOOL_CASES)( + 'validates and dispatches %s', + async (toolId, input, operation, operationKind) => { + const controller = new AbortController() + operation.mockResolvedValue({ success: true, output: { toolId } }) + + const response = await executeOutlookTool( + createRequest({ toolId, input, signal: controller.signal }) + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ success: true, output: { toolId } }) + if (operationKind === 'mail') { + expect(operation).toHaveBeenCalledWith(input, { + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + }) + } else { + expect(operation).toHaveBeenCalledWith(input, controller.signal) + } + } + ) + + it('returns the canonical validation envelope before provider work', async () => { + const response = await executeOutlookTool( + createRequest({ input: { accessToken: '', messageId: '' } }) + ) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(operationMocks.executeOutlookCopy).not.toHaveBeenCalled() + }) + + it('rejects non-object operation input', async () => { + const response = await executeOutlookTool(createRequest({ input: '{' })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(operationMocks.executeOutlookCopy).not.toHaveBeenCalled() + }) + + it('rejects oversized bodies before parsing or provider work', async () => { + const response = await executeOutlookTool( + createRequest({ input: { body: ' '.repeat(DEFAULT_MAX_JSON_BODY_BYTES + 1) } }) + ) + + expect(response.status).toBe(413) + await expect(response.json()).resolves.toEqual({ + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }) + expect(operationMocks.executeOutlookCopy).not.toHaveBeenCalled() + }) + + it('preserves provider status and error envelopes', async () => { + operationMocks.executeOutlookCopy.mockRejectedValue( + new OutlookOperationError('Message not found', 404) + ) + + const response = await executeOutlookTool(createRequest()) + + expect(response.status).toBe(404) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Message not found', + }) + }) + + it('preserves unexpected errors', async () => { + operationMocks.executeOutlookCopy.mockRejectedValue(new Error('Microsoft unavailable')) + + const response = await executeOutlookTool(createRequest()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Microsoft unavailable', + }) + }) + + it('rejects unsupported Outlook IDs without provider work', async () => { + const response = await executeOutlookTool(createRequest({ toolId: 'outlook_unknown' })) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Unsupported Outlook tool: outlook_unknown', + }) + expect(operationMocks.executeOutlookCopy).not.toHaveBeenCalled() + }) + + it('propagates cancellation without starting provider work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeOutlookTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(operationMocks.executeOutlookCopy).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/outlook/execute-tool.ts b/apps/sim/lib/internal/outlook/execute-tool.ts new file mode 100644 index 00000000000..b209433362c --- /dev/null +++ b/apps/sim/lib/internal/outlook/execute-tool.ts @@ -0,0 +1,117 @@ +import { getErrorMessage } from '@sim/utils/errors' +import type { AnyApiRouteContract, ContractBody } from '@/lib/api/contracts' +import { + outlookCopyContract, + outlookDeleteContract, + outlookDraftContract, + outlookMarkReadContract, + outlookMarkUnreadContract, + outlookMoveContract, + outlookSendContract, +} from '@/lib/api/contracts/tools/microsoft' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { OutlookOperationError } from '@/lib/internal/outlook/errors' +import { + executeOutlookCopy, + executeOutlookDelete, + executeOutlookDraft, + executeOutlookMarkRead, + executeOutlookMarkUnread, + executeOutlookMove, + executeOutlookSend, + type OutlookMailOperationContext, +} from '@/lib/internal/outlook/operations' +import { parseInternalToolInput } from '@/lib/internal/tool-operations/parse-input' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +async function executeOperation( + contract: C, + input: unknown, + execute: (input: ContractBody) => Promise, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const parsed = parseInternalToolInput(contract, input, { + maxInputBytes: DEFAULT_MAX_JSON_BODY_BYTES, + }) + if (!parsed.success) return parsed.response + try { + const result = await execute(parsed.data) + signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + signal?.throwIfAborted() + if (error instanceof OutlookOperationError) { + return Response.json(error.body, { status: error.status }) + } + return Response.json( + { success: false, error: getErrorMessage(error, 'Unknown error occurred') }, + { status: 500 } + ) + } +} + +export const executeOutlookTool: InternalToolOperationHandler = async (request) => { + const { input, context, requestId, signal, toolId } = request + const mailContext: OutlookMailOperationContext = { + requestId, + signal, + userId: context.userId, + } + switch (toolId) { + case 'outlook_copy': + return executeOperation( + outlookCopyContract, + input, + (input) => executeOutlookCopy(input, signal), + signal + ) + case 'outlook_delete': + return executeOperation( + outlookDeleteContract, + input, + (input) => executeOutlookDelete(input, signal), + signal + ) + case 'outlook_draft': + return executeOperation( + outlookDraftContract, + input, + (input) => executeOutlookDraft(input, mailContext), + signal + ) + case 'outlook_mark_read': + return executeOperation( + outlookMarkReadContract, + input, + (input) => executeOutlookMarkRead(input, signal), + signal + ) + case 'outlook_mark_unread': + return executeOperation( + outlookMarkUnreadContract, + input, + (input) => executeOutlookMarkUnread(input, signal), + signal + ) + case 'outlook_move': + return executeOperation( + outlookMoveContract, + input, + (input) => executeOutlookMove(input, signal), + signal + ) + case 'outlook_send': + return executeOperation( + outlookSendContract, + input, + (input) => executeOutlookSend(input, mailContext), + signal + ) + default: + return Response.json( + { success: false, error: `Unsupported Outlook tool: ${toolId}` }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/outlook/operations.test.ts b/apps/sim/lib/internal/outlook/operations.test.ts new file mode 100644 index 00000000000..2587dc1acca --- /dev/null +++ b/apps/sim/lib/internal/outlook/operations.test.ts @@ -0,0 +1,335 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + assertToolFileAccess: vi.fn(), + downloadServableFilesWithinBudget: vi.fn(), + empty: vi.fn(), + json: vi.fn(), + processFilesToUserFiles: vi.fn(), +})) + +vi.mock('@/lib/internal/outlook/client', () => ({ + OutlookClient: class { + json(...args: unknown[]) { + return mocks.json(...args) + } + + empty(...args: unknown[]) { + return mocks.empty(...args) + } + }, +})) +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertToolFileAccess, +})) +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + processFilesToUserFiles: mocks.processFilesToUserFiles, +})) +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFilesWithinBudget: mocks.downloadServableFilesWithinBudget, +})) + +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { OutlookOperationError } from '@/lib/internal/outlook/errors' +import { + executeOutlookCopy, + executeOutlookDelete, + executeOutlookDraft, + executeOutlookMarkRead, + executeOutlookMarkUnread, + executeOutlookMove, + executeOutlookSend, +} from '@/lib/internal/outlook/operations' + +const MAIL_INPUT = { + accessToken: 'access-token', + to: 'first@example.com, second@example.com', + subject: 'Hello', + body: 'Message body', + contentType: 'html' as const, + cc: 'cc@example.com', + bcc: 'bcc@example.com', +} + +const MAIL_CONTEXT = { + requestId: 'request-1', + userId: 'user-1', +} + +const RAW_ATTACHMENT = { + id: 'file-1', + key: 'workspace/file-1', + name: 'report.pdf', + size: 6, + type: 'application/pdf', +} + +const USER_FILE = { + ...RAW_ATTACHMENT, + url: '/api/files/serve?key=workspace/file-1', + context: 'workspace', +} + +describe('Outlook operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.assertToolFileAccess.mockResolvedValue(null) + mocks.downloadServableFilesWithinBudget.mockResolvedValue([ + { buffer: Buffer.from('report'), contentType: 'application/pdf' }, + ]) + mocks.empty.mockResolvedValue(undefined) + mocks.json.mockResolvedValue({}) + mocks.processFilesToUserFiles.mockReturnValue([]) + }) + + it('copies messages using encoded Graph paths and preserves outputs', async () => { + const controller = new AbortController() + mocks.json.mockResolvedValue({ id: 'copied-1', parentFolderId: 'folder-1' }) + + const result = await executeOutlookCopy( + { + accessToken: 'access-token', + messageId: 'message/1', + destinationId: 'folder-1', + }, + controller.signal + ) + + expect(mocks.json).toHaveBeenCalledWith( + '/me/messages/message%2F1/copy', + { method: 'POST', body: JSON.stringify({ destinationId: 'folder-1' }) }, + 'Failed to copy email', + controller.signal + ) + expect(result).toEqual({ + success: true, + output: { + message: 'Email copied successfully', + originalMessageId: 'message/1', + copiedMessageId: 'copied-1', + destinationFolderId: 'folder-1', + }, + }) + }) + + it('deletes messages and preserves the route output contract', async () => { + const result = await executeOutlookDelete({ + accessToken: 'access-token', + messageId: 'message-1', + }) + + expect(mocks.empty).toHaveBeenCalledWith( + '/me/messages/message-1', + { method: 'DELETE' }, + 'Failed to delete email', + undefined + ) + expect(result.output).toEqual({ + message: 'Email moved to Deleted Items successfully', + messageId: 'message-1', + status: 'deleted', + }) + }) + + it.each([ + [executeOutlookMarkRead, true, 'read'], + [executeOutlookMarkUnread, false, 'unread'], + ] as const)('updates message read state', async (execute, isRead, label) => { + mocks.json.mockResolvedValue({ id: 'message-1', isRead }) + + const result = await execute({ accessToken: 'access-token', messageId: 'message-1' }) + + expect(mocks.json).toHaveBeenCalledWith( + '/me/messages/message-1', + { method: 'PATCH', body: JSON.stringify({ isRead }) }, + `Failed to mark email as ${label}`, + undefined + ) + expect(result.output).toMatchObject({ messageId: 'message-1', isRead }) + }) + + it('moves messages and returns the new canonical IDs', async () => { + mocks.json.mockResolvedValue({ id: 'moved-1', parentFolderId: 'folder-2' }) + + const result = await executeOutlookMove({ + accessToken: 'access-token', + messageId: 'message-1', + destinationId: 'folder-2', + }) + + expect(mocks.json).toHaveBeenCalledWith( + '/me/messages/message-1/move', + { method: 'POST', body: JSON.stringify({ destinationId: 'folder-2' }) }, + 'Failed to move email', + undefined + ) + expect(result.output).toMatchObject({ messageId: 'moved-1', newFolderId: 'folder-2' }) + }) + + it('creates drafts with recipients and exact output fields', async () => { + mocks.json.mockResolvedValue({ id: 'draft-1', subject: 'Hello' }) + + const result = await executeOutlookDraft(MAIL_INPUT, MAIL_CONTEXT) + + expect(mocks.json).toHaveBeenCalledWith( + '/me/messages', + { + method: 'POST', + body: JSON.stringify({ + subject: 'Hello', + body: { contentType: 'html', content: 'Message body' }, + toRecipients: [ + { emailAddress: { address: 'first@example.com' } }, + { emailAddress: { address: 'second@example.com' } }, + ], + ccRecipients: [{ emailAddress: { address: 'cc@example.com' } }], + bccRecipients: [{ emailAddress: { address: 'bcc@example.com' } }], + }), + }, + 'Failed to create draft', + undefined + ) + expect(result.output).toEqual({ + message: 'Draft created successfully', + messageId: 'draft-1', + subject: 'Hello', + attachmentCount: 0, + }) + }) + + it('sends new messages with the Graph sendMail envelope', async () => { + const result = await executeOutlookSend(MAIL_INPUT, MAIL_CONTEXT) + + const [path, init, fallback] = mocks.empty.mock.calls[0] + expect(path).toBe('/me/sendMail') + expect(fallback).toBe('Failed to send email') + expect(JSON.parse(init.body)).toMatchObject({ + saveToSentItems: true, + message: { + subject: 'Hello', + body: { contentType: 'html', content: 'Message body' }, + }, + }) + expect(result.output).toMatchObject({ + message: 'Email sent successfully', + status: 'sent', + attachmentCount: 0, + timestamp: expect.any(String), + }) + }) + + it('preserves reply envelopes and encodes reply message IDs', async () => { + await executeOutlookSend({ ...MAIL_INPUT, replyToMessageId: 'message/1' }, MAIL_CONTEXT) + + const [path, init] = mocks.empty.mock.calls[0] + expect(path).toBe('/me/messages/message%2F1/reply') + expect(JSON.parse(init.body)).toMatchObject({ + comment: 'Message body', + message: { subject: 'Hello' }, + }) + }) + + it.each([ + [executeOutlookSend, 3 * 1024 * 1024, '3MB', 'Microsoft Graph API limit'], + [executeOutlookDraft, 4 * 1024 * 1024, '4MB', "Outlook's limit"], + ] as const)( + 'enforces the operation attachment cap before file access', + async (execute, maxBytes, limitLabel, providerLabel) => { + mocks.processFilesToUserFiles.mockReturnValue([{ ...USER_FILE, size: maxBytes + 1 }]) + + await expect( + execute({ ...MAIL_INPUT, attachments: [RAW_ATTACHMENT] }, MAIL_CONTEXT) + ).rejects.toMatchObject({ + status: 400, + message: expect.stringContaining(`${providerLabel} of ${limitLabel} per request`), + }) + expect(mocks.assertToolFileAccess).not.toHaveBeenCalled() + } + ) + + it('authorizes, bounds, and attaches resolved servable bytes', async () => { + const controller = new AbortController() + mocks.processFilesToUserFiles.mockReturnValue([USER_FILE]) + + const result = await executeOutlookDraft( + { ...MAIL_INPUT, attachments: [RAW_ATTACHMENT] }, + { ...MAIL_CONTEXT, signal: controller.signal } + ) + + expect(mocks.assertToolFileAccess).toHaveBeenCalledWith( + 'workspace/file-1', + 'user-1', + 'request-1', + expect.any(Object) + ) + expect(mocks.downloadServableFilesWithinBudget).toHaveBeenCalledWith( + [USER_FILE], + 'request-1', + expect.any(Object), + { + totalMaxBytes: 4 * 1024 * 1024, + label: 'Total attachment size', + signal: controller.signal, + } + ) + const message = JSON.parse(mocks.json.mock.calls[0][1].body) + expect(message.attachments).toEqual([ + { + '@odata.type': '#microsoft.graph.fileAttachment', + name: 'report.pdf', + contentType: 'application/pdf', + contentBytes: Buffer.from('report').toString('base64'), + }, + ]) + expect(result.output.attachmentCount).toBe(1) + }) + + it('fails closed when file access is denied', async () => { + mocks.processFilesToUserFiles.mockReturnValue([USER_FILE]) + mocks.assertToolFileAccess.mockResolvedValue(new Response(null, { status: 404 })) + + await expect( + executeOutlookSend({ ...MAIL_INPUT, attachments: [RAW_ATTACHMENT] }, MAIL_CONTEXT) + ).rejects.toEqual(new OutlookOperationError('File not found', 404)) + expect(mocks.downloadServableFilesWithinBudget).not.toHaveBeenCalled() + }) + + it('maps delivered-byte overruns to the exact Outlook size envelope', async () => { + mocks.processFilesToUserFiles.mockReturnValue([USER_FILE]) + mocks.downloadServableFilesWithinBudget.mockRejectedValue( + new PayloadSizeLimitError({ + label: 'Total attachment size', + maxBytes: 4 * 1024 * 1024, + observedBytes: 5 * 1024 * 1024, + }) + ) + + await expect( + executeOutlookDraft({ ...MAIL_INPUT, attachments: [RAW_ATTACHMENT] }, MAIL_CONTEXT) + ).rejects.toMatchObject({ + status: 400, + message: "Total attachment size (5.00MB) exceeds Outlook's limit of 4MB per request", + }) + }) + + it('requires an authenticated user for send and draft operations', async () => { + await expect(executeOutlookSend(MAIL_INPUT, { requestId: 'request-1' })).rejects.toEqual( + new OutlookOperationError('Authentication required', 401) + ) + expect(mocks.empty).not.toHaveBeenCalled() + }) + + it('propagates cancellation before file or provider work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeOutlookDraft(MAIL_INPUT, { ...MAIL_CONTEXT, signal: controller.signal }) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mocks.json).not.toHaveBeenCalled() + expect(mocks.assertToolFileAccess).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/outlook/operations.ts b/apps/sim/lib/internal/outlook/operations.ts new file mode 100644 index 00000000000..1f6594c1f0a --- /dev/null +++ b/apps/sim/lib/internal/outlook/operations.ts @@ -0,0 +1,294 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { + OutlookCopyBody, + OutlookDeleteBody, + OutlookDraftBody, + OutlookMarkReadBody, + OutlookMarkUnreadBody, + OutlookMoveBody, + OutlookSendBody, +} from '@/lib/api/contracts/tools/microsoft' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { OutlookClient, type OutlookJsonObject } from '@/lib/internal/outlook/client' +import { OutlookOperationError } from '@/lib/internal/outlook/errors' +import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/doc-not-ready' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFilesWithinBudget } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +const logger = createLogger('OutlookOperations') +const OUTLOOK_SEND_ATTACHMENT_MAX_BYTES = 3 * 1024 * 1024 +const OUTLOOK_DRAFT_ATTACHMENT_MAX_BYTES = 4 * 1024 * 1024 + +interface OutlookMailOperationContext { + requestId: string + signal?: AbortSignal + userId?: string +} + +interface OutlookRecipient { + emailAddress: { address: string } +} + +interface OutlookFileAttachment { + '@odata.type': '#microsoft.graph.fileAttachment' + name: string + contentType: string + contentBytes: string +} + +interface OutlookMessagePayload { + subject: string + body: { contentType: 'text' | 'html'; content: string } + toRecipients: OutlookRecipient[] + ccRecipients?: OutlookRecipient[] + bccRecipients?: OutlookRecipient[] + attachments?: OutlookFileAttachment[] +} + +function optionalString(data: OutlookJsonObject, key: string): string | undefined { + return typeof data[key] === 'string' ? data[key] : undefined +} + +function optionalBoolean(data: OutlookJsonObject, key: string): boolean | undefined { + return typeof data[key] === 'boolean' ? data[key] : undefined +} + +function recipients(value: string): OutlookRecipient[] { + return value.split(',').map((email) => ({ emailAddress: { address: email.trim() } })) +} + +function requireUser(context: OutlookMailOperationContext): string { + context.signal?.throwIfAborted() + if (!context.userId) throw new OutlookOperationError('Authentication required', 401) + return context.userId +} + +function attachmentSizeError(observedBytes: number, kind: 'send' | 'draft'): OutlookOperationError { + const sizeMB = (observedBytes / (1024 * 1024)).toFixed(2) + const message = + kind === 'send' + ? `Total attachment size (${sizeMB}MB) exceeds Microsoft Graph API limit of 3MB per request` + : `Total attachment size (${sizeMB}MB) exceeds Outlook's limit of 4MB per request` + return new OutlookOperationError(message, 400) +} + +async function resolveAttachments( + rawAttachments: OutlookSendBody['attachments'] | OutlookDraftBody['attachments'], + context: OutlookMailOperationContext, + kind: 'send' | 'draft' +): Promise { + if (!rawAttachments?.length) return [] + const userId = requireUser(context) + const attachments = processFilesToUserFiles(rawAttachments, context.requestId, logger) + if (attachments.length === 0) return [] + const maxBytes = + kind === 'send' ? OUTLOOK_SEND_ATTACHMENT_MAX_BYTES : OUTLOOK_DRAFT_ATTACHMENT_MAX_BYTES + const declaredSize = attachments.reduce((total, file) => total + file.size, 0) + if (declaredSize > maxBytes) throw attachmentSizeError(declaredSize, kind) + + for (const file of attachments) { + context.signal?.throwIfAborted() + const denied = await assertToolFileAccess(file.key, userId, context.requestId, logger) + context.signal?.throwIfAborted() + if (denied) throw new OutlookOperationError('File not found', denied.status) + } + + let resolved: Awaited> + try { + resolved = await downloadServableFilesWithinBudget(attachments, context.requestId, logger, { + totalMaxBytes: maxBytes, + label: 'Total attachment size', + signal: context.signal, + }) + } catch (error) { + context.signal?.throwIfAborted() + if (isDocNotReadyError(error)) { + throw new OutlookOperationError(docNotReadyMessage(), 409) + } + if (isPayloadSizeLimitError(error)) { + throw attachmentSizeError(error.observedBytes ?? declaredSize, kind) + } + throw new OutlookOperationError( + `Failed to download attachment: ${getErrorMessage(error, 'Unknown error')}`, + 500 + ) + } + context.signal?.throwIfAborted() + + return attachments.map((file, index) => { + const resolvedFile = resolved[index] + if (!resolvedFile) { + throw new OutlookOperationError('Failed to download attachment: Missing file data', 500) + } + return { + '@odata.type': '#microsoft.graph.fileAttachment', + name: file.name, + contentType: resolvedFile.contentType || file.type || 'application/octet-stream', + contentBytes: resolvedFile.buffer.toString('base64'), + } + }) +} + +async function buildMessage( + input: OutlookSendBody | OutlookDraftBody, + context: OutlookMailOperationContext, + kind: 'send' | 'draft' +): Promise { + const message: OutlookMessagePayload = { + subject: input.subject, + body: { contentType: input.contentType || 'text', content: input.body }, + toRecipients: recipients(input.to), + } + if (input.cc) message.ccRecipients = recipients(input.cc) + if (input.bcc) message.bccRecipients = recipients(input.bcc) + const attachments = await resolveAttachments(input.attachments, context, kind) + if (attachments.length > 0) message.attachments = attachments + return message +} + +export async function executeOutlookCopy(input: OutlookCopyBody, signal?: AbortSignal) { + const client = new OutlookClient(input.accessToken) + const data = await client.json( + `/me/messages/${encodeURIComponent(input.messageId)}/copy`, + { method: 'POST', body: JSON.stringify({ destinationId: input.destinationId }) }, + 'Failed to copy email', + signal + ) + return { + success: true as const, + output: { + message: 'Email copied successfully', + originalMessageId: input.messageId, + copiedMessageId: optionalString(data, 'id'), + destinationFolderId: optionalString(data, 'parentFolderId'), + }, + } +} + +export async function executeOutlookDelete(input: OutlookDeleteBody, signal?: AbortSignal) { + const client = new OutlookClient(input.accessToken) + await client.empty( + `/me/messages/${encodeURIComponent(input.messageId)}`, + { method: 'DELETE' }, + 'Failed to delete email', + signal + ) + return { + success: true as const, + output: { + message: 'Email moved to Deleted Items successfully', + messageId: input.messageId, + status: 'deleted', + }, + } +} + +export async function executeOutlookDraft( + input: OutlookDraftBody, + context: OutlookMailOperationContext +) { + requireUser(context) + const message = await buildMessage(input, context, 'draft') + const client = new OutlookClient(input.accessToken) + const data = await client.json( + '/me/messages', + { method: 'POST', body: JSON.stringify(message) }, + 'Failed to create draft', + context.signal + ) + return { + success: true as const, + output: { + message: 'Draft created successfully', + messageId: optionalString(data, 'id'), + subject: optionalString(data, 'subject'), + attachmentCount: message.attachments?.length || 0, + }, + } +} + +async function executeOutlookReadState( + input: OutlookMarkReadBody | OutlookMarkUnreadBody, + isRead: boolean, + signal?: AbortSignal +) { + const client = new OutlookClient(input.accessToken) + const fallback = isRead ? 'Failed to mark email as read' : 'Failed to mark email as unread' + const data = await client.json( + `/me/messages/${encodeURIComponent(input.messageId)}`, + { method: 'PATCH', body: JSON.stringify({ isRead }) }, + fallback, + signal + ) + return { + success: true as const, + output: { + message: isRead ? 'Email marked as read successfully' : 'Email marked as unread successfully', + messageId: optionalString(data, 'id'), + isRead: optionalBoolean(data, 'isRead'), + }, + } +} + +export function executeOutlookMarkRead(input: OutlookMarkReadBody, signal?: AbortSignal) { + return executeOutlookReadState(input, true, signal) +} + +export function executeOutlookMarkUnread(input: OutlookMarkUnreadBody, signal?: AbortSignal) { + return executeOutlookReadState(input, false, signal) +} + +export async function executeOutlookMove(input: OutlookMoveBody, signal?: AbortSignal) { + const client = new OutlookClient(input.accessToken) + const data = await client.json( + `/me/messages/${encodeURIComponent(input.messageId)}/move`, + { method: 'POST', body: JSON.stringify({ destinationId: input.destinationId }) }, + 'Failed to move email', + signal + ) + return { + success: true as const, + output: { + message: 'Email moved successfully', + messageId: optionalString(data, 'id'), + newFolderId: optionalString(data, 'parentFolderId'), + }, + } +} + +export async function executeOutlookSend( + input: OutlookSendBody, + context: OutlookMailOperationContext +) { + requireUser(context) + const message = await buildMessage(input, context, 'send') + const client = new OutlookClient(input.accessToken) + const replyToMessageId = input.replyToMessageId + await client.empty( + replyToMessageId + ? `/me/messages/${encodeURIComponent(replyToMessageId)}/reply` + : '/me/sendMail', + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify( + replyToMessageId ? { comment: input.body, message } : { message, saveToSentItems: true } + ), + }, + 'Failed to send email', + context.signal + ) + return { + success: true as const, + output: { + message: 'Email sent successfully', + status: 'sent', + timestamp: new Date().toISOString(), + attachmentCount: message.attachments?.length || 0, + }, + } +} + +export type { OutlookMailOperationContext } diff --git a/apps/sim/lib/internal/persona/errors.ts b/apps/sim/lib/internal/persona/errors.ts new file mode 100644 index 00000000000..02e5651ccaf --- /dev/null +++ b/apps/sim/lib/internal/persona/errors.ts @@ -0,0 +1,9 @@ +export class PersonaOperationError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'PersonaOperationError' + } +} diff --git a/apps/sim/lib/internal/persona/execute-tool.test.ts b/apps/sim/lib/internal/persona/execute-tool.test.ts new file mode 100644 index 00000000000..2861e91d5ac --- /dev/null +++ b/apps/sim/lib/internal/persona/execute-tool.test.ts @@ -0,0 +1,44 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ importPersonaAccounts: vi.fn() })) + +vi.mock('@/lib/internal/persona/operations', () => ({ + importPersonaAccounts: mocks.importPersonaAccounts, +})) + +import { executePersonaTool } from '@/lib/internal/persona/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +describe('executePersonaTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.importPersonaAccounts.mockResolvedValue({ success: true, output: {} }) + }) + + it('uses the trusted execution user for stored-file authorization', async () => { + const controller = new AbortController() + const input = { + apiKey: 'token', + file: { key: 'workspace/file.csv', name: 'file.csv', size: 3 }, + } + const request: InternalToolOperationCall = { + toolId: 'persona_import_accounts', + input, + headers: new Headers(), + context: { ...createExecutionContext(), userId: 'user-1' }, + requestId: 'request-1', + signal: controller.signal, + } + + expect((await executePersonaTool(request)).status).toBe(200) + expect(mocks.importPersonaAccounts).toHaveBeenCalledWith(input, { + userId: 'user-1', + requestId: 'request-1', + signal: controller.signal, + }) + }) +}) diff --git a/apps/sim/lib/internal/persona/execute-tool.ts b/apps/sim/lib/internal/persona/execute-tool.ts new file mode 100644 index 00000000000..c791bbc84ef --- /dev/null +++ b/apps/sim/lib/internal/persona/execute-tool.ts @@ -0,0 +1,53 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { z } from 'zod' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { PersonaOperationError } from '@/lib/internal/persona/errors' +import { importPersonaAccounts } from '@/lib/internal/persona/operations' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' + +const inputSchema = z.object({ + apiKey: z.string().min(1, 'Persona API key is required'), + file: RawFileInputSchema, +}) + +export const executePersonaTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'persona_import_accounts') { + return Response.json( + { success: false, error: `Unsupported Persona tool: ${request.toolId}` }, + { status: 500 } + ) + } + const userId = request.context.userId + if (!userId) { + return Response.json({ success: false, error: 'Unauthorized' }, { status: 401 }) + } + const parsed = inputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + try { + return Response.json( + await importPersonaAccounts(parsed.data, { + userId, + requestId: request.requestId, + signal: request.signal, + }) + ) + } catch (error) { + request.signal?.throwIfAborted() + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + const status = isPayloadSizeLimitError(error) + ? 413 + : error instanceof PersonaOperationError + ? error.status + : 500 + return Response.json( + { success: false, error: getErrorMessage(error, 'Internal server error') }, + { status } + ) + } +} diff --git a/apps/sim/lib/internal/persona/operations.test.ts b/apps/sim/lib/internal/persona/operations.test.ts new file mode 100644 index 00000000000..c5eaf1ce5b9 --- /dev/null +++ b/apps/sim/lib/internal/persona/operations.test.ts @@ -0,0 +1,75 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + assertToolFileAccess: vi.fn(), + downloadServableFileFromStorage: vi.fn(), + fetch: vi.fn(), +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertToolFileAccess, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mocks.downloadServableFileFromStorage, +})) + +import { importPersonaAccounts } from '@/lib/internal/persona/operations' + +describe('importPersonaAccounts', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mocks.fetch) + mocks.assertToolFileAccess.mockResolvedValue(null) + mocks.downloadServableFileFromStorage.mockResolvedValue({ buffer: Buffer.from('a,b\n1,2') }) + mocks.fetch.mockResolvedValue( + Response.json({ + data: { + id: 'impr_1', + attributes: { status: 'pending', 'successful-count': 0, 'error-count': 0 }, + }, + }) + ) + }) + + it('authorizes and materializes the stored file before one provider submission', async () => { + const controller = new AbortController() + const result = await importPersonaAccounts( + { + apiKey: 'token', + file: { key: 'workspace/file.csv', name: 'file.csv', size: 7 }, + }, + { userId: 'user-1', requestId: 'request-1', signal: controller.signal } + ) + + expect(mocks.assertToolFileAccess).toHaveBeenCalledWith( + 'workspace/file.csv', + 'user-1', + 'request-1', + expect.anything() + ) + expect(mocks.fetch).toHaveBeenCalledTimes(1) + expect(mocks.fetch.mock.calls[0][1]).toEqual( + expect.objectContaining({ signal: controller.signal }) + ) + expect(result.output.importer.id).toBe('impr_1') + }) + + it('fails closed before provider work when file access is denied', async () => { + mocks.assertToolFileAccess.mockResolvedValue(new Response(null, { status: 404 })) + + await expect( + importPersonaAccounts( + { + apiKey: 'token', + file: { key: 'workspace/file.csv', name: 'file.csv', size: 7 }, + }, + { userId: 'user-1', requestId: 'request-1' } + ) + ).rejects.toMatchObject({ status: 404 }) + expect(mocks.fetch).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/persona/operations.ts b/apps/sim/lib/internal/persona/operations.ts new file mode 100644 index 00000000000..3d8ec065e74 --- /dev/null +++ b/apps/sim/lib/internal/persona/operations.ts @@ -0,0 +1,80 @@ +import { createLogger } from '@sim/logger' +import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' +import { PersonaOperationError } from '@/lib/internal/persona/errors' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' +import type { PersonaImportAccountsResponse } from '@/tools/persona/types' +import { + buildPersonaHeaders, + extractPersonaErrorMessage, + mapImporter, + PERSONA_API_BASE, + type PersonaResourceData, +} from '@/tools/persona/utils' + +const logger = createLogger('PersonaImportAccountsOperation') +const MAX_PERSONA_RESPONSE_BYTES = 2 * 1024 * 1024 + +export interface PersonaImportAccountsInput { + apiKey: string + file: RawFileInput +} + +export interface PersonaOperationContext { + userId: string + requestId: string + signal?: AbortSignal +} + +export async function importPersonaAccounts( + input: PersonaImportAccountsInput, + context: PersonaOperationContext +): Promise { + context.signal?.throwIfAborted() + const userFile = processFilesToUserFiles([input.file], context.requestId, logger)[0] + if (!userFile) { + throw new PersonaOperationError('Invalid file input: a stored CSV file is required', 400) + } + const denied = await assertToolFileAccess(userFile.key, context.userId, context.requestId, logger) + if (denied) throw new PersonaOperationError('File not found', denied.status) + + const resolved = await downloadServableFileFromStorage(userFile, context.requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) + const buffer = resolved.buffer + context.signal?.throwIfAborted() + const response = await fetch(`${PERSONA_API_BASE}/importer/accounts`, { + method: 'POST', + headers: buildPersonaHeaders(input.apiKey), + body: JSON.stringify({ + data: { + attributes: { + file: { data: buffer.toString('base64'), filename: userFile.name }, + }, + }, + }), + signal: context.signal, + }) + const data = await readResponseJsonWithLimit<{ data?: PersonaResourceData } | null>(response, { + maxBytes: MAX_PERSONA_RESPONSE_BYTES, + label: 'Persona import accounts response', + signal: context.signal, + }).catch(() => null) + if (!response.ok) { + throw new PersonaOperationError( + extractPersonaErrorMessage(data, `Persona API error: ${response.statusText}`), + response.status + ) + } + const importer = mapImporter(data?.data ?? {}) + if (!importer.id) { + throw new PersonaOperationError( + 'Persona returned an unexpected response for the account import', + 502 + ) + } + return { success: true, output: { importer } } +} diff --git a/apps/sim/lib/internal/pipedrive/client.ts b/apps/sim/lib/internal/pipedrive/client.ts new file mode 100644 index 00000000000..c4427e3f26c --- /dev/null +++ b/apps/sim/lib/internal/pipedrive/client.ts @@ -0,0 +1,120 @@ +import { isRecordLike } from '@sim/utils/object' +import { + MAX_JSON_API_RESPONSE_BYTES, + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { + readResponseJsonWithLimit, + readResponseToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' +import { PipedriveOperationError } from '@/lib/internal/pipedrive/errors' +import { getPipedriveAuthHeaders } from '@/tools/pipedrive/utils' + +export interface PipedriveFile { + id?: number + name?: string + url?: string + [key: string]: unknown +} + +export interface PipedriveFilesPage { + files: PipedriveFile[] + hasMore: boolean + nextStart: number | null +} + +function isPipedriveHost(url: string): boolean { + try { + const hostname = new URL(url).hostname.toLowerCase() + return hostname === 'pipedrive.com' || hostname.endsWith('.pipedrive.com') + } catch { + return false + } +} + +export async function listPipedriveFiles( + input: { + accessToken: string + authStyle?: 'x-api-token' + limit?: string | null + sort?: 'id' | 'update_time' | null + start?: string | null + }, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const url = new URL('https://api.pipedrive.com/v1/files') + if (input.sort) url.searchParams.set('sort', input.sort) + if (input.limit) url.searchParams.set('limit', input.limit) + if (input.start) url.searchParams.set('start', input.start) + const validation = await validateUrlWithDNS(url.toString(), 'apiUrl') + signal?.throwIfAborted() + if (!validation.isValid || !validation.resolvedIP) { + throw new PipedriveOperationError(validation.error || 'Invalid Pipedrive API URL', 400) + } + const response = await secureFetchWithPinnedIP(url.toString(), validation.resolvedIP, { + method: 'GET', + headers: getPipedriveAuthHeaders(input), + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + signal, + }) + const data = await readResponseJsonWithLimit(response, { + maxBytes: MAX_JSON_API_RESPONSE_BYTES, + label: 'Pipedrive files response', + signal, + }) + if (!isRecordLike(data) || data.success !== true) { + throw new PipedriveOperationError( + isRecordLike(data) && typeof data.error === 'string' + ? data.error + : 'Failed to fetch files from Pipedrive', + 400 + ) + } + const files = Array.isArray(data.data) + ? data.data.filter((file): file is PipedriveFile => isRecordLike(file)) + : [] + const additionalData = isRecordLike(data.additional_data) ? data.additional_data : null + const pagination = + additionalData && isRecordLike(additionalData.pagination) ? additionalData.pagination : null + return { + files, + hasMore: pagination?.more_items_in_collection === true, + nextStart: typeof pagination?.next_start === 'number' ? pagination.next_start : null, + } +} + +export async function downloadPipedriveFile( + fileUrl: string, + input: { accessToken: string; authStyle?: 'x-api-token' }, + maxBytes: number, + signal?: AbortSignal +): Promise<{ buffer: Buffer; contentType: string | null } | null> { + signal?.throwIfAborted() + const validation = await validateUrlWithDNS(fileUrl, 'fileUrl') + signal?.throwIfAborted() + if (!validation.isValid || !validation.resolvedIP) return null + const authHeaders: Record = + input.authStyle === 'x-api-token' + ? { 'x-api-token': input.accessToken } + : { Authorization: `Bearer ${input.accessToken}` } + const response = await secureFetchWithPinnedIP(fileUrl, validation.resolvedIP, { + method: 'GET', + headers: isPipedriveHost(fileUrl) ? authHeaders : {}, + maxResponseBytes: maxBytes, + signal, + }) + if (!response.ok) { + await response.body?.cancel().catch(() => {}) + return null + } + return { + buffer: await readResponseToBufferWithLimit(response, { + maxBytes, + label: 'Pipedrive file download', + signal, + }), + contentType: response.headers.get('content-type'), + } +} diff --git a/apps/sim/lib/internal/pipedrive/errors.ts b/apps/sim/lib/internal/pipedrive/errors.ts new file mode 100644 index 00000000000..bfbb325ad11 --- /dev/null +++ b/apps/sim/lib/internal/pipedrive/errors.ts @@ -0,0 +1,10 @@ +export class PipedriveOperationError extends Error { + constructor( + message: string, + readonly status: number, + readonly body: Record = { success: false, error: message } + ) { + super(message) + this.name = 'PipedriveOperationError' + } +} diff --git a/apps/sim/lib/internal/pipedrive/execute-tool.ts b/apps/sim/lib/internal/pipedrive/execute-tool.ts new file mode 100644 index 00000000000..78394ababff --- /dev/null +++ b/apps/sim/lib/internal/pipedrive/execute-tool.ts @@ -0,0 +1,65 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { PipedriveOperationError } from '@/lib/internal/pipedrive/errors' +import { executePipedriveGetFiles } from '@/lib/internal/pipedrive/operations' +import { pipedriveGetFilesInputSchema } from '@/lib/internal/pipedrive/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const logger = createLogger('PipedriveToolExecution') + +function inputSizeError(input: unknown): Response | null { + let serialized: string + try { + serialized = JSON.stringify(input) ?? '' + } catch { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + return Buffer.byteLength(serialized) > DEFAULT_MAX_JSON_BODY_BYTES + ? Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + : null +} + +export const executePipedriveTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'pipedrive_get_files') { + return Response.json( + { success: false, error: `Unsupported Pipedrive tool: ${request.toolId}` }, + { status: 500 } + ) + } + if (!request.context.userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + const sizeError = inputSizeError(request.input) + if (sizeError) return sizeError + const parsed = pipedriveGetFilesInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { success: false, error: getValidationErrorMessage(parsed.error, 'Invalid request data') }, + { status: 400 } + ) + } + try { + const result = await executePipedriveGetFiles(parsed.data, { + requestId: request.requestId, + signal: request.signal, + }) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof PipedriveOperationError) { + return Response.json(error.body, { status: error.status }) + } + const message = getErrorMessage(error, 'Unknown error occurred') + logger.error('Pipedrive get files failed', { error: message, requestId: request.requestId }) + return Response.json({ success: false, error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/pipedrive/operations.ts b/apps/sim/lib/internal/pipedrive/operations.ts new file mode 100644 index 00000000000..0418852eeca --- /dev/null +++ b/apps/sim/lib/internal/pipedrive/operations.ts @@ -0,0 +1,70 @@ +import { createLogger } from '@sim/logger' +import { downloadPipedriveFile, listPipedriveFiles } from '@/lib/internal/pipedrive/client' +import type { PipedriveGetFilesInput } from '@/lib/internal/pipedrive/schema' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { getFileExtension, getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils' + +const logger = createLogger('PipedriveOperations') + +export interface PipedriveOperationContext { + requestId: string + signal?: AbortSignal +} + +export async function executePipedriveGetFiles( + input: PipedriveGetFilesInput, + context: PipedriveOperationContext +) { + context.signal?.throwIfAborted() + const page = await listPipedriveFiles(input, context.signal) + const downloadedFiles: Array<{ + data: string + mimeType: string + name: string + size: number + }> = [] + let downloadedBytes = 0 + + if (input.downloadFiles) { + for (const file of page.files) { + context.signal?.throwIfAborted() + if (!file.url || downloadedBytes >= MAX_BUFFERED_TRANSFER_BYTES) continue + try { + const downloaded = await downloadPipedriveFile( + file.url, + input, + MAX_BUFFERED_TRANSFER_BYTES - downloadedBytes, + context.signal + ) + if (!downloaded) continue + downloadedBytes += downloaded.buffer.length + const name = file.name || `pipedrive-file-${file.id || Date.now()}` + const extension = getFileExtension(name) + downloadedFiles.push({ + name, + mimeType: downloaded.contentType || getMimeTypeFromExtension(extension), + data: downloaded.buffer.toString('base64'), + size: downloaded.buffer.length, + }) + } catch (error) { + context.signal?.throwIfAborted() + logger.warn('Failed to download Pipedrive file', { + fileId: file.id, + requestId: context.requestId, + }) + } + } + } + context.signal?.throwIfAborted() + return { + success: true, + output: { + files: page.files, + downloadedFiles: downloadedFiles.length > 0 ? downloadedFiles : undefined, + total_items: page.files.length, + has_more: page.hasMore, + next_start: page.nextStart, + success: true, + }, + } +} diff --git a/apps/sim/lib/internal/pipedrive/schema.ts b/apps/sim/lib/internal/pipedrive/schema.ts new file mode 100644 index 00000000000..783a968ebd5 --- /dev/null +++ b/apps/sim/lib/internal/pipedrive/schema.ts @@ -0,0 +1,12 @@ +import { z } from 'zod' + +export const pipedriveGetFilesInputSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + authStyle: z.enum(['x-api-token']).optional(), + sort: z.enum(['id', 'update_time']).optional().nullable(), + limit: z.string().optional().nullable(), + start: z.string().optional().nullable(), + downloadFiles: z.boolean().optional().default(false), +}) + +export type PipedriveGetFilesInput = z.output diff --git a/apps/sim/lib/internal/postgresql/client.test.ts b/apps/sim/lib/internal/postgresql/client.test.ts new file mode 100644 index 00000000000..e37a123d842 --- /dev/null +++ b/apps/sim/lib/internal/postgresql/client.test.ts @@ -0,0 +1,102 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { PostgresConnectionConfig } from '@/tools/postgresql/types' + +const { mockValidateDatabaseHost, mockPostgres } = vi.hoisted(() => ({ + mockValidateDatabaseHost: vi.fn(), + mockPostgres: vi.fn(() => ({})), +})) + +vi.mock('postgres', () => ({ default: mockPostgres })) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + validateDatabaseHost: mockValidateDatabaseHost, +})) + +import { createPostgresClient, executePostgresQuery } from '@/lib/internal/postgresql/client' + +function makeConfig(overrides: Partial = {}): PostgresConnectionConfig { + return { + host: 'db.example.com', + port: 5432, + database: 'app', + username: 'app', + password: 'secret', + ssl: 'required', + ...overrides, + } +} + +describe('PostgreSQL client', () => { + beforeEach(() => { + vi.clearAllMocks() + mockValidateDatabaseHost.mockResolvedValue({ + isValid: true, + resolvedIP: '93.184.216.34', + originalHostname: 'db.example.com', + }) + }) + + it('does not create a connection when host validation fails', async () => { + mockValidateDatabaseHost.mockResolvedValue({ + isValid: false, + error: 'host resolves to a blocked IP address', + }) + + await expect( + createPostgresClient(makeConfig({ host: 'rebind.attacker.example' })) + ).rejects.toThrow('host resolves to a blocked IP address') + expect(mockPostgres).not.toHaveBeenCalled() + }) + + it.each(['disabled', 'required', 'preferred'] as const)( + 'pins the validated IP for ssl=%s', + async (ssl) => { + await createPostgresClient(makeConfig({ host: 'rebind.attacker.example', ssl })) + + expect(mockValidateDatabaseHost).toHaveBeenCalledWith('rebind.attacker.example', 'host') + expect(mockPostgres.mock.calls[0][0]).toMatchObject({ host: '93.184.216.34' }) + } + ) + + it('preserves the original hostname as the TLS servername', async () => { + await createPostgresClient(makeConfig({ host: 'db.example.com', ssl: 'required' })) + + expect(mockPostgres.mock.calls[0][0]).toMatchObject({ + host: '93.184.216.34', + ssl: { rejectUnauthorized: false, servername: 'db.example.com' }, + }) + }) + + it('does not validate or connect when already cancelled', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect(createPostgresClient(makeConfig(), controller.signal)).rejects.toMatchObject({ + name: 'AbortError', + }) + expect(mockValidateDatabaseHost).not.toHaveBeenCalled() + expect(mockPostgres).not.toHaveBeenCalled() + }) + + it('cancels an in-flight query when the signal aborts', async () => { + const controller = new AbortController() + let rejectQuery: (reason: Error) => void = () => undefined + const pendingQuery = Object.assign( + new Promise((_resolve, reject) => { + rejectQuery = reject + }), + { + cancel: vi.fn(() => rejectQuery(new Error('query cancelled'))), + } + ) + + const execution = executePostgresQuery(pendingQuery, controller.signal) + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect(execution).rejects.toThrow('query cancelled') + expect(pendingQuery.cancel).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/internal/postgresql/client.ts b/apps/sim/lib/internal/postgresql/client.ts new file mode 100644 index 00000000000..1de71b4cd19 --- /dev/null +++ b/apps/sim/lib/internal/postgresql/client.ts @@ -0,0 +1,65 @@ +import postgres from 'postgres' +import { validateDatabaseHost } from '@/lib/core/security/input-validation.server' +import type { PostgresConnectionConfig } from '@/tools/postgresql/types' + +export type PostgresClient = ReturnType + +interface PendingPostgresQuery extends PromiseLike { + cancel(): void +} + +export async function createPostgresClient( + config: PostgresConnectionConfig, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const hostValidation = await validateDatabaseHost(config.host, 'host') + signal?.throwIfAborted() + + if (!hostValidation.isValid) { + throw new Error(hostValidation.error) + } + + const resolvedHost = hostValidation.resolvedIP ?? config.host + const sslConfig: boolean | 'prefer' | { rejectUnauthorized: boolean; servername?: string } = + config.ssl === 'disabled' + ? false + : config.ssl === 'preferred' + ? 'prefer' + : { rejectUnauthorized: false, servername: config.host } + + return postgres({ + host: resolvedHost, + port: config.port, + database: config.database, + username: config.username, + password: config.password, + ssl: sslConfig, + connect_timeout: 10, + idle_timeout: 20, + max_lifetime: 60 * 30, + max: 1, + }) +} + +export async function executePostgresQuery( + query: PendingPostgresQuery, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + + const cancelQuery = () => query.cancel() + signal?.addEventListener('abort', cancelQuery, { once: true }) + + try { + if (signal?.aborted) { + cancelQuery() + signal.throwIfAborted() + } + const result = await query + signal?.throwIfAborted() + return result + } finally { + signal?.removeEventListener('abort', cancelQuery) + } +} diff --git a/apps/sim/lib/internal/postgresql/execute-tool.test.ts b/apps/sim/lib/internal/postgresql/execute-tool.test.ts new file mode 100644 index 00000000000..13ff84a901a --- /dev/null +++ b/apps/sim/lib/internal/postgresql/execute-tool.test.ts @@ -0,0 +1,142 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operationMocks = vi.hoisted(() => { + class PostgresqlOperationInputError extends Error {} + + return { + PostgresqlOperationInputError, + executePostgresqlDelete: vi.fn(), + executePostgresqlInsert: vi.fn(), + executePostgresqlIntrospection: vi.fn(), + executePostgresqlQuery: vi.fn(), + executePostgresqlStatement: vi.fn(), + executePostgresqlUpdate: vi.fn(), + } +}) + +vi.mock('@/lib/internal/postgresql/operations', () => operationMocks) + +import { executePostgresqlTool } from '@/lib/internal/postgresql/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const VALID_BODY = { + host: 'db.example.com', + port: 5432, + database: 'application', + username: 'application', + password: 'secret', + ssl: 'required', + query: 'SELECT 1', +} as const + +const SUPPORTED_TOOL_IDS = [ + 'postgresql_query', + 'postgresql_execute', + 'postgresql_insert', + 'postgresql_update', + 'postgresql_delete', + 'postgresql_introspect', +] as const + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'postgresql_query', + input: VALID_BODY, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executePostgresqlTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('validates and executes the matching operation with cancellation', async () => { + const controller = new AbortController() + operationMocks.executePostgresqlQuery.mockResolvedValue({ + message: 'Query executed successfully. 1 row(s) returned.', + rows: [{ value: 1 }], + rowCount: 1, + }) + + const response = await executePostgresqlTool(createRequest({ signal: controller.signal })) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + message: 'Query executed successfully. 1 row(s) returned.', + rows: [{ value: 1 }], + rowCount: 1, + }) + expect(operationMocks.executePostgresqlQuery).toHaveBeenCalledWith( + VALID_BODY, + controller.signal + ) + }) + + it('returns the canonical contract validation envelope before database work', async () => { + const response = await executePostgresqlTool( + createRequest({ input: { host: 'db.example.com' } }) + ) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(operationMocks.executePostgresqlQuery).not.toHaveBeenCalled() + }) + + it.each(SUPPORTED_TOOL_IDS)('recognizes the canonical tool ID %s', async (toolId) => { + const response = await executePostgresqlTool(createRequest({ toolId, input: {} })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ error: 'Invalid request data' }) + }) + + it('preserves the route-compatible provider error envelope', async () => { + operationMocks.executePostgresqlQuery.mockRejectedValue(new Error('database unavailable')) + + const response = await executePostgresqlTool(createRequest()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'PostgreSQL query failed: database unavailable', + }) + }) + + it('preserves execute query validation as a 400 error', async () => { + operationMocks.executePostgresqlStatement.mockRejectedValue( + new operationMocks.PostgresqlOperationInputError('Query validation failed: invalid query') + ) + + const response = await executePostgresqlTool(createRequest({ toolId: 'postgresql_execute' })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ + error: 'Query validation failed: invalid query', + }) + }) + + it('propagates cancellation without converting it into a database failure', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executePostgresqlTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(operationMocks.executePostgresqlQuery).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/postgresql/execute-tool.ts b/apps/sim/lib/internal/postgresql/execute-tool.ts new file mode 100644 index 00000000000..b235318e436 --- /dev/null +++ b/apps/sim/lib/internal/postgresql/execute-tool.ts @@ -0,0 +1,112 @@ +import { getErrorMessage } from '@sim/utils/errors' +import type { z } from 'zod' +import { + executePostgresqlDelete, + executePostgresqlInsert, + executePostgresqlIntrospection, + executePostgresqlQuery, + executePostgresqlStatement, + executePostgresqlUpdate, + PostgresqlOperationInputError, +} from '@/lib/internal/postgresql/operations' +import { + postgresqlDeleteInputSchema, + postgresqlExecuteInputSchema, + postgresqlInsertInputSchema, + postgresqlIntrospectInputSchema, + postgresqlQueryInputSchema, + postgresqlUpdateInputSchema, +} from '@/lib/internal/postgresql/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +async function executeOperation( + schema: z.ZodType, + input: unknown, + execute: (input: TInput, signal?: AbortSignal) => Promise, + errorMessage: string, + signal?: AbortSignal +): Promise { + const parsed = schema.safeParse(input) + if (!parsed.success) { + return Response.json( + { error: 'Invalid request data', details: parsed.error.issues }, + { status: 400 } + ) + } + + try { + const result = await execute(parsed.data, signal) + signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + signal?.throwIfAborted() + if (error instanceof PostgresqlOperationInputError) { + return Response.json({ error: error.message }, { status: 400 }) + } + return Response.json( + { error: `${errorMessage}: ${getErrorMessage(error, 'Unknown error occurred')}` }, + { status: 500 } + ) + } +} + +export const executePostgresqlTool: InternalToolOperationHandler = async ({ + toolId, + input, + signal, +}) => { + signal?.throwIfAborted() + + switch (toolId) { + case 'postgresql_query': + return executeOperation( + postgresqlQueryInputSchema, + input, + executePostgresqlQuery, + 'PostgreSQL query failed', + signal + ) + case 'postgresql_execute': + return executeOperation( + postgresqlExecuteInputSchema, + input, + executePostgresqlStatement, + 'PostgreSQL execute failed', + signal + ) + case 'postgresql_insert': + return executeOperation( + postgresqlInsertInputSchema, + input, + executePostgresqlInsert, + 'PostgreSQL insert failed', + signal + ) + case 'postgresql_update': + return executeOperation( + postgresqlUpdateInputSchema, + input, + executePostgresqlUpdate, + 'PostgreSQL update failed', + signal + ) + case 'postgresql_delete': + return executeOperation( + postgresqlDeleteInputSchema, + input, + executePostgresqlDelete, + 'PostgreSQL delete failed', + signal + ) + case 'postgresql_introspect': + return executeOperation( + postgresqlIntrospectInputSchema, + input, + executePostgresqlIntrospection, + 'PostgreSQL introspection failed', + signal + ) + default: + return Response.json({ error: `Unsupported PostgreSQL tool: ${toolId}` }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/postgresql/operations.test.ts b/apps/sim/lib/internal/postgresql/operations.test.ts new file mode 100644 index 00000000000..2effbe9b2ce --- /dev/null +++ b/apps/sim/lib/internal/postgresql/operations.test.ts @@ -0,0 +1,114 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const clientMocks = vi.hoisted(() => ({ + createPostgresClient: vi.fn(), +})) + +const queryMocks = vi.hoisted(() => ({ + deletePostgresRows: vi.fn(), + insertPostgresRows: vi.fn(), + introspectPostgresSchema: vi.fn(), + queryPostgres: vi.fn(), + updatePostgresRows: vi.fn(), + validatePostgresQuery: vi.fn(), +})) + +vi.mock('@/lib/internal/postgresql/client', () => clientMocks) +vi.mock('@/lib/internal/postgresql/queries', () => queryMocks) + +import { + executePostgresqlIntrospection, + executePostgresqlQuery, + executePostgresqlStatement, + PostgresqlOperationInputError, +} from '@/lib/internal/postgresql/operations' + +const CONNECTION = { + host: 'db.example.com', + port: 5432, + database: 'application', + username: 'application', + password: 'secret', + ssl: 'required', +} as const + +describe('PostgreSQL operations', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('passes cancellation to the query and closes the client after success', async () => { + const controller = new AbortController() + const client = { end: vi.fn().mockResolvedValue(undefined) } + clientMocks.createPostgresClient.mockResolvedValue(client) + queryMocks.queryPostgres.mockResolvedValue({ rows: [{ value: 1 }], rowCount: 1 }) + + await expect( + executePostgresqlQuery({ ...CONNECTION, query: 'SELECT 1' }, controller.signal) + ).resolves.toEqual({ + message: 'Query executed successfully. 1 row(s) returned.', + rows: [{ value: 1 }], + rowCount: 1, + }) + + expect(clientMocks.createPostgresClient).toHaveBeenCalledWith( + { ...CONNECTION, query: 'SELECT 1' }, + controller.signal + ) + expect(queryMocks.queryPostgres).toHaveBeenCalledWith(client, 'SELECT 1', [], controller.signal) + expect(client.end).toHaveBeenCalledOnce() + }) + + it('closes the client when a database query rejects', async () => { + const client = { end: vi.fn().mockResolvedValue(undefined) } + clientMocks.createPostgresClient.mockResolvedValue(client) + queryMocks.queryPostgres.mockRejectedValue(new Error('database unavailable')) + + await expect(executePostgresqlQuery({ ...CONNECTION, query: 'SELECT 1' })).rejects.toThrow( + 'database unavailable' + ) + expect(client.end).toHaveBeenCalledOnce() + }) + + it('rejects disallowed execute statements before creating a connection', () => { + queryMocks.validatePostgresQuery.mockReturnValue({ + isValid: false, + error: + 'Only SELECT, INSERT, UPDATE, DELETE, WITH, EXPLAIN, ANALYZE, and SHOW statements are allowed', + }) + + expect(() => executePostgresqlStatement({ ...CONNECTION, query: 'DROP TABLE users' })).toThrow( + new PostgresqlOperationInputError( + 'Query validation failed: Only SELECT, INSERT, UPDATE, DELETE, WITH, EXPLAIN, ANALYZE, and SHOW statements are allowed' + ) + ) + expect(clientMocks.createPostgresClient).not.toHaveBeenCalled() + }) + + it('preserves introspection output and cancellation', async () => { + const controller = new AbortController() + const client = { end: vi.fn().mockResolvedValue(undefined) } + clientMocks.createPostgresClient.mockResolvedValue(client) + queryMocks.introspectPostgresSchema.mockResolvedValue({ + tables: [], + schemas: ['public'], + }) + + await expect( + executePostgresqlIntrospection({ ...CONNECTION, schema: 'public' }, controller.signal) + ).resolves.toEqual({ + message: "Schema introspection completed. Found 0 table(s) in schema 'public'.", + tables: [], + schemas: ['public'], + }) + expect(queryMocks.introspectPostgresSchema).toHaveBeenCalledWith( + client, + 'public', + controller.signal + ) + expect(client.end).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/internal/postgresql/operations.ts b/apps/sim/lib/internal/postgresql/operations.ts new file mode 100644 index 00000000000..f07ac3f9bf0 --- /dev/null +++ b/apps/sim/lib/internal/postgresql/operations.ts @@ -0,0 +1,109 @@ +import { createPostgresClient, type PostgresClient } from '@/lib/internal/postgresql/client' +import { + deletePostgresRows, + insertPostgresRows, + introspectPostgresSchema, + queryPostgres, + updatePostgresRows, + validatePostgresQuery, +} from '@/lib/internal/postgresql/queries' +import type { + PostgresqlDeleteInput, + PostgresqlExecuteInput, + PostgresqlInsertInput, + PostgresqlIntrospectInput, + PostgresqlQueryInput, + PostgresqlUpdateInput, +} from '@/lib/internal/postgresql/schema' +import type { PostgresConnectionConfig } from '@/tools/postgresql/types' + +export class PostgresqlOperationInputError extends Error {} + +async function withPostgresClient( + input: PostgresConnectionConfig, + signal: AbortSignal | undefined, + execute: (client: PostgresClient) => Promise +): Promise { + const client = await createPostgresClient(input, signal) + try { + return await execute(client) + } finally { + await client.end() + } +} + +export function executePostgresqlQuery(input: PostgresqlQueryInput, signal?: AbortSignal) { + return withPostgresClient(input, signal, async (client) => { + const result = await queryPostgres(client, input.query, [], signal) + return { + message: `Query executed successfully. ${result.rowCount} row(s) returned.`, + rows: result.rows, + rowCount: result.rowCount, + } + }) +} + +export function executePostgresqlStatement(input: PostgresqlExecuteInput, signal?: AbortSignal) { + const validation = validatePostgresQuery(input.query) + if (!validation.isValid) { + throw new PostgresqlOperationInputError( + `Query validation failed: ${validation.error ?? 'Invalid query'}` + ) + } + + return withPostgresClient(input, signal, async (client) => { + const result = await queryPostgres(client, input.query, [], signal) + return { + message: `SQL executed successfully. ${result.rowCount} row(s) affected.`, + rows: result.rows, + rowCount: result.rowCount, + } + }) +} + +export function executePostgresqlInsert(input: PostgresqlInsertInput, signal?: AbortSignal) { + return withPostgresClient(input, signal, async (client) => { + const result = await insertPostgresRows(client, input.table, input.data, signal) + return { + message: `Data inserted successfully. ${result.rowCount} row(s) affected.`, + rows: result.rows, + rowCount: result.rowCount, + } + }) +} + +export function executePostgresqlUpdate(input: PostgresqlUpdateInput, signal?: AbortSignal) { + return withPostgresClient(input, signal, async (client) => { + const result = await updatePostgresRows(client, input.table, input.data, input.where, signal) + return { + message: `Data updated successfully. ${result.rowCount} row(s) affected.`, + rows: result.rows, + rowCount: result.rowCount, + } + }) +} + +export function executePostgresqlDelete(input: PostgresqlDeleteInput, signal?: AbortSignal) { + return withPostgresClient(input, signal, async (client) => { + const result = await deletePostgresRows(client, input.table, input.where, signal) + return { + message: `Data deleted successfully. ${result.rowCount} row(s) affected.`, + rows: result.rows, + rowCount: result.rowCount, + } + }) +} + +export function executePostgresqlIntrospection( + input: PostgresqlIntrospectInput, + signal?: AbortSignal +) { + return withPostgresClient(input, signal, async (client) => { + const result = await introspectPostgresSchema(client, input.schema, signal) + return { + message: `Schema introspection completed. Found ${result.tables.length} table(s) in schema '${input.schema}'.`, + tables: result.tables, + schemas: result.schemas, + } + }) +} diff --git a/apps/sim/lib/internal/postgresql/queries.test.ts b/apps/sim/lib/internal/postgresql/queries.test.ts new file mode 100644 index 00000000000..85ecf21195e --- /dev/null +++ b/apps/sim/lib/internal/postgresql/queries.test.ts @@ -0,0 +1,147 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockExecutePostgresQuery } = vi.hoisted(() => ({ + mockExecutePostgresQuery: vi.fn(), +})) + +vi.mock('@/lib/internal/postgresql/client', () => ({ + executePostgresQuery: mockExecutePostgresQuery, +})) + +import type { PostgresClient } from '@/lib/internal/postgresql/client' +import { + deletePostgresRows, + insertPostgresRows, + introspectPostgresSchema, + sanitizePostgresIdentifier, + updatePostgresRows, + validatePostgresQuery, +} from '@/lib/internal/postgresql/queries' + +const pendingQuery = { cancel: vi.fn() } +const mockClient = { + unsafe: vi.fn(() => pendingQuery), +} as unknown as PostgresClient + +describe('PostgreSQL queries', () => { + beforeEach(() => { + vi.clearAllMocks() + mockExecutePostgresQuery.mockResolvedValue([{ id: 1 }]) + }) + + it('builds a parameterized INSERT and preserves its result', async () => { + const controller = new AbortController() + + await expect( + insertPostgresRows( + mockClient, + 'public.users', + { email: 'person@example.com', active: true }, + controller.signal + ) + ).resolves.toEqual({ rows: [{ id: 1 }], rowCount: 1 }) + + expect(mockClient.unsafe).toHaveBeenCalledWith( + 'INSERT INTO "public"."users" ("email", "active") VALUES ($1, $2) RETURNING *', + ['person@example.com', true] + ) + expect(mockExecutePostgresQuery).toHaveBeenCalledWith(pendingQuery, controller.signal) + }) + + it('builds parameterized UPDATE values while preserving the existing WHERE syntax', async () => { + await updatePostgresRows(mockClient, 'users', { active: false }, 'id = 42') + + expect(mockClient.unsafe).toHaveBeenCalledWith( + 'UPDATE "users" SET "active" = $1 WHERE id = 42 RETURNING *', + [false] + ) + }) + + it('rejects dangerous WHERE clauses before issuing an UPDATE or DELETE', async () => { + await expect( + updatePostgresRows(mockClient, 'users', { active: false }, 'id = 42; DROP TABLE users') + ).rejects.toThrow('WHERE clause contains potentially dangerous operation') + await expect(deletePostgresRows(mockClient, 'users', "id = 42 OR 'x'='x'")).rejects.toThrow( + 'WHERE clause contains potentially dangerous operation' + ) + expect(mockClient.unsafe).not.toHaveBeenCalled() + }) + + it('preserves identifier validation and query allowlisting', () => { + expect(sanitizePostgresIdentifier('public.users')).toBe('"public"."users"') + expect(() => sanitizePostgresIdentifier('users; DROP TABLE users')).toThrow( + 'Invalid identifier' + ) + expect(validatePostgresQuery('ANALYZE users')).toEqual({ isValid: true }) + expect(validatePostgresQuery('DROP TABLE users')).toEqual({ + isValid: false, + error: + 'Only SELECT, INSERT, UPDATE, DELETE, WITH, EXPLAIN, ANALYZE, and SHOW statements are allowed', + }) + }) + + it('preserves introspection shaping and cancellation for every database query', async () => { + const controller = new AbortController() + const pendingQueries = Array.from({ length: 6 }, (_, index) => ({ + cancel: vi.fn(), + index, + })) + const taggedClient = vi.fn(() => pendingQueries.shift()) as unknown as PostgresClient + mockExecutePostgresQuery + .mockResolvedValueOnce([{ schema_name: 'public' }]) + .mockResolvedValueOnce([{ table_name: 'users', table_schema: 'public' }]) + .mockResolvedValueOnce([ + { + column_name: 'role', + data_type: 'USER-DEFINED', + is_nullable: 'NO', + column_default: null, + udt_name: 'user_role', + }, + ]) + .mockResolvedValueOnce([{ column_name: 'role' }]) + .mockResolvedValueOnce([ + { + column_name: 'role', + foreign_table_name: 'roles', + foreign_column_name: 'name', + }, + ]) + .mockResolvedValueOnce([ + { index_name: 'users_role_idx', column_name: 'role', is_unique: false }, + ]) + + await expect( + introspectPostgresSchema(taggedClient, 'public', controller.signal) + ).resolves.toEqual({ + schemas: ['public'], + tables: [ + { + name: 'users', + schema: 'public', + columns: [ + { + name: 'role', + type: 'user_role', + nullable: false, + default: null, + isPrimaryKey: true, + isForeignKey: true, + references: { table: 'roles', column: 'name' }, + }, + ], + primaryKey: ['role'], + foreignKeys: [{ column: 'role', referencesTable: 'roles', referencesColumn: 'name' }], + indexes: [{ name: 'users_role_idx', columns: ['role'], unique: false }], + }, + ], + }) + expect(mockExecutePostgresQuery).toHaveBeenCalledTimes(6) + for (const call of mockExecutePostgresQuery.mock.calls) { + expect(call[1]).toBe(controller.signal) + } + }) +}) diff --git a/apps/sim/lib/internal/postgresql/queries.ts b/apps/sim/lib/internal/postgresql/queries.ts new file mode 100644 index 00000000000..11939fad59c --- /dev/null +++ b/apps/sim/lib/internal/postgresql/queries.ts @@ -0,0 +1,361 @@ +import type { PostgresClient } from '@/lib/internal/postgresql/client' +import { executePostgresQuery } from '@/lib/internal/postgresql/client' + +export interface PostgresRowsResult { + rows: unknown[] + rowCount: number +} + +export interface PostgresIntrospectionResult { + tables: Array<{ + name: string + schema: string + columns: Array<{ + name: string + type: string + nullable: boolean + default: string | null + isPrimaryKey: boolean + isForeignKey: boolean + references?: { + table: string + column: string + } + }> + primaryKey: string[] + foreignKeys: Array<{ + column: string + referencesTable: string + referencesColumn: string + }> + indexes: Array<{ + name: string + columns: string[] + unique: boolean + }> + }> + schemas: string[] +} + +interface SchemaRow { + schema_name: string +} + +interface TableRow { + table_name: string + table_schema: string +} + +interface ColumnRow { + column_name: string + data_type: string + is_nullable: string + column_default: string | null + udt_name: string +} + +interface PrimaryKeyRow { + column_name: string +} + +interface ForeignKeyRow { + column_name: string + foreign_table_name: string + foreign_column_name: string +} + +interface IndexRow { + index_name: string + column_name: string + is_unique: boolean +} + +export async function queryPostgres( + client: PostgresClient, + query: string, + params: unknown[] = [], + signal?: AbortSignal +): Promise { + type PostgresParameters = NonNullable[1]> + const result = await executePostgresQuery( + client.unsafe(query, params as PostgresParameters), + signal + ) + return { + rows: result, + rowCount: result.count ?? result.length ?? 0, + } +} + +export function validatePostgresQuery(query: string): { isValid: boolean; error?: string } { + const trimmedQuery = query.trim().toLowerCase() + const allowedStatements = /^(select|insert|update|delete|with|explain|analyze|show)\s+/i + + if (!allowedStatements.test(trimmedQuery)) { + return { + isValid: false, + error: + 'Only SELECT, INSERT, UPDATE, DELETE, WITH, EXPLAIN, ANALYZE, and SHOW statements are allowed', + } + } + + return { isValid: true } +} + +export function sanitizePostgresIdentifier(identifier: string): string { + if (identifier.includes('.')) { + return identifier + .split('.') + .map((part) => sanitizeSingleIdentifier(part)) + .join('.') + } + + return sanitizeSingleIdentifier(identifier) +} + +export async function insertPostgresRows( + client: PostgresClient, + table: string, + data: Record, + signal?: AbortSignal +): Promise { + const sanitizedTable = sanitizePostgresIdentifier(table) + const columns = Object.keys(data) + const sanitizedColumns = columns.map((column) => sanitizePostgresIdentifier(column)) + const placeholders = columns.map((_, index) => `$${index + 1}`) + const values = columns.map((column) => data[column]) + const query = `INSERT INTO ${sanitizedTable} (${sanitizedColumns.join(', ')}) VALUES (${placeholders.join(', ')}) RETURNING *` + + return queryPostgres(client, query, values, signal) +} + +export async function updatePostgresRows( + client: PostgresClient, + table: string, + data: Record, + where: string, + signal?: AbortSignal +): Promise { + validateWhereClause(where) + + const sanitizedTable = sanitizePostgresIdentifier(table) + const columns = Object.keys(data) + const sanitizedColumns = columns.map((column) => sanitizePostgresIdentifier(column)) + const setClause = sanitizedColumns.map((column, index) => `${column} = $${index + 1}`).join(', ') + const values = columns.map((column) => data[column]) + const query = `UPDATE ${sanitizedTable} SET ${setClause} WHERE ${where} RETURNING *` + + return queryPostgres(client, query, values, signal) +} + +export async function deletePostgresRows( + client: PostgresClient, + table: string, + where: string, + signal?: AbortSignal +): Promise { + validateWhereClause(where) + + const sanitizedTable = sanitizePostgresIdentifier(table) + const query = `DELETE FROM ${sanitizedTable} WHERE ${where} RETURNING *` + + return queryPostgres(client, query, [], signal) +} + +export async function introspectPostgresSchema( + client: PostgresClient, + schemaName = 'public', + signal?: AbortSignal +): Promise { + const schemasResult = await executePostgresQuery( + client` + SELECT schema_name + FROM information_schema.schemata + WHERE schema_name NOT IN ('pg_catalog', 'information_schema', 'pg_toast') + ORDER BY schema_name + `, + signal + ) + const schemas = schemasResult.map((row) => row.schema_name) + + const tablesResult = await executePostgresQuery( + client` + SELECT table_name, table_schema + FROM information_schema.tables + WHERE table_schema = ${schemaName} + AND table_type = 'BASE TABLE' + ORDER BY table_name + `, + signal + ) + + const tables: PostgresIntrospectionResult['tables'] = [] + + for (const tableRow of tablesResult) { + signal?.throwIfAborted() + const tableName = tableRow.table_name + const tableSchema = tableRow.table_schema + + const columnsResult = await executePostgresQuery( + client` + SELECT + c.column_name, + c.data_type, + c.is_nullable, + c.column_default, + c.udt_name + FROM information_schema.columns c + WHERE c.table_schema = ${tableSchema} + AND c.table_name = ${tableName} + ORDER BY c.ordinal_position + `, + signal + ) + + const primaryKeyResult = await executePostgresQuery( + client` + SELECT kcu.column_name + FROM information_schema.table_constraints tc + JOIN information_schema.key_column_usage kcu + ON tc.constraint_name = kcu.constraint_name + AND tc.table_schema = kcu.table_schema + WHERE tc.constraint_type = 'PRIMARY KEY' + AND tc.table_schema = ${tableSchema} + AND tc.table_name = ${tableName} + `, + signal + ) + const primaryKey = primaryKeyResult.map((row) => row.column_name) + + const foreignKeyResult = await executePostgresQuery( + client` + SELECT + kcu.column_name, + ccu.table_name AS foreign_table_name, + ccu.column_name AS foreign_column_name + FROM information_schema.table_constraints tc + JOIN information_schema.key_column_usage kcu + ON tc.constraint_name = kcu.constraint_name + AND tc.table_schema = kcu.table_schema + JOIN information_schema.constraint_column_usage ccu + ON ccu.constraint_name = tc.constraint_name + AND ccu.table_schema = tc.table_schema + WHERE tc.constraint_type = 'FOREIGN KEY' + AND tc.table_schema = ${tableSchema} + AND tc.table_name = ${tableName} + `, + signal + ) + const foreignKeys = foreignKeyResult.map((row) => ({ + column: row.column_name, + referencesTable: row.foreign_table_name, + referencesColumn: row.foreign_column_name, + })) + const foreignKeyColumns = new Set(foreignKeys.map((foreignKey) => foreignKey.column)) + + const indexesResult = await executePostgresQuery( + client` + SELECT + i.relname AS index_name, + a.attname AS column_name, + ix.indisunique AS is_unique + FROM pg_class t + JOIN pg_index ix ON t.oid = ix.indrelid + JOIN pg_class i ON i.oid = ix.indexrelid + JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(ix.indkey) + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE t.relkind = 'r' + AND n.nspname = ${tableSchema} + AND t.relname = ${tableName} + AND NOT ix.indisprimary + ORDER BY i.relname, a.attnum + `, + signal + ) + + const indexesByName = new Map() + for (const row of indexesResult) { + const index = indexesByName.get(row.index_name) ?? { + name: row.index_name, + columns: [], + unique: row.is_unique, + } + index.columns.push(row.column_name) + indexesByName.set(row.index_name, index) + } + + const columns = columnsResult.map((column) => { + const foreignKey = foreignKeys.find((candidate) => candidate.column === column.column_name) + return { + name: column.column_name, + type: column.data_type === 'USER-DEFINED' ? column.udt_name : column.data_type, + nullable: column.is_nullable === 'YES', + default: column.column_default, + isPrimaryKey: primaryKey.includes(column.column_name), + isForeignKey: foreignKeyColumns.has(column.column_name), + ...(foreignKey && { + references: { + table: foreignKey.referencesTable, + column: foreignKey.referencesColumn, + }, + }), + } + }) + + tables.push({ + name: tableName, + schema: tableSchema, + columns, + primaryKey, + foreignKeys, + indexes: Array.from(indexesByName.values()), + }) + } + + return { tables, schemas } +} + +function sanitizeSingleIdentifier(identifier: string): string { + const cleaned = identifier.replace(/"/g, '') + + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(cleaned)) { + throw new Error( + `Invalid identifier: ${identifier}. Identifiers must start with a letter or underscore and contain only letters, numbers, and underscores.` + ) + } + + return `"${cleaned}"` +} + +function validateWhereClause(where: string): void { + const dangerousPatterns = [ + /;\s*(drop|delete|insert|update|create|alter|grant|revoke)/i, + /union\s+(all\s+)?select/i, + /into\s+outfile/i, + /load_file\s*\(/i, + /pg_read_file/i, + /--/, + /\/\*/, + /\*\//, + /\bor\s+(['"]?)(\w+)\1\s*=\s*\1\2\1/i, + /\bor\s+true\b/i, + /\bor\s+false\b/i, + /\band\s+(['"]?)(\w+)\1\s*=\s*\1\2\1/i, + /\band\s+true\b/i, + /\band\s+false\b/i, + /\bsleep\s*\(/i, + /\bwaitfor\s+delay/i, + /\bpg_sleep\s*\(/i, + /\bbenchmark\s*\(/i, + /;\s*\w+/, + /information_schema/i, + /pg_catalog/i, + /\bxp_cmdshell/i, + ] + + for (const pattern of dangerousPatterns) { + if (pattern.test(where)) { + throw new Error('WHERE clause contains potentially dangerous operation') + } + } +} diff --git a/apps/sim/lib/internal/postgresql/schema.ts b/apps/sim/lib/internal/postgresql/schema.ts new file mode 100644 index 00000000000..00dbe358aa6 --- /dev/null +++ b/apps/sim/lib/internal/postgresql/schema.ts @@ -0,0 +1,73 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { z } from 'zod' + +const sslModeSchema = z.enum(['disabled', 'required', 'preferred']).default('preferred') + +const nonEmptyRecordSchema = (message: string) => + z.record(z.string(), z.unknown()).refine((value) => Object.keys(value).length > 0, { message }) + +const jsonObjectStringSchema = (message: string, includeReceivedValue = false) => + z + .string() + .min(1) + .transform((value) => { + try { + const parsed = JSON.parse(value) + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw new Error('Data must be a JSON object') + } + return parsed + } catch (error) { + if (!includeReceivedValue) throw new Error(message) + throw new Error( + `${message}: ${getErrorMessage(error, 'Unknown error')}. Received: ${value.substring(0, 100)}...` + ) + } + }) + +const connectionInputSchema = z.object({ + host: z.string().min(1, 'Host is required'), + port: z.coerce.number().int().positive('Port must be a positive integer'), + database: z.string().min(1, 'Database name is required'), + username: z.string().min(1, 'Username is required'), + password: z.string().min(1, 'Password is required'), + ssl: sslModeSchema, +}) + +const queryInputSchema = connectionInputSchema.extend({ + query: z.string().min(1, 'Query is required'), +}) +const insertDataSchema = z.union([ + nonEmptyRecordSchema('Data object cannot be empty'), + jsonObjectStringSchema('Invalid JSON format in data field', true), +]) +const updateDataSchema = z.union([ + nonEmptyRecordSchema('Data object cannot be empty'), + jsonObjectStringSchema('Invalid JSON format in data field'), +]) + +export const postgresqlQueryInputSchema = queryInputSchema +export const postgresqlExecuteInputSchema = queryInputSchema +export const postgresqlInsertInputSchema = connectionInputSchema.extend({ + table: z.string().min(1, 'Table name is required'), + data: insertDataSchema, +}) +export const postgresqlUpdateInputSchema = connectionInputSchema.extend({ + table: z.string().min(1, 'Table name is required'), + data: updateDataSchema, + where: z.string().min(1, 'WHERE clause is required'), +}) +export const postgresqlDeleteInputSchema = connectionInputSchema.extend({ + table: z.string().min(1, 'Table name is required'), + where: z.string().min(1, 'WHERE clause is required'), +}) +export const postgresqlIntrospectInputSchema = connectionInputSchema.extend({ + schema: z.string().default('public'), +}) + +export type PostgresqlQueryInput = z.output +export type PostgresqlExecuteInput = z.output +export type PostgresqlInsertInput = z.output +export type PostgresqlUpdateInput = z.output +export type PostgresqlDeleteInput = z.output +export type PostgresqlIntrospectInput = z.output diff --git a/apps/sim/lib/internal/principals/executor.test.ts b/apps/sim/lib/internal/principals/executor.test.ts new file mode 100644 index 00000000000..064b418b34c --- /dev/null +++ b/apps/sim/lib/internal/principals/executor.test.ts @@ -0,0 +1,220 @@ +/** + * @vitest-environment node + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { ExecutionContext } from '@/executor/types' + +const { mockBindInternalExecutorDelegation } = vi.hoisted(() => ({ + mockBindInternalExecutorDelegation: vi.fn(), +})) + +vi.mock('@/lib/auth/internal-delegation', () => ({ + bindInternalExecutorDelegation: mockBindInternalExecutorDelegation, +})) + +import { createExecutorPrincipalFromExecutionContext } from '@/lib/internal/principals/executor' + +function executionContext(overrides: Partial = {}): ExecutionContext { + return { + workflowId: 'workflow-current', + executionId: 'execution-current', + userId: 'user-current', + ...overrides, + } as ExecutionContext +} + +describe('createExecutorPrincipalFromExecutionContext', () => { + beforeEach(() => { + vi.clearAllMocks() + mockBindInternalExecutorDelegation.mockImplementation(async (claims, options) => ({ + kind: 'delegated', + serviceId: 'executor', + ...(claims.subjectUserId ? { subjectUserId: claims.subjectUserId } : {}), + workspaceId: 'workspace-canonical', + delegationId: claims.delegationId, + audience: options.audience, + issuedAt: claims.issuedAt, + expiresAt: claims.expiresAt, + resourceScope: options.resourceScope, + delegationContext: { + kind: 'workflow_execution', + workflowId: claims.workflowId, + ...(claims.executionId ? { executionId: claims.executionId } : {}), + ...(claims.principal ? { principal: claims.principal } : {}), + ...(claims.currentWorkflow ? { currentWorkflow: claims.currentWorkflow } : {}), + }, + })) + }) + + it('uses the signed delegation origin ahead of nested execution identity', async () => { + await createExecutorPrincipalFromExecutionContext({ + context: executionContext({ + executorDelegationOrigin: { + subjectUserId: 'user-origin', + workflowId: 'workflow-origin', + executionId: 'execution-origin', + }, + }), + audience: 'sim:tables', + resourceScope: { tableId: 'table-1' }, + }) + + expect(mockBindInternalExecutorDelegation).toHaveBeenCalledWith( + expect.objectContaining({ + subjectUserId: 'user-origin', + workflowId: 'workflow-origin', + executionId: 'execution-origin', + }), + { audience: 'sim:tables', resourceScope: { tableId: 'table-1' } } + ) + }) + + it('uses an explicit trusted execution deadline as the delegation expiry', async () => { + const expiresAt = new Date('2026-01-01T01:00:00.000Z') + + await createExecutorPrincipalFromExecutionContext({ + context: executionContext({ + executorDelegationOrigin: { + subjectUserId: 'user-origin', + workflowId: 'workflow-origin', + executionId: 'execution-origin', + }, + }), + audience: 'sim:function-executions', + expiresAt, + }) + + expect(mockBindInternalExecutorDelegation).toHaveBeenCalledWith( + expect.objectContaining({ expiresAt }), + { audience: 'sim:function-executions' } + ) + }) + + it.each([ + { + name: 'schedule', + principal: { + kind: 'system' as const, + serviceId: 'schedule' as const, + workspaceId: 'workspace-canonical', + workflowId: 'workflow-origin', + }, + }, + { + name: 'workspace API key', + principal: { + kind: 'workspace_api_key' as const, + workspaceId: 'workspace-canonical', + keyId: 'workspace-key-1', + }, + }, + { + name: 'webhook external subject', + principal: { + kind: 'system' as const, + serviceId: 'webhook' as const, + workspaceId: 'workspace-canonical', + workflowId: 'workflow-origin', + webhookId: 'webhook-1', + provider: 'slack', + subject: { + kind: 'external_user' as const, + provider: 'slack', + tenantId: 'team-1', + subjectId: 'external-user-1', + }, + }, + }, + ])('preserves an actorless $name principal and deployment authority', async ({ principal }) => { + const currentWorkflow = { + workflowId: 'workflow-origin', + mode: 'deployment' as const, + deploymentVersionId: 'deployment-1', + } + + await createExecutorPrincipalFromExecutionContext({ + context: executionContext({ + executorDelegationOrigin: { + workflowId: 'workflow-origin', + executionId: 'execution-origin', + principal, + currentWorkflow, + }, + }), + audience: 'sim:tables', + }) + + expect(mockBindInternalExecutorDelegation).toHaveBeenCalledWith( + expect.objectContaining({ + workflowId: 'workflow-origin', + executionId: 'execution-origin', + principal, + currentWorkflow, + }), + { audience: 'sim:tables' } + ) + expect(mockBindInternalExecutorDelegation.mock.calls[0]?.[0]).not.toHaveProperty( + 'subjectUserId' + ) + }) + + it('derives the subject from the preserved human principal', async () => { + const principal = { + kind: 'session' as const, + userId: 'user-origin', + sessionId: 'session-origin', + } + + await createExecutorPrincipalFromExecutionContext({ + context: executionContext({ + executorDelegationOrigin: { + workflowId: 'workflow-origin', + executionId: 'execution-origin', + principal, + currentWorkflow: { workflowId: 'workflow-origin', mode: 'draft' }, + }, + }), + audience: 'sim:tables', + }) + + expect(mockBindInternalExecutorDelegation).toHaveBeenCalledWith( + expect.objectContaining({ + subjectUserId: 'user-origin', + principal, + currentWorkflow: { workflowId: 'workflow-origin', mode: 'draft' }, + }), + { audience: 'sim:tables' } + ) + }) + + it('rejects a supplied subject that disagrees with the preserved principal', async () => { + await expect( + createExecutorPrincipalFromExecutionContext({ + context: executionContext({ + executorDelegationOrigin: { + subjectUserId: 'forged-user', + workflowId: 'workflow-origin', + principal: { + kind: 'session', + userId: 'user-origin', + sessionId: 'session-origin', + }, + }, + }), + audience: 'sim:tables', + }) + ).rejects.toThrow('Executor subject does not match its workflow principal') + expect(mockBindInternalExecutorDelegation).not.toHaveBeenCalled() + }) + + it('fails closed without a canonical delegation origin', async () => { + await expect( + createExecutorPrincipalFromExecutionContext({ + context: executionContext(), + audience: 'sim:tables', + }) + ).rejects.toThrow('Executor delegation origin is required') + expect(mockBindInternalExecutorDelegation).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/principals/executor.ts b/apps/sim/lib/internal/principals/executor.ts new file mode 100644 index 00000000000..9295ef88fb5 --- /dev/null +++ b/apps/sim/lib/internal/principals/executor.ts @@ -0,0 +1,75 @@ +import { type DelegatedPrincipal, resolvePrincipalSubject } from '@sim/auth/principal' +import { generateId } from '@sim/utils/id' +import { bindInternalExecutorDelegation } from '@/lib/auth/internal-delegation' +import type { InternalToolOperationContext } from '@/lib/internal/tool-operations/types' +import type { ExecutorDelegationOrigin } from '@/executor/types' + +const EXECUTOR_DELEGATION_TTL_MS = 5 * 60 * 1000 + +export function resolveExecutorOriginSubject(origin: ExecutorDelegationOrigin): string | undefined { + const principalSubject = origin.principal ? resolvePrincipalSubject(origin.principal) : null + if (principalSubject?.kind === 'external_user' && origin.subjectUserId) { + throw new Error('External workflow subjects cannot be represented as Sim users') + } + if (!principalSubject && origin.principal && origin.subjectUserId) { + throw new Error('Actorless workflow principals cannot be represented as Sim users') + } + if ( + principalSubject?.kind === 'sim_user' && + origin.subjectUserId && + origin.subjectUserId !== principalSubject.userId + ) { + throw new Error('Executor subject does not match its workflow principal') + } + + const subjectUserId = + principalSubject?.kind === 'sim_user' ? principalSubject.userId : origin.subjectUserId + if (!subjectUserId && !origin.principal) throw new Error('Authentication required') + return subjectUserId +} + +async function bindExecutorPrincipal( + origin: ExecutorDelegationOrigin, + audience: string, + resourceScope?: DelegatedPrincipal['resourceScope'], + expiresAt?: Date +) { + if (!origin.workflowId.trim()) throw new Error('Authentication required') + const subjectUserId = resolveExecutorOriginSubject(origin) + const issuedAt = new Date() + return bindInternalExecutorDelegation( + { + serviceId: 'executor', + ...(subjectUserId ? { subjectUserId } : {}), + workflowId: origin.workflowId, + ...(origin.executionId ? { executionId: origin.executionId } : {}), + ...(origin.principal ? { principal: origin.principal } : {}), + ...(origin.currentWorkflow ? { currentWorkflow: origin.currentWorkflow } : {}), + delegationId: generateId(), + issuedAt, + expiresAt: expiresAt ?? new Date(issuedAt.getTime() + EXECUTOR_DELEGATION_TTL_MS), + }, + { + audience, + ...(resourceScope ? { resourceScope } : {}), + } + ) +} + +export interface CreateExecutorPrincipalFromExecutionContextInput { + context: InternalToolOperationContext + audience: string + resourceScope?: DelegatedPrincipal['resourceScope'] + expiresAt?: Date +} + +export async function createExecutorPrincipalFromExecutionContext({ + context, + audience, + resourceScope, + expiresAt, +}: CreateExecutorPrincipalFromExecutionContextInput) { + const origin = context.executorDelegationOrigin + if (!origin) throw new Error('Executor delegation origin is required') + return bindExecutorPrincipal(origin, audience, resourceScope, expiresAt) +} diff --git a/apps/sim/lib/internal/pulse/client.ts b/apps/sim/lib/internal/pulse/client.ts new file mode 100644 index 00000000000..c2deac2b4a2 --- /dev/null +++ b/apps/sim/lib/internal/pulse/client.ts @@ -0,0 +1,54 @@ +import { createLogger } from '@sim/logger' +import { + DEFAULT_MAX_RESPONSE_BYTES, + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + readResponseTextWithLimit, +} from '@/lib/core/utils/stream-limits' +import { PulseOperationError } from '@/lib/internal/pulse/errors' + +const logger = createLogger('PulseClient') +const PULSE_ENDPOINT = 'https://api.runpulse.com/extract' + +export async function submitPulseParse( + apiKey: string, + formData: FormData, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const validation = await validateUrlWithDNS(PULSE_ENDPOINT, 'Pulse API URL') + signal?.throwIfAborted() + if (!validation.isValid || !validation.resolvedIP) { + throw new PulseOperationError(502, { success: false, error: 'Failed to reach Pulse API' }) + } + + const payload = new Response(formData) + const contentType = payload.headers.get('content-type') || 'multipart/form-data' + const body = Buffer.from(await payload.arrayBuffer()) + signal?.throwIfAborted() + const response = await secureFetchWithPinnedIP(PULSE_ENDPOINT, validation.resolvedIP, { + method: 'POST', + headers: { 'x-api-key': apiKey, 'Content-Type': contentType }, + body, + maxResponseBytes: DEFAULT_MAX_RESPONSE_BYTES, + signal, + }) + signal?.throwIfAborted() + + if (!response.ok) { + const diagnostic = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Pulse API error response', + signal, + }) + logger.error('Pulse API error', { status: response.status, diagnostic }) + throw new PulseOperationError(response.status, { + success: false, + error: `Pulse API error: ${response.statusText}`, + }) + } + return response.json() +} diff --git a/apps/sim/lib/internal/pulse/errors.ts b/apps/sim/lib/internal/pulse/errors.ts new file mode 100644 index 00000000000..76c0d2ac96b --- /dev/null +++ b/apps/sim/lib/internal/pulse/errors.ts @@ -0,0 +1,9 @@ +export class PulseOperationError extends Error { + constructor( + readonly status: number, + readonly body: unknown + ) { + super('Pulse operation failed') + this.name = 'PulseOperationError' + } +} diff --git a/apps/sim/lib/internal/pulse/execute-tool.test.ts b/apps/sim/lib/internal/pulse/execute-tool.test.ts new file mode 100644 index 00000000000..cfa713d9cf4 --- /dev/null +++ b/apps/sim/lib/internal/pulse/execute-tool.test.ts @@ -0,0 +1,31 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operation = vi.hoisted(() => vi.fn()) +vi.mock('@/lib/internal/pulse/operations', () => ({ executePulseParse: operation })) + +import { executePulseTool } from '@/lib/internal/pulse/execute-tool' + +describe('executePulseTool', () => { + beforeEach(() => { + vi.clearAllMocks() + operation.mockResolvedValue({ success: true, output: { job_id: 'job-1' } }) + }) + + it('dispatches both canonical IDs with trusted context', async () => { + for (const toolId of ['pulse_parser', 'pulse_parser_v2']) { + const response = await executePulseTool({ + toolId, + input: { apiKey: 'key', filePath: 'https://example.com/file.pdf' }, + headers: new Headers(), + context: { ...createExecutionContext(), userId: 'user-1' }, + requestId: 'request-1', + }) + expect(response.status).toBe(200) + } + expect(operation).toHaveBeenCalledTimes(2) + }) +}) diff --git a/apps/sim/lib/internal/pulse/execute-tool.ts b/apps/sim/lib/internal/pulse/execute-tool.ts new file mode 100644 index 00000000000..f2d81a9c775 --- /dev/null +++ b/apps/sim/lib/internal/pulse/execute-tool.ts @@ -0,0 +1,71 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { z } from 'zod' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { PulseOperationError } from '@/lib/internal/pulse/errors' +import { pulseParseInputSchema } from '@/lib/internal/pulse/input' +import { executePulseParse } from '@/lib/internal/pulse/operations' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const logger = createLogger('PulseToolExecution') + +function validationResponse(error: z.ZodError): Response { + return Response.json( + { + success: false, + error: error.issues[0]?.message || 'Invalid request data', + details: error.issues, + }, + { status: 400 } + ) +} + +export const executePulseTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!['pulse_parser', 'pulse_parser_v2'].includes(request.toolId)) { + return Response.json( + { success: false, error: `Unsupported Pulse tool: ${request.toolId}` }, + { status: 500 } + ) + } + let serialized: string + try { + serialized = JSON.stringify(request.input) ?? '' + } catch { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + if (Buffer.byteLength(serialized, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + success: false, + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + const parsed = pulseParseInputSchema.safeParse(request.input) + if (!parsed.success) return validationResponse(parsed.error) + + try { + const result = await executePulseParse(parsed.data, { + headers: request.headers, + requestId: request.requestId, + signal: request.signal, + userId: request.context.userId, + }) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof PulseOperationError) { + return Response.json(error.body, { status: error.status }) + } + const message = getErrorMessage(error, 'Internal server error') + logger.error('Pulse operation failed', { + error: message, + requestId: request.requestId, + toolId: request.toolId, + }) + return Response.json({ success: false, error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/pulse/input.ts b/apps/sim/lib/internal/pulse/input.ts new file mode 100644 index 00000000000..23c7ea35248 --- /dev/null +++ b/apps/sim/lib/internal/pulse/input.ts @@ -0,0 +1,19 @@ +import { z } from 'zod' +import { resolvedSecretTraceProvenanceSchema } from '@/lib/api/contracts/primitives' +import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' + +export const pulseParseInputSchema = z.object({ + apiKey: z.string().min(1, 'API key is required'), + filePath: z.string().optional(), + file: RawFileInputSchema.optional(), + pages: z.string().max(10_000).optional(), + extractFigure: z.boolean().optional(), + figureDescription: z.boolean().optional(), + returnHtml: z.boolean().optional(), + chunking: z.string().max(10_000).optional(), + chunkSize: z.number().finite().optional(), + [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), +}) + +export type PulseParseInput = z.infer diff --git a/apps/sim/lib/internal/pulse/operations.test.ts b/apps/sim/lib/internal/pulse/operations.test.ts new file mode 100644 index 00000000000..e74ab6219cf --- /dev/null +++ b/apps/sim/lib/internal/pulse/operations.test.ts @@ -0,0 +1,63 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + resolveFileInputToUrl: vi.fn(), + submitPulseParse: vi.fn(), + validateProvenance: vi.fn(), +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + resolveFileInputToUrl: mocks.resolveFileInputToUrl, +})) +vi.mock('@/lib/internal/pulse/client', () => ({ submitPulseParse: mocks.submitPulseParse })) +vi.mock('@/lib/execution/model-input-provenance', () => ({ + validateOpaqueModelInputProvenance: mocks.validateProvenance, +})) + +import { executePulseParse } from '@/lib/internal/pulse/operations' + +describe('executePulseParse', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.validateProvenance.mockReturnValue({ success: true }) + mocks.resolveFileInputToUrl.mockResolvedValue({ fileUrl: 'https://example.com/file.pdf' }) + mocks.submitPulseParse.mockResolvedValue({ job_id: 'job-1' }) + }) + + it('preserves multipart parser options and forwards cancellation', async () => { + const controller = new AbortController() + await executePulseParse( + { + apiKey: 'key', + filePath: 'https://example.com/file.pdf', + pages: '1-2,5', + extractFigure: true, + chunking: 'semantic', + chunkSize: 2000, + }, + { + headers: new Headers(), + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + } + ) + + const [, formData, signal] = mocks.submitPulseParse.mock.calls[0] as [ + string, + FormData, + AbortSignal, + ] + expect(Object.fromEntries(formData.entries())).toEqual({ + file_url: 'https://example.com/file.pdf', + pages: '1-2,5', + extract_figure: 'true', + chunking: 'semantic', + chunk_size: '2000', + }) + expect(signal).toBe(controller.signal) + }) +}) diff --git a/apps/sim/lib/internal/pulse/operations.ts b/apps/sim/lib/internal/pulse/operations.ts new file mode 100644 index 00000000000..4f8c4520692 --- /dev/null +++ b/apps/sim/lib/internal/pulse/operations.ts @@ -0,0 +1,72 @@ +import { createLogger } from '@sim/logger' +import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' +import { submitPulseParse } from '@/lib/internal/pulse/client' +import { PulseOperationError } from '@/lib/internal/pulse/errors' +import type { PulseParseInput } from '@/lib/internal/pulse/input' +import { resolveFileInputToUrl } from '@/lib/uploads/utils/file-utils.server' + +const logger = createLogger('PulseOperations') + +export interface PulseOperationContext { + headers: Headers + requestId: string + signal?: AbortSignal + userId?: string +} + +export async function executePulseParse( + input: PulseParseInput, + context: PulseOperationContext +): Promise<{ success: true; output: unknown }> { + context.signal?.throwIfAborted() + if (!context.userId) { + throw new PulseOperationError(401, { success: false, error: 'Unauthorized' }) + } + const provenance = validateOpaqueModelInputProvenance({ + headers: context.headers, + payload: input, + isInternalRequest: true, + }) + if (!provenance.success) { + throw new PulseOperationError(provenance.status, { + success: false, + error: provenance.error, + }) + } + + const resolution = await resolveFileInputToUrl({ + file: input.file, + filePath: input.filePath, + userId: context.userId, + requestId: context.requestId, + logger, + modelEgress: true, + }) + context.signal?.throwIfAborted() + if (resolution.error) { + throw new PulseOperationError(resolution.error.status, { + success: false, + error: resolution.error.message, + }) + } + if (!resolution.fileUrl) { + throw new PulseOperationError(400, { success: false, error: 'File input is required' }) + } + + const formData = new FormData() + formData.append('file_url', resolution.fileUrl) + if (input.pages) formData.append('pages', input.pages) + if (input.extractFigure !== undefined) { + formData.append('extract_figure', String(input.extractFigure)) + } + if (input.figureDescription !== undefined) { + formData.append('figure_description', String(input.figureDescription)) + } + if (input.returnHtml !== undefined) formData.append('return_html', String(input.returnHtml)) + if (input.chunking) formData.append('chunking', input.chunking) + if (input.chunkSize !== undefined) formData.append('chunk_size', String(input.chunkSize)) + + const output = await submitPulseParse(input.apiKey, formData, context.signal) + context.signal?.throwIfAborted() + return { success: true, output } +} diff --git a/apps/sim/lib/internal/quiver/client.test.ts b/apps/sim/lib/internal/quiver/client.test.ts new file mode 100644 index 00000000000..08190bcc0ca --- /dev/null +++ b/apps/sim/lib/internal/quiver/client.test.ts @@ -0,0 +1,71 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { requestQuiverSvg } from '@/lib/internal/quiver/client' + +describe('Quiver client', () => { + beforeEach(() => { + vi.restoreAllMocks() + vi.clearAllMocks() + }) + + it('preserves provider URL, authentication, payload, and cancellation', async () => { + const controller = new AbortController() + const fetchMock = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValue(Response.json({ data: [{ svg: '' }], id: 'generation-1' })) + + await expect( + requestQuiverSvg( + 'generations', + 'secret', + { model: 'arrow-preview', prompt: 'A compass' }, + controller.signal + ) + ).resolves.toEqual({ data: [{ svg: '' }], id: 'generation-1' }) + expect(fetchMock).toHaveBeenCalledWith('https://api.quiver.ai/v1/svgs/generations', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Bearer secret', + }, + body: JSON.stringify({ model: 'arrow-preview', prompt: 'A compass' }), + signal: controller.signal, + }) + }) + + it('preserves provider error status and text', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response('invalid model', { status: 422 })) + + await expect( + requestQuiverSvg('vectorizations', 'secret', { model: 'bad' }) + ).rejects.toMatchObject({ + status: 422, + body: { success: false, error: 'Quiver API error: 422 - invalid model' }, + }) + }) + + it('bounds provider success and error bodies before buffering', async () => { + vi.spyOn(globalThis, 'fetch') + .mockResolvedValueOnce( + new Response('{}', { + headers: { 'content-length': String(MAX_JSON_API_RESPONSE_BYTES + 1) }, + }) + ) + .mockResolvedValueOnce( + new Response('error', { + status: 500, + headers: { 'content-length': String(64 * 1024 + 1) }, + }) + ) + + await expect(requestQuiverSvg('generations', 'secret', {})).rejects.toMatchObject({ + name: 'PayloadSizeLimitError', + }) + await expect(requestQuiverSvg('generations', 'secret', {})).rejects.toMatchObject({ + name: 'PayloadSizeLimitError', + }) + }) +}) diff --git a/apps/sim/lib/internal/quiver/client.ts b/apps/sim/lib/internal/quiver/client.ts new file mode 100644 index 00000000000..acb7d512020 --- /dev/null +++ b/apps/sim/lib/internal/quiver/client.ts @@ -0,0 +1,54 @@ +import { createLogger } from '@sim/logger' +import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { + readResponseJsonWithLimit, + readResponseTextWithLimit, +} from '@/lib/core/utils/stream-limits' +import { QuiverOperationError } from '@/lib/internal/quiver/errors' + +const logger = createLogger('QuiverClient') +const QUIVER_API_BASE_URL = 'https://api.quiver.ai/v1/svgs' +const MAX_QUIVER_ERROR_BYTES = 64 * 1024 + +export type QuiverOperationPath = 'generations' | 'vectorizations' + +export async function requestQuiverSvg( + path: QuiverOperationPath, + apiKey: string, + body: Record, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const response = await fetch(`${QUIVER_API_BASE_URL}/${path}`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify(body), + signal, + }) + signal?.throwIfAborted() + + if (!response.ok) { + const errorText = await readResponseTextWithLimit(response, { + maxBytes: MAX_QUIVER_ERROR_BYTES, + label: 'Quiver error response', + signal, + }) + signal?.throwIfAborted() + logger.error('Quiver API request failed', { path, status: response.status, error: errorText }) + throw new QuiverOperationError( + `Quiver API error: ${response.status} - ${errorText}`, + response.status + ) + } + + const result = await readResponseJsonWithLimit(response, { + maxBytes: MAX_JSON_API_RESPONSE_BYTES, + label: 'Quiver SVG response', + signal, + }) + signal?.throwIfAborted() + return result +} diff --git a/apps/sim/lib/internal/quiver/errors.ts b/apps/sim/lib/internal/quiver/errors.ts new file mode 100644 index 00000000000..fbd6b224418 --- /dev/null +++ b/apps/sim/lib/internal/quiver/errors.ts @@ -0,0 +1,10 @@ +export class QuiverOperationError extends Error { + constructor( + message: string, + readonly status: number, + readonly body: Record = { success: false, error: message } + ) { + super(message) + this.name = 'QuiverOperationError' + } +} diff --git a/apps/sim/lib/internal/quiver/execute-tool.test.ts b/apps/sim/lib/internal/quiver/execute-tool.test.ts new file mode 100644 index 00000000000..d89b4394be8 --- /dev/null +++ b/apps/sim/lib/internal/quiver/execute-tool.test.ts @@ -0,0 +1,122 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' + +const mocks = vi.hoisted(() => ({ + executeImage: vi.fn(), + executeText: vi.fn(), +})) + +vi.mock('@/lib/internal/quiver/operations', () => ({ + executeQuiverImageToSvg: mocks.executeImage, + executeQuiverTextToSvg: mocks.executeText, +})) + +import { QuiverOperationError } from '@/lib/internal/quiver/errors' +import { executeQuiverTool } from '@/lib/internal/quiver/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +function request(overrides: Partial = {}) { + return { + toolId: 'quiver_text_to_svg', + input: { apiKey: 'secret', model: 'arrow-preview', prompt: 'A compass' }, + headers: new Headers(), + context: { ...createExecutionContext({ workflowId: 'workflow-1' }), userId: 'user-1' }, + requestId: 'request-1', + ...overrides, + } as InternalToolOperationCall +} + +describe('executeQuiverTool', () => { + beforeEach(() => { + vi.clearAllMocks() + const result = { + success: true, + output: { + file: { name: 'generated.svg' }, + files: [{ name: 'generated.svg' }], + svgContent: '', + id: 'generation-1', + usage: null, + }, + } + mocks.executeText.mockResolvedValue(result) + mocks.executeImage.mockResolvedValue(result) + }) + + it.each([ + ['quiver_text_to_svg', mocks.executeText], + ['quiver_image_to_svg', mocks.executeImage], + ])('dispatches %s to the typed operation', async (toolId, execute) => { + const input = + toolId === 'quiver_image_to_svg' + ? { apiKey: 'secret', model: 'arrow-preview', image: 'https://example.com/image.png' } + : { apiKey: 'secret', model: 'arrow-preview', prompt: 'A compass' } + const response = await executeQuiverTool(request({ toolId, input })) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ success: true }) + expect(execute).toHaveBeenCalledWith( + expect.objectContaining({ apiKey: 'secret', model: 'arrow-preview' }), + expect.objectContaining({ userId: 'user-1', requestId: 'request-1' }) + ) + }) + + it('authenticates before parsing input', async () => { + const response = await executeQuiverTool( + request({ input: null, context: createExecutionContext({ workflowId: 'workflow-1' }) }) + ) + + expect(response.status).toBe(401) + await expect(response.json()).resolves.toEqual({ success: false, error: 'Unauthorized' }) + expect(mocks.executeText).not.toHaveBeenCalled() + }) + + it('preserves validation envelopes', async () => { + const response = await executeQuiverTool(request({ input: {} })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + success: false, + error: expect.any(String), + details: expect.any(Array), + }) + }) + + it('preserves the route input byte ceiling', async () => { + const response = await executeQuiverTool( + request({ + input: { + apiKey: 'secret', + model: 'arrow-preview', + prompt: 'x'.repeat(DEFAULT_MAX_JSON_BODY_BYTES), + }, + }) + ) + + expect(response.status).toBe(413) + expect(mocks.executeText).not.toHaveBeenCalled() + }) + + it('projects exact operation errors', async () => { + mocks.executeText.mockRejectedValueOnce(new QuiverOperationError('invalid model', 422)) + + const response = await executeQuiverTool(request()) + + expect(response.status).toBe(422) + await expect(response.json()).resolves.toEqual({ success: false, error: 'invalid model' }) + }) + + it('stops before dispatch when execution is already aborted', async () => { + const controller = new AbortController() + controller.abort(new Error('Execution aborted')) + + await expect(executeQuiverTool(request({ signal: controller.signal }))).rejects.toThrow( + 'Execution aborted' + ) + expect(mocks.executeText).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/quiver/execute-tool.ts b/apps/sim/lib/internal/quiver/execute-tool.ts new file mode 100644 index 00000000000..f4e4663e4d1 --- /dev/null +++ b/apps/sim/lib/internal/quiver/execute-tool.ts @@ -0,0 +1,108 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { z } from 'zod' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { QuiverOperationError } from '@/lib/internal/quiver/errors' +import { + executeQuiverImageToSvg, + executeQuiverTextToSvg, + type QuiverOperationContext, +} from '@/lib/internal/quiver/operations' +import { + quiverImageToSvgInputSchema, + quiverTextToSvgInputSchema, +} from '@/lib/internal/quiver/schema' +import type { + InternalToolOperationCall, + InternalToolOperationHandler, +} from '@/lib/internal/tool-operations/types' + +const logger = createLogger('QuiverToolExecution') + +function validateInputSize(input: unknown): Response | null { + let serializedInput: string + try { + serializedInput = JSON.stringify(input) ?? '' + } catch { + return Response.json( + { success: false, error: 'Invalid request data', details: [] }, + { status: 400 } + ) + } + if (Buffer.byteLength(serializedInput, 'utf8') <= DEFAULT_MAX_JSON_BODY_BYTES) return null + return Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) +} + +async function executeOperation( + request: InternalToolOperationCall, + schema: z.ZodType, + execute: (input: Input, context: QuiverOperationContext) => Promise +): Promise { + request.signal?.throwIfAborted() + const sizeError = validateInputSize(request.input) + if (sizeError) return sizeError + const parsed = schema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { + success: false, + error: getValidationErrorMessage(parsed.error, 'Invalid request data'), + details: parsed.error.issues, + }, + { status: 400 } + ) + } + const userId = request.context.userId + if (!userId) return Response.json({ success: false, error: 'Unauthorized' }, { status: 401 }) + + try { + const result = await execute(parsed.data, { + headers: request.headers, + requestId: request.requestId, + signal: request.signal, + userId, + }) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof QuiverOperationError) { + return Response.json(error.body, { status: error.status }) + } + const message = getErrorMessage(error, 'Unknown error') + logger.error('Quiver operation failed', { + error: message, + requestId: request.requestId, + toolId: request.toolId, + }) + return Response.json( + { success: false, error: message }, + { status: isPayloadSizeLimitError(error) ? 413 : 500 } + ) + } +} + +export const executeQuiverTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!request.context.userId) { + return Response.json({ success: false, error: 'Unauthorized' }, { status: 401 }) + } + switch (request.toolId) { + case 'quiver_text_to_svg': + return executeOperation(request, quiverTextToSvgInputSchema, executeQuiverTextToSvg) + case 'quiver_image_to_svg': + return executeOperation(request, quiverImageToSvgInputSchema, executeQuiverImageToSvg) + default: + return Response.json( + { success: false, error: `Unsupported Quiver tool: ${request.toolId}` }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/quiver/operations.test.ts b/apps/sim/lib/internal/quiver/operations.test.ts new file mode 100644 index 00000000000..65fc8133fda --- /dev/null +++ b/apps/sim/lib/internal/quiver/operations.test.ts @@ -0,0 +1,229 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { PRIVATE_MODEL_INPUT_PROVENANCE_HEADER } from '@/lib/execution/model-input-provenance' +import { + RESOLVED_SECRET_PROVENANCE_FIELD, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, +} from '@/lib/execution/private-tool-metadata' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' + +const mocks = vi.hoisted(() => ({ + assertToolFileAccess: vi.fn(), + downloadFileFromStorage: vi.fn(), + isModelSafeWorkspaceFileKey: vi.fn(), + processFilesToUserFiles: vi.fn(), + requestQuiverSvg: vi.fn(), +})) + +vi.mock('@/lib/internal/quiver/client', () => ({ requestQuiverSvg: mocks.requestQuiverSvg })) +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertToolFileAccess, +})) +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + processFilesToUserFiles: mocks.processFilesToUserFiles, +})) +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadFileFromStorage: mocks.downloadFileFromStorage, +})) +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({ + isModelSafeWorkspaceFileKey: mocks.isModelSafeWorkspaceFileKey, + MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE: + 'File cannot be sent to a model because its secret provenance is unavailable', +})) + +import { executeQuiverImageToSvg, executeQuiverTextToSvg } from '@/lib/internal/quiver/operations' + +const rawFile = { + key: 'workspace/workspace-1/image.png', + name: 'image.png', + size: 3, + type: 'image/png', +} + +const context = { + headers: new Headers(), + requestId: 'request-1', + userId: 'user-1', +} + +describe('Quiver operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.assertToolFileAccess.mockResolvedValue(null) + mocks.downloadFileFromStorage.mockResolvedValue(Buffer.from([1, 2, 3])) + mocks.isModelSafeWorkspaceFileKey.mockResolvedValue(true) + mocks.processFilesToUserFiles.mockImplementation((files: unknown[]) => files) + mocks.requestQuiverSvg.mockResolvedValue({ + data: [{ svg: 'one' }, { svg: 'two' }], + id: 'generation-1', + usage: { total_tokens: 9, input_tokens: 4, output_tokens: 5 }, + }) + }) + + it('authorizes, checks, and cumulatively bounds stored references', async () => { + mocks.downloadFileFromStorage + .mockResolvedValueOnce(Buffer.from([1, 2, 3])) + .mockResolvedValueOnce(Buffer.from([4, 5])) + const controller = new AbortController() + + const result = await executeQuiverTextToSvg( + { + apiKey: 'secret', + model: 'arrow-preview', + prompt: 'A compass', + instructions: 'Minimal', + references: [rawFile, { ...rawFile, key: 'workspace/workspace-1/second.png' }], + n: 2, + temperature: 0.5, + }, + { ...context, signal: controller.signal } + ) + + expect(mocks.assertToolFileAccess).toHaveBeenCalledTimes(2) + expect(mocks.isModelSafeWorkspaceFileKey).toHaveBeenCalledTimes(2) + expect(mocks.downloadFileFromStorage).toHaveBeenNthCalledWith( + 1, + expect.anything(), + 'request-1', + expect.anything(), + { maxBytes: MAX_BUFFERED_TRANSFER_BYTES } + ) + expect(mocks.downloadFileFromStorage).toHaveBeenNthCalledWith( + 2, + expect.anything(), + 'request-1', + expect.anything(), + { maxBytes: MAX_BUFFERED_TRANSFER_BYTES - 3 } + ) + expect(mocks.requestQuiverSvg).toHaveBeenCalledWith( + 'generations', + 'secret', + { + model: 'arrow-preview', + prompt: 'A compass', + instructions: 'Minimal', + references: [{ base64: 'AQID' }, { base64: 'BAU=' }], + n: 2, + temperature: 0.5, + }, + controller.signal + ) + expect(result.output).toMatchObject({ + file: { name: 'generated-1.svg', mimeType: 'image/svg+xml' }, + files: [{ name: 'generated-1.svg' }, { name: 'generated-2.svg' }], + svgContent: 'one', + id: 'generation-1', + usage: { totalTokens: 9, inputTokens: 4, outputTokens: 5 }, + }) + }) + + it('preserves image URL inputs without reading local files', async () => { + const result = await executeQuiverImageToSvg( + { + apiKey: 'secret', + model: 'arrow-preview', + image: 'https://images.example.com/source.png', + auto_crop: false, + target_size: 512, + }, + context + ) + + expect(mocks.assertToolFileAccess).not.toHaveBeenCalled() + expect(mocks.requestQuiverSvg).toHaveBeenCalledWith( + 'vectorizations', + 'secret', + { + model: 'arrow-preview', + image: { url: 'https://images.example.com/source.png' }, + auto_crop: false, + target_size: 512, + }, + undefined + ) + expect(result.output.file.name).toBe('vectorized.svg') + expect(result.output.files).toHaveLength(1) + expect(result.output.svgContent).toBe('one') + }) + + it('authorizes stored image inputs and sends their bytes', async () => { + await executeQuiverImageToSvg( + { apiKey: 'secret', model: 'arrow-preview', image: rawFile }, + context + ) + + expect(mocks.assertToolFileAccess).toHaveBeenCalledWith( + rawFile.key, + 'user-1', + 'request-1', + expect.anything() + ) + expect(mocks.downloadFileFromStorage).toHaveBeenCalledWith( + expect.anything(), + 'request-1', + expect.anything(), + { maxBytes: MAX_BUFFERED_TRANSFER_BYTES } + ) + expect(mocks.requestQuiverSvg).toHaveBeenCalledWith( + 'vectorizations', + 'secret', + { model: 'arrow-preview', image: { base64: 'AQID' } }, + undefined + ) + }) + + it('fails closed on incomplete private model-input provenance', async () => { + const headers = new Headers({ + [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1, + }) + + await expect( + executeQuiverImageToSvg( + { + apiKey: 'secret', + model: 'arrow-preview', + image: rawFile, + [RESOLVED_SECRET_PROVENANCE_FIELD]: { version: 1, complete: false, entries: [] }, + }, + { ...context, headers } + ) + ).rejects.toMatchObject({ + status: 400, + body: { success: false, error: 'Model input provenance is unavailable' }, + }) + expect(mocks.assertToolFileAccess).not.toHaveBeenCalled() + expect(mocks.requestQuiverSvg).not.toHaveBeenCalled() + }) + + it('rejects model-unsafe stored files before downloading them', async () => { + mocks.isModelSafeWorkspaceFileKey.mockResolvedValue(false) + + await expect( + executeQuiverImageToSvg({ apiKey: 'secret', model: 'arrow-preview', image: rawFile }, context) + ).rejects.toMatchObject({ + status: 400, + body: { + success: false, + error: 'File cannot be sent to a model because its secret provenance is unavailable', + }, + }) + expect(mocks.downloadFileFromStorage).not.toHaveBeenCalled() + expect(mocks.requestQuiverSvg).not.toHaveBeenCalled() + }) + + it('preserves the empty-provider-result error', async () => { + mocks.requestQuiverSvg.mockResolvedValue({ data: [] }) + + await expect( + executeQuiverTextToSvg( + { apiKey: 'secret', model: 'arrow-preview', prompt: 'A compass' }, + context + ) + ).rejects.toMatchObject({ + status: 500, + body: { success: false, error: 'No SVG data returned from Quiver API' }, + }) + }) +}) diff --git a/apps/sim/lib/internal/quiver/operations.ts b/apps/sim/lib/internal/quiver/operations.ts new file mode 100644 index 00000000000..809830b7bb1 --- /dev/null +++ b/apps/sim/lib/internal/quiver/operations.ts @@ -0,0 +1,249 @@ +import { createLogger } from '@sim/logger' +import { isRecordLike } from '@sim/utils/object' +import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' +import { requestQuiverSvg } from '@/lib/internal/quiver/client' +import { QuiverOperationError } from '@/lib/internal/quiver/errors' +import type { QuiverImageToSvgInput, QuiverTextToSvgInput } from '@/lib/internal/quiver/schema' +import { + isModelSafeWorkspaceFileKey, + MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, +} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +const logger = createLogger('QuiverOperations') + +export interface QuiverOperationContext { + headers: Headers + requestId: string + signal?: AbortSignal + userId: string +} + +interface QuiverFile { + name: string + mimeType: 'image/svg+xml' + data: string + size: number +} + +interface QuiverUsage { + totalTokens: number + inputTokens: number + outputTokens: number +} + +export interface QuiverSvgOutput { + success: true + output: { + file: QuiverFile + files: QuiverFile[] + svgContent: string + id: string | null + usage: QuiverUsage | null + } +} + +type ApiImage = { url: string } | { base64: string } + +function fail(message: string, status: number, body?: Record): never { + throw new QuiverOperationError(message, status, body) +} + +function record(value: unknown): Record { + return isRecordLike(value) ? value : {} +} + +function optionalNumber(value: unknown): number { + return typeof value === 'number' ? value : 0 +} + +function validateProvenance(input: Record, context: QuiverOperationContext): void { + const provenance = validateOpaqueModelInputProvenance({ + headers: context.headers, + payload: input, + isInternalRequest: true, + }) + if (!provenance.success) fail(provenance.error, provenance.status) +} + +async function deniedBody(response: Response): Promise> { + const body: unknown = await response.json() + return isRecordLike(body) ? body : { success: false, error: 'File not found' } +} + +async function storedFileToBase64( + input: RawFileInput, + context: QuiverOperationContext, + maxBytes: number +): Promise<{ base64: string; size: number } | null> { + const files = processFilesToUserFiles([input], context.requestId, logger) + const file = files[0] + if (!file) return null + + context.signal?.throwIfAborted() + const denied = await assertToolFileAccess(file.key, context.userId, context.requestId, logger) + context.signal?.throwIfAborted() + if (denied) fail('File not found', denied.status, await deniedBody(denied)) + if (!(await isModelSafeWorkspaceFileKey(file.key))) { + fail(MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, 400) + } + context.signal?.throwIfAborted() + const buffer = await downloadFileFromStorage(file, context.requestId, logger, { maxBytes }) + context.signal?.throwIfAborted() + return { base64: buffer.toString('base64'), size: buffer.length } +} + +async function resolveReference( + reference: unknown, + context: QuiverOperationContext, + maxBytes: number +): Promise<{ image?: ApiImage; size: number }> { + if (typeof reference === 'string') { + try { + const parsed: unknown = JSON.parse(reference) + if (parsed && typeof parsed === 'object') { + const file = await storedFileToBase64(parsed as RawFileInput, context, maxBytes) + return file ? { image: { base64: file.base64 }, size: file.size } : { size: 0 } + } + return { size: 0 } + } catch (error) { + context.signal?.throwIfAborted() + if (error instanceof QuiverOperationError) throw error + return { image: { url: reference }, size: 0 } + } + } + if (reference && typeof reference === 'object') { + const file = await storedFileToBase64(reference as RawFileInput, context, maxBytes) + return file ? { image: { base64: file.base64 }, size: file.size } : { size: 0 } + } + return { size: 0 } +} + +async function resolveImage( + image: QuiverImageToSvgInput['image'], + context: QuiverOperationContext +): Promise { + if (typeof image === 'string') { + try { + const parsed: unknown = JSON.parse(image) + if (parsed && typeof parsed === 'object') { + const file = await storedFileToBase64( + parsed as RawFileInput, + context, + MAX_BUFFERED_TRANSFER_BYTES + ) + if (!file) fail('Invalid file input', 400) + return { base64: file.base64 } + } + return { url: image } + } catch (error) { + context.signal?.throwIfAborted() + if (error instanceof QuiverOperationError) throw error + return { url: image } + } + } + + const file = await storedFileToBase64(image, context, MAX_BUFFERED_TRANSFER_BYTES) + if (!file) fail('Invalid file input', 400) + return { base64: file.base64 } +} + +function projectResult( + result: unknown, + fileName: (index: number, total: number) => string, + firstOnly = false +): QuiverSvgOutput { + const root = record(result) + const data = root.data + if (!Array.isArray(data) || data.length === 0) { + fail('No SVG data returned from Quiver API', 500) + } + + const projectedData = firstOnly ? data.slice(0, 1) : data + const files = projectedData.map((value, index) => { + const svg = record(value).svg + if (typeof svg !== 'string') fail('No SVG data returned from Quiver API', 500) + const buffer = Buffer.from(svg, 'utf8') + return { + name: fileName(index, projectedData.length), + mimeType: 'image/svg+xml' as const, + data: buffer.toString('base64'), + size: buffer.length, + } + }) + const usage = isRecordLike(root.usage) + ? { + totalTokens: optionalNumber(root.usage.total_tokens), + inputTokens: optionalNumber(root.usage.input_tokens), + outputTokens: optionalNumber(root.usage.output_tokens), + } + : null + + return { + success: true, + output: { + file: files[0], + files, + svgContent: record(data[0]).svg as string, + id: typeof root.id === 'string' ? root.id : null, + usage, + }, + } +} + +export async function executeQuiverTextToSvg( + input: QuiverTextToSvgInput, + context: QuiverOperationContext +): Promise { + context.signal?.throwIfAborted() + validateProvenance(input, context) + const references: ApiImage[] = [] + let referenceBudget = MAX_BUFFERED_TRANSFER_BYTES + if (input.references) { + const rawReferences = Array.isArray(input.references) ? input.references : [input.references] + for (const reference of rawReferences) { + const resolved = await resolveReference(reference, context, referenceBudget) + referenceBudget -= resolved.size + if (resolved.image) references.push(resolved.image) + } + } + + const body: Record = { model: input.model, prompt: input.prompt } + if (input.instructions) body.instructions = input.instructions + if (references.length > 0) body.references = references.slice(0, 4) + if (input.n != null) body.n = input.n + if (input.temperature != null) body.temperature = input.temperature + if (input.top_p != null) body.top_p = input.top_p + if (input.max_output_tokens != null) body.max_output_tokens = input.max_output_tokens + if (input.presence_penalty != null) body.presence_penalty = input.presence_penalty + + const result = await requestQuiverSvg('generations', input.apiKey, body, context.signal) + context.signal?.throwIfAborted() + return projectResult(result, (index, total) => + total > 1 ? `generated-${index + 1}.svg` : 'generated.svg' + ) +} + +export async function executeQuiverImageToSvg( + input: QuiverImageToSvgInput, + context: QuiverOperationContext +): Promise { + context.signal?.throwIfAborted() + validateProvenance(input, context) + const image = await resolveImage(input.image, context) + const body: Record = { model: input.model, image } + if (input.temperature != null) body.temperature = input.temperature + if (input.top_p != null) body.top_p = input.top_p + if (input.max_output_tokens != null) body.max_output_tokens = input.max_output_tokens + if (input.presence_penalty != null) body.presence_penalty = input.presence_penalty + if (input.auto_crop != null) body.auto_crop = input.auto_crop + if (input.target_size != null) body.target_size = input.target_size + + const result = await requestQuiverSvg('vectorizations', input.apiKey, body, context.signal) + context.signal?.throwIfAborted() + return projectResult(result, () => 'vectorized.svg', true) +} diff --git a/apps/sim/lib/internal/quiver/schema.ts b/apps/sim/lib/internal/quiver/schema.ts new file mode 100644 index 00000000000..4d0e2259630 --- /dev/null +++ b/apps/sim/lib/internal/quiver/schema.ts @@ -0,0 +1,33 @@ +import { z } from 'zod' +import { resolvedSecretTraceProvenanceSchema } from '@/lib/api/contracts/primitives' +import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' +import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' + +const quiverCommonInputSchema = z.object({ + apiKey: z.string().min(1), + model: z.string().min(1), + temperature: z.number().min(0).max(2).optional().nullable(), + top_p: z.number().min(0).max(1).optional().nullable(), + max_output_tokens: z.number().int().min(1).max(131072).optional().nullable(), + presence_penalty: z.number().min(-2).max(2).optional().nullable(), + [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), +}) + +export const quiverTextToSvgInputSchema = quiverCommonInputSchema.extend({ + prompt: z.string().min(1), + instructions: z.string().optional().nullable(), + references: z + .union([z.array(FileInputSchema), FileInputSchema, z.string()]) + .optional() + .nullable(), + n: z.number().int().min(1).max(16).optional().nullable(), +}) + +export const quiverImageToSvgInputSchema = quiverCommonInputSchema.extend({ + image: z.union([FileInputSchema, z.string()]), + auto_crop: z.boolean().optional().nullable(), + target_size: z.number().int().min(128).max(4096).optional().nullable(), +}) + +export type QuiverTextToSvgInput = z.output +export type QuiverImageToSvgInput = z.output diff --git a/apps/sim/lib/internal/rds/client.ts b/apps/sim/lib/internal/rds/client.ts new file mode 100644 index 00000000000..458266a9ee3 --- /dev/null +++ b/apps/sim/lib/internal/rds/client.ts @@ -0,0 +1,752 @@ +import { + ExecuteStatementCommand, + type ExecuteStatementCommandOutput, + type Field, + RDSDataClient, + type SqlParameter, +} from '@aws-sdk/client-rds-data' +import type { RdsConnectionConfig } from '@/tools/rds/types' + +export function createRdsClient(config: RdsConnectionConfig): RDSDataClient { + return new RDSDataClient({ + region: config.region, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + }) +} + +export async function executeStatement( + client: RDSDataClient, + resourceArn: string, + secretArn: string, + database: string | undefined, + sql: string, + parameters?: SqlParameter[], + signal?: AbortSignal +): Promise<{ rows: Record[]; rowCount: number }> { + const command = new ExecuteStatementCommand({ + resourceArn, + secretArn, + ...(database && { database }), + sql, + ...(parameters && parameters.length > 0 && { parameters }), + includeResultMetadata: true, + }) + + const response = await client.send(command, { abortSignal: signal }) + const rows = parseRdsResponse(response) + + return { + rows, + rowCount: response.numberOfRecordsUpdated ?? rows.length, + } +} + +function parseRdsResponse(response: ExecuteStatementCommandOutput): Record[] { + if (!response.records || !response.columnMetadata) { + return [] + } + + const columnNames = response.columnMetadata.map((col) => col.name || col.label || 'unknown') + + return response.records.map((record) => { + const row: Record = {} + record.forEach((field, index) => { + const columnName = columnNames[index] || `column_${index}` + row[columnName] = parseFieldValue(field) + }) + return row + }) +} + +function parseFieldValue(field: Field): unknown { + if (field.isNull) return null + if (field.stringValue !== undefined) return field.stringValue + if (field.longValue !== undefined) return field.longValue + if (field.doubleValue !== undefined) return field.doubleValue + if (field.booleanValue !== undefined) return field.booleanValue + if (field.blobValue !== undefined) return Buffer.from(field.blobValue).toString('base64') + if (field.arrayValue !== undefined) { + const arr = field.arrayValue + if (arr.stringValues) return arr.stringValues + if (arr.longValues) return arr.longValues + if (arr.doubleValues) return arr.doubleValues + if (arr.booleanValues) return arr.booleanValues + if (arr.arrayValues) return arr.arrayValues.map((f) => parseFieldValue({ arrayValue: f })) + return [] + } + return null +} + +export function validateQuery(query: string): { isValid: boolean; error?: string } { + const trimmedQuery = query.trim().toLowerCase() + + const allowedStatements = /^(select|insert|update|delete|with|explain|show)\s+/i + if (!allowedStatements.test(trimmedQuery)) { + return { + isValid: false, + error: 'Only SELECT, INSERT, UPDATE, DELETE, WITH, EXPLAIN, and SHOW statements are allowed', + } + } + + return { isValid: true } +} + +export function sanitizeIdentifier(identifier: string): string { + if (identifier.includes('.')) { + const parts = identifier.split('.') + return parts.map((part) => sanitizeSingleIdentifier(part)).join('.') + } + + return sanitizeSingleIdentifier(identifier) +} + +function sanitizeSingleIdentifier(identifier: string): string { + const cleaned = identifier.replace(/`/g, '').replace(/"/g, '') + + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(cleaned)) { + throw new Error( + `Invalid identifier: ${identifier}. Identifiers must start with a letter or underscore and contain only letters, numbers, and underscores.` + ) + } + + return cleaned +} + +/** + * Convert a JS value to an RDS Data API SqlParameter value + */ +function toSqlParameterValue(value: unknown): SqlParameter['value'] { + if (value === null || value === undefined) { + return { isNull: true } + } + if (typeof value === 'boolean') { + return { booleanValue: value } + } + if (typeof value === 'number') { + if (Number.isInteger(value)) { + return { longValue: value } + } + return { doubleValue: value } + } + if (typeof value === 'string') { + return { stringValue: value } + } + if (value instanceof Uint8Array || Buffer.isBuffer(value)) { + return { blobValue: value } + } + // Objects/arrays as JSON strings + return { stringValue: JSON.stringify(value) } +} + +/** + * Build parameterized INSERT query + */ +export async function executeInsert( + client: RDSDataClient, + resourceArn: string, + secretArn: string, + database: string | undefined, + table: string, + data: Record, + signal?: AbortSignal +): Promise<{ rows: Record[]; rowCount: number }> { + const sanitizedTable = sanitizeIdentifier(table) + const columns = Object.keys(data) + const sanitizedColumns = columns.map((col) => sanitizeIdentifier(col)) + + const placeholders = columns.map((col) => `:${col}`) + const parameters: SqlParameter[] = columns.map((col) => ({ + name: col, + value: toSqlParameterValue(data[col]), + })) + + const sql = `INSERT INTO ${sanitizedTable} (${sanitizedColumns.join(', ')}) VALUES (${placeholders.join(', ')})` + + return executeStatement(client, resourceArn, secretArn, database, sql, parameters, signal) +} + +/** + * Build parameterized UPDATE query with conditions + */ +export async function executeUpdate( + client: RDSDataClient, + resourceArn: string, + secretArn: string, + database: string | undefined, + table: string, + data: Record, + conditions: Record, + signal?: AbortSignal +): Promise<{ rows: Record[]; rowCount: number }> { + const sanitizedTable = sanitizeIdentifier(table) + + // Build SET clause with parameters + const dataColumns = Object.keys(data) + const setClause = dataColumns.map((col) => `${sanitizeIdentifier(col)} = :set_${col}`).join(', ') + + // Build WHERE clause with parameters + const conditionColumns = Object.keys(conditions) + if (conditionColumns.length === 0) { + throw new Error('At least one condition is required for UPDATE operations') + } + const whereClause = conditionColumns + .map((col) => `${sanitizeIdentifier(col)} = :where_${col}`) + .join(' AND ') + + // Build parameters array (prefixed to avoid name collisions) + const parameters: SqlParameter[] = [ + ...dataColumns.map((col) => ({ + name: `set_${col}`, + value: toSqlParameterValue(data[col]), + })), + ...conditionColumns.map((col) => ({ + name: `where_${col}`, + value: toSqlParameterValue(conditions[col]), + })), + ] + + const sql = `UPDATE ${sanitizedTable} SET ${setClause} WHERE ${whereClause}` + + return executeStatement(client, resourceArn, secretArn, database, sql, parameters, signal) +} + +/** + * Build parameterized DELETE query with conditions + */ +export async function executeDelete( + client: RDSDataClient, + resourceArn: string, + secretArn: string, + database: string | undefined, + table: string, + conditions: Record, + signal?: AbortSignal +): Promise<{ rows: Record[]; rowCount: number }> { + const sanitizedTable = sanitizeIdentifier(table) + + // Build WHERE clause with parameters + const conditionColumns = Object.keys(conditions) + if (conditionColumns.length === 0) { + throw new Error('At least one condition is required for DELETE operations') + } + const whereClause = conditionColumns + .map((col) => `${sanitizeIdentifier(col)} = :${col}`) + .join(' AND ') + + const parameters: SqlParameter[] = conditionColumns.map((col) => ({ + name: col, + value: toSqlParameterValue(conditions[col]), + })) + + const sql = `DELETE FROM ${sanitizedTable} WHERE ${whereClause}` + + return executeStatement(client, resourceArn, secretArn, database, sql, parameters, signal) +} + +export type RdsEngine = 'aurora-postgresql' | 'aurora-mysql' + +export interface RdsIntrospectionResult { + engine: RdsEngine + tables: Array<{ + name: string + schema: string + columns: Array<{ + name: string + type: string + nullable: boolean + default: string | null + isPrimaryKey: boolean + isForeignKey: boolean + references?: { + table: string + column: string + } + }> + primaryKey: string[] + foreignKeys: Array<{ + column: string + referencesTable: string + referencesColumn: string + }> + indexes: Array<{ + name: string + columns: string[] + unique: boolean + }> + }> + schemas: string[] +} + +/** + * Detects the database engine by querying SELECT VERSION() + */ +async function detectEngine( + client: RDSDataClient, + resourceArn: string, + secretArn: string, + database: string | undefined, + signal?: AbortSignal +): Promise { + const result = await executeStatement( + client, + resourceArn, + secretArn, + database, + 'SELECT VERSION()', + undefined, + signal + ) + + if (result.rows.length > 0) { + const versionRow = result.rows[0] as Record + const versionValue = Object.values(versionRow)[0] + const versionString = String(versionValue).toLowerCase() + + if (versionString.includes('postgresql') || versionString.includes('postgres')) { + return 'aurora-postgresql' + } + if (versionString.includes('mysql') || versionString.includes('mariadb')) { + return 'aurora-mysql' + } + } + + throw new Error('Unable to detect database engine. Please specify the engine parameter.') +} + +/** + * Introspects PostgreSQL schema using INFORMATION_SCHEMA + */ +async function introspectPostgresql( + client: RDSDataClient, + resourceArn: string, + secretArn: string, + database: string | undefined, + schemaName: string, + signal?: AbortSignal +): Promise { + const schemasResult = await executeStatement( + client, + resourceArn, + secretArn, + database, + `SELECT schema_name FROM information_schema.schemata + WHERE schema_name NOT IN ('pg_catalog', 'information_schema', 'pg_toast') + ORDER BY schema_name`, + undefined, + signal + ) + const schemas = schemasResult.rows.map((row) => (row as { schema_name: string }).schema_name) + + const tablesResult = await executeStatement( + client, + resourceArn, + secretArn, + database, + `SELECT table_name, table_schema + FROM information_schema.tables + WHERE table_schema = :schemaName + AND table_type = 'BASE TABLE' + ORDER BY table_name`, + [{ name: 'schemaName', value: { stringValue: schemaName } }], + signal + ) + + const tables = [] + + for (const tableRow of tablesResult.rows) { + const row = tableRow as { table_name: string; table_schema: string } + const tableName = row.table_name + const tableSchema = row.table_schema + + const columnsResult = await executeStatement( + client, + resourceArn, + secretArn, + database, + `SELECT + c.column_name, + c.data_type, + c.is_nullable, + c.column_default, + c.udt_name + FROM information_schema.columns c + WHERE c.table_schema = :tableSchema + AND c.table_name = :tableName + ORDER BY c.ordinal_position`, + [ + { name: 'tableSchema', value: { stringValue: tableSchema } }, + { name: 'tableName', value: { stringValue: tableName } }, + ], + signal + ) + + const pkResult = await executeStatement( + client, + resourceArn, + secretArn, + database, + `SELECT kcu.column_name + FROM information_schema.table_constraints tc + JOIN information_schema.key_column_usage kcu + ON tc.constraint_name = kcu.constraint_name + AND tc.table_schema = kcu.table_schema + WHERE tc.constraint_type = 'PRIMARY KEY' + AND tc.table_schema = :tableSchema + AND tc.table_name = :tableName`, + [ + { name: 'tableSchema', value: { stringValue: tableSchema } }, + { name: 'tableName', value: { stringValue: tableName } }, + ], + signal + ) + const primaryKeyColumns = pkResult.rows.map((r) => (r as { column_name: string }).column_name) + + const fkResult = await executeStatement( + client, + resourceArn, + secretArn, + database, + `SELECT + kcu.column_name, + ccu.table_name AS foreign_table_name, + ccu.column_name AS foreign_column_name + FROM information_schema.table_constraints tc + JOIN information_schema.key_column_usage kcu + ON tc.constraint_name = kcu.constraint_name + AND tc.table_schema = kcu.table_schema + JOIN information_schema.constraint_column_usage ccu + ON ccu.constraint_name = tc.constraint_name + AND ccu.table_schema = tc.table_schema + WHERE tc.constraint_type = 'FOREIGN KEY' + AND tc.table_schema = :tableSchema + AND tc.table_name = :tableName`, + [ + { name: 'tableSchema', value: { stringValue: tableSchema } }, + { name: 'tableName', value: { stringValue: tableName } }, + ], + signal + ) + + const foreignKeys = fkResult.rows.map((r) => { + const fkRow = r as { + column_name: string + foreign_table_name: string + foreign_column_name: string + } + return { + column: fkRow.column_name, + referencesTable: fkRow.foreign_table_name, + referencesColumn: fkRow.foreign_column_name, + } + }) + + const fkColumnSet = new Set(foreignKeys.map((fk) => fk.column)) + + const indexesResult = await executeStatement( + client, + resourceArn, + secretArn, + database, + `SELECT + i.relname AS index_name, + a.attname AS column_name, + ix.indisunique AS is_unique + FROM pg_class t + JOIN pg_index ix ON t.oid = ix.indrelid + JOIN pg_class i ON i.oid = ix.indexrelid + JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(ix.indkey) + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE t.relkind = 'r' + AND n.nspname = :tableSchema + AND t.relname = :tableName + AND NOT ix.indisprimary + ORDER BY i.relname, a.attnum`, + [ + { name: 'tableSchema', value: { stringValue: tableSchema } }, + { name: 'tableName', value: { stringValue: tableName } }, + ], + signal + ) + + const indexMap = new Map() + for (const idxRow of indexesResult.rows) { + const idx = idxRow as { index_name: string; column_name: string; is_unique: boolean } + const indexName = idx.index_name + if (!indexMap.has(indexName)) { + indexMap.set(indexName, { + name: indexName, + columns: [], + unique: idx.is_unique, + }) + } + indexMap.get(indexName)!.columns.push(idx.column_name) + } + const indexes = Array.from(indexMap.values()) + + const columns = columnsResult.rows.map((colRow) => { + const col = colRow as { + column_name: string + data_type: string + is_nullable: string + column_default: string | null + udt_name: string + } + const columnName = col.column_name + const fk = foreignKeys.find((f) => f.column === columnName) + + return { + name: columnName, + type: col.data_type === 'USER-DEFINED' ? col.udt_name : col.data_type, + nullable: col.is_nullable === 'YES', + default: col.column_default, + isPrimaryKey: primaryKeyColumns.includes(columnName), + isForeignKey: fkColumnSet.has(columnName), + ...(fk && { + references: { + table: fk.referencesTable, + column: fk.referencesColumn, + }, + }), + } + }) + + tables.push({ + name: tableName, + schema: tableSchema, + columns, + primaryKey: primaryKeyColumns, + foreignKeys, + indexes, + }) + } + + return { engine: 'aurora-postgresql', tables, schemas } +} + +/** + * Introspects MySQL schema using INFORMATION_SCHEMA + */ +async function introspectMysql( + client: RDSDataClient, + resourceArn: string, + secretArn: string, + database: string | undefined, + schemaName: string, + signal?: AbortSignal +): Promise { + const schemasResult = await executeStatement( + client, + resourceArn, + secretArn, + database, + `SELECT SCHEMA_NAME as schema_name FROM information_schema.SCHEMATA + WHERE SCHEMA_NAME NOT IN ('mysql', 'information_schema', 'performance_schema', 'sys') + ORDER BY SCHEMA_NAME`, + undefined, + signal + ) + const schemas = schemasResult.rows.map((row) => (row as { schema_name: string }).schema_name) + + const tablesResult = await executeStatement( + client, + resourceArn, + secretArn, + database, + `SELECT TABLE_NAME as table_name, TABLE_SCHEMA as table_schema + FROM information_schema.TABLES + WHERE TABLE_SCHEMA = :schemaName + AND TABLE_TYPE = 'BASE TABLE' + ORDER BY TABLE_NAME`, + [{ name: 'schemaName', value: { stringValue: schemaName } }], + signal + ) + + const tables = [] + + for (const tableRow of tablesResult.rows) { + const row = tableRow as { table_name: string; table_schema: string } + const tableName = row.table_name + const tableSchema = row.table_schema + + const columnsResult = await executeStatement( + client, + resourceArn, + secretArn, + database, + `SELECT + COLUMN_NAME as column_name, + DATA_TYPE as data_type, + IS_NULLABLE as is_nullable, + COLUMN_DEFAULT as column_default, + COLUMN_TYPE as column_type, + COLUMN_KEY as column_key + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = :tableSchema + AND TABLE_NAME = :tableName + ORDER BY ORDINAL_POSITION`, + [ + { name: 'tableSchema', value: { stringValue: tableSchema } }, + { name: 'tableName', value: { stringValue: tableName } }, + ], + signal + ) + + const pkResult = await executeStatement( + client, + resourceArn, + secretArn, + database, + `SELECT COLUMN_NAME as column_name + FROM information_schema.KEY_COLUMN_USAGE + WHERE TABLE_SCHEMA = :tableSchema + AND TABLE_NAME = :tableName + AND CONSTRAINT_NAME = 'PRIMARY' + ORDER BY ORDINAL_POSITION`, + [ + { name: 'tableSchema', value: { stringValue: tableSchema } }, + { name: 'tableName', value: { stringValue: tableName } }, + ], + signal + ) + const primaryKeyColumns = pkResult.rows.map((r) => (r as { column_name: string }).column_name) + + const fkResult = await executeStatement( + client, + resourceArn, + secretArn, + database, + `SELECT + kcu.COLUMN_NAME as column_name, + kcu.REFERENCED_TABLE_NAME as foreign_table_name, + kcu.REFERENCED_COLUMN_NAME as foreign_column_name + FROM information_schema.KEY_COLUMN_USAGE kcu + WHERE kcu.TABLE_SCHEMA = :tableSchema + AND kcu.TABLE_NAME = :tableName + AND kcu.REFERENCED_TABLE_NAME IS NOT NULL`, + [ + { name: 'tableSchema', value: { stringValue: tableSchema } }, + { name: 'tableName', value: { stringValue: tableName } }, + ], + signal + ) + + const foreignKeys = fkResult.rows.map((r) => { + const fkRow = r as { + column_name: string + foreign_table_name: string + foreign_column_name: string + } + return { + column: fkRow.column_name, + referencesTable: fkRow.foreign_table_name, + referencesColumn: fkRow.foreign_column_name, + } + }) + + const fkColumnSet = new Set(foreignKeys.map((fk) => fk.column)) + + const indexesResult = await executeStatement( + client, + resourceArn, + secretArn, + database, + `SELECT + INDEX_NAME as index_name, + COLUMN_NAME as column_name, + NON_UNIQUE as non_unique + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = :tableSchema + AND TABLE_NAME = :tableName + AND INDEX_NAME != 'PRIMARY' + ORDER BY INDEX_NAME, SEQ_IN_INDEX`, + [ + { name: 'tableSchema', value: { stringValue: tableSchema } }, + { name: 'tableName', value: { stringValue: tableName } }, + ], + signal + ) + + const indexMap = new Map() + for (const idxRow of indexesResult.rows) { + const idx = idxRow as { index_name: string; column_name: string; non_unique: number } + const indexName = idx.index_name + if (!indexMap.has(indexName)) { + indexMap.set(indexName, { + name: indexName, + columns: [], + unique: idx.non_unique === 0, + }) + } + indexMap.get(indexName)!.columns.push(idx.column_name) + } + const indexes = Array.from(indexMap.values()) + + const columns = columnsResult.rows.map((colRow) => { + const col = colRow as { + column_name: string + data_type: string + is_nullable: string + column_default: string | null + column_type: string + column_key: string + } + const columnName = col.column_name + const fk = foreignKeys.find((f) => f.column === columnName) + + return { + name: columnName, + type: col.column_type || col.data_type, + nullable: col.is_nullable === 'YES', + default: col.column_default, + isPrimaryKey: col.column_key === 'PRI', + isForeignKey: fkColumnSet.has(columnName), + ...(fk && { + references: { + table: fk.referencesTable, + column: fk.referencesColumn, + }, + }), + } + }) + + tables.push({ + name: tableName, + schema: tableSchema, + columns, + primaryKey: primaryKeyColumns, + foreignKeys, + indexes, + }) + } + + return { engine: 'aurora-mysql', tables, schemas } +} + +/** + * Introspects RDS Aurora database schema with auto-detection of engine type + */ +export async function executeIntrospect( + client: RDSDataClient, + resourceArn: string, + secretArn: string, + database: string | undefined, + schemaName?: string, + engine?: RdsEngine, + signal?: AbortSignal +): Promise { + const detectedEngine = + engine || (await detectEngine(client, resourceArn, secretArn, database, signal)) + + if (detectedEngine === 'aurora-postgresql') { + const schema = schemaName || 'public' + return introspectPostgresql(client, resourceArn, secretArn, database, schema, signal) + } + const schema = schemaName || database || '' + if (!schema) { + throw new Error('Schema or database name is required for MySQL introspection') + } + return introspectMysql(client, resourceArn, secretArn, database, schema, signal) +} diff --git a/apps/sim/lib/internal/rds/execute-tool.test.ts b/apps/sim/lib/internal/rds/execute-tool.test.ts new file mode 100644 index 00000000000..8985ed32561 --- /dev/null +++ b/apps/sim/lib/internal/rds/execute-tool.test.ts @@ -0,0 +1,135 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operationMocks = vi.hoisted(() => { + class RdsOperationInputError extends Error {} + + return { + RdsOperationInputError, + executeRdsDelete: vi.fn(), + executeRdsInsert: vi.fn(), + executeRdsIntrospection: vi.fn(), + executeRdsQuery: vi.fn(), + executeRdsStatement: vi.fn(), + executeRdsUpdate: vi.fn(), + } +}) + +vi.mock('@/lib/internal/rds/operations', () => operationMocks) + +import { executeRdsTool } from '@/lib/internal/rds/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const VALID_BODY = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', + resourceArn: 'arn:aws:rds:us-east-1:123456789012:cluster:database', + secretArn: 'arn:aws:secretsmanager:us-east-1:123456789012:secret:database', + database: 'application', + query: 'SELECT 1', +} as const + +const SUPPORTED_TOOL_IDS = [ + 'rds_query', + 'rds_execute', + 'rds_insert', + 'rds_update', + 'rds_delete', + 'rds_introspect', +] as const + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'rds_query', + input: VALID_BODY, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeRdsTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('validates and executes the matching RDS operation with cancellation', async () => { + const controller = new AbortController() + operationMocks.executeRdsQuery.mockResolvedValue({ + message: 'Query executed successfully. 1 row(s) returned.', + rows: [{ value: 1 }], + rowCount: 1, + }) + + const response = await executeRdsTool(createRequest({ signal: controller.signal })) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + message: 'Query executed successfully. 1 row(s) returned.', + rows: [{ value: 1 }], + rowCount: 1, + }) + expect(operationMocks.executeRdsQuery).toHaveBeenCalledWith(VALID_BODY, controller.signal) + }) + + it('returns the route-compatible contract validation envelope before provider work', async () => { + const response = await executeRdsTool(createRequest({ input: { region: 'us-east-1' } })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(operationMocks.executeRdsQuery).not.toHaveBeenCalled() + }) + + it.each(SUPPORTED_TOOL_IDS)('recognizes the canonical tool ID %s', async (toolId) => { + const response = await executeRdsTool(createRequest({ toolId, input: {} })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ error: 'Invalid request data' }) + }) + + it('preserves the provider error envelope', async () => { + operationMocks.executeRdsQuery.mockRejectedValue(new Error('AWS rejected credentials')) + + const response = await executeRdsTool(createRequest()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'RDS query failed: AWS rejected credentials', + }) + }) + + it('preserves query validation as a 400 error', async () => { + operationMocks.executeRdsQuery.mockRejectedValue( + new operationMocks.RdsOperationInputError('Only SELECT statements are allowed') + ) + + const response = await executeRdsTool(createRequest()) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ error: 'Only SELECT statements are allowed' }) + }) + + it('propagates cancellation without converting it into a provider failure', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeRdsTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(operationMocks.executeRdsQuery).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/rds/execute-tool.ts b/apps/sim/lib/internal/rds/execute-tool.ts new file mode 100644 index 00000000000..48d136714ad --- /dev/null +++ b/apps/sim/lib/internal/rds/execute-tool.ts @@ -0,0 +1,108 @@ +import { getErrorMessage } from '@sim/utils/errors' +import type { z } from 'zod' +import { + executeRdsDelete, + executeRdsInsert, + executeRdsIntrospection, + executeRdsQuery, + executeRdsStatement, + executeRdsUpdate, + RdsOperationInputError, +} from '@/lib/internal/rds/operations' +import { + rdsDeleteInputSchema, + rdsExecuteInputSchema, + rdsInsertInputSchema, + rdsIntrospectInputSchema, + rdsQueryInputSchema, + rdsUpdateInputSchema, +} from '@/lib/internal/rds/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +async function executeOperation( + schema: z.ZodType, + input: unknown, + execute: (input: TInput, signal?: AbortSignal) => Promise, + errorMessage: string, + signal?: AbortSignal +): Promise { + const parsed = schema.safeParse(input) + if (!parsed.success) { + return Response.json( + { error: 'Invalid request data', details: parsed.error.issues }, + { status: 400 } + ) + } + + try { + const result = await execute(parsed.data, signal) + signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + signal?.throwIfAborted() + if (error instanceof RdsOperationInputError) { + return Response.json({ error: error.message }, { status: 400 }) + } + return Response.json( + { error: `${errorMessage}: ${getErrorMessage(error, 'Unknown error occurred')}` }, + { status: 500 } + ) + } +} + +export const executeRdsTool: InternalToolOperationHandler = async ({ toolId, input, signal }) => { + signal?.throwIfAborted() + + switch (toolId) { + case 'rds_query': + return executeOperation( + rdsQueryInputSchema, + input, + executeRdsQuery, + 'RDS query failed', + signal + ) + case 'rds_execute': + return executeOperation( + rdsExecuteInputSchema, + input, + executeRdsStatement, + 'RDS execute failed', + signal + ) + case 'rds_insert': + return executeOperation( + rdsInsertInputSchema, + input, + executeRdsInsert, + 'RDS insert failed', + signal + ) + case 'rds_update': + return executeOperation( + rdsUpdateInputSchema, + input, + executeRdsUpdate, + 'RDS update failed', + signal + ) + case 'rds_delete': + return executeOperation( + rdsDeleteInputSchema, + input, + executeRdsDelete, + 'RDS delete failed', + signal + ) + case 'rds_introspect': + return executeOperation( + rdsIntrospectInputSchema, + input, + executeRdsIntrospection, + 'RDS introspection failed', + signal + ) + default: + return Response.json({ error: `Unsupported RDS tool: ${toolId}` }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/rds/operations.test.ts b/apps/sim/lib/internal/rds/operations.test.ts new file mode 100644 index 00000000000..67ebb5139de --- /dev/null +++ b/apps/sim/lib/internal/rds/operations.test.ts @@ -0,0 +1,128 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const clientMocks = vi.hoisted(() => ({ + createRdsClient: vi.fn(), + executeDelete: vi.fn(), + executeInsert: vi.fn(), + executeIntrospect: vi.fn(), + executeStatement: vi.fn(), + executeUpdate: vi.fn(), + validateQuery: vi.fn(), +})) + +vi.mock('@/lib/internal/rds/client', () => clientMocks) + +import { + executeRdsIntrospection, + executeRdsQuery, + RdsOperationInputError, +} from '@/lib/internal/rds/operations' + +const CONNECTION = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', + resourceArn: 'arn:aws:rds:us-east-1:123456789012:cluster:database', + secretArn: 'arn:aws:secretsmanager:us-east-1:123456789012:secret:database', + database: 'application', +} as const + +describe('RDS operations', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('passes the abort signal to RDS and destroys the client after success', async () => { + const controller = new AbortController() + const client = { destroy: vi.fn() } + clientMocks.createRdsClient.mockReturnValue(client) + clientMocks.validateQuery.mockReturnValue({ isValid: true }) + clientMocks.executeStatement.mockResolvedValue({ rows: [{ value: 1 }], rowCount: 1 }) + + await expect( + executeRdsQuery({ ...CONNECTION, query: 'SELECT 1' }, controller.signal) + ).resolves.toEqual({ + message: 'Query executed successfully. 1 row(s) returned.', + rows: [{ value: 1 }], + rowCount: 1, + }) + + expect(clientMocks.createRdsClient).toHaveBeenCalledWith({ + ...CONNECTION, + query: 'SELECT 1', + }) + expect(clientMocks.executeStatement).toHaveBeenCalledWith( + client, + CONNECTION.resourceArn, + CONNECTION.secretArn, + CONNECTION.database, + 'SELECT 1', + undefined, + controller.signal + ) + expect(client.destroy).toHaveBeenCalledOnce() + }) + + it('destroys the RDS client when the provider rejects', async () => { + const client = { destroy: vi.fn() } + clientMocks.createRdsClient.mockReturnValue(client) + clientMocks.validateQuery.mockReturnValue({ isValid: true }) + clientMocks.executeStatement.mockRejectedValue(new Error('provider failed')) + + await expect(executeRdsQuery({ ...CONNECTION, query: 'SELECT 1' })).rejects.toThrow( + 'provider failed' + ) + expect(client.destroy).toHaveBeenCalledOnce() + }) + + it('rejects disallowed query statements before creating an RDS client', () => { + clientMocks.validateQuery.mockReturnValue({ + isValid: false, + error: 'Only SELECT, INSERT, UPDATE, DELETE, WITH, EXPLAIN, and SHOW statements are allowed', + }) + + expect(() => executeRdsQuery({ ...CONNECTION, query: 'DROP TABLE users' })).toThrow( + new RdsOperationInputError( + 'Only SELECT, INSERT, UPDATE, DELETE, WITH, EXPLAIN, and SHOW statements are allowed' + ) + ) + expect(clientMocks.createRdsClient).not.toHaveBeenCalled() + }) + + it('passes cancellation into multi-query introspection and preserves its response', async () => { + const controller = new AbortController() + const client = { destroy: vi.fn() } + clientMocks.createRdsClient.mockReturnValue(client) + clientMocks.executeIntrospect.mockResolvedValue({ + engine: 'aurora-postgresql', + tables: [], + schemas: ['public'], + }) + + await expect( + executeRdsIntrospection( + { ...CONNECTION, schema: 'public', engine: 'aurora-postgresql' }, + controller.signal + ) + ).resolves.toEqual({ + message: 'Schema introspection completed. Engine: aurora-postgresql. Found 0 table(s).', + engine: 'aurora-postgresql', + tables: [], + schemas: ['public'], + }) + + expect(clientMocks.executeIntrospect).toHaveBeenCalledWith( + client, + CONNECTION.resourceArn, + CONNECTION.secretArn, + CONNECTION.database, + 'public', + 'aurora-postgresql', + controller.signal + ) + expect(client.destroy).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/internal/rds/operations.ts b/apps/sim/lib/internal/rds/operations.ts new file mode 100644 index 00000000000..b4214d7a2ab --- /dev/null +++ b/apps/sim/lib/internal/rds/operations.ts @@ -0,0 +1,154 @@ +import type { RDSDataClient } from '@aws-sdk/client-rds-data' +import { + createRdsClient, + executeDelete, + executeInsert, + executeIntrospect, + executeStatement, + executeUpdate, + validateQuery, +} from '@/lib/internal/rds/client' +import type { + RdsDeleteInput, + RdsExecuteInput, + RdsInsertInput, + RdsIntrospectInput, + RdsQueryInput, + RdsUpdateInput, +} from '@/lib/internal/rds/schema' +import type { RdsConnectionConfig } from '@/tools/rds/types' + +export class RdsOperationInputError extends Error {} + +async function withRdsClient( + input: RdsConnectionConfig, + execute: (client: RDSDataClient) => Promise +): Promise { + const client = createRdsClient(input) + try { + return await execute(client) + } finally { + client.destroy() + } +} + +export function executeRdsQuery(input: RdsQueryInput, signal?: AbortSignal) { + const validation = validateQuery(input.query) + if (!validation.isValid) { + throw new RdsOperationInputError(validation.error ?? 'Invalid query') + } + + return withRdsClient(input, async (client) => { + const result = await executeStatement( + client, + input.resourceArn, + input.secretArn, + input.database, + input.query, + undefined, + signal + ) + return { + message: `Query executed successfully. ${result.rowCount} row(s) returned.`, + rows: result.rows, + rowCount: result.rowCount, + } + }) +} + +export function executeRdsStatement(input: RdsExecuteInput, signal?: AbortSignal) { + return withRdsClient(input, async (client) => { + const result = await executeStatement( + client, + input.resourceArn, + input.secretArn, + input.database, + input.query, + undefined, + signal + ) + return { + message: `Query executed successfully. ${result.rowCount} row(s) affected.`, + rows: result.rows, + rowCount: result.rowCount, + } + }) +} + +export function executeRdsInsert(input: RdsInsertInput, signal?: AbortSignal) { + return withRdsClient(input, async (client) => { + const result = await executeInsert( + client, + input.resourceArn, + input.secretArn, + input.database, + input.table, + input.data, + signal + ) + return { + message: `Insert executed successfully. ${result.rowCount} row(s) inserted.`, + rows: result.rows, + rowCount: result.rowCount, + } + }) +} + +export function executeRdsUpdate(input: RdsUpdateInput, signal?: AbortSignal) { + return withRdsClient(input, async (client) => { + const result = await executeUpdate( + client, + input.resourceArn, + input.secretArn, + input.database, + input.table, + input.data, + input.conditions, + signal + ) + return { + message: `Update executed successfully. ${result.rowCount} row(s) updated.`, + rows: result.rows, + rowCount: result.rowCount, + } + }) +} + +export function executeRdsDelete(input: RdsDeleteInput, signal?: AbortSignal) { + return withRdsClient(input, async (client) => { + const result = await executeDelete( + client, + input.resourceArn, + input.secretArn, + input.database, + input.table, + input.conditions, + signal + ) + return { + message: `Delete executed successfully. ${result.rowCount} row(s) deleted.`, + rows: result.rows, + rowCount: result.rowCount, + } + }) +} + +export function executeRdsIntrospection(input: RdsIntrospectInput, signal?: AbortSignal) { + return withRdsClient(input, async (client) => { + const result = await executeIntrospect( + client, + input.resourceArn, + input.secretArn, + input.database, + input.schema, + input.engine, + signal + ) + return { + message: `Schema introspection completed. Engine: ${result.engine}. Found ${result.tables.length} table(s).`, + engine: result.engine, + tables: result.tables, + schemas: result.schemas, + } + }) +} diff --git a/apps/sim/lib/internal/rds/schema.ts b/apps/sim/lib/internal/rds/schema.ts new file mode 100644 index 00000000000..6c8574377b6 --- /dev/null +++ b/apps/sim/lib/internal/rds/schema.ts @@ -0,0 +1,42 @@ +import { z } from 'zod' + +const nonEmptyRecordSchema = (message: string) => + z.record(z.string(), z.unknown()).refine((value) => Object.keys(value).length > 0, { message }) + +const rdsConnectionInputSchema = z.object({ + region: z.string().min(1, 'AWS region is required'), + accessKeyId: z.string().min(1, 'AWS access key ID is required'), + secretAccessKey: z.string().min(1, 'AWS secret access key is required'), + resourceArn: z.string().min(1, 'Resource ARN is required'), + secretArn: z.string().min(1, 'Secret ARN is required'), + database: z.string().optional(), +}) + +export const rdsQueryInputSchema = rdsConnectionInputSchema.extend({ + query: z.string().min(1, 'Query is required'), +}) +export const rdsExecuteInputSchema = rdsQueryInputSchema +export const rdsInsertInputSchema = rdsConnectionInputSchema.extend({ + table: z.string().min(1, 'Table name is required'), + data: nonEmptyRecordSchema('Data object must have at least one field'), +}) +export const rdsUpdateInputSchema = rdsConnectionInputSchema.extend({ + table: z.string().min(1, 'Table name is required'), + data: nonEmptyRecordSchema('Data object must have at least one field'), + conditions: nonEmptyRecordSchema('At least one condition is required'), +}) +export const rdsDeleteInputSchema = rdsConnectionInputSchema.extend({ + table: z.string().min(1, 'Table name is required'), + conditions: nonEmptyRecordSchema('At least one condition is required'), +}) +export const rdsIntrospectInputSchema = rdsConnectionInputSchema.extend({ + schema: z.string().optional(), + engine: z.enum(['aurora-postgresql', 'aurora-mysql']).optional(), +}) + +export type RdsQueryInput = z.output +export type RdsExecuteInput = z.output +export type RdsInsertInput = z.output +export type RdsUpdateInput = z.output +export type RdsDeleteInput = z.output +export type RdsIntrospectInput = z.output diff --git a/apps/sim/lib/internal/redis/client.test.ts b/apps/sim/lib/internal/redis/client.test.ts new file mode 100644 index 00000000000..65958343ee5 --- /dev/null +++ b/apps/sim/lib/internal/redis/client.test.ts @@ -0,0 +1,72 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const redisMocks = vi.hoisted(() => ({ + configs: [] as Array>, + call: vi.fn(), +})) + +vi.mock('ioredis', () => ({ + default: class Redis { + constructor(config: Record) { + redisMocks.configs.push(config) + } + + call = redisMocks.call + }, +})) + +import { createRedisClient, executeRedisClientCommand } from '@/lib/internal/redis/client' + +describe('Redis client', () => { + beforeEach(() => { + vi.clearAllMocks() + redisMocks.configs.length = 0 + }) + + it('preserves the connection timeout, command timeout, and retry policy', () => { + createRedisClient({ + host: '203.0.113.10', + port: 6380, + username: 'user', + password: 'password', + db: 2, + family: 4, + tlsServername: 'cache.example.com', + }) + + expect(redisMocks.configs).toEqual([ + { + host: '203.0.113.10', + port: 6380, + username: 'user', + password: 'password', + db: 2, + family: 4, + tls: { servername: 'cache.example.com' }, + connectTimeout: 10000, + commandTimeout: 10000, + maxRetriesPerRequest: 1, + lazyConnect: true, + }, + ]) + }) + + it('forwards raw string and numeric arguments without coercion', async () => { + redisMocks.call.mockResolvedValue(['0', ['key:1']]) + const client = createRedisClient({ + host: '203.0.113.10', + port: 6379, + db: 0, + family: 4, + }) + + await expect(executeRedisClientCommand(client, 'SCAN', ['0', 'COUNT', 100])).resolves.toEqual([ + '0', + ['key:1'], + ]) + expect(redisMocks.call).toHaveBeenCalledWith('SCAN', '0', 'COUNT', 100) + }) +}) diff --git a/apps/sim/lib/internal/redis/client.ts b/apps/sim/lib/internal/redis/client.ts new file mode 100644 index 00000000000..2db86fff6ff --- /dev/null +++ b/apps/sim/lib/internal/redis/client.ts @@ -0,0 +1,35 @@ +import Redis from 'ioredis' + +export interface RedisClientConfig { + host: string + port: number + username?: string + password?: string + db: number + family: 4 | 6 + tlsServername?: string +} + +export function createRedisClient(config: RedisClientConfig): Redis { + return new Redis({ + host: config.host, + port: config.port, + username: config.username, + password: config.password, + db: config.db, + family: config.family, + tls: config.tlsServername ? { servername: config.tlsServername } : undefined, + connectTimeout: 10000, + commandTimeout: 10000, + maxRetriesPerRequest: 1, + lazyConnect: true, + }) +} + +export function executeRedisClientCommand( + client: Redis, + command: string, + args: Array +): Promise { + return client.call(command, ...args) +} diff --git a/apps/sim/lib/internal/redis/execute-tool.test.ts b/apps/sim/lib/internal/redis/execute-tool.test.ts new file mode 100644 index 00000000000..c5985371fa0 --- /dev/null +++ b/apps/sim/lib/internal/redis/execute-tool.test.ts @@ -0,0 +1,158 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operationMocks = vi.hoisted(() => { + class RedisOperationInputError extends Error { + constructor(readonly responseError: string | undefined) { + super(responseError) + } + } + + return { + executeRedisCommand: vi.fn(), + RedisOperationInputError, + } +}) + +vi.mock('@/lib/internal/redis/operations', () => operationMocks) + +import { executeRedisTool } from '@/lib/internal/redis/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const REDIS_TOOL_IDS = [ + 'redis_command', + 'redis_delete', + 'redis_exists', + 'redis_expire', + 'redis_get', + 'redis_hdel', + 'redis_hget', + 'redis_hgetall', + 'redis_hset', + 'redis_incr', + 'redis_incrby', + 'redis_keys', + 'redis_llen', + 'redis_lpop', + 'redis_lpush', + 'redis_lrange', + 'redis_persist', + 'redis_rpop', + 'redis_rpush', + 'redis_set', + 'redis_setnx', + 'redis_ttl', +] as const + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'redis_command', + input: { + url: 'redis://cache.example.com/2', + command: 'scan', + args: ['0', 'MATCH', 'user:*', 100], + }, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeRedisTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('validates the canonical body and preserves raw Redis response shapes', async () => { + const controller = new AbortController() + operationMocks.executeRedisCommand.mockResolvedValue({ + result: ['0', ['user:1', 'user:2']], + }) + + const response = await executeRedisTool(createRequest({ signal: controller.signal })) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + result: ['0', ['user:1', 'user:2']], + }) + expect(operationMocks.executeRedisCommand).toHaveBeenCalledWith( + { + url: 'redis://cache.example.com/2', + command: 'scan', + args: ['0', 'MATCH', 'user:*', 100], + }, + controller.signal + ) + }) + + it('returns the route-compatible first validation error', async () => { + const response = await executeRedisTool(createRequest({ input: { url: '', command: '' } })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ + error: 'Redis connection URL is required', + }) + expect(operationMocks.executeRedisCommand).not.toHaveBeenCalled() + }) + + it('rejects non-object operation input', async () => { + const response = await executeRedisTool(createRequest({ input: '{' })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ + error: 'Invalid input: expected object, received string', + }) + expect(operationMocks.executeRedisCommand).not.toHaveBeenCalled() + }) + + it.each(REDIS_TOOL_IDS)('recognizes the canonical tool ID %s', async (toolId) => { + const response = await executeRedisTool(createRequest({ toolId, input: {} })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ error: expect.any(String) }) + }) + + it('preserves Redis input errors as HTTP 400 responses', async () => { + operationMocks.executeRedisCommand.mockRejectedValue( + new operationMocks.RedisOperationInputError( + "Invalid Redis database index in URL path: 'invalid'" + ) + ) + + const response = await executeRedisTool(createRequest()) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ + error: "Invalid Redis database index in URL path: 'invalid'", + }) + }) + + it('preserves the provider error envelope', async () => { + operationMocks.executeRedisCommand.mockRejectedValue(new Error('connection refused')) + + const response = await executeRedisTool(createRequest()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ error: 'connection refused' }) + }) + + it('propagates cancellation without converting it into a Redis failure', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeRedisTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(operationMocks.executeRedisCommand).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/redis/execute-tool.ts b/apps/sim/lib/internal/redis/execute-tool.ts new file mode 100644 index 00000000000..731d4b075cf --- /dev/null +++ b/apps/sim/lib/internal/redis/execute-tool.ts @@ -0,0 +1,67 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { executeRedisCommand, RedisOperationInputError } from '@/lib/internal/redis/operations' +import { type RedisExecuteInput, redisExecuteInputSchema } from '@/lib/internal/redis/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const REDIS_TOOL_IDS = new Set([ + 'redis_command', + 'redis_delete', + 'redis_exists', + 'redis_expire', + 'redis_get', + 'redis_hdel', + 'redis_hget', + 'redis_hgetall', + 'redis_hset', + 'redis_incr', + 'redis_incrby', + 'redis_keys', + 'redis_llen', + 'redis_lpop', + 'redis_lpush', + 'redis_lrange', + 'redis_persist', + 'redis_rpop', + 'redis_rpush', + 'redis_set', + 'redis_setnx', + 'redis_ttl', +]) + +function parseRedisInput( + input: unknown +): { success: true; data: RedisExecuteInput } | { success: false; response: Response } { + const parsed = redisExecuteInputSchema.safeParse(input) + if (!parsed.success) { + return { + success: false, + response: Response.json( + { error: parsed.error.issues[0]?.message ?? 'Invalid request' }, + { status: 400 } + ), + } + } + return { success: true, data: parsed.data } +} + +export const executeRedisTool: InternalToolOperationHandler = async ({ toolId, input, signal }) => { + signal?.throwIfAborted() + if (!REDIS_TOOL_IDS.has(toolId)) { + return Response.json({ error: `Unsupported Redis tool: ${toolId}` }, { status: 500 }) + } + + const parsed = parseRedisInput(input) + if (!parsed.success) return parsed.response + + try { + const result = await executeRedisCommand(parsed.data, signal) + signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + signal?.throwIfAborted() + if (error instanceof RedisOperationInputError) { + return Response.json({ error: error.responseError }, { status: 400 }) + } + return Response.json({ error: getErrorMessage(error, 'Redis command failed') }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/redis/operations.test.ts b/apps/sim/lib/internal/redis/operations.test.ts new file mode 100644 index 00000000000..de3f19693a7 --- /dev/null +++ b/apps/sim/lib/internal/redis/operations.test.ts @@ -0,0 +1,185 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const clientMocks = vi.hoisted(() => ({ + createRedisClient: vi.fn(), + executeRedisClientCommand: vi.fn(), +})) + +const validationMocks = vi.hoisted(() => ({ + validateDatabaseHost: vi.fn(), +})) + +vi.mock('@/lib/internal/redis/client', () => clientMocks) +vi.mock('@/lib/core/security/input-validation.server', () => validationMocks) + +import { executeRedisCommand, RedisOperationInputError } from '@/lib/internal/redis/operations' + +function createClient() { + return { + connect: vi.fn().mockResolvedValue(undefined), + quit: vi.fn().mockResolvedValue('OK'), + disconnect: vi.fn(), + } +} + +describe('Redis operations', () => { + beforeEach(() => { + vi.clearAllMocks() + validationMocks.validateDatabaseHost.mockResolvedValue({ + isValid: true, + resolvedIP: '203.0.113.10', + }) + }) + + it('resolves the host, parses connection options, and preserves raw command results', async () => { + const controller = new AbortController() + const client = createClient() + clientMocks.createRedisClient.mockReturnValue(client) + clientMocks.executeRedisClientCommand.mockResolvedValue(['0', ['user:1', 'user:2']]) + + await expect( + executeRedisCommand( + { + url: 'rediss://user%40name:pass%20word@cache.example.com:6380/2', + command: 'scan', + args: ['0', 'MATCH', 'user:*', 100], + }, + controller.signal + ) + ).resolves.toEqual({ result: ['0', ['user:1', 'user:2']] }) + + expect(validationMocks.validateDatabaseHost).toHaveBeenCalledWith('cache.example.com', 'host') + expect(clientMocks.createRedisClient).toHaveBeenCalledWith({ + host: '203.0.113.10', + port: 6380, + username: 'user@name', + password: 'pass word', + db: 2, + family: 4, + tlsServername: 'cache.example.com', + }) + expect(client.connect).toHaveBeenCalledOnce() + expect(clientMocks.executeRedisClientCommand).toHaveBeenCalledWith(client, 'SCAN', [ + '0', + 'MATCH', + 'user:*', + 100, + ]) + expect(client.quit).toHaveBeenCalledOnce() + expect(client.disconnect).not.toHaveBeenCalled() + }) + + it('uses the validated IPv6 address and default connection values', async () => { + validationMocks.validateDatabaseHost.mockResolvedValue({ + isValid: true, + resolvedIP: '2001:db8::10', + }) + const client = createClient() + clientMocks.createRedisClient.mockReturnValue(client) + clientMocks.executeRedisClientCommand.mockResolvedValue('value') + + await executeRedisCommand({ + url: 'redis://[2001:db8::1]', + command: 'get', + args: ['key'], + }) + + expect(validationMocks.validateDatabaseHost).toHaveBeenCalledWith('2001:db8::1', 'host') + expect(clientMocks.createRedisClient).toHaveBeenCalledWith({ + host: '2001:db8::10', + port: 6379, + username: undefined, + password: undefined, + db: 0, + family: 6, + tlsServername: undefined, + }) + }) + + it('rejects an unsafe host before creating a Redis client', async () => { + validationMocks.validateDatabaseHost.mockResolvedValue({ + isValid: false, + error: 'Private network addresses are not allowed', + }) + + await expect( + executeRedisCommand({ + url: 'redis://127.0.0.1', + command: 'get', + args: ['key'], + }) + ).rejects.toEqual(new RedisOperationInputError('Private network addresses are not allowed')) + expect(clientMocks.createRedisClient).not.toHaveBeenCalled() + }) + + it('preserves the exact invalid database index error', async () => { + await expect( + executeRedisCommand({ + url: 'redis://cache.example.com/01', + command: 'get', + args: ['key'], + }) + ).rejects.toEqual( + new RedisOperationInputError("Invalid Redis database index in URL path: '01'") + ) + expect(clientMocks.createRedisClient).not.toHaveBeenCalled() + }) + + it('quits the client after a provider failure and preserves the primary error', async () => { + const client = createClient() + clientMocks.createRedisClient.mockReturnValue(client) + clientMocks.executeRedisClientCommand.mockRejectedValue(new Error('provider failed')) + + await expect( + executeRedisCommand({ + url: 'redis://cache.example.com', + command: 'get', + args: ['key'], + }) + ).rejects.toThrow('provider failed') + expect(client.quit).toHaveBeenCalledOnce() + }) + + it('disconnects and surfaces a cleanup failure after successful provider work', async () => { + const client = createClient() + client.quit.mockRejectedValue(new Error('quit failed')) + clientMocks.createRedisClient.mockReturnValue(client) + clientMocks.executeRedisClientCommand.mockResolvedValue('value') + + await expect( + executeRedisCommand({ + url: 'redis://cache.example.com', + command: 'get', + args: ['key'], + }) + ).rejects.toThrow('quit failed') + expect(client.quit).toHaveBeenCalledTimes(2) + expect(client.disconnect).toHaveBeenCalledOnce() + }) + + it('disconnects on cancellation and propagates the abort reason', async () => { + const controller = new AbortController() + const client = createClient() + clientMocks.createRedisClient.mockReturnValue(client) + clientMocks.executeRedisClientCommand.mockImplementation(async () => { + controller.abort(new DOMException('cancelled', 'AbortError')) + throw new Error('connection closed') + }) + + await expect( + executeRedisCommand( + { + url: 'redis://cache.example.com', + command: 'get', + args: ['key'], + }, + controller.signal + ) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(client.disconnect).toHaveBeenCalledOnce() + expect(client.quit).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/internal/redis/operations.ts b/apps/sim/lib/internal/redis/operations.ts new file mode 100644 index 00000000000..a87aca5392e --- /dev/null +++ b/apps/sim/lib/internal/redis/operations.ts @@ -0,0 +1,74 @@ +import { validateDatabaseHost } from '@/lib/core/security/input-validation.server' +import { createRedisClient, executeRedisClientCommand } from '@/lib/internal/redis/client' +import type { RedisExecuteInput } from '@/lib/internal/redis/schema' + +export class RedisOperationInputError extends Error { + constructor(readonly responseError: string | undefined) { + super(responseError) + } +} + +function getDatabaseIndex(parsedUrl: URL): number { + if (!parsedUrl.pathname || parsedUrl.pathname.length <= 1) return 0 + + const dbSegment = parsedUrl.pathname.slice(1) + const parsedDb = Number.parseInt(dbSegment, 10) + if (!Number.isFinite(parsedDb) || String(parsedDb) !== dbSegment) { + throw new RedisOperationInputError(`Invalid Redis database index in URL path: '${dbSegment}'`) + } + return parsedDb +} + +export async function executeRedisCommand( + input: RedisExecuteInput, + signal?: AbortSignal +): Promise<{ result: unknown }> { + signal?.throwIfAborted() + + const parsedUrl = new URL(input.url) + const hostname = + parsedUrl.hostname.startsWith('[') && parsedUrl.hostname.endsWith(']') + ? parsedUrl.hostname.slice(1, -1) + : parsedUrl.hostname + const hostValidation = await validateDatabaseHost(hostname, 'host') + signal?.throwIfAborted() + if (!hostValidation.isValid) { + throw new RedisOperationInputError(hostValidation.error) + } + + const resolvedIP = hostValidation.resolvedIP ?? hostname + const client = createRedisClient({ + host: resolvedIP, + port: parsedUrl.port ? Number(parsedUrl.port) : 6379, + username: parsedUrl.username ? decodeURIComponent(parsedUrl.username) : undefined, + password: parsedUrl.password ? decodeURIComponent(parsedUrl.password) : undefined, + db: getDatabaseIndex(parsedUrl), + family: resolvedIP.includes(':') ? 6 : 4, + tlsServername: parsedUrl.protocol === 'rediss:' ? hostname : undefined, + }) + + const disconnectOnAbort = () => client.disconnect() + signal?.addEventListener('abort', disconnectOnAbort, { once: true }) + let clientClosed = false + + try { + await client.connect() + signal?.throwIfAborted() + const result = await executeRedisClientCommand(client, input.command.toUpperCase(), input.args) + signal?.throwIfAborted() + + await client.quit() + clientClosed = true + return { result } + } finally { + signal?.removeEventListener('abort', disconnectOnAbort) + if (!clientClosed) { + try { + await client.quit() + } catch { + client.disconnect() + } + } + signal?.throwIfAborted() + } +} diff --git a/apps/sim/lib/internal/redis/schema.ts b/apps/sim/lib/internal/redis/schema.ts new file mode 100644 index 00000000000..e5a10ca6bd0 --- /dev/null +++ b/apps/sim/lib/internal/redis/schema.ts @@ -0,0 +1,9 @@ +import { z } from 'zod' + +export const redisExecuteInputSchema = z.object({ + url: z.string().min(1, 'Redis connection URL is required'), + command: z.string().min(1, 'Redis command is required'), + args: z.array(z.union([z.string(), z.number()])).default([]), +}) + +export type RedisExecuteInput = z.output diff --git a/apps/sim/lib/internal/reducto/client.ts b/apps/sim/lib/internal/reducto/client.ts new file mode 100644 index 00000000000..b5a79fb7383 --- /dev/null +++ b/apps/sim/lib/internal/reducto/client.ts @@ -0,0 +1,58 @@ +import { createLogger } from '@sim/logger' +import { + DEFAULT_MAX_RESPONSE_BYTES, + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + readResponseTextWithLimit, +} from '@/lib/core/utils/stream-limits' +import { ReductoOperationError } from '@/lib/internal/reducto/errors' + +const logger = createLogger('ReductoClient') +const REDUCTO_ENDPOINT = 'https://platform.reducto.ai/parse' + +export async function submitReductoParse( + apiKey: string, + body: Record, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const validation = await validateUrlWithDNS(REDUCTO_ENDPOINT, 'Reducto API URL') + signal?.throwIfAborted() + if (!validation.isValid || !validation.resolvedIP) { + throw new ReductoOperationError(502, { + success: false, + error: 'Failed to reach Reducto API', + }) + } + + const response = await secureFetchWithPinnedIP(REDUCTO_ENDPOINT, validation.resolvedIP, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify(body), + maxResponseBytes: DEFAULT_MAX_RESPONSE_BYTES, + signal, + }) + signal?.throwIfAborted() + + if (!response.ok) { + const diagnostic = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Reducto API error response', + signal, + }) + logger.error('Reducto API error', { status: response.status, diagnostic }) + throw new ReductoOperationError(response.status, { + success: false, + error: `Reducto API error: ${response.statusText}`, + }) + } + + return response.json() +} diff --git a/apps/sim/lib/internal/reducto/errors.ts b/apps/sim/lib/internal/reducto/errors.ts new file mode 100644 index 00000000000..fe7d4385f8b --- /dev/null +++ b/apps/sim/lib/internal/reducto/errors.ts @@ -0,0 +1,9 @@ +export class ReductoOperationError extends Error { + constructor( + readonly status: number, + readonly body: unknown + ) { + super('Reducto operation failed') + this.name = 'ReductoOperationError' + } +} diff --git a/apps/sim/lib/internal/reducto/execute-tool.test.ts b/apps/sim/lib/internal/reducto/execute-tool.test.ts new file mode 100644 index 00000000000..a86b74db384 --- /dev/null +++ b/apps/sim/lib/internal/reducto/execute-tool.test.ts @@ -0,0 +1,38 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operation = vi.hoisted(() => vi.fn()) +vi.mock('@/lib/internal/reducto/operations', () => ({ executeReductoParse: operation })) + +import { executeReductoTool } from '@/lib/internal/reducto/execute-tool' + +describe('executeReductoTool', () => { + beforeEach(() => { + vi.clearAllMocks() + operation.mockResolvedValue({ success: true, output: { job_id: 'job-1' } }) + }) + + it('dispatches both canonical IDs with trusted context', async () => { + for (const toolId of ['reducto_parser', 'reducto_parser_v2']) { + const controller = new AbortController() + const response = await executeReductoTool({ + toolId, + input: { apiKey: 'key', filePath: 'https://example.com/file.pdf' }, + headers: new Headers(), + context: { ...createExecutionContext(), userId: 'user-1' }, + requestId: 'request-1', + signal: controller.signal, + }) + expect(response.status).toBe(200) + expect(operation).toHaveBeenLastCalledWith(expect.any(Object), { + headers: expect.any(Headers), + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + }) + } + }) +}) diff --git a/apps/sim/lib/internal/reducto/execute-tool.ts b/apps/sim/lib/internal/reducto/execute-tool.ts new file mode 100644 index 00000000000..c2961e4195d --- /dev/null +++ b/apps/sim/lib/internal/reducto/execute-tool.ts @@ -0,0 +1,71 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { z } from 'zod' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { ReductoOperationError } from '@/lib/internal/reducto/errors' +import { reductoParseInputSchema } from '@/lib/internal/reducto/input' +import { executeReductoParse } from '@/lib/internal/reducto/operations' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const logger = createLogger('ReductoToolExecution') + +function validationResponse(error: z.ZodError): Response { + return Response.json( + { + success: false, + error: error.issues[0]?.message || 'Invalid request data', + details: error.issues, + }, + { status: 400 } + ) +} + +export const executeReductoTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!['reducto_parser', 'reducto_parser_v2'].includes(request.toolId)) { + return Response.json( + { success: false, error: `Unsupported Reducto tool: ${request.toolId}` }, + { status: 500 } + ) + } + let serialized: string + try { + serialized = JSON.stringify(request.input) ?? '' + } catch { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + if (Buffer.byteLength(serialized, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + success: false, + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + const parsed = reductoParseInputSchema.safeParse(request.input) + if (!parsed.success) return validationResponse(parsed.error) + + try { + const result = await executeReductoParse(parsed.data, { + headers: request.headers, + requestId: request.requestId, + signal: request.signal, + userId: request.context.userId, + }) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof ReductoOperationError) { + return Response.json(error.body, { status: error.status }) + } + const message = getErrorMessage(error, 'Internal server error') + logger.error('Reducto operation failed', { + error: message, + requestId: request.requestId, + toolId: request.toolId, + }) + return Response.json({ success: false, error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/reducto/input.ts b/apps/sim/lib/internal/reducto/input.ts new file mode 100644 index 00000000000..52ff6ec3d24 --- /dev/null +++ b/apps/sim/lib/internal/reducto/input.ts @@ -0,0 +1,15 @@ +import { z } from 'zod' +import { resolvedSecretTraceProvenanceSchema } from '@/lib/api/contracts/primitives' +import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' + +export const reductoParseInputSchema = z.object({ + apiKey: z.string().min(1, 'API key is required'), + filePath: z.string().optional(), + file: RawFileInputSchema.optional(), + pages: z.array(z.number()).max(10_000).optional(), + tableOutputFormat: z.enum(['html', 'md']).optional(), + [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), +}) + +export type ReductoParseInput = z.infer diff --git a/apps/sim/lib/internal/reducto/operations.test.ts b/apps/sim/lib/internal/reducto/operations.test.ts new file mode 100644 index 00000000000..af2019e8f7d --- /dev/null +++ b/apps/sim/lib/internal/reducto/operations.test.ts @@ -0,0 +1,55 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + resolveFileInputToUrl: vi.fn(), + submitReductoParse: vi.fn(), + validateProvenance: vi.fn(), +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + resolveFileInputToUrl: mocks.resolveFileInputToUrl, +})) +vi.mock('@/lib/internal/reducto/client', () => ({ + submitReductoParse: mocks.submitReductoParse, +})) +vi.mock('@/lib/execution/model-input-provenance', () => ({ + validateOpaqueModelInputProvenance: mocks.validateProvenance, +})) + +import { executeReductoParse } from '@/lib/internal/reducto/operations' + +describe('executeReductoParse', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.validateProvenance.mockReturnValue({ success: true }) + mocks.resolveFileInputToUrl.mockResolvedValue({ fileUrl: 'https://example.com/file.pdf' }) + mocks.submitReductoParse.mockResolvedValue({ job_id: 'job-1' }) + }) + + it('submits one bounded page range and forwards cancellation', async () => { + const controller = new AbortController() + await expect( + executeReductoParse( + { apiKey: 'key', filePath: 'https://example.com/file.pdf', pages: [9, 2, 4] }, + { + headers: new Headers(), + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + } + ) + ).resolves.toEqual({ success: true, output: { job_id: 'job-1' } }) + + expect(mocks.submitReductoParse).toHaveBeenCalledWith( + 'key', + { + input: 'https://example.com/file.pdf', + settings: { page_range: { start: 2, end: 9 } }, + }, + controller.signal + ) + }) +}) diff --git a/apps/sim/lib/internal/reducto/operations.ts b/apps/sim/lib/internal/reducto/operations.ts new file mode 100644 index 00000000000..c246fbfcb75 --- /dev/null +++ b/apps/sim/lib/internal/reducto/operations.ts @@ -0,0 +1,73 @@ +import { createLogger } from '@sim/logger' +import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' +import { submitReductoParse } from '@/lib/internal/reducto/client' +import { ReductoOperationError } from '@/lib/internal/reducto/errors' +import type { ReductoParseInput } from '@/lib/internal/reducto/input' +import { resolveFileInputToUrl } from '@/lib/uploads/utils/file-utils.server' + +const logger = createLogger('ReductoOperations') + +export interface ReductoOperationContext { + headers: Headers + requestId: string + signal?: AbortSignal + userId?: string +} + +export async function executeReductoParse( + input: ReductoParseInput, + context: ReductoOperationContext +): Promise<{ success: true; output: unknown }> { + context.signal?.throwIfAborted() + if (!context.userId) { + throw new ReductoOperationError(401, { success: false, error: 'Unauthorized' }) + } + const provenance = validateOpaqueModelInputProvenance({ + headers: context.headers, + payload: input, + isInternalRequest: true, + }) + if (!provenance.success) { + throw new ReductoOperationError(provenance.status, { + success: false, + error: provenance.error, + }) + } + + const resolution = await resolveFileInputToUrl({ + file: input.file, + filePath: input.filePath, + userId: context.userId, + requestId: context.requestId, + logger, + modelEgress: true, + }) + context.signal?.throwIfAborted() + if (resolution.error) { + throw new ReductoOperationError(resolution.error.status, { + success: false, + error: resolution.error.message, + }) + } + if (!resolution.fileUrl) { + throw new ReductoOperationError(400, { success: false, error: 'File input is required' }) + } + + const body: Record = { input: resolution.fileUrl } + if (input.pages?.length) { + let start = input.pages[0] + let end = input.pages[0] + for (const page of input.pages) { + if (page < start) start = page + if (page > end) end = page + } + body.settings = { page_range: { start, end } } + } + if (input.tableOutputFormat) { + body.formatting = { table_output_format: input.tableOutputFormat } + } + + const output = await submitReductoParse(input.apiKey, body, context.signal) + context.signal?.throwIfAborted() + return { success: true, output } +} diff --git a/apps/sim/lib/internal/resend/client.ts b/apps/sim/lib/internal/resend/client.ts new file mode 100644 index 00000000000..3298db84a10 --- /dev/null +++ b/apps/sim/lib/internal/resend/client.ts @@ -0,0 +1,58 @@ +import { isRecordLike } from '@sim/utils/object' +import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' +import { ResendOperationError } from '@/lib/internal/resend/errors' + +const MAX_RESEND_ERROR_BYTES = 64 * 1024 + +function record(value: unknown): Record { + return isRecordLike(value) ? value : {} +} + +function message(value: unknown): string { + const data = record(value) + return typeof data.message === 'string' ? data.message : 'Unknown error' +} + +export async function sendResendEmail( + apiKey: string, + body: Record, + signal?: AbortSignal +): Promise> { + signal?.throwIfAborted() + const response = await fetch('https://api.resend.com/emails', { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(body), + signal, + }) + signal?.throwIfAborted() + + if (!response.ok) { + const error = await readResponseJsonWithLimit(response, { + maxBytes: MAX_RESEND_ERROR_BYTES, + label: 'Resend error response', + signal, + }).catch(() => { + signal?.throwIfAborted() + return {} + }) + signal?.throwIfAborted() + const errorMessage = `Failed to send email: ${message(error)}` + throw new ResendOperationError(errorMessage, 500, { + success: false, + message: errorMessage, + }) + } + + const result = await readResponseJsonWithLimit(response, { + maxBytes: MAX_JSON_API_RESPONSE_BYTES, + label: 'Resend send response', + signal, + }) + signal?.throwIfAborted() + return record(result) +} diff --git a/apps/sim/lib/internal/resend/errors.ts b/apps/sim/lib/internal/resend/errors.ts new file mode 100644 index 00000000000..1d7a2190a39 --- /dev/null +++ b/apps/sim/lib/internal/resend/errors.ts @@ -0,0 +1,10 @@ +export class ResendOperationError extends Error { + constructor( + message: string, + readonly status: number, + readonly body: Record + ) { + super(message) + this.name = 'ResendOperationError' + } +} diff --git a/apps/sim/lib/internal/resend/execute-tool.ts b/apps/sim/lib/internal/resend/execute-tool.ts new file mode 100644 index 00000000000..50ae8d59682 --- /dev/null +++ b/apps/sim/lib/internal/resend/execute-tool.ts @@ -0,0 +1,66 @@ +import { createLogger } from '@sim/logger' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { ResendOperationError } from '@/lib/internal/resend/errors' +import { executeResendSend } from '@/lib/internal/resend/operations' +import { resendSendInputSchema } from '@/lib/internal/resend/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const logger = createLogger('ResendToolExecution') + +export const executeResendTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'resend_send') { + return Response.json( + { success: false, message: `Unsupported Resend tool: ${request.toolId}` }, + { status: 500 } + ) + } + if (!request.context.userId) { + return Response.json({ success: false, message: 'Authentication required' }, { status: 401 }) + } + let serializedInput: string + try { + serializedInput = JSON.stringify(request.input) ?? '' + } catch { + return Response.json( + { success: false, message: 'Invalid request data', errors: [] }, + { status: 400 } + ) + } + if (Buffer.byteLength(serializedInput, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + const parsed = resendSendInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { + success: false, + message: getValidationErrorMessage(parsed.error, 'Invalid request data'), + errors: parsed.error.issues, + }, + { status: 400 } + ) + } + + try { + const result = await executeResendSend(parsed.data, request.signal) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof ResendOperationError) { + return Response.json(error.body, { status: error.status }) + } + logger.error('Resend send failed', { requestId: request.requestId }) + return Response.json( + { success: false, message: 'Internal server error while sending email', data: {} }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/resend/operations.ts b/apps/sim/lib/internal/resend/operations.ts new file mode 100644 index 00000000000..b51d829cf3d --- /dev/null +++ b/apps/sim/lib/internal/resend/operations.ts @@ -0,0 +1,49 @@ +import { convert } from 'html-to-text' +import { sendResendEmail } from '@/lib/internal/resend/client' +import type { ResendSendInput } from '@/lib/internal/resend/schema' + +interface ResendTag { + name: string + value: string +} + +function tags(value: string): ResendTag[] { + return value + .split(',') + .map((pair) => { + const trimmed = pair.trim() + const colonIndex = trimmed.indexOf(':') + if (colonIndex === -1) return null + const name = trimmed.substring(0, colonIndex).trim() + const tagValue = trimmed.substring(colonIndex + 1).trim() + return name ? { name, value: tagValue || '' } : null + }) + .filter((tag): tag is ResendTag => tag !== null) +} + +export async function executeResendSend(input: ResendSendInput, signal?: AbortSignal) { + signal?.throwIfAborted() + const body: Record = { + from: input.fromAddress, + to: input.to, + subject: input.subject, + } + if ((input.contentType || 'text') === 'html') { + body.html = input.body + body.text = convert(input.body, { wordwrap: false }) + } else { + body.text = input.body + } + if (input.cc) body.cc = input.cc + if (input.bcc) body.bcc = input.bcc + if (input.replyTo) body.reply_to = input.replyTo + if (input.scheduledAt) body.scheduled_at = input.scheduledAt + if (input.tags) body.tags = tags(input.tags) + + const data = await sendResendEmail(input.resendApiKey, body, signal) + return { + success: true, + message: 'Email sent successfully via Resend', + data, + } +} diff --git a/apps/sim/lib/internal/resend/resend.test.ts b/apps/sim/lib/internal/resend/resend.test.ts new file mode 100644 index 00000000000..ee6a209f487 --- /dev/null +++ b/apps/sim/lib/internal/resend/resend.test.ts @@ -0,0 +1,71 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { executeResendSend } from '@/lib/internal/resend/operations' + +describe('Resend operation', () => { + beforeEach(() => { + vi.restoreAllMocks() + }) + + it('preserves HTML fallback text, tags, optional recipients, and cancellation', async () => { + const controller = new AbortController() + const fetchMock = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValue(Response.json({ id: 'mail-1' })) + await expect( + executeResendSend( + { + resendApiKey: 'secret', + fromAddress: 'from@example.com', + to: 'to@example.com', + subject: 'Hello', + body: 'Hello', + contentType: 'html', + cc: 'cc@example.com', + replyTo: 'reply@example.com', + tags: 'category:welcome, invalid, empty:', + }, + controller.signal + ) + ).resolves.toEqual({ + success: true, + message: 'Email sent successfully via Resend', + data: { id: 'mail-1' }, + }) + const init = fetchMock.mock.calls[0][1] + expect(init?.signal).toBe(controller.signal) + expect(JSON.parse(init?.body as string)).toMatchObject({ + from: 'from@example.com', + to: 'to@example.com', + subject: 'Hello', + html: 'Hello', + text: 'Hello', + cc: 'cc@example.com', + reply_to: 'reply@example.com', + tags: [ + { name: 'category', value: 'welcome' }, + { name: 'empty', value: '' }, + ], + }) + }) + + it('preserves the legacy 500 error envelope for provider failures', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + Response.json({ message: 'Invalid API key' }, { status: 401 }) + ) + await expect( + executeResendSend({ + resendApiKey: 'bad', + fromAddress: 'from@example.com', + to: 'to@example.com', + subject: 'Hello', + body: 'Hello', + }) + ).rejects.toMatchObject({ + status: 500, + body: { success: false, message: 'Failed to send email: Invalid API key' }, + }) + }) +}) diff --git a/apps/sim/lib/internal/resend/schema.ts b/apps/sim/lib/internal/resend/schema.ts new file mode 100644 index 00000000000..b818c98da8c --- /dev/null +++ b/apps/sim/lib/internal/resend/schema.ts @@ -0,0 +1,26 @@ +import { z } from 'zod' + +export const resendSendInputSchema = z.object({ + fromAddress: z.string().min(1, 'From address is required'), + to: z.string().min(1, 'To email is required'), + subject: z.string().min(1, 'Subject is required'), + body: z.string().min(1, 'Email body is required'), + contentType: z.enum(['text', 'html']).optional().nullable(), + resendApiKey: z.string().min(1, 'Resend API key is required'), + cc: z + .union([z.string().min(1), z.array(z.string().min(1))]) + .optional() + .nullable(), + bcc: z + .union([z.string().min(1), z.array(z.string().min(1))]) + .optional() + .nullable(), + replyTo: z + .union([z.string().min(1), z.array(z.string().min(1))]) + .optional() + .nullable(), + scheduledAt: z.string().datetime().optional().nullable(), + tags: z.string().optional().nullable(), +}) + +export type ResendSendInput = z.output diff --git a/apps/sim/lib/internal/s3/client.ts b/apps/sim/lib/internal/s3/client.ts new file mode 100644 index 00000000000..95d336d332a --- /dev/null +++ b/apps/sim/lib/internal/s3/client.ts @@ -0,0 +1,17 @@ +import { S3Client } from '@aws-sdk/client-s3' + +export interface S3ConnectionConfig { + region: string + accessKeyId: string + secretAccessKey: string +} + +export function createS3Client(config: S3ConnectionConfig): S3Client { + return new S3Client({ + region: config.region, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + }) +} diff --git a/apps/sim/lib/internal/s3/errors.ts b/apps/sim/lib/internal/s3/errors.ts new file mode 100644 index 00000000000..ad9ea0dda10 --- /dev/null +++ b/apps/sim/lib/internal/s3/errors.ts @@ -0,0 +1,9 @@ +export class S3OperationError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'S3OperationError' + } +} diff --git a/apps/sim/lib/internal/s3/execute-tool.test.ts b/apps/sim/lib/internal/s3/execute-tool.test.ts new file mode 100644 index 00000000000..3de376955c7 --- /dev/null +++ b/apps/sim/lib/internal/s3/execute-tool.test.ts @@ -0,0 +1,156 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mockOperations = vi.hoisted(() => ({ + executeS3CopyObject: vi.fn(), + executeS3CreateBucket: vi.fn(), + executeS3DeleteBucket: vi.fn(), + executeS3DeleteObject: vi.fn(), + executeS3DeleteObjects: vi.fn(), + executeS3HeadObject: vi.fn(), + executeS3ListBuckets: vi.fn(), + executeS3ListObjects: vi.fn(), + executeS3PresignedUrl: vi.fn(), + executeS3PutObject: vi.fn(), +})) + +vi.mock('@/lib/internal/s3/operations', () => mockOperations) + +import { S3OperationError } from '@/lib/internal/s3/errors' +import { executeS3Tool } from '@/lib/internal/s3/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const CONNECTION = { + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', + region: 'us-east-1', +} + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 's3_list_buckets', + input: CONNECTION, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + userId: 'user-1', + metadata: {}, + }, + requestId: 'request-1', + ...overrides, + } +} + +const OBJECT = { ...CONNECTION, bucketName: 'bucket', objectKey: 'folder/file.txt' } + +const TOOL_CASES = [ + [ + 's3_copy_object', + { + ...CONNECTION, + sourceBucket: 'source', + sourceKey: 'source.txt', + destinationBucket: 'destination', + destinationKey: 'destination.txt', + }, + mockOperations.executeS3CopyObject, + ], + [ + 's3_create_bucket', + { ...CONNECTION, bucketName: 'bucket' }, + mockOperations.executeS3CreateBucket, + ], + [ + 's3_delete_bucket', + { ...CONNECTION, bucketName: 'bucket' }, + mockOperations.executeS3DeleteBucket, + ], + ['s3_delete_object', OBJECT, mockOperations.executeS3DeleteObject], + [ + 's3_delete_objects', + { ...CONNECTION, bucketName: 'bucket', keys: ['one'] }, + mockOperations.executeS3DeleteObjects, + ], + ['s3_head_object', OBJECT, mockOperations.executeS3HeadObject], + ['s3_list_buckets', CONNECTION, mockOperations.executeS3ListBuckets], + ['s3_list_objects', { ...CONNECTION, bucketName: 'bucket' }, mockOperations.executeS3ListObjects], + [ + 's3_presigned_url', + { ...OBJECT, method: 'get', expiresIn: 300 }, + mockOperations.executeS3PresignedUrl, + ], + ['s3_put_object', { ...OBJECT, content: 'hello' }, mockOperations.executeS3PutObject], +] as const + +describe('executeS3Tool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it.each(TOOL_CASES)('validates and dispatches %s', async (toolId, input, operation) => { + const controller = new AbortController() + operation.mockResolvedValue({ success: true, output: { toolId } }) + + const response = await executeS3Tool( + createRequest({ toolId, input, signal: controller.signal }) + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ success: true, output: { toolId } }) + if (toolId === 's3_put_object') { + expect(operation).toHaveBeenCalledWith(input, { + headers: expect.any(Headers), + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + }) + } else { + expect(operation).toHaveBeenCalledWith(input, controller.signal) + } + }) + + it('returns the canonical validation envelope before provider work', async () => { + const response = await executeS3Tool(createRequest({ input: { region: '' } })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(mockOperations.executeS3ListBuckets).not.toHaveBeenCalled() + }) + + it('preserves S3 operation status and error envelopes', async () => { + mockOperations.executeS3PutObject.mockRejectedValue( + new S3OperationError('Either file or content must be provided', 400) + ) + + const response = await executeS3Tool( + createRequest({ + toolId: 's3_put_object', + input: { ...OBJECT, content: 'hello' }, + }) + ) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Either file or content must be provided', + }) + }) + + it('propagates cancellation before provider work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect(executeS3Tool(createRequest({ signal: controller.signal }))).rejects.toMatchObject( + { name: 'AbortError' } + ) + expect(mockOperations.executeS3ListBuckets).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/s3/execute-tool.ts b/apps/sim/lib/internal/s3/execute-tool.ts new file mode 100644 index 00000000000..23331d6936a --- /dev/null +++ b/apps/sim/lib/internal/s3/execute-tool.ts @@ -0,0 +1,139 @@ +import { getErrorMessage } from '@sim/utils/errors' +import type { AnyApiRouteContract, ContractBody } from '@/lib/api/contracts' +import { awsS3CopyObjectContract } from '@/lib/api/contracts/tools/aws/s3-copy-object' +import { awsS3CreateBucketContract } from '@/lib/api/contracts/tools/aws/s3-create-bucket' +import { awsS3DeleteBucketContract } from '@/lib/api/contracts/tools/aws/s3-delete-bucket' +import { awsS3DeleteObjectContract } from '@/lib/api/contracts/tools/aws/s3-delete-object' +import { awsS3DeleteObjectsContract } from '@/lib/api/contracts/tools/aws/s3-delete-objects' +import { awsS3HeadObjectContract } from '@/lib/api/contracts/tools/aws/s3-head-object' +import { awsS3ListBucketsContract } from '@/lib/api/contracts/tools/aws/s3-list-buckets' +import { awsS3ListObjectsContract } from '@/lib/api/contracts/tools/aws/s3-list-objects' +import { awsS3PresignedUrlContract } from '@/lib/api/contracts/tools/aws/s3-presigned-url' +import { awsS3PutObjectContract } from '@/lib/api/contracts/tools/aws/s3-put-object' +import { S3OperationError } from '@/lib/internal/s3/errors' +import { + executeS3CopyObject, + executeS3CreateBucket, + executeS3DeleteBucket, + executeS3DeleteObject, + executeS3DeleteObjects, + executeS3HeadObject, + executeS3ListBuckets, + executeS3ListObjects, + executeS3PresignedUrl, + executeS3PutObject, + type S3OperationContext, +} from '@/lib/internal/s3/operations' +import { parseInternalToolInput } from '@/lib/internal/tool-operations/parse-input' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +async function executeOperation( + contract: C, + input: unknown, + execute: (input: ContractBody) => Promise, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const parsed = parseInternalToolInput(contract, input) + if (!parsed.success) return parsed.response + + try { + const result = await execute(parsed.data) + signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + signal?.throwIfAborted() + const status = error instanceof S3OperationError ? error.status : 500 + return Response.json( + { success: false, error: getErrorMessage(error, 'Internal server error') }, + { status } + ) + } +} + +export const executeS3Tool: InternalToolOperationHandler = async (request) => { + const { toolId, input, context, headers, requestId, signal } = request + const operationContext: S3OperationContext = { + headers, + requestId, + signal, + userId: context.userId, + } + switch (toolId) { + case 's3_copy_object': + return executeOperation( + awsS3CopyObjectContract, + input, + (input) => executeS3CopyObject(input, signal), + signal + ) + case 's3_create_bucket': + return executeOperation( + awsS3CreateBucketContract, + input, + (input) => executeS3CreateBucket(input, signal), + signal + ) + case 's3_delete_bucket': + return executeOperation( + awsS3DeleteBucketContract, + input, + (input) => executeS3DeleteBucket(input, signal), + signal + ) + case 's3_delete_object': + return executeOperation( + awsS3DeleteObjectContract, + input, + (input) => executeS3DeleteObject(input, signal), + signal + ) + case 's3_delete_objects': + return executeOperation( + awsS3DeleteObjectsContract, + input, + (input) => executeS3DeleteObjects(input, signal), + signal + ) + case 's3_head_object': + return executeOperation( + awsS3HeadObjectContract, + input, + (input) => executeS3HeadObject(input, signal), + signal + ) + case 's3_list_buckets': + return executeOperation( + awsS3ListBucketsContract, + input, + (input) => executeS3ListBuckets(input, signal), + signal + ) + case 's3_list_objects': + return executeOperation( + awsS3ListObjectsContract, + input, + (input) => executeS3ListObjects(input, signal), + signal + ) + case 's3_presigned_url': + return executeOperation( + awsS3PresignedUrlContract, + input, + (input) => executeS3PresignedUrl(input, signal), + signal + ) + case 's3_put_object': + return executeOperation( + awsS3PutObjectContract, + input, + (input) => executeS3PutObject(input, operationContext), + signal + ) + default: + return Response.json( + { success: false, error: `Unsupported S3 tool: ${toolId}` }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/s3/operations.test.ts b/apps/sim/lib/internal/s3/operations.test.ts new file mode 100644 index 00000000000..7a54def81af --- /dev/null +++ b/apps/sim/lib/internal/s3/operations.test.ts @@ -0,0 +1,267 @@ +/** + * @vitest-environment node + */ +import { + CopyObjectCommand, + HeadObjectCommand, + ListBucketsCommand, + PutObjectCommand, +} from '@aws-sdk/client-s3' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + assertToolFileAccess: vi.fn(), + createS3Client: vi.fn(), + destroy: vi.fn(), + downloadServableFileFromStorage: vi.fn(), + getSignedUrl: vi.fn(), + processSingleFileToUserFile: vi.fn(), + send: vi.fn(), +})) + +vi.mock('@/lib/internal/s3/client', () => ({ + createS3Client: mocks.createS3Client, +})) +vi.mock('@aws-sdk/s3-request-presigner', () => ({ + getSignedUrl: mocks.getSignedUrl, +})) +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertToolFileAccess, +})) +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + processSingleFileToUserFile: mocks.processSingleFileToUserFile, +})) +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mocks.downloadServableFileFromStorage, +})) + +import { S3OperationError } from '@/lib/internal/s3/errors' +import { + executeS3CopyObject, + executeS3HeadObject, + executeS3ListBuckets, + executeS3PresignedUrl, + executeS3PutObject, +} from '@/lib/internal/s3/operations' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' + +const CONNECTION = { + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', + region: 'us-east-1', +} + +const CONTEXT = { + headers: new Headers(), + requestId: 'request-1', + userId: 'user-1', +} + +describe('S3 operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.createS3Client.mockReturnValue({ send: mocks.send, destroy: mocks.destroy }) + mocks.assertToolFileAccess.mockResolvedValue(null) + mocks.processSingleFileToUserFile.mockReturnValue({ + key: 'workspace/file-key', + name: 'file.txt', + size: 5, + type: 'text/plain', + }) + mocks.downloadServableFileFromStorage.mockResolvedValue({ + buffer: Buffer.from('hello'), + contentType: 'text/plain', + }) + }) + + it('copies encoded object keys, forwards cancellation, and destroys the client', async () => { + const controller = new AbortController() + mocks.send.mockResolvedValue({ + CopyObjectResult: { ETag: 'etag' }, + CopySourceVersionId: 'source-version', + VersionId: 'version', + }) + + const result = await executeS3CopyObject( + { + ...CONNECTION, + sourceBucket: 'source', + sourceKey: 'folder/source file.txt', + destinationBucket: 'destination', + destinationKey: 'folder/destination file.txt', + }, + controller.signal + ) + + const [command, options] = mocks.send.mock.calls[0] + expect(command).toBeInstanceOf(CopyObjectCommand) + expect(command.input).toMatchObject({ + CopySource: 'source/folder/source%20file.txt', + Key: 'folder/destination file.txt', + }) + expect(options).toEqual({ abortSignal: controller.signal }) + expect(result.output).toMatchObject({ + url: 'https://destination.s3.us-east-1.amazonaws.com/folder/destination%20file.txt', + uri: 's3://destination/folder/destination file.txt', + etag: 'etag', + }) + expect(mocks.destroy).toHaveBeenCalledOnce() + }) + + it('maps an S3 not-found head response to exists false', async () => { + mocks.send.mockRejectedValue({ name: 'NotFound', $metadata: { httpStatusCode: 404 } }) + + const result = await executeS3HeadObject({ + ...CONNECTION, + bucketName: 'bucket', + objectKey: 'missing.txt', + }) + + expect(mocks.send.mock.calls[0][0]).toBeInstanceOf(HeadObjectCommand) + expect(result.output).toEqual({ + exists: false, + contentLength: null, + contentType: null, + etag: null, + lastModified: null, + versionId: null, + storageClass: null, + serverSideEncryption: null, + deleteMarker: null, + metadata: {}, + }) + expect(mocks.destroy).toHaveBeenCalledOnce() + }) + + it('preserves list-buckets pagination and nullable fields', async () => { + mocks.send.mockResolvedValue({ + Buckets: [{ Name: 'bucket', CreationDate: new Date('2026-01-01T00:00:00.000Z') }], + ContinuationToken: 'next', + Prefix: 'prod-', + }) + + const result = await executeS3ListBuckets({ + ...CONNECTION, + prefix: 'prod-', + maxBuckets: 25, + continuationToken: 'token', + }) + + const command = mocks.send.mock.calls[0][0] + expect(command).toBeInstanceOf(ListBucketsCommand) + expect(command.input).toEqual({ + Prefix: 'prod-', + MaxBuckets: 25, + ContinuationToken: 'token', + }) + expect(result.output).toEqual({ + buckets: [ + { + name: 'bucket', + creationDate: '2026-01-01T00:00:00.000Z', + region: null, + }, + ], + owner: null, + continuationToken: 'next', + prefix: 'prod-', + }) + }) + + it('generates presigned URLs without leaking the client', async () => { + mocks.getSignedUrl.mockResolvedValue('https://signed.example/object') + + const result = await executeS3PresignedUrl({ + ...CONNECTION, + bucketName: 'bucket', + objectKey: 'object.txt', + method: 'put', + expiresIn: 300, + contentType: 'text/plain', + }) + + expect(mocks.getSignedUrl).toHaveBeenCalledWith( + expect.objectContaining({ send: mocks.send }), + expect.any(PutObjectCommand), + { expiresIn: 300 } + ) + expect(result.output.url).toBe('https://signed.example/object') + expect(mocks.destroy).toHaveBeenCalledOnce() + }) + + it('uploads inline content with the same public URL and content-type semantics', async () => { + const controller = new AbortController() + mocks.send.mockResolvedValue({ ETag: 'etag' }) + + const result = await executeS3PutObject( + { + ...CONNECTION, + bucketName: 'bucket', + objectKey: 'folder/file name.txt', + content: 'hello', + }, + { ...CONTEXT, signal: controller.signal } + ) + + const [command, options] = mocks.send.mock.calls[0] + expect(command).toBeInstanceOf(PutObjectCommand) + expect(command.input).toMatchObject({ + Bucket: 'bucket', + Key: 'folder/file name.txt', + ContentType: 'text/plain', + }) + expect(Buffer.from(command.input.Body).toString()).toBe('hello') + expect(options).toEqual({ abortSignal: controller.signal }) + expect(result.output.url).toBe( + 'https://bucket.s3.us-east-1.amazonaws.com/folder/file%20name.txt' + ) + expect(mocks.destroy).toHaveBeenCalledOnce() + }) + + it('authorizes and bounds stored files before sending them to S3', async () => { + mocks.send.mockResolvedValue({}) + const file = { key: 'workspace/file-key', name: 'file.txt', size: 5, type: 'text/plain' } + + await executeS3PutObject( + { ...CONNECTION, bucketName: 'bucket', objectKey: 'file.txt', file }, + CONTEXT + ) + + expect(mocks.assertToolFileAccess).toHaveBeenCalledWith( + 'workspace/file-key', + 'user-1', + 'request-1', + expect.anything() + ) + expect(mocks.downloadServableFileFromStorage).toHaveBeenCalledWith( + expect.objectContaining({ key: 'workspace/file-key' }), + 'request-1', + expect.anything(), + { maxBytes: MAX_BUFFERED_TRANSFER_BYTES } + ) + }) + + it('fails closed when stored-file access is denied', async () => { + mocks.assertToolFileAccess.mockResolvedValue(new Response(null, { status: 404 })) + + await expect( + executeS3PutObject( + { + ...CONNECTION, + bucketName: 'bucket', + objectKey: 'file.txt', + file: { key: 'workspace/file-key', name: 'file.txt', size: 5 }, + }, + CONTEXT + ) + ).rejects.toEqual(new S3OperationError('File not found', 404)) + expect(mocks.createS3Client).not.toHaveBeenCalled() + }) + + it('destroys the client when AWS rejects the operation', async () => { + mocks.send.mockRejectedValue(new Error('S3 rejected')) + + await expect(executeS3ListBuckets(CONNECTION)).rejects.toThrow('S3 rejected') + expect(mocks.destroy).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/internal/s3/operations.ts b/apps/sim/lib/internal/s3/operations.ts new file mode 100644 index 00000000000..dface170c3c --- /dev/null +++ b/apps/sim/lib/internal/s3/operations.ts @@ -0,0 +1,377 @@ +import { + type BucketCannedACL, + type BucketLocationConstraint, + CopyObjectCommand, + CreateBucketCommand, + DeleteBucketCommand, + DeleteObjectCommand, + DeleteObjectsCommand, + GetObjectCommand, + HeadObjectCommand, + ListBucketsCommand, + ListObjectsV2Command, + type ObjectCannedACL, + PutObjectCommand, + type S3Client, +} from '@aws-sdk/client-s3' +import { getSignedUrl } from '@aws-sdk/s3-request-presigner' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { AwsS3CopyObjectBody } from '@/lib/api/contracts/tools/aws/s3-copy-object' +import type { AwsS3CreateBucketBody } from '@/lib/api/contracts/tools/aws/s3-create-bucket' +import type { AwsS3DeleteBucketBody } from '@/lib/api/contracts/tools/aws/s3-delete-bucket' +import type { AwsS3DeleteObjectBody } from '@/lib/api/contracts/tools/aws/s3-delete-object' +import type { AwsS3DeleteObjectsBody } from '@/lib/api/contracts/tools/aws/s3-delete-objects' +import type { AwsS3HeadObjectBody } from '@/lib/api/contracts/tools/aws/s3-head-object' +import type { AwsS3ListBucketsBody } from '@/lib/api/contracts/tools/aws/s3-list-buckets' +import type { AwsS3ListObjectsBody } from '@/lib/api/contracts/tools/aws/s3-list-objects' +import type { AwsS3PresignedUrlBody } from '@/lib/api/contracts/tools/aws/s3-presigned-url' +import type { AwsS3PutObjectBody } from '@/lib/api/contracts/tools/aws/s3-put-object' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { createS3Client, type S3ConnectionConfig } from '@/lib/internal/s3/client' +import { S3OperationError } from '@/lib/internal/s3/errors' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/doc-not-ready' +import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +const logger = createLogger('S3Operations') + +export interface S3OperationContext { + headers: Headers + requestId: string + signal?: AbortSignal + userId?: string +} + +async function withS3Client( + input: S3ConnectionConfig, + execute: (client: S3Client) => Promise +): Promise { + const client = createS3Client(input) + try { + return await execute(client) + } finally { + client.destroy() + } +} + +function encodeObjectKey(key: string): string { + return key.split('/').map(encodeURIComponent).join('/') +} + +export async function executeS3CopyObject(input: AwsS3CopyObjectBody, signal?: AbortSignal) { + return withS3Client(input, async (client) => { + const encodedSourceKey = encodeObjectKey(input.sourceKey) + const response = await client.send( + new CopyObjectCommand({ + Bucket: input.destinationBucket, + Key: input.destinationKey, + CopySource: `${input.sourceBucket}/${encodedSourceKey}`, + ACL: input.acl as ObjectCannedACL | undefined, + }), + { abortSignal: signal } + ) + const encodedDestinationKey = encodeObjectKey(input.destinationKey) + const url = `https://${input.destinationBucket}.s3.${input.region}.amazonaws.com/${encodedDestinationKey}` + return { + success: true as const, + output: { + url, + uri: `s3://${input.destinationBucket}/${input.destinationKey}`, + copySourceVersionId: response.CopySourceVersionId, + versionId: response.VersionId, + etag: response.CopyObjectResult?.ETag, + }, + } + }) +} + +export async function executeS3CreateBucket(input: AwsS3CreateBucketBody, signal?: AbortSignal) { + return withS3Client(input, async (client) => { + const response = await client.send( + new CreateBucketCommand({ + Bucket: input.bucketName, + ACL: (input.acl as BucketCannedACL | undefined) || undefined, + CreateBucketConfiguration: + input.region === 'us-east-1' + ? undefined + : { LocationConstraint: input.region as BucketLocationConstraint }, + }), + { abortSignal: signal } + ) + return { + success: true as const, + output: { + bucket: input.bucketName, + location: response.Location ?? null, + bucketArn: response.BucketArn ?? null, + }, + } + }) +} + +export async function executeS3DeleteBucket(input: AwsS3DeleteBucketBody, signal?: AbortSignal) { + return withS3Client(input, async (client) => { + await client.send(new DeleteBucketCommand({ Bucket: input.bucketName }), { + abortSignal: signal, + }) + return { + success: true as const, + output: { deleted: true as const, bucket: input.bucketName }, + } + }) +} + +export async function executeS3DeleteObject(input: AwsS3DeleteObjectBody, signal?: AbortSignal) { + return withS3Client(input, async (client) => { + const response = await client.send( + new DeleteObjectCommand({ Bucket: input.bucketName, Key: input.objectKey }), + { abortSignal: signal } + ) + return { + success: true as const, + output: { + key: input.objectKey, + deleteMarker: response.DeleteMarker, + versionId: response.VersionId, + }, + } + }) +} + +export async function executeS3DeleteObjects(input: AwsS3DeleteObjectsBody, signal?: AbortSignal) { + return withS3Client(input, async (client) => { + const response = await client.send( + new DeleteObjectsCommand({ + Bucket: input.bucketName, + Delete: { + Objects: input.keys.map((key) => ({ Key: key })), + Quiet: input.quiet ?? false, + }, + }), + { abortSignal: signal } + ) + return { + success: true as const, + output: { + deleted: (response.Deleted ?? []).map((object) => ({ + key: object.Key ?? null, + versionId: object.VersionId ?? null, + deleteMarker: object.DeleteMarker ?? null, + })), + errors: (response.Errors ?? []).map((error) => ({ + key: error.Key ?? null, + code: error.Code ?? null, + message: error.Message ?? null, + })), + }, + } + }) +} + +export async function executeS3HeadObject(input: AwsS3HeadObjectBody, signal?: AbortSignal) { + return withS3Client(input, async (client) => { + try { + const response = await client.send( + new HeadObjectCommand({ + Bucket: input.bucketName, + Key: input.objectKey, + VersionId: input.versionId || undefined, + }), + { abortSignal: signal } + ) + return { + success: true as const, + output: { + exists: true, + contentLength: response.ContentLength ?? null, + contentType: response.ContentType ?? null, + etag: response.ETag ?? null, + lastModified: response.LastModified?.toISOString() ?? null, + versionId: response.VersionId ?? null, + storageClass: response.StorageClass ?? null, + serverSideEncryption: response.ServerSideEncryption ?? null, + deleteMarker: response.DeleteMarker ?? null, + metadata: response.Metadata ?? {}, + }, + } + } catch (error) { + signal?.throwIfAborted() + const metadata = error as { name?: string; $metadata?: { httpStatusCode?: number } } + if (metadata.name !== 'NotFound' && metadata.$metadata?.httpStatusCode !== 404) throw error + return { + success: true as const, + output: { + exists: false, + contentLength: null, + contentType: null, + etag: null, + lastModified: null, + versionId: null, + storageClass: null, + serverSideEncryption: null, + deleteMarker: null, + metadata: {}, + }, + } + } + }) +} + +export async function executeS3ListBuckets(input: AwsS3ListBucketsBody, signal?: AbortSignal) { + return withS3Client(input, async (client) => { + const response = await client.send( + new ListBucketsCommand({ + Prefix: input.prefix || undefined, + MaxBuckets: input.maxBuckets || undefined, + ContinuationToken: input.continuationToken || undefined, + }), + { abortSignal: signal } + ) + return { + success: true as const, + output: { + buckets: (response.Buckets ?? []).map((bucket) => ({ + name: bucket.Name || '', + creationDate: bucket.CreationDate?.toISOString() ?? null, + region: bucket.BucketRegion ?? null, + })), + owner: response.Owner + ? { + displayName: response.Owner.DisplayName ?? null, + id: response.Owner.ID ?? null, + } + : null, + continuationToken: response.ContinuationToken ?? null, + prefix: response.Prefix ?? null, + }, + } + }) +} + +export async function executeS3ListObjects(input: AwsS3ListObjectsBody, signal?: AbortSignal) { + return withS3Client(input, async (client) => { + const response = await client.send( + new ListObjectsV2Command({ + Bucket: input.bucketName, + Prefix: input.prefix || undefined, + MaxKeys: input.maxKeys || undefined, + ContinuationToken: input.continuationToken || undefined, + }), + { abortSignal: signal } + ) + return { + success: true as const, + output: { + objects: (response.Contents ?? []).map((object) => ({ + key: object.Key || '', + size: object.Size || 0, + lastModified: object.LastModified?.toISOString() || '', + etag: object.ETag || '', + })), + isTruncated: response.IsTruncated, + nextContinuationToken: response.NextContinuationToken, + keyCount: response.KeyCount, + prefix: input.prefix, + }, + } + }) +} + +export async function executeS3PresignedUrl(input: AwsS3PresignedUrlBody, signal?: AbortSignal) { + return withS3Client(input, async (client) => { + signal?.throwIfAborted() + const command = + input.method === 'put' + ? new PutObjectCommand({ + Bucket: input.bucketName, + Key: input.objectKey, + ContentType: input.contentType || undefined, + }) + : new GetObjectCommand({ Bucket: input.bucketName, Key: input.objectKey }) + const url = await getSignedUrl(client, command, { expiresIn: input.expiresIn }) + signal?.throwIfAborted() + return { + success: true as const, + output: { + url, + method: input.method, + expiresIn: input.expiresIn, + expiresAt: new Date(Date.now() + input.expiresIn * 1000).toISOString(), + }, + } + }) +} + +export async function executeS3PutObject(input: AwsS3PutObjectBody, context: S3OperationContext) { + const { requestId, signal, userId } = context + signal?.throwIfAborted() + if (!userId) throw new S3OperationError('Authentication required', 401) + + let uploadBody: Buffer | string + let uploadContentType: string | undefined + if (input.file) { + let userFile: ReturnType + try { + userFile = processSingleFileToUserFile(input.file, requestId, logger) + } catch (error) { + throw new S3OperationError(getErrorMessage(error, 'Failed to process file'), 400) + } + + const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) + if (denied) throw new S3OperationError('File not found', 404) + signal?.throwIfAborted() + + try { + const downloaded = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) + signal?.throwIfAborted() + uploadBody = downloaded.buffer + uploadContentType = + input.contentType || downloaded.contentType || userFile.type || 'application/octet-stream' + } catch (error) { + signal?.throwIfAborted() + if (isDocNotReadyError(error)) { + throw new S3OperationError(docNotReadyMessage(), 409) + } + throw new S3OperationError( + getErrorMessage(error, 'Failed to download file'), + isPayloadSizeLimitError(error) ? 413 : 500 + ) + } + } else if (input.content) { + uploadBody = Buffer.from(input.content, 'utf-8') + uploadContentType = input.contentType || 'text/plain' + } else { + throw new S3OperationError('Either file or content must be provided', 400) + } + + return withS3Client(input, async (client) => { + const response = await client.send( + new PutObjectCommand({ + Bucket: input.bucketName, + Key: input.objectKey, + Body: uploadBody, + ContentType: uploadContentType, + ACL: input.acl as ObjectCannedACL | undefined, + }), + { abortSignal: signal } + ) + const encodedKey = encodeObjectKey(input.objectKey) + const url = `https://${input.bucketName}.s3.${input.region}.amazonaws.com/${encodedKey}` + return { + success: true as const, + output: { + url, + uri: `s3://${input.bucketName}/${input.objectKey}`, + etag: response.ETag, + location: url, + key: input.objectKey, + bucket: input.bucketName, + }, + } + }) +} diff --git a/apps/sim/lib/internal/sap-concur/client.test.ts b/apps/sim/lib/internal/sap-concur/client.test.ts new file mode 100644 index 00000000000..ccf939059d3 --- /dev/null +++ b/apps/sim/lib/internal/sap-concur/client.test.ts @@ -0,0 +1,984 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockSecureFetch, MOCK_MAX_JSON_BYTES } = vi.hoisted(() => ({ + mockSecureFetch: vi.fn(), + MOCK_MAX_JSON_BYTES: 10 * 1024 * 1024, +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + secureFetchWithValidation: mockSecureFetch, + MAX_JSON_API_RESPONSE_BYTES: MOCK_MAX_JSON_BYTES, +})) + +import { + assertSafeExternalUrl, + extractSapConcurError, + fetchSapConcurAccessToken, + forwardedSapConcurHeaders, + invokeSapConcurMultipart, +} from '@/lib/internal/sap-concur/client' +import { + SAP_CONCUR_ALLOWED_DATACENTERS, + type SapConcurAuth, + sapConcurApiInputSchema, + sapConcurApiPathSchema, + sapConcurDatacenterSchema, +} from '@/lib/internal/sap-concur/schema' + +const CLIENT_SECRET = 'super-secret-client-value' +const PASSWORD = 'hunter2-plaintext-password' + +/** + * `TOKEN_CACHE` in the client is module-global and survives every test in this file, so + * each case takes its own `clientId`. That guarantees a cold cache key for the case and + * keeps one test's cached token from silently satisfying the next test's assertions. + */ +let clientIdCounter = 0 +function freshClientId(): string { + clientIdCounter += 1 + return `client-${clientIdCounter}` +} + +function auth(overrides: Partial & { clientId: string }): SapConcurAuth { + return { + datacenter: 'us.api.concursolutions.com', + grantType: 'client_credentials', + clientSecret: CLIENT_SECRET, + ...overrides, + } +} + +function tokenResponse( + body: Record = { access_token: 'token-1', expires_in: 3600 }, + status = 200 +) { + return { + ok: status >= 200 && status < 300, + status, + headers: new Headers(), + json: async () => body, + text: async () => JSON.stringify(body), + } +} + +beforeEach(() => { + vi.clearAllMocks() + // mockReset also drains any `mockResolvedValueOnce` a failing test left queued. + mockSecureFetch.mockReset() + mockSecureFetch.mockResolvedValue(tokenResponse()) +}) + +describe('fetchSapConcurAccessToken token cache key isolation', () => { + /** + * Regression test for the auth-bypass: with the password absent from the cache key, a + * request carrying the wrong password was served a token minted from the correct one. + */ + it('does not share a cache entry across differing passwords', async () => { + const clientId = freshClientId() + const base = auth({ + clientId, + grantType: 'password', + username: 'alice@example.com', + }) + + mockSecureFetch.mockResolvedValueOnce( + tokenResponse({ access_token: 'token-correct', expires_in: 3600 }) + ) + const first = await fetchSapConcurAccessToken({ ...base, password: PASSWORD }, 'req-1') + + mockSecureFetch.mockResolvedValueOnce( + tokenResponse({ access_token: 'token-other', expires_in: 3600 }) + ) + const second = await fetchSapConcurAccessToken({ ...base, password: 'a-different-pw' }, 'req-2') + + expect(mockSecureFetch).toHaveBeenCalledTimes(2) + expect(first.accessToken).toBe('token-correct') + expect(second.accessToken).toBe('token-other') + }) + + it('does not share a cache entry across differing client secrets', async () => { + const clientId = freshClientId() + const base = auth({ clientId }) + + mockSecureFetch.mockResolvedValueOnce( + tokenResponse({ access_token: 'token-first-secret', expires_in: 3600 }) + ) + const first = await fetchSapConcurAccessToken(base, 'req-1') + + mockSecureFetch.mockResolvedValueOnce( + tokenResponse({ access_token: 'token-second-secret', expires_in: 3600 }) + ) + const second = await fetchSapConcurAccessToken( + { ...base, clientSecret: 'a-different-client-secret' }, + 'req-2' + ) + + expect(mockSecureFetch).toHaveBeenCalledTimes(2) + expect(first.accessToken).toBe('token-first-secret') + expect(second.accessToken).toBe('token-second-secret') + }) + + it('does not share a cache entry across differing companyUuid', async () => { + const clientId = freshClientId() + const base = auth({ clientId }) + + await fetchSapConcurAccessToken({ ...base, companyUuid: 'company-a' }, 'req-1') + await fetchSapConcurAccessToken({ ...base, companyUuid: 'company-b' }, 'req-2') + + expect(mockSecureFetch).toHaveBeenCalledTimes(2) + }) + + it('does not share a cache entry across differing credtype', async () => { + const clientId = freshClientId() + const base = auth({ + clientId, + grantType: 'password', + username: 'alice@example.com', + password: PASSWORD, + }) + + await fetchSapConcurAccessToken({ ...base, credtype: 'password' }, 'req-1') + await fetchSapConcurAccessToken({ ...base, credtype: 'authtoken' }, 'req-2') + + expect(mockSecureFetch).toHaveBeenCalledTimes(2) + }) + + it('shares the cache for two fully identical requests', async () => { + const clientId = freshClientId() + const base = auth({ + clientId, + grantType: 'password', + username: 'alice@example.com', + password: PASSWORD, + companyUuid: 'company-a', + credtype: 'password', + }) + + mockSecureFetch.mockResolvedValueOnce( + tokenResponse({ access_token: 'token-cached', expires_in: 3600 }) + ) + const first = await fetchSapConcurAccessToken({ ...base }, 'req-1') + const second = await fetchSapConcurAccessToken({ ...base }, 'req-2') + + expect(mockSecureFetch).toHaveBeenCalledTimes(1) + expect(first.accessToken).toBe('token-cached') + expect(second.accessToken).toBe('token-cached') + }) + + it('refetches once a cached token falls inside the 60s safety window', async () => { + vi.useFakeTimers() + try { + vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z')) + const base = auth({ clientId: freshClientId() }) + + mockSecureFetch.mockResolvedValueOnce( + tokenResponse({ access_token: 'token-first', expires_in: 120 }) + ) + const first = await fetchSapConcurAccessToken(base, 'req-1') + expect(first.accessToken).toBe('token-first') + + // 30s in: still outside the 60s safety window, so the cache answers. + vi.setSystemTime(new Date('2026-01-01T00:00:30.000Z')) + const cached = await fetchSapConcurAccessToken(base, 'req-2') + expect(cached.accessToken).toBe('token-first') + expect(mockSecureFetch).toHaveBeenCalledTimes(1) + + // 90s in: expiry (120s) minus the 60s window has passed, so it refetches. + vi.setSystemTime(new Date('2026-01-01T00:01:30.000Z')) + mockSecureFetch.mockResolvedValueOnce( + tokenResponse({ access_token: 'token-second', expires_in: 3600 }) + ) + const refreshed = await fetchSapConcurAccessToken(base, 'req-3') + expect(refreshed.accessToken).toBe('token-second') + expect(mockSecureFetch).toHaveBeenCalledTimes(2) + } finally { + vi.useRealTimers() + } + }) +}) + +/** + * A parallel block fanning out many Concur calls, or a cold container after a deploy, + * misses the token cache on every branch at once. Without coalescing each branch fires + * its own `POST /oauth2/v0/token` into an endpoint Concur rate-limits hard. + */ +describe('fetchSapConcurAccessToken in-flight coalescing', () => { + it('collapses concurrent misses for one key into a single token fetch', async () => { + const base = auth({ clientId: freshClientId() }) + + let release: () => void = () => {} + const gate = new Promise((resolve) => { + release = resolve + }) + mockSecureFetch.mockImplementation(async () => { + await gate + return tokenResponse({ access_token: 'token-shared', expires_in: 3600 }) + }) + + const inFlight = Array.from({ length: 8 }, (_, index) => + fetchSapConcurAccessToken(base, `req-${index}`) + ) + release() + const results = await Promise.all(inFlight) + + expect(mockSecureFetch).toHaveBeenCalledTimes(1) + for (const result of results) { + expect(result.accessToken).toBe('token-shared') + } + }) + + it('does not collapse concurrent misses for different keys', async () => { + const first = auth({ clientId: freshClientId() }) + const second = auth({ clientId: freshClientId() }) + + await Promise.all([ + fetchSapConcurAccessToken(first, 'req-1'), + fetchSapConcurAccessToken(second, 'req-2'), + ]) + + expect(mockSecureFetch).toHaveBeenCalledTimes(2) + }) + + it('does not poison the key when the in-flight request rejects', async () => { + const base = auth({ clientId: freshClientId() }) + + mockSecureFetch.mockRejectedValueOnce(new Error('socket hang up')) + const first = fetchSapConcurAccessToken(base, 'req-1') + const joiner = fetchSapConcurAccessToken(base, 'req-2') + + await expect(first).rejects.toThrow('socket hang up') + await expect(joiner).rejects.toThrow('socket hang up') + + mockSecureFetch.mockResolvedValueOnce( + tokenResponse({ access_token: 'token-after-retry', expires_in: 3600 }) + ) + const retried = await fetchSapConcurAccessToken(base, 'req-3') + + expect(retried.accessToken).toBe('token-after-retry') + expect(mockSecureFetch).toHaveBeenCalledTimes(2) + }) + + it('keeps a shared token request alive until every waiting execution cancels', async () => { + const base = auth({ clientId: freshClientId() }) + const firstController = new AbortController() + const secondController = new AbortController() + let providerSignal: AbortSignal | undefined + + mockSecureFetch.mockImplementationOnce( + async (_url: string, options: { signal?: AbortSignal }) => { + providerSignal = options.signal + return await new Promise((_, reject) => { + options.signal?.addEventListener( + 'abort', + () => reject(options.signal?.reason ?? new DOMException('Aborted', 'AbortError')), + { once: true } + ) + }) + } + ) + + const first = fetchSapConcurAccessToken(base, 'req-1', firstController.signal) + const second = fetchSapConcurAccessToken(base, 'req-2', secondController.signal) + firstController.abort(new DOMException('First cancelled', 'AbortError')) + + await expect(first).rejects.toMatchObject({ name: 'AbortError' }) + expect(providerSignal?.aborted).toBe(false) + + secondController.abort(new DOMException('Second cancelled', 'AbortError')) + await expect(second).rejects.toMatchObject({ name: 'AbortError' }) + expect(providerSignal?.aborted).toBe(true) + expect(mockSecureFetch).toHaveBeenCalledOnce() + }) +}) + +describe('fetchSapConcurAccessToken geolocation validation', () => { + const accepted = [ + 'https://us.api.concursolutions.com', + 'https://www-us2.api.concursolutions.com', + 'https://apj1.api.concursolutions.com', + 'https://emea-impl.api.concursolutions.com', + ] + + it.each(accepted)('accepts the Concur geolocation %s', async (geolocation) => { + mockSecureFetch.mockResolvedValueOnce( + tokenResponse({ access_token: 'token-1', expires_in: 3600, geolocation }) + ) + const result = await fetchSapConcurAccessToken(auth({ clientId: freshClientId() }), 'req-1') + expect(result.geolocation).toBe(geolocation) + }) + + const rejected: Array<[string, string]> = [ + ['an unrelated host', 'https://evil.com'], + ['a suffix-confusion host', 'https://concursolutions.com.evil.com'], + ['a subdomain-confusion host', 'https://us.api.concursolutions.com.evil.com'], + ] + + it.each(rejected)('rejects %s', async (_label, geolocation) => { + mockSecureFetch.mockResolvedValueOnce( + tokenResponse({ access_token: 'token-1', expires_in: 3600, geolocation }) + ) + await expect( + fetchSapConcurAccessToken(auth({ clientId: freshClientId() }), 'req-1') + ).rejects.toThrow('not a valid Concur API host') + }) + + it('rejects a plain-http geolocation', async () => { + mockSecureFetch.mockResolvedValueOnce( + tokenResponse({ + access_token: 'token-1', + expires_in: 3600, + geolocation: 'http://us.api.concursolutions.com', + }) + ) + await expect( + fetchSapConcurAccessToken(auth({ clientId: freshClientId() }), 'req-1') + ).rejects.toThrow('geolocation must use https://') + }) + + it('rejects a loopback geolocation', async () => { + mockSecureFetch.mockResolvedValueOnce( + tokenResponse({ access_token: 'token-1', expires_in: 3600, geolocation: 'https://127.0.0.1' }) + ) + await expect( + fetchSapConcurAccessToken(auth({ clientId: freshClientId() }), 'req-1') + ).rejects.toThrow('geolocation host is not allowed') + }) + + it('normalizes a bare hostname to https and still validates it', async () => { + mockSecureFetch.mockResolvedValueOnce( + tokenResponse({ + access_token: 'token-1', + expires_in: 3600, + geolocation: 'us2.api.concursolutions.com', + }) + ) + const result = await fetchSapConcurAccessToken(auth({ clientId: freshClientId() }), 'req-1') + expect(result.geolocation).toBe('https://us2.api.concursolutions.com') + }) + + it('rejects a bare hostname that normalizes to a non-Concur host', async () => { + mockSecureFetch.mockResolvedValueOnce( + tokenResponse({ access_token: 'token-1', expires_in: 3600, geolocation: 'evil.com' }) + ) + await expect( + fetchSapConcurAccessToken(auth({ clientId: freshClientId() }), 'req-1') + ).rejects.toThrow('not a valid Concur API host') + }) + + /** + * DOCUMENTED TRUST ASSUMPTION, asserted as current behavior on purpose: the geolocation + * check validates the *shape* `[label].api.concursolutions.com`, not membership in + * {@link SAP_CONCUR_ALLOWED_DATACENTERS}. That is deliberate — Concur's docs instruct + * clients to store and reuse whatever geolocation the token response returns, and SAP + * adds datacenters (GLZ was one) without clients redeploying, so pinning the response + * to the selectable set would break tenants on a new datacenter. + * + * The consequence is that an attacker-flavored label like `evil-us` is accepted. Such a + * host can only exist if SAP itself creates it under concursolutions.com, which puts it + * inside the same trust boundary as every other Concur host. Narrowing this to the + * allowlist is a deliberate product decision, not a bug fix — do not "harden" it + * without re-reading the geolocation guidance in the authentication docs. + */ + it('accepts any SAP-created label under api.concursolutions.com (trust boundary is the domain)', async () => { + const geolocation = 'https://evil-us.api.concursolutions.com' + mockSecureFetch.mockResolvedValueOnce( + tokenResponse({ access_token: 'token-1', expires_in: 3600, geolocation }) + ) + const result = await fetchSapConcurAccessToken(auth({ clientId: freshClientId() }), 'req-1') + expect(result.geolocation).toBe(geolocation) + }) + + it('rejects a userinfo-form geolocation whose real hostname is attacker-controlled', async () => { + mockSecureFetch.mockResolvedValueOnce( + tokenResponse({ + access_token: 'token-1', + expires_in: 3600, + geolocation: 'https://us.api.concursolutions.com@evil.com', + }) + ) + await expect( + fetchSapConcurAccessToken(auth({ clientId: freshClientId() }), 'req-1') + ).rejects.toThrow('not a valid Concur API host') + }) + + it('accepts a Concur host carrying an explicit port and preserves it', async () => { + const geolocation = 'https://us.api.concursolutions.com:8443' + mockSecureFetch.mockResolvedValueOnce( + tokenResponse({ access_token: 'token-1', expires_in: 3600, geolocation }) + ) + const result = await fetchSapConcurAccessToken(auth({ clientId: freshClientId() }), 'req-1') + expect(result.geolocation).toBe(geolocation) + }) + + it('rejects a non-Concur host even when the port looks Concur-shaped', async () => { + mockSecureFetch.mockResolvedValueOnce( + tokenResponse({ + access_token: 'token-1', + expires_in: 3600, + geolocation: 'https://evil.com:443', + }) + ) + await expect( + fetchSapConcurAccessToken(auth({ clientId: freshClientId() }), 'req-1') + ).rejects.toThrow('not a valid Concur API host') + }) +}) + +/** + * Concur's company-level flow is a password grant that carries the company UUID in + * `username`, the 24-hour App Center request token in `password`, and `credtype=authtoken`. + */ +describe('fetchSapConcurAccessToken company-level auth', () => { + function submittedParams(): URLSearchParams { + const [, init] = mockSecureFetch.mock.calls[0] + return new URLSearchParams(init.body as string) + } + + it('submits the companyUuid as username and defaults credtype to authtoken', async () => { + await fetchSapConcurAccessToken( + auth({ + clientId: freshClientId(), + grantType: 'password', + password: 'company-request-token', + companyUuid: '08BCCA1E-0D4F-4261-9F1B-F778D96617D6', + }), + 'req-1' + ) + + const params = submittedParams() + expect(params.get('grant_type')).toBe('password') + expect(params.get('username')).toBe('08BCCA1E-0D4F-4261-9F1B-F778D96617D6') + expect(params.get('password')).toBe('company-request-token') + expect(params.get('credtype')).toBe('authtoken') + }) + + it('lets an explicit credtype override the company-flow default', async () => { + await fetchSapConcurAccessToken( + auth({ + clientId: freshClientId(), + grantType: 'password', + password: 'company-request-token', + companyUuid: 'company-uuid-1', + credtype: 'password', + }), + 'req-1' + ) + + expect(submittedParams().get('credtype')).toBe('password') + }) + + it('prefers the companyUuid over a supplied username', async () => { + await fetchSapConcurAccessToken( + auth({ + clientId: freshClientId(), + grantType: 'password', + username: 'alice@example.com', + password: 'company-request-token', + companyUuid: 'company-uuid-2', + }), + 'req-1' + ) + + expect(submittedParams().get('username')).toBe('company-uuid-2') + }) + + it('leaves the user-level password grant untouched (no credtype, real username)', async () => { + await fetchSapConcurAccessToken( + auth({ + clientId: freshClientId(), + grantType: 'password', + username: 'alice@example.com', + password: PASSWORD, + }), + 'req-1' + ) + + const params = submittedParams() + expect(params.get('username')).toBe('alice@example.com') + expect(params.has('credtype')).toBe(false) + }) + + it('requires a username or a companyUuid for a password grant', async () => { + await expect( + fetchSapConcurAccessToken( + auth({ clientId: freshClientId(), grantType: 'password', password: PASSWORD }), + 'req-1' + ) + ).rejects.toThrow('username is required for password grant') + }) +}) + +describe('fetchSapConcurAccessToken secret handling', () => { + it('never puts the clientSecret or password into a token-fetch error message', async () => { + mockSecureFetch.mockResolvedValueOnce( + tokenResponse({ error: 'invalid_grant', error_description: 'Bad credentials' }, 401) + ) + + const promise = fetchSapConcurAccessToken( + auth({ + clientId: freshClientId(), + grantType: 'password', + username: 'alice@example.com', + password: PASSWORD, + }), + 'req-1' + ) + + await expect(promise).rejects.toThrow('Concur token request failed: invalid_grant') + const error = await promise.catch((e: Error) => e) + expect(error.message).not.toContain(CLIENT_SECRET) + expect(error.message).not.toContain(PASSWORD) + }) + + /** + * The token request body is form-encoded `client_id=…&client_secret=…&password=…`, so an + * intermediary that rejects the request and echoes it back would otherwise have its page + * surfaced verbatim. The raw fallback is capped on the token path for that reason. + */ + it('truncates an unstructured token-error body instead of echoing it back', async () => { + const echoedRequest = `Request blocked by proxy. Your request was: POST /oauth2/v0/token client_id=abc&client_secret=${CLIENT_SECRET}&grant_type=password&username=alice@example.com&password=${PASSWORD}&credtype=password. Contact your administrator with reference id 0000-1111-2222-3333 for further assistance with this policy decision.` + + mockSecureFetch.mockResolvedValueOnce({ + ok: false, + status: 403, + headers: new Headers(), + json: async () => ({}), + text: async () => echoedRequest, + }) + + const error = await fetchSapConcurAccessToken( + auth({ + clientId: freshClientId(), + grantType: 'password', + username: 'alice@example.com', + password: PASSWORD, + }), + 'req-1' + ).catch((e: Error) => e) + + expect(error.message).not.toContain(CLIENT_SECRET) + expect(error.message).not.toContain(PASSWORD) + expect(error.message.length).toBeLessThan(echoedRequest.length) + expect(error.message).toContain('Request blocked by proxy') + }) + + it('never leaks credentials when the outbound fetch itself throws', async () => { + mockSecureFetch.mockRejectedValueOnce(new Error('socket hang up')) + + const error = await fetchSapConcurAccessToken( + auth({ + clientId: freshClientId(), + grantType: 'password', + username: 'alice@example.com', + password: PASSWORD, + }), + 'req-1' + ).catch((e: Error) => e) + + expect(error.message).not.toContain(CLIENT_SECRET) + expect(error.message).not.toContain(PASSWORD) + }) +}) + +describe('sapConcurApiPathSchema', () => { + const accepted = [ + '/expensereports/v4/reports/abc123', + 'expensereports/v4/reports/abc123', + '/profile/v1/principals/1234-5678', + ] + + it.each(accepted)('accepts the ordinary path %s', (path) => { + expect(sapConcurApiPathSchema.safeParse(path).success).toBe(true) + }) + + const rejected = [ + '/expensereports/../../etc/passwd', + '/expensereports/./v4/reports', + '..', + '/expensereports\\..\\v4', + '/expensereports/v4#fragment', + '/expensereports/%2e%2e/v4', + '/expensereports/%2E%2E/v4', + '/expensereports%2fv4', + '/expensereports%5cv4', + '/expensereports%23v4', + ] + + it.each(rejected)('rejects the traversal-shaped path %s', (path) => { + expect(sapConcurApiPathSchema.safeParse(path).success).toBe(false) + }) + + /** + * KNOWN LIMITATION, asserted as current behavior on purpose: the refine only inspects + * one layer of percent-encoding, so a double-encoded `%252e%252e` passes. That is + * acceptable because the refine is defense-in-depth — the request host is still pinned + * by `assertSafeExternalUrl` against the validated Concur geolocation, so a decoded + * `..` can at worst walk within concursolutions.com and cannot reach another origin. + * Do not "fix" this here without re-checking the host pinning that backs it. + */ + it('does not reject double-encoded traversal (defense-in-depth, host is pinned elsewhere)', () => { + expect(sapConcurApiPathSchema.safeParse('/expensereports/%252e%252e/v4').success).toBe(true) + }) +}) + +describe('sapConcurApiInputSchema accept', () => { + function parse(overrides: Record) { + return sapConcurApiInputSchema.parse({ + clientId: 'client-1', + clientSecret: CLIENT_SECRET, + path: '/expensereports/v4/reports', + ...overrides, + }) + } + + it('leaves accept undefined so callConcur applies its application/json default', () => { + expect(parse({}).accept).toBeUndefined() + }) + + /** Itinerary and Travel Profile are XML-only and 406 an Accept they cannot satisfy. */ + it('carries an explicit XML accept through', () => { + expect(parse({ accept: 'application/xml' }).accept).toBe('application/xml') + }) +}) + +/** + * The executor retries 429/5xx for a block with a retry config and paces itself off + * `Retry-After`; dropping the header downgrades a precise wait to blind backoff. + */ +describe('forwardedSapConcurHeaders', () => { + it('forwards Retry-After, Location, and Link', () => { + expect( + forwardedSapConcurHeaders( + new Headers({ + 'Retry-After': '30', + Location: 'https://us.api.concursolutions.com/receipts/v4/receipts/abc', + Link: '; rel="next"', + }) + ) + ).toEqual({ + 'retry-after': '30', + location: 'https://us.api.concursolutions.com/receipts/v4/receipts/abc', + link: '; rel="next"', + }) + }) + + it('omits headers Concur did not send', () => { + expect(forwardedSapConcurHeaders(new Headers({ 'Retry-After': '5' }))).toEqual({ + 'retry-after': '5', + }) + }) + + it('forwards nothing when no interesting header is present', () => { + expect(forwardedSapConcurHeaders(new Headers({ 'Content-Type': 'application/json' }))).toEqual( + {} + ) + }) +}) + +describe('invokeSapConcurMultipart request cap', () => { + it('rejects the serialized multipart body before provider I/O when it exceeds the cap', async () => { + const formData = new FormData() + formData.append('file', new Blob(['receipt']), 'receipt.txt') + + await expect( + invokeSapConcurMultipart( + 'https://us.api.concursolutions.com/receipts/v4/upload', + 'access-token', + formData, + 1 + ) + ).rejects.toMatchObject({ + name: 'PayloadSizeLimitError', + label: 'Concur multipart request', + maxBytes: 1, + }) + expect(mockSecureFetch).not.toHaveBeenCalled() + }) +}) + +describe('sapConcurDatacenterSchema', () => { + /** Every host in the published Base URIs table, plus the legacy `eu`/`emea` aliases. */ + const accepted = [ + 'us.api.concursolutions.com', + 'www-us.api.concursolutions.com', + 'us2.api.concursolutions.com', + 'www-us2.api.concursolutions.com', + 'eu.api.concursolutions.com', + 'eu2.api.concursolutions.com', + 'www-eu2.api.concursolutions.com', + 'emea.api.concursolutions.com', + 'www-emea.api.concursolutions.com', + 'apj1.api.concursolutions.com', + 'www-apj1.api.concursolutions.com', + 'usg.api.concursolutions.com', + 'www-usg.api.concursolutions.com', + 'glz.api.concursolutions.com', + 'us-impl.api.concursolutions.com', + 'www-us-impl.api.concursolutions.com', + 'emea-impl.api.concursolutions.com', + 'www-emea-impl.api.concursolutions.com', + ] + + it.each(accepted)('accepts the documented datacenter %s', (datacenter) => { + expect(sapConcurDatacenterSchema.safeParse(datacenter).success).toBe(true) + }) + + it('covers exactly the documented set with no extras', () => { + expect([...SAP_CONCUR_ALLOWED_DATACENTERS].sort()).toEqual([...accepted].sort()) + }) + + /** GLZ is the one production row the Base URIs table publishes without a `www-` twin. */ + it('does not offer a www- twin for GLZ', () => { + expect(sapConcurDatacenterSchema.safeParse('www-glz.api.concursolutions.com').success).toBe( + false + ) + }) + + const rejected = [ + 'evil.com', + 'us.api.concursolutions.com.evil.com', + 'evil-us.api.concursolutions.com', + 'https://us.api.concursolutions.com', + ] + + it.each(rejected)('rejects the non-selectable datacenter %s', (datacenter) => { + expect(sapConcurDatacenterSchema.safeParse(datacenter).success).toBe(false) + }) +}) + +describe('assertSafeExternalUrl', () => { + it('accepts a normal Concur https URL', () => { + const url = assertSafeExternalUrl('https://us.api.concursolutions.com/expense/v4', 'apiUrl') + expect(url.hostname).toBe('us.api.concursolutions.com') + }) + + it('rejects a non-URL', () => { + expect(() => assertSafeExternalUrl('not a url', 'apiUrl')).toThrow('must be a valid URL') + }) + + it('rejects a non-https scheme', () => { + expect(() => assertSafeExternalUrl('http://us.api.concursolutions.com', 'apiUrl')).toThrow( + 'must use https://' + ) + }) + + const forbiddenHosts = [ + 'https://localhost/x', + 'https://0.0.0.0/x', + 'https://127.0.0.1/x', + 'https://169.254.169.254/latest/meta-data/', + 'https://metadata.google.internal/x', + 'https://[::1]/x', + ] + + it.each(forbiddenHosts)('rejects the metadata/loopback host %s', (url) => { + expect(() => assertSafeExternalUrl(url, 'apiUrl')).toThrow('is not allowed') + }) + + const privateIps = ['https://10.0.0.5/x', 'https://192.168.1.10/x', 'https://172.16.4.4/x'] + + it.each(privateIps)('rejects the private IP %s', (url) => { + expect(() => assertSafeExternalUrl(url, 'apiUrl')).toThrow('private/loopback range') + }) +}) + +describe('extractSapConcurError', () => { + it('combines the OAuth error and error_description', () => { + expect( + extractSapConcurError({ error: 'invalid_client', error_description: 'Bad client id' }, 401) + ).toBe('invalid_client: Bad client id') + }) + + it('includes the Expense v4 errorMessage with its validation details', () => { + const message = extractSapConcurError( + { + errorMessage: 'Report is not valid', + validationErrors: [{ message: 'purpose is required' }, { message: 'amount must be > 0' }], + }, + 400 + ) + expect(message).toContain('Report is not valid') + expect(message).toContain('purpose is required') + expect(message).toContain('amount must be > 0') + }) + + it('returns a bare Expense v4 errorMessage when there are no validation errors', () => { + expect(extractSapConcurError({ errorMessage: 'Report is not valid' }, 400)).toBe( + 'Report is not valid' + ) + }) + + it('prefixes the SCIM detail with the scimType', () => { + expect( + extractSapConcurError({ scimType: 'invalidValue', detail: 'userName already exists' }, 409) + ).toBe('[invalidValue] userName already exists') + }) + + it('returns a SCIM detail without a scimType', () => { + expect(extractSapConcurError({ detail: 'userName already exists' }, 409)).toBe( + 'userName already exists' + ) + }) + + it('reads the legacy nested Content.Error.Message envelope', () => { + expect( + extractSapConcurError({ Content: { Error: { Message: 'Invalid report key' } } }, 400) + ).toBe('Invalid report key') + }) + + it('reads the legacy top-level Error.Message envelope', () => { + expect(extractSapConcurError({ Error: { Message: 'Invalid itinerary' } }, 400)).toBe( + 'Invalid itinerary' + ) + }) + + it('includes the token-error code alongside the OAuth error', () => { + expect( + extractSapConcurError( + { code: 16, error: 'invalid_request', error_description: 'user lives elsewhere' }, + 400 + ) + ).toBe('[16] invalid_request: user lives elsewhere') + }) + + it('accepts a string-typed token-error code', () => { + expect(extractSapConcurError({ code: '53', error: 'invalid_grant' }, 400)).toBe( + '[53] invalid_grant' + ) + }) + + /** Budget v4 (Budget Category) failure response, verbatim from the API reference. */ + it('joins the Budget v4 errorMessageList with its types and codes', () => { + expect( + extractSapConcurError( + { + status: false, + errorMessageList: [ + { + errorType: 'ERROR', + errorCode: 'BUDGET.BUDGET_CATEGORY_NAME_REQUIRED', + errorMessage: 'Budget category name is required', + }, + { + errorType: 'ERROR', + errorCode: 'BUDGET.BUDGET_CATEGORY_NAME_UNIQUE_ERROR', + errorMessage: 'Budget category must have a unique name', + }, + ], + }, + 400 + ) + ).toBe( + '[ERROR BUDGET.BUDGET_CATEGORY_NAME_REQUIRED] Budget category name is required; ' + + '[ERROR BUDGET.BUDGET_CATEGORY_NAME_UNIQUE_ERROR] Budget category must have a unique name' + ) + }) + + /** Budget Adjustments v4 nests the same object one level down under `message`. */ + it('unwraps an object-valued message to reach a nested errorMessageList', () => { + expect( + extractSapConcurError( + { + message: { + status: false, + errorMessageList: [ + { + errorType: 'ERROR', + errorCode: 'BUDGET.BUDGET_PERIOD_REQUIRED', + errorMessage: 'Record 1) Budget period is missing', + }, + ], + }, + }, + 400 + ) + ).toBe('[ERROR BUDGET.BUDGET_PERIOD_REQUIRED] Record 1) Budget period is missing') + }) + + it('still prefers a string-valued message over the legacy envelope', () => { + expect(extractSapConcurError({ message: 'Report not found' }, 404)).toBe('Report not found') + }) + + it('falls back to the Concur SCIM messages extension when detail is absent', () => { + expect( + extractSapConcurError( + { + schemas: ['urn:ietf:params:scim:api:messages:2.0:Error'], + status: '400', + 'urn:ietf:params:scim:api:messages:concur:2.0:Error': { + messages: [ + { + code: 'ATTRIBUTE_REQUIRED', + message: 'userName is required', + schemaPath: 'userName', + type: 'error', + }, + ], + }, + }, + 400 + ) + ).toBe('[ATTRIBUTE_REQUIRED] userName is required (userName)') + }) + + it('prefers the SCIM detail over the messages extension when both are present', () => { + expect( + extractSapConcurError( + { + detail: 'userName already exists', + 'urn:ietf:params:scim:api:messages:concur:2.0:Error': { + messages: [{ code: 'DUP', message: 'duplicate', type: 'error' }], + }, + }, + 409 + ) + ).toBe('userName already exists') + }) + + it('joins an errors list with its error codes', () => { + expect( + extractSapConcurError( + { + errors: [ + { errorCode: 'E1', errorMessage: 'first problem' }, + { errorCode: 'E2', errorMessage: 'second problem' }, + ], + }, + 400 + ) + ).toBe('[E1] first problem; [E2] second problem') + }) + + it('passes a raw string body through when no cap is set', () => { + expect(extractSapConcurError('Service Unavailable', 503)).toBe('Service Unavailable') + }) + + it('caps a raw string body when maxRawBodyLength is set', () => { + expect(extractSapConcurError('x'.repeat(500), 503, { maxRawBodyLength: 20 })).toBe( + `${'x'.repeat(20)}...` + ) + }) + + it('leaves a structured body uncapped even when maxRawBodyLength is set', () => { + expect(extractSapConcurError({ error: 'invalid_client' }, 401, { maxRawBodyLength: 5 })).toBe( + 'invalid_client' + ) + }) + + it('falls back to the generic HTTP message for an unrecognized shape', () => { + expect(extractSapConcurError({ unexpected: true }, 418)).toBe( + 'Concur request failed with HTTP 418' + ) + }) + + it('falls back to the generic HTTP message for an empty body', () => { + expect(extractSapConcurError('', 500)).toBe('Concur request failed with HTTP 500') + }) +}) + +afterEach(() => { + vi.useRealTimers() +}) diff --git a/apps/sim/lib/internal/sap-concur/client.ts b/apps/sim/lib/internal/sap-concur/client.ts new file mode 100644 index 00000000000..31c436e85fc --- /dev/null +++ b/apps/sim/lib/internal/sap-concur/client.ts @@ -0,0 +1,725 @@ +import { createHmac } from 'node:crypto' +import { createLogger } from '@sim/logger' +import { isPrivateIpHost } from '@sim/security/ssrf' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { truncate } from '@sim/utils/string' +import { env } from '@/lib/core/config/env' +import { + MAX_JSON_API_RESPONSE_BYTES, + secureFetchWithValidation, +} from '@/lib/core/security/input-validation.server' +import { PayloadSizeLimitError, readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' +import type { SapConcurApiInput, SapConcurAuth } from '@/lib/internal/sap-concur/schema' + +const logger = createLogger('SapConcurClient') + +/** Documented host form for a Concur geolocation, including `www-` prefixed variants. */ +const SAP_CONCUR_GEOLOCATION_HOST_PATTERN = /^(www-)?[a-z0-9-]+\.api\.concursolutions\.com$/ + +const FORBIDDEN_HOSTS = new Set([ + 'localhost', + '0.0.0.0', + '127.0.0.1', + '169.254.169.254', + 'metadata.google.internal', + 'metadata', + '[::1]', + '[::]', + '[::ffff:127.0.0.1]', + '[fd00:ec2::254]', +]) + +/** Validate a URL is https and not pointing to a private/loopback host. */ +export function assertSafeExternalUrl(rawUrl: string, label: string): URL { + let parsed: URL + try { + parsed = new URL(rawUrl) + } catch { + throw new Error(`${label} must be a valid URL`) + } + if (parsed.protocol !== 'https:') { + throw new Error(`${label} must use https://`) + } + const host = parsed.hostname.toLowerCase() + if (FORBIDDEN_HOSTS.has(host) || FORBIDDEN_HOSTS.has(`[${host}]`)) { + throw new Error(`${label} host is not allowed`) + } + if (isPrivateIpHost(host)) { + throw new Error(`${label} host is not allowed (private/loopback range)`) + } + return parsed +} + +interface CachedToken { + accessToken: string + geolocation: string + expiresAt: number +} + +/** Access token plus the geolocation every subsequent API call for it must be sent to. */ +export interface SapConcurToken { + accessToken: string + geolocation: string +} + +const TOKEN_CACHE = new Map() +const TOKEN_CACHE_MAX_ENTRIES = 500 +const TOKEN_SAFETY_WINDOW_MS = 60_000 +export const SAP_CONCUR_OUTBOUND_FETCH_TIMEOUT_MS = 30_000 + +interface InFlightTokenRequest { + controller: AbortController + promise: Promise + state: { + settled: boolean + waiters: number + } +} + +const TOKEN_REQUESTS = new Map() +const TOKEN_REQUESTS_MAX_ENTRIES = 500 + +/** Cached token for `key`, or `undefined` when absent or inside the expiry safety window. */ +function readCachedToken(key: string): SapConcurToken | undefined { + const cached = TOKEN_CACHE.get(key) + if (!cached || cached.expiresAt - TOKEN_SAFETY_WINDOW_MS <= Date.now()) return undefined + return { accessToken: cached.accessToken, geolocation: cached.geolocation } +} + +/** + * Cache key covering every factor that authenticates the token request. The password + * and company UUID must participate: without them a cache hit skips the token endpoint + * entirely, so a request carrying the wrong password would be served a token minted from + * someone else's correct credentials out of this module-global cache. + * + * The whole tuple is JSON-encoded before hashing rather than concatenated with a + * separator, so a free-form field (clientId, companyUuid) cannot span a field boundary + * and collide with a different tuple. + * + * Keyed with a server-side secret rather than a bare digest. The inputs include a + * user-chosen password, which is low-entropy enough to brute-force from a plain SHA-256 + * if a key ever reached a heap dump or a debug log; an HMAC makes the key useless without + * the secret. A password-hashing KDF would be the wrong tool — this runs on every token + * fetch and the goal is collision-free partitioning, not verification of a stored + * credential. + */ +function tokenCacheKey(req: SapConcurAuth): string { + const payload = JSON.stringify([ + req.datacenter, + req.grantType, + req.clientId, + req.clientSecret, + req.username ?? '', + req.password ?? '', + req.companyUuid ?? '', + req.credtype ?? '', + ]) + const hmac = createHmac('sha256', env.INTERNAL_API_SECRET) + hmac.update(payload, 'utf8') + return hmac.digest('hex') +} + +/** + * Insert a token and evict from the front once the cache is over its cap. + * + * Eviction is FIFO by insertion order, not LRU — a cache *read* does not move an entry + * back. At 500 entries that is deliberate: a token is short-lived and re-minted on the + * next miss, so the extra bookkeeping an LRU needs buys nothing here. + */ +function rememberToken(key: string, token: CachedToken): void { + if (TOKEN_CACHE.has(key)) TOKEN_CACHE.delete(key) + TOKEN_CACHE.set(key, token) + while (TOKEN_CACHE.size > TOKEN_CACHE_MAX_ENTRIES) { + const oldestKey = TOKEN_CACHE.keys().next().value + if (oldestKey === undefined) break + TOKEN_CACHE.delete(oldestKey) + } +} + +function normalizeGeolocation(raw: string | undefined, fallback: string): string { + if (!raw) return `https://${fallback}` + const trimmed = raw.replace(/\/+$/, '') + if (trimmed.startsWith('http://') || trimmed.startsWith('https://')) return trimmed + return `https://${trimmed}` +} + +/** + * Cap for an unstructured token-endpoint error body, applied both to the log line and to + * the surfaced message. The token request body is form-encoded + * `client_id=…&client_secret=…&password=…`, so an intermediary (WAF, proxy, captive + * portal) that echoes the request it rejected would otherwise have its page returned to + * the caller verbatim. Structured Concur error shapes are unaffected — they are matched + * before the raw fallback is reached. + */ +const TOKEN_ERROR_RAW_BODY_MAX_LENGTH = 200 + +/** + * Blank out the credential values this module just submitted, wherever they appear in an + * error body. + * + * Truncation alone bounds the exposure but does not remove it — a secret can sit inside + * the surviving prefix. Because the exact values are known at the callsite, they can be + * substituted out precisely. A genuine Concur error body never contains them, so this is + * a no-op for every documented shape and only bites on an intermediary echoing our + * request back at us. + */ +function redactTokenSecrets(text: string, auth: SapConcurAuth): string { + if (!text) return text + let redacted = text + for (const secret of [auth.clientSecret, auth.password]) { + if (secret && secret.length > 0) redacted = redacted.split(secret).join('[redacted]') + } + return redacted +} + +/** Best-effort JSON parse of an error body, falling back to the raw text. */ +function parseMaybeJson(text: string): unknown { + if (!text) return '' + try { + return JSON.parse(text) + } catch { + return text + } +} + +/** + * Acquire a Concur access token, sharing a cache across direct tool operations. + * Validates that the geolocation returned by Concur is a safe external URL. + * + * Misses are coalesced per cache key: a parallel block fanning out many Concur calls, or + * a cold container after a deploy, would otherwise fire one `POST /oauth2/v0/token` per + * branch into an endpoint Concur rate-limits hard. Coalescing also removes an + * interleaving hazard — with concurrent mints, a slow response settling last could cache + * an earlier-expiring token over a fresher one. + * + * Two password-grant shapes are supported: + * + * - User-level: `username` is the user's login and `password` their password. `credtype` + * is omitted, which Concur reads as its `password` default. + * - Company-level: when `companyUuid` is set, Concur's documented company flow is + * `grant_type=password&username=&password=&credtype=authtoken`. + * The company UUID is submitted as `username` and `credtype` defaults to `authtoken`. + * An explicitly supplied `credtype` always wins. + * + * KNOWN LIMITATION of the company flow: the company request token obtained from the App + * Center is valid for 24 hours only, and Concur returns a `refresh_token` alongside the + * access token so the connection can outlive it. Refresh-token exchange is not + * implemented here, so a company connection stops working once the request token expires + * and a fresh one must be issued. + * + * Token-endpoint failures carry `{ code, error, error_description, geolocation? }`. + * Code 16 ("user lives elsewhere") additionally returns the correct geolocation for the + * tenant; retrying the token request against that host is not implemented here. + */ +export async function fetchSapConcurAccessToken( + auth: SapConcurAuth, + requestId: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + if (auth.grantType === 'password') { + if (!auth.username && !auth.companyUuid) { + throw new Error( + 'username is required for password grant (or companyUuid for company-level auth)' + ) + } + if (!auth.password) throw new Error('password is required for password grant') + } + + const cacheKey = tokenCacheKey(auth) + const cached = readCachedToken(cacheKey) + if (cached) return cached + + const existing = TOKEN_REQUESTS.get(cacheKey) + if (!existing && TOKEN_REQUESTS.size >= TOKEN_REQUESTS_MAX_ENTRIES) { + return requestAccessToken(auth, requestId, cacheKey, signal) + } + const request = existing ?? createTokenRequest(auth, requestId, cacheKey) + return waitForTokenRequest(request, signal) +} + +function createTokenRequest( + auth: SapConcurAuth, + requestId: string, + cacheKey: string +): InFlightTokenRequest { + const controller = new AbortController() + const state = { settled: false, waiters: 0 } + const promise = requestAccessToken(auth, requestId, cacheKey, controller.signal).finally(() => { + state.settled = true + if (TOKEN_REQUESTS.get(cacheKey)?.controller === controller) TOKEN_REQUESTS.delete(cacheKey) + }) + const request = { controller, promise, state } + TOKEN_REQUESTS.set(cacheKey, request) + return request +} + +async function waitForTokenRequest( + request: InFlightTokenRequest, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + request.state.waiters += 1 + let onAbort: (() => void) | undefined + const aborted = signal + ? new Promise((_resolve, reject) => { + onAbort = () => reject(signal.reason ?? new DOMException('Aborted', 'AbortError')) + signal.addEventListener('abort', onAbort, { once: true }) + }) + : undefined + + try { + return await (aborted ? Promise.race([request.promise, aborted]) : request.promise) + } finally { + if (onAbort) signal?.removeEventListener('abort', onAbort) + request.state.waiters -= 1 + if (!request.state.settled && request.state.waiters === 0) { + request.controller.abort(signal?.reason) + } + } +} + +/** Mint a fresh token from the Concur token endpoint and cache it under `cacheKey`. */ +async function requestAccessToken( + auth: SapConcurAuth, + requestId: string, + cacheKey: string, + signal?: AbortSignal +): Promise { + const tokenUrl = assertSafeExternalUrl( + `https://${auth.datacenter}/oauth2/v0/token`, + 'tokenUrl' + ).toString() + + const params = new URLSearchParams() + params.set('client_id', auth.clientId) + params.set('client_secret', auth.clientSecret) + params.set('grant_type', auth.grantType) + if (auth.grantType === 'password') { + const companyUuid = auth.companyUuid + params.set('username', companyUuid ?? auth.username ?? '') + params.set('password', auth.password ?? '') + const credtype = auth.credtype ?? (companyUuid ? 'authtoken' : undefined) + if (credtype) params.set('credtype', credtype) + } + + const response = await secureFetchWithValidation( + tokenUrl, + { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + Accept: 'application/json', + }, + body: params.toString(), + timeout: SAP_CONCUR_OUTBOUND_FETCH_TIMEOUT_MS, + maxRedirects: 0, + stripAuthOnRedirect: true, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + signal, + }, + 'tokenUrl' + ) + signal?.throwIfAborted() + + if (!response.ok) { + const text = redactTokenSecrets(await response.text().catch(() => ''), auth) + signal?.throwIfAborted() + logger.warn( + `[${requestId}] Concur token fetch failed (${response.status}): ${truncate( + text, + TOKEN_ERROR_RAW_BODY_MAX_LENGTH + )}` + ) + throw new Error( + `Concur token request failed: ${extractSapConcurError(parseMaybeJson(text), response.status, { + maxRawBodyLength: TOKEN_ERROR_RAW_BODY_MAX_LENGTH, + })}` + ) + } + + const data = (await response.json()) as { + access_token?: string + expires_in?: number + geolocation?: string + } + signal?.throwIfAborted() + + if (!data.access_token) { + throw new Error('Concur token response missing access_token') + } + + const geolocation = normalizeGeolocation(data.geolocation, auth.datacenter) + const geolocationUrl = assertSafeExternalUrl(geolocation, 'geolocation') + if (!SAP_CONCUR_GEOLOCATION_HOST_PATTERN.test(geolocationUrl.hostname.toLowerCase())) { + throw new Error( + `Concur geolocation host is not a valid Concur API host: ${geolocationUrl.hostname}` + ) + } + + const expiresInMs = (data.expires_in ?? 3600) * 1000 + rememberToken(cacheKey, { + accessToken: data.access_token, + geolocation, + expiresAt: Date.now() + expiresInMs, + }) + return { accessToken: data.access_token, geolocation } +} + +export interface SapConcurInvocation { + status: number + body: unknown + headers: Record +} + +function buildApiUrl(geolocation: string, input: SapConcurApiInput): string { + const base = geolocation.replace(/\/+$/, '') + const subPath = input.path.startsWith('/') ? input.path : `/${input.path}` + const url = `${base}${subPath}` + if (!input.query || Object.keys(input.query).length === 0) return url + + const search = new URLSearchParams() + for (const [key, value] of Object.entries(input.query)) { + if (value === undefined || value === null) continue + search.append(key, String(value)) + } + const queryString = search.toString() + if (!queryString) return url + return url.includes('?') ? `${url}&${queryString}` : `${url}?${queryString}` +} + +export async function readConcurApiBody(response: { + status: number + text: () => Promise +}): Promise { + const read = response.text() + if (response.status >= 200 && response.status < 300) return read + return read.catch(() => '') +} + +function parseResponseBody(raw: string): unknown { + if (raw.length === 0) return null + try { + return JSON.parse(raw) + } catch { + return raw + } +} + +export async function invokeSapConcur( + input: SapConcurApiInput, + accessToken: string, + geolocation: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const url = assertSafeExternalUrl(buildApiUrl(geolocation, input), 'apiUrl').toString() + const hasBody = input.body !== undefined && input.body !== null + const headers: Record = { + Authorization: `Bearer ${accessToken}`, + Accept: input.accept ?? 'application/json', + 'concur-correlationid': generateId(), + } + if (hasBody) headers['Content-Type'] = input.contentType ?? 'application/json' + + const response = await secureFetchWithValidation( + url, + { + method: input.method, + headers, + body: hasBody + ? typeof input.body === 'string' + ? input.body + : JSON.stringify(input.body) + : undefined, + timeout: SAP_CONCUR_OUTBOUND_FETCH_TIMEOUT_MS, + maxRedirects: 0, + stripAuthOnRedirect: true, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + signal, + }, + 'apiUrl' + ) + signal?.throwIfAborted() + const raw = await readConcurApiBody(response) + signal?.throwIfAborted() + return { + status: response.status, + body: parseResponseBody(raw), + headers: forwardedSapConcurHeaders(response.headers), + } +} + +export async function readConcurUploadBody(response: { + status: number + headers?: { get(name: string): string | null } + body?: ReadableStream | null + arrayBuffer?: () => Promise + text?: () => Promise +}): Promise { + const read = readResponseTextWithLimit(response, { + maxBytes: MAX_JSON_API_RESPONSE_BYTES, + label: 'Concur upload response', + }) + if (response.status >= 200 && response.status < 300) return read + return read.catch(() => '') +} + +export async function invokeSapConcurMultipart( + url: string, + accessToken: string, + formData: FormData, + maxBodyBytes: number, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const headers: Record = { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + 'concur-correlationid': generateId(), + } + const serialized = new Request('http://localhost/internal-multipart-serializer', { + method: 'POST', + body: formData, + }) + const contentType = serialized.headers.get('content-type') + if (contentType) headers['Content-Type'] = contentType + const bodyBuffer = Buffer.from(await serialized.arrayBuffer()) + signal?.throwIfAborted() + if (bodyBuffer.length > maxBodyBytes) { + throw new PayloadSizeLimitError({ + label: 'Concur multipart request', + maxBytes: maxBodyBytes, + observedBytes: bodyBuffer.length, + }) + } + + const response = await secureFetchWithValidation( + url, + { + method: 'POST', + headers, + body: bodyBuffer, + timeout: SAP_CONCUR_OUTBOUND_FETCH_TIMEOUT_MS, + maxRedirects: 0, + stripAuthOnRedirect: true, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + signal, + }, + 'apiUrl' + ) + signal?.throwIfAborted() + const raw = await readConcurUploadBody(response) + signal?.throwIfAborted() + let body = parseResponseBody(raw) + if ( + body === null || + (typeof body === 'object' && body !== null && Object.keys(body).length === 0) + ) { + const location = response.headers.get('Location') + const link = response.headers.get('Link') + if (location || link) body = { location, link } + } + return { + status: response.status, + body, + headers: forwardedSapConcurHeaders(response.headers), + } +} + +/** + * Concur response headers carried through onto the direct operation response. + * + * `Retry-After` is the load-bearing one: the executor retries 429/5xx for a block with a + * retry config and paces itself off this header, so dropping it downgrades a precise wait + * into blind exponential backoff against an endpoint that just told us how long to wait. + * `Location` and `Link` identify the resource created by, or the next page of, an + * accepted request. + */ +const FORWARDED_CONCUR_HEADERS = ['retry-after', 'location', 'link'] as const + +/** + * Pick the {@link FORWARDED_CONCUR_HEADERS} present on a Concur response. + * + * Typed structurally rather than as `Headers` so it accepts both a DOM `Headers` and the + * `SecureFetchHeaders` returned by `secureFetchWithValidation`, which exposes only `get`. + */ +export function forwardedSapConcurHeaders(source: { + get(name: string): string | null +}): Record { + const forwarded: Record = {} + for (const name of FORWARDED_CONCUR_HEADERS) { + const value = source.get(name) + if (value) forwarded[name] = value + } + return forwarded +} + +/** + * Turn an outbound-fetch rejection into a message a caller can act on. + * + * `secureFetchWithValidation` runs with `maxRedirects: 0`, so any Concur response that is + * a redirect *with* a `Location` header rejects with `Too many redirects (max: 0)` rather + * than returning a status. That is a deliberate refusal (the bearer token must never be + * replayed to another origin), but the bare message reads like an internal fault, so it + * is restated in terms of what actually happened. + */ +export function describeSapConcurFetchError(error: unknown): string { + const message = getErrorMessage(error, 'Unknown error') + if (message.startsWith('Too many redirects')) { + return 'Concur returned a redirect, which is not followed because the access token must not be replayed to another origin. Check the datacenter/geolocation and the request path.' + } + return message +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function nonEmptyString(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined +} + +/** + * Message from the legacy nested envelope used by Expense v3 and Travel: + * `{ Content: { Error: { Message } } }` or `{ Error: { Message } }`. + */ +function legacyEnvelopeMessage(obj: Record): string | undefined { + const container = isRecord(obj.Content) ? obj.Content : obj + const error = isRecord(container.Error) ? container.Error : undefined + return error ? nonEmptyString(error.Message) : undefined +} + +/** URN of the Concur SCIM error extension carrying per-attribute `messages[]`. */ +const SCIM_CONCUR_ERROR_URN = 'urn:ietf:params:scim:api:messages:concur:2.0:Error' + +/** + * Render one Budget v4 `errorMessageList` entry (`{ errorType, errorCode, errorMessage }`). + * `errorType` is kept because it distinguishes a hard `ERROR` from a `WARNING`. + */ +function formatErrorMessageListEntry(entry: unknown): string { + if (!isRecord(entry)) return String(entry) + const label = [nonEmptyString(entry.errorType), nonEmptyString(entry.errorCode)] + .filter(Boolean) + .join(' ') + const message = nonEmptyString(entry.errorMessage) ?? '' + return label ? `[${label}] ${message}`.trim() : message +} + +/** Render one Concur SCIM extension message (`{ code, message, schemaPath, type }`). */ +function formatScimExtensionMessage(entry: unknown): string { + if (!isRecord(entry)) return String(entry) + const code = nonEmptyString(entry.code) + const message = nonEmptyString(entry.message) ?? '' + const schemaPath = nonEmptyString(entry.schemaPath) + const head = code ? `[${code}] ` : '' + const tail = schemaPath ? ` (${schemaPath})` : '' + return `${head}${message}${tail}`.trim() +} + +function joinNonEmpty(values: unknown[], format: (value: unknown) => string): string | undefined { + const joined = values.map(format).filter(Boolean).join('; ') + return joined.length > 0 ? joined : undefined +} + +/** + * Match a Concur error record against the documented shapes, returning `undefined` when + * none apply so the caller can fall through to its own default. + * + * `depth` bounds the single documented level of nesting: Budget Adjustments v4 wraps the + * same `{ status, errorMessageList }` object under a `message` key, so an object-valued + * `message` is unwrapped once before the string-valued `message` shape is considered. + */ +function extractFromRecord(obj: Record, depth: number): string | undefined { + if (depth === 0 && isRecord(obj.message)) { + const nested = extractFromRecord(obj.message, depth + 1) + if (nested) return nested + } + + const error = nonEmptyString(obj.error) + if (error) { + const description = nonEmptyString(obj.error_description) + const code = obj.code + const codePrefix = typeof code === 'string' || typeof code === 'number' ? `[${code}] ` : '' + return `${codePrefix}${error}${description ? `: ${description}` : ''}` + } + + const errorMessage = nonEmptyString(obj.errorMessage) + if (errorMessage) { + const validationErrors = Array.isArray(obj.validationErrors) + ? obj.validationErrors + .map((v) => (isRecord(v) ? nonEmptyString(v.message) : undefined)) + .filter((m): m is string => Boolean(m)) + : [] + return validationErrors.length > 0 + ? `${errorMessage}: ${validationErrors.join('; ')}` + : errorMessage + } + + if (Array.isArray(obj.errorMessageList) && obj.errorMessageList.length > 0) { + const joined = joinNonEmpty(obj.errorMessageList, formatErrorMessageListEntry) + if (joined) return joined + } + + const detail = nonEmptyString(obj.detail) + if (detail) { + const scimType = nonEmptyString(obj.scimType) + return scimType ? `[${scimType}] ${detail}` : detail + } + + const scimExtension = obj[SCIM_CONCUR_ERROR_URN] + if (isRecord(scimExtension) && Array.isArray(scimExtension.messages)) { + const joined = joinNonEmpty(scimExtension.messages, formatScimExtensionMessage) + if (joined) return joined + } + + const message = nonEmptyString(obj.message) + if (message) return message + + const legacy = legacyEnvelopeMessage(obj) + if (legacy) return legacy + + if (Array.isArray(obj.errors) && obj.errors.length > 0) { + return joinNonEmpty(obj.errors, (e) => { + if (!isRecord(e)) return String(e) + const code = nonEmptyString(e.errorCode) + const msg = nonEmptyString(e.errorMessage) ?? '' + return `${code ? `[${code}] ` : ''}${msg}`.trim() + }) + } + + return undefined +} + +interface ExtractSapConcurErrorOptions { + /** + * Cap applied to an unstructured string body before it is surfaced. Set on the token + * path, where the request body carries credentials an intermediary might echo back. + * Left unset elsewhere so ordinary API errors surface in full. + */ + maxRawBodyLength?: number +} + +/** + * Extract a meaningful error message from a Concur error response body, covering the + * OAuth `{ code, error, error_description }` shape, the Expense v4 `ErrorMessage` schema, + * the Budget v4 `errorMessageList` shape (including the Budget Adjustments v4 variant + * that nests it under `message`), the SCIM (Identity v4.1) `detail` shape and its Concur + * `messages[]` extension, the legacy nested `Content.Error.Message` envelope, and an + * undocumented `{ errors: [...] }` list kept for tolerance. + */ +export function extractSapConcurError( + body: unknown, + status: number, + options: ExtractSapConcurErrorOptions = {} +): string { + if (isRecord(body)) { + const message = extractFromRecord(body, 0) + if (message) return message + } + if (typeof body === 'string' && body.length > 0) { + return options.maxRawBodyLength === undefined ? body : truncate(body, options.maxRawBodyLength) + } + return `Concur request failed with HTTP ${status}` +} diff --git a/apps/sim/lib/internal/sap-concur/execute-tool.test.ts b/apps/sim/lib/internal/sap-concur/execute-tool.test.ts new file mode 100644 index 00000000000..8851bc32d6a --- /dev/null +++ b/apps/sim/lib/internal/sap-concur/execute-tool.test.ts @@ -0,0 +1,231 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockExecuteApi, mockExecuteUpload } = vi.hoisted(() => ({ + mockExecuteApi: vi.fn(), + mockExecuteUpload: vi.fn(), +})) + +vi.mock('@/lib/internal/sap-concur/operations', () => { + class SapConcurOperationError extends Error { + constructor( + readonly status: number, + readonly body: { success: false; error: string; status?: number }, + readonly headers: HeadersInit = {} + ) { + super(body.error) + this.name = 'SapConcurOperationError' + } + } + return { + executeSapConcurApiOperation: mockExecuteApi, + executeSapConcurUploadOperation: mockExecuteUpload, + SapConcurOperationError, + } +}) + +vi.mock('@/lib/api/server/validation', () => ({ + DEFAULT_MAX_JSON_BODY_BYTES: 1024, +})) + +import { executeSapConcurTool, SAP_CONCUR_TOOL_IDS } from '@/lib/internal/sap-concur/execute-tool' +import { SapConcurOperationError } from '@/lib/internal/sap-concur/operations' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' +import * as sapConcurTools from '@/tools/sap_concur' + +function call(overrides: Partial = {}): InternalToolOperationCall { + return { + toolId: 'sap_concur_get_budget', + input: { + clientId: 'client-id', + clientSecret: 'client-secret', + path: '/budget/v4/budgets/budget-1', + method: 'GET', + }, + headers: new Headers(), + context: { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +beforeEach(() => { + vi.clearAllMocks() + mockExecuteApi.mockResolvedValue({ + body: { success: true, output: { status: 200, data: { id: 'budget-1' } } }, + headers: { 'retry-after': '5' }, + }) + mockExecuteUpload.mockResolvedValue({ + body: { success: true, output: { status: 201, data: { id: 'receipt-1' } } }, + headers: { location: 'https://us.api.concursolutions.com/receipts/receipt-1' }, + }) +}) + +describe('executeSapConcurTool', () => { + it('publishes exactly the 70 canonical SAP Concur IDs', () => { + const declaredIds = Object.values(sapConcurTools).map((tool) => tool.id) + expect(SAP_CONCUR_TOOL_IDS).toHaveLength(70) + expect(new Set(SAP_CONCUR_TOOL_IDS).size).toBe(70) + expect([...SAP_CONCUR_TOOL_IDS].sort()).toEqual(declaredIds.sort()) + }) + + it('validates and dispatches API operations with the trusted cancellation signal', async () => { + const controller = new AbortController() + const response = await executeSapConcurTool(call({ signal: controller.signal })) + + expect(response.status).toBe(200) + expect(response.headers.get('retry-after')).toBe('5') + expect(await response.json()).toEqual({ + success: true, + output: { status: 200, data: { id: 'budget-1' } }, + }) + expect(mockExecuteApi).toHaveBeenCalledWith( + expect.objectContaining({ + clientId: 'client-id', + datacenter: 'us.api.concursolutions.com', + grantType: 'client_credentials', + path: '/budget/v4/budgets/budget-1', + }), + { + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + } + ) + }) + + it('requires trusted user identity before dispatching a protected upload', async () => { + const response = await executeSapConcurTool( + call({ + toolId: 'sap_concur_upload_receipt_image', + context: { workflowId: 'workflow-1', workspaceId: 'workspace-1' }, + input: { + clientId: 'client-id', + clientSecret: 'client-secret', + operation: 'upload_receipt_image', + userId: 'concur-user-1', + receipt: { key: 'workspace-file-key', name: 'receipt.pdf', size: 10 }, + }, + }) + ) + + expect(response.status).toBe(401) + expect(await response.json()).toEqual({ + success: false, + error: 'Authentication required', + }) + expect(mockExecuteUpload).not.toHaveBeenCalled() + }) + + it('dispatches protected uploads without losing their provider response headers', async () => { + const response = await executeSapConcurTool( + call({ + toolId: 'sap_concur_upload_receipt_image', + input: { + clientId: 'client-id', + clientSecret: 'client-secret', + operation: 'upload_receipt_image', + userId: 'concur-user-1', + receipt: { key: 'workspace-file-key', name: 'receipt.pdf', size: 10 }, + }, + }) + ) + + expect(response.status).toBe(200) + expect(response.headers.get('location')).toContain('/receipts/receipt-1') + expect(mockExecuteUpload).toHaveBeenCalledOnce() + }) + + it('preserves provider status, body, and retry headers', async () => { + mockExecuteApi.mockRejectedValueOnce( + new SapConcurOperationError( + 429, + { success: false, error: 'Rate limited', status: 429 }, + { 'retry-after': '30' } + ) + ) + + const response = await executeSapConcurTool(call()) + + expect(response.status).toBe(429) + expect(response.headers.get('retry-after')).toBe('30') + expect(await response.json()).toEqual({ + success: false, + error: 'Rate limited', + status: 429, + }) + }) + + it('rejects invalid operation input before provider work', async () => { + const response = await executeSapConcurTool(call({ input: { clientId: 'client-id' } })) + + expect(response.status).toBe(400) + expect(await response.json()).toEqual({ + success: false, + error: 'Invalid input: expected string, received undefined', + }) + expect(mockExecuteApi).not.toHaveBeenCalled() + }) + + it('rejects oversized operation input before schema traversal or provider work', async () => { + const response = await executeSapConcurTool( + call({ + input: { + clientId: 'client-id', + clientSecret: 'client-secret', + path: '/budget/v4/budgets/budget-1', + body: 'x'.repeat(1024), + }, + }) + ) + + expect(response.status).toBe(413) + expect(await response.json()).toEqual({ + success: false, + error: 'Request body exceeds the maximum allowed size of 1024 bytes', + }) + expect(mockExecuteApi).not.toHaveBeenCalled() + }) + + it('rejects IDs outside the canonical family', async () => { + const response = await executeSapConcurTool(call({ toolId: 'sap_concur_not_real' })) + + expect(response.status).toBe(500) + expect(await response.json()).toEqual({ + success: false, + error: 'Unsupported SAP Concur tool: sap_concur_not_real', + }) + expect(mockExecuteApi).not.toHaveBeenCalled() + }) + + it('aborts before validation or provider dispatch', async () => { + const controller = new AbortController() + controller.abort(new DOMException('Cancelled', 'AbortError')) + + await expect(executeSapConcurTool(call({ signal: controller.signal }))).rejects.toMatchObject({ + name: 'AbortError', + }) + expect(mockExecuteApi).not.toHaveBeenCalled() + }) + + it('does not convert an abort after provider dispatch into a tool error', async () => { + const controller = new AbortController() + mockExecuteApi.mockImplementationOnce(async () => { + controller.abort(new DOMException('Cancelled', 'AbortError')) + return { + body: { success: true, output: { status: 200, data: null } }, + headers: {}, + } + }) + + await expect(executeSapConcurTool(call({ signal: controller.signal }))).rejects.toMatchObject({ + name: 'AbortError', + }) + }) +}) diff --git a/apps/sim/lib/internal/sap-concur/execute-tool.ts b/apps/sim/lib/internal/sap-concur/execute-tool.ts new file mode 100644 index 00000000000..5708d1aa404 --- /dev/null +++ b/apps/sim/lib/internal/sap-concur/execute-tool.ts @@ -0,0 +1,191 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { z } from 'zod' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { describeSapConcurFetchError } from '@/lib/internal/sap-concur/client' +import { + executeSapConcurApiOperation, + executeSapConcurUploadOperation, + type SapConcurOperationContext, + SapConcurOperationError, + type SapConcurOperationResult, +} from '@/lib/internal/sap-concur/operations' +import { + sapConcurApiInputSchema, + sapConcurUploadInputSchema, +} from '@/lib/internal/sap-concur/schema' +import type { + InternalToolOperationCall, + InternalToolOperationHandler, +} from '@/lib/internal/tool-operations/types' + +const logger = createLogger('SapConcurToolExecution') + +export const SAP_CONCUR_TOOL_IDS = [ + 'sap_concur_approve_expense_report', + 'sap_concur_associate_attendees', + 'sap_concur_create_cash_advance', + 'sap_concur_create_expected_expense', + 'sap_concur_create_expense_report', + 'sap_concur_create_list_item', + 'sap_concur_create_purchase_request', + 'sap_concur_create_quick_expense', + 'sap_concur_create_quick_expense_with_image', + 'sap_concur_create_report_comment', + 'sap_concur_create_travel_request', + 'sap_concur_create_user', + 'sap_concur_delete_expected_expense', + 'sap_concur_delete_expense', + 'sap_concur_delete_expense_report', + 'sap_concur_delete_list_item', + 'sap_concur_delete_travel_request', + 'sap_concur_delete_user', + 'sap_concur_get_allocation', + 'sap_concur_get_budget', + 'sap_concur_get_cash_advance', + 'sap_concur_get_expected_expense', + 'sap_concur_get_expense', + 'sap_concur_get_expense_report', + 'sap_concur_get_itemizations', + 'sap_concur_get_itinerary', + 'sap_concur_get_list', + 'sap_concur_get_list_item', + 'sap_concur_get_purchase_request', + 'sap_concur_get_receipt', + 'sap_concur_get_receipt_status', + 'sap_concur_get_request_cash_advance', + 'sap_concur_get_travel_profile', + 'sap_concur_get_travel_request', + 'sap_concur_get_user', + 'sap_concur_issue_cash_advance', + 'sap_concur_list_allocations', + 'sap_concur_list_attendee_associations', + 'sap_concur_list_budget_categories', + 'sap_concur_list_budgets', + 'sap_concur_list_exceptions', + 'sap_concur_list_expected_expenses', + 'sap_concur_list_expense_reports', + 'sap_concur_list_expenses', + 'sap_concur_list_itineraries', + 'sap_concur_list_list_items', + 'sap_concur_list_lists', + 'sap_concur_list_receipts', + 'sap_concur_list_report_comments', + 'sap_concur_list_reports_to_approve', + 'sap_concur_list_travel_profiles_summary', + 'sap_concur_list_travel_request_comments', + 'sap_concur_list_travel_requests', + 'sap_concur_list_users', + 'sap_concur_move_travel_request', + 'sap_concur_recall_expense_report', + 'sap_concur_remove_all_attendees', + 'sap_concur_search_locations', + 'sap_concur_search_users', + 'sap_concur_send_back_expense_report', + 'sap_concur_submit_expense_report', + 'sap_concur_update_allocation', + 'sap_concur_update_expected_expense', + 'sap_concur_update_expense', + 'sap_concur_update_expense_report', + 'sap_concur_update_list_item', + 'sap_concur_update_travel_request', + 'sap_concur_update_user', + 'sap_concur_upload_exchange_rates', + 'sap_concur_upload_receipt_image', +] as const + +const SAP_CONCUR_TOOL_ID_SET = new Set(SAP_CONCUR_TOOL_IDS) + +const SAP_CONCUR_UPLOAD_TOOL_IDS = new Set([ + 'sap_concur_upload_receipt_image', + 'sap_concur_create_quick_expense_with_image', +]) + +function validationResponse(error: z.ZodError): Response { + return Response.json( + { + success: false, + error: error.issues[0]?.message || 'Validation failed', + }, + { status: 400 } + ) +} + +function inputSizeResponse(input: unknown): Response | null { + let serialized: string + try { + serialized = JSON.stringify(input) ?? '' + } catch { + return Response.json({ success: false, error: 'Validation failed' }, { status: 400 }) + } + if (Buffer.byteLength(serialized, 'utf8') <= DEFAULT_MAX_JSON_BODY_BYTES) return null + return Response.json( + { + success: false, + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) +} + +function isSapConcurToolId(toolId: string): boolean { + return SAP_CONCUR_TOOL_ID_SET.has(toolId) +} + +async function dispatch( + request: InternalToolOperationCall, + context: SapConcurOperationContext +): Promise { + if (!isSapConcurToolId(request.toolId)) { + return Response.json( + { success: false, error: `Unsupported SAP Concur tool: ${request.toolId}` }, + { status: 500 } + ) + } + + if (SAP_CONCUR_UPLOAD_TOOL_IDS.has(request.toolId)) { + if (!request.context.userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + const parsed = sapConcurUploadInputSchema.safeParse(request.input) + if (!parsed.success) return validationResponse(parsed.error) + return executeSapConcurUploadOperation(parsed.data, context) + } + + const parsed = sapConcurApiInputSchema.safeParse(request.input) + if (!parsed.success) return validationResponse(parsed.error) + return executeSapConcurApiOperation(parsed.data, context) +} + +export const executeSapConcurTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + const tooLarge = inputSizeResponse(request.input) + if (tooLarge) return tooLarge + try { + const result = await dispatch(request, { + requestId: request.requestId, + signal: request.signal, + userId: request.context.userId, + }) + request.signal?.throwIfAborted() + return result instanceof Response + ? result + : Response.json(result.body, { headers: result.headers }) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof SapConcurOperationError) { + return Response.json(error.body, { status: error.status, headers: error.headers }) + } + if (isPayloadSizeLimitError(error)) { + return Response.json({ success: false, error: error.message }, { status: 413 }) + } + const message = describeSapConcurFetchError(error) + logger.error('SAP Concur operation failed', { + error: getErrorMessage(error), + requestId: request.requestId, + toolId: request.toolId, + }) + return Response.json({ success: false, error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/sap-concur/operations.test.ts b/apps/sim/lib/internal/sap-concur/operations.test.ts new file mode 100644 index 00000000000..bfa483e1158 --- /dev/null +++ b/apps/sim/lib/internal/sap-concur/operations.test.ts @@ -0,0 +1,237 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { MAX_MULTIPART_OVERHEAD_BYTES } from '@/lib/core/utils/stream-limits' + +const { + mockAssertFileAccess, + mockDownloadFile, + mockFetchToken, + mockInvokeApi, + mockInvokeMultipart, + mockProcessFiles, +} = vi.hoisted(() => ({ + mockAssertFileAccess: vi.fn(), + mockDownloadFile: vi.fn(), + mockFetchToken: vi.fn(), + mockInvokeApi: vi.fn(), + mockInvokeMultipart: vi.fn(), + mockProcessFiles: vi.fn(), +})) + +vi.mock('@/lib/internal/sap-concur/client', () => ({ + assertSafeExternalUrl: (url: string) => new URL(url), + extractSapConcurError: (body: unknown, status: number) => + typeof body === 'object' && body !== null && 'message' in body + ? String(body.message) + : `Concur request failed with HTTP ${status}`, + fetchSapConcurAccessToken: mockFetchToken, + invokeSapConcur: mockInvokeApi, + invokeSapConcurMultipart: mockInvokeMultipart, +})) + +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + processFilesToUserFiles: mockProcessFiles, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mockDownloadFile, +})) + +vi.mock('@/lib/uploads/utils/servable-file-response', () => ({ + docNotReadyResponse: () => null, +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mockAssertFileAccess, +})) + +import { + executeSapConcurApiOperation, + executeSapConcurUploadOperation, + SapConcurOperationError, +} from '@/lib/internal/sap-concur/operations' +import { + sapConcurApiInputSchema, + sapConcurUploadInputSchema, +} from '@/lib/internal/sap-concur/schema' + +const context = { + requestId: 'request-1', + userId: 'user-1', +} + +beforeEach(() => { + vi.clearAllMocks() + mockFetchToken.mockResolvedValue({ + accessToken: 'access-token', + geolocation: 'https://us.api.concursolutions.com', + }) + mockInvokeApi.mockResolvedValue({ status: 200, body: { id: 'budget-1' }, headers: {} }) + mockInvokeMultipart.mockResolvedValue({ + status: 201, + body: { id: 'receipt-1' }, + headers: { location: 'https://us.api.concursolutions.com/receipts/receipt-1' }, + }) + mockProcessFiles.mockReturnValue([ + { + key: 'workspace-file-key', + name: 'receipt.pdf', + size: 1024, + type: 'application/pdf', + }, + ]) + mockAssertFileAccess.mockResolvedValue(null) + mockDownloadFile.mockResolvedValue({ + buffer: Buffer.from('receipt'), + contentType: 'application/pdf', + }) +}) + +describe('executeSapConcurApiOperation', () => { + it('threads the AbortSignal through token acquisition and the provider request', async () => { + const controller = new AbortController() + const input = sapConcurApiInputSchema.parse({ + clientId: 'client-id', + clientSecret: 'client-secret', + path: '/budget/v4/budgets/budget-1', + method: 'GET', + }) + + const result = await executeSapConcurApiOperation(input, { + ...context, + signal: controller.signal, + }) + + expect(mockFetchToken).toHaveBeenCalledWith(input, 'request-1', controller.signal) + expect(mockInvokeApi).toHaveBeenCalledWith( + input, + 'access-token', + 'https://us.api.concursolutions.com', + controller.signal + ) + expect(result.body).toEqual({ + success: true, + output: { status: 200, data: { id: 'budget-1' } }, + }) + }) + + it('preserves provider failures and response headers', async () => { + const input = sapConcurApiInputSchema.parse({ + clientId: 'client-id', + clientSecret: 'client-secret', + path: '/budget/v4/budgets/budget-1', + }) + mockInvokeApi.mockResolvedValueOnce({ + status: 429, + body: { message: 'Slow down' }, + headers: { 'retry-after': '30' }, + }) + + const error = await executeSapConcurApiOperation(input, context).catch((caught) => caught) + + expect(error).toBeInstanceOf(SapConcurOperationError) + expect(error).toMatchObject({ + status: 429, + body: { success: false, error: 'Slow down', status: 429 }, + headers: { 'retry-after': '30' }, + }) + }) +}) + +describe('executeSapConcurUploadOperation', () => { + function uploadInput(size = 1024) { + return sapConcurUploadInputSchema.parse({ + clientId: 'client-id', + clientSecret: 'client-secret', + operation: 'upload_receipt_image', + userId: 'concur-user-1', + receipt: { + key: 'workspace-file-key', + name: 'receipt.pdf', + size, + type: 'application/pdf', + }, + }) + } + + it('authorizes the protected file before storage or provider access', async () => { + mockAssertFileAccess.mockResolvedValueOnce( + Response.json({ success: false, error: 'File not found' }, { status: 404 }) + ) + + const error = await executeSapConcurUploadOperation(uploadInput(), context).catch( + (caught) => caught + ) + + expect(error).toBeInstanceOf(SapConcurOperationError) + expect(error).toMatchObject({ + status: 404, + body: { success: false, error: 'File not found' }, + }) + expect(mockDownloadFile).not.toHaveBeenCalled() + expect(mockFetchToken).not.toHaveBeenCalled() + }) + + it('rejects declared receipt sizes above 25MB before materializing the file', async () => { + mockProcessFiles.mockReturnValueOnce([ + { + key: 'workspace-file-key', + name: 'receipt.pdf', + size: 25 * 1024 * 1024 + 1, + type: 'application/pdf', + }, + ]) + + const error = await executeSapConcurUploadOperation(uploadInput(), context).catch( + (caught) => caught + ) + + expect(error).toBeInstanceOf(SapConcurOperationError) + expect(error.body.error).toContain('exceeds Concur upload limit of 25MB') + expect(mockDownloadFile).not.toHaveBeenCalled() + }) + + it('threads cancellation through the bounded file download and multipart request', async () => { + const controller = new AbortController() + const input = uploadInput() + + const result = await executeSapConcurUploadOperation(input, { + ...context, + signal: controller.signal, + }) + + expect(mockDownloadFile).toHaveBeenCalledWith( + expect.objectContaining({ key: 'workspace-file-key' }), + 'request-1', + expect.anything(), + { maxBytes: 25 * 1024 * 1024, signal: controller.signal } + ) + expect(mockFetchToken).toHaveBeenCalledWith(input, 'request-1', controller.signal) + expect(mockInvokeMultipart).toHaveBeenCalledWith( + 'https://us.api.concursolutions.com/receipts/v4/users/concur-user-1/image-only-receipts', + 'access-token', + expect.any(FormData), + 25 * 1024 * 1024 + DEFAULT_MAX_JSON_BODY_BYTES + MAX_MULTIPART_OVERHEAD_BYTES, + controller.signal + ) + expect(result.headers).toEqual({ + location: 'https://us.api.concursolutions.com/receipts/receipt-1', + }) + }) + + it('rejects an abort before protected file resolution', async () => { + const controller = new AbortController() + controller.abort(new DOMException('Cancelled', 'AbortError')) + + await expect( + executeSapConcurUploadOperation(uploadInput(), { + ...context, + signal: controller.signal, + }) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mockAssertFileAccess).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/sap-concur/operations.ts b/apps/sim/lib/internal/sap-concur/operations.ts new file mode 100644 index 00000000000..51d5c9612cb --- /dev/null +++ b/apps/sim/lib/internal/sap-concur/operations.ts @@ -0,0 +1,316 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { + isPayloadSizeLimitError, + MAX_MULTIPART_OVERHEAD_BYTES, + type PayloadSizeLimitError, +} from '@/lib/core/utils/stream-limits' +import { + assertSafeExternalUrl, + extractSapConcurError, + fetchSapConcurAccessToken, + invokeSapConcur, + invokeSapConcurMultipart, + type SapConcurInvocation, +} from '@/lib/internal/sap-concur/client' +import type { SapConcurApiInput, SapConcurUploadInput } from '@/lib/internal/sap-concur/schema' +import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +const logger = createLogger('SapConcurOperations') + +const RECEIPT_ALLOWED_MIME_TYPES = new Set([ + 'application/pdf', + 'image/png', + 'image/jpeg', + 'image/jpg', + 'image/gif', + 'image/tiff', + 'image/tif', +]) + +const QUICK_EXPENSE_ALLOWED_MIME_TYPES = new Set([ + 'application/pdf', + 'image/png', + 'image/jpeg', + 'image/jpg', + 'image/tiff', + 'image/tif', +]) + +const ALL_ALLOWED_MIME_TYPES = RECEIPT_ALLOWED_MIME_TYPES +const MAX_RECEIPT_IMAGE_BYTES = 25 * 1024 * 1024 +const MAX_QUICK_EXPENSE_IMAGE_BYTES = 50 * 1024 * 1024 +const UNKNOWN_MIME_TYPE = 'application/octet-stream' +const MIME_TYPE_ALIASES: Record = { + 'image/jpg': 'image/jpeg', + 'image/tif': 'image/tiff', +} + +export interface SapConcurOperationContext { + requestId: string + signal?: AbortSignal + userId?: string +} + +export interface SapConcurOperationErrorBody { + success: false + error: string + status?: number +} + +export class SapConcurOperationError extends Error { + constructor( + readonly status: number, + readonly body: SapConcurOperationErrorBody, + readonly headers: HeadersInit = {} + ) { + super(body.error) + this.name = 'SapConcurOperationError' + } +} + +function clampErrorStatus(status: number): number { + return status >= 400 ? status : 502 +} + +export interface SapConcurOperationResult { + body: { + success: true + output: { status: number; data: unknown } + } + headers: HeadersInit +} + +function resultFromInvocation(invocation: SapConcurInvocation): SapConcurOperationResult { + if (invocation.status >= 200 && invocation.status < 300) { + return { + body: { + success: true, + output: { + status: invocation.status, + data: invocation.status === 204 ? null : invocation.body, + }, + }, + headers: invocation.headers, + } + } + + const message = extractSapConcurError(invocation.body, invocation.status) + throw new SapConcurOperationError( + clampErrorStatus(invocation.status), + { success: false, error: message, status: invocation.status }, + invocation.headers + ) +} + +export async function executeSapConcurApiOperation( + input: SapConcurApiInput, + context: SapConcurOperationContext +) { + context.signal?.throwIfAborted() + const token = await fetchSapConcurAccessToken(input, context.requestId, context.signal) + context.signal?.throwIfAborted() + const invocation = await invokeSapConcur( + input, + token.accessToken, + token.geolocation, + context.signal + ) + context.signal?.throwIfAborted() + return resultFromInvocation(invocation) +} + +function maxImageBytesForOperation(operation: SapConcurUploadInput['operation']): number { + return operation === 'create_quick_expense_with_image' + ? MAX_QUICK_EXPENSE_IMAGE_BYTES + : MAX_RECEIPT_IMAGE_BYTES +} + +function uploadSizeError(bytes: number, maxBytes: number): SapConcurOperationError { + const sizeMB = (bytes / (1024 * 1024)).toFixed(2) + const limitMB = Math.round(maxBytes / (1024 * 1024)) + return new SapConcurOperationError(400, { + success: false, + error: `File size (${sizeMB}MB) exceeds Concur upload limit of ${limitMB}MB`, + }) +} + +function unsupportedMimeTypeError(mimeType: string, allowedLabel: string): SapConcurOperationError { + return new SapConcurOperationError(400, { + success: false, + error: `Unsupported receipt mime type: ${mimeType}. Allowed: ${allowedLabel}`, + }) +} + +function inferMimeType(name: string, declared?: string): string { + if (declared && ALL_ALLOWED_MIME_TYPES.has(declared.toLowerCase())) { + const lowerDeclared = declared.toLowerCase() + return MIME_TYPE_ALIASES[lowerDeclared] ?? lowerDeclared + } + const lower = name.toLowerCase() + if (lower.endsWith('.pdf')) return 'application/pdf' + if (lower.endsWith('.png')) return 'image/png' + if (lower.endsWith('.jpg') || lower.endsWith('.jpeg')) return 'image/jpeg' + if (lower.endsWith('.gif')) return 'image/gif' + if (lower.endsWith('.tif') || lower.endsWith('.tiff')) return 'image/tiff' + return UNKNOWN_MIME_TYPE +} + +async function responseBody(response: Response): Promise { + try { + return (await response.json()) as SapConcurOperationErrorBody + } catch { + return { success: false, error: response.statusText || 'File operation failed' } + } +} + +async function operationErrorFromResponse(response: Response): Promise { + return new SapConcurOperationError( + response.status, + await responseBody(response), + response.headers + ) +} + +async function resolveUploadFile( + input: SapConcurUploadInput, + context: SapConcurOperationContext +): Promise<{ buffer: Buffer; name: string; mimeType: string }> { + if (!context.userId) { + throw new SapConcurOperationError(401, { + success: false, + error: 'Authentication required', + }) + } + context.signal?.throwIfAborted() + const userFiles = processFilesToUserFiles( + [input.receipt as RawFileInput], + context.requestId, + logger + ) + if (userFiles.length === 0) { + throw new SapConcurOperationError(400, { + success: false, + error: 'Invalid receipt file input', + }) + } + const userFile = userFiles[0] + const denied = await assertToolFileAccess(userFile.key, context.userId, context.requestId, logger) + context.signal?.throwIfAborted() + if (denied) throw await operationErrorFromResponse(denied) + + const maxBytes = maxImageBytesForOperation(input.operation) + const allowedForOperation = + input.operation === 'create_quick_expense_with_image' + ? QUICK_EXPENSE_ALLOWED_MIME_TYPES + : RECEIPT_ALLOWED_MIME_TYPES + const allowedLabel = + input.operation === 'create_quick_expense_with_image' + ? 'pdf, png, jpeg, tiff' + : 'pdf, png, jpeg, gif, tiff' + + if (userFile.size > maxBytes) throw uploadSizeError(userFile.size, maxBytes) + const declaredMimeType = inferMimeType(userFile.name, userFile.type) + if (declaredMimeType !== UNKNOWN_MIME_TYPE && !allowedForOperation.has(declaredMimeType)) { + throw unsupportedMimeTypeError(declaredMimeType, allowedLabel) + } + + try { + const resolved = await downloadServableFileFromStorage(userFile, context.requestId, logger, { + maxBytes, + signal: context.signal, + }) + context.signal?.throwIfAborted() + if (resolved.buffer.length > maxBytes) { + throw uploadSizeError(resolved.buffer.length, maxBytes) + } + const mimeType = inferMimeType(userFile.name, resolved.contentType || userFile.type) + if (!allowedForOperation.has(mimeType)) { + throw unsupportedMimeTypeError(mimeType, allowedLabel) + } + return { buffer: resolved.buffer, name: userFile.name, mimeType } + } catch (error) { + context.signal?.throwIfAborted() + if (error instanceof SapConcurOperationError) throw error + const notReady = docNotReadyResponse(error) + if (notReady) throw await operationErrorFromResponse(notReady) + if (isPayloadSizeLimitError(error)) { + throw uploadSizeError( + (error as PayloadSizeLimitError).observedBytes ?? userFile.size, + maxBytes + ) + } + logger.error('Failed to download Concur receipt file', { + error: getErrorMessage(error), + requestId: context.requestId, + }) + throw new SapConcurOperationError(500, { + success: false, + error: getErrorMessage(error, 'Unknown error'), + }) + } +} + +function stringifyMaybeJson(value: unknown): string { + return typeof value === 'string' ? value : JSON.stringify(value ?? {}) +} + +function buildUploadForm( + input: SapConcurUploadInput, + file: { buffer: Buffer; name: string; mimeType: string } +): { urlPath: string; formData: FormData } { + const fileBytes = new Uint8Array(file.buffer) + const formData = new FormData() + if (input.operation === 'upload_receipt_image') { + formData.append('image', new Blob([fileBytes], { type: file.mimeType }), file.name) + return { + urlPath: `/receipts/v4/users/${encodeURIComponent(input.userId)}/image-only-receipts`, + formData, + } + } + + const contextType = input.contextType?.trim() || 'TRAVELER' + formData.append('quickExpenseRequest', stringifyMaybeJson(input.body ?? {})) + formData.append('fileContent', new Blob([fileBytes], { type: file.mimeType }), file.name) + return { + urlPath: `/quickexpense/v4/users/${encodeURIComponent(input.userId)}/context/${encodeURIComponent(contextType)}/quickexpenses/image`, + formData, + } +} + +export async function executeSapConcurUploadOperation( + input: SapConcurUploadInput, + context: SapConcurOperationContext +) { + context.signal?.throwIfAborted() + const file = await resolveUploadFile(input, context) + context.signal?.throwIfAborted() + const token = await fetchSapConcurAccessToken(input, context.requestId, context.signal) + context.signal?.throwIfAborted() + const upload = buildUploadForm(input, file) + const url = assertSafeExternalUrl( + `${token.geolocation.replace(/\/+$/, '')}${upload.urlPath}`, + 'apiUrl' + ).toString() + const invocation = await invokeSapConcurMultipart( + url, + token.accessToken, + upload.formData, + maxImageBytesForOperation(input.operation) + + DEFAULT_MAX_JSON_BODY_BYTES + + MAX_MULTIPART_OVERHEAD_BYTES, + context.signal + ) + context.signal?.throwIfAborted() + logger.info('Concur upload succeeded', { + operation: input.operation, + requestId: context.requestId, + status: invocation.status, + }) + return resultFromInvocation(invocation) +} diff --git a/apps/sim/app/api/tools/sap_concur/response-body.test.ts b/apps/sim/lib/internal/sap-concur/response-body.test.ts similarity index 90% rename from apps/sim/app/api/tools/sap_concur/response-body.test.ts rename to apps/sim/lib/internal/sap-concur/response-body.test.ts index ed5389bb7c3..1a5801f16cd 100644 --- a/apps/sim/app/api/tools/sap_concur/response-body.test.ts +++ b/apps/sim/lib/internal/sap-concur/response-body.test.ts @@ -29,8 +29,7 @@ vi.mock('@/lib/core/security/input-validation.server', () => ({ MAX_JSON_API_RESPONSE_BYTES: MOCK_MAX_JSON_BYTES, })) -import { readConcurProxyBody } from '@/app/api/tools/sap_concur/proxy/route' -import { readConcurUploadBody } from '@/app/api/tools/sap_concur/upload/route' +import { readConcurApiBody, readConcurUploadBody } from '@/lib/internal/sap-concur/client' /** Minimal response shape both helpers accept. */ function uploadResponse(status: number): Parameters[0] { @@ -41,10 +40,10 @@ function uploadResponse(status: number): Parameters } } -function proxyResponse( +function apiResponse( status: number, text: () => Promise -): Parameters[0] { +): Parameters[0] { return { status, text } } @@ -56,7 +55,7 @@ beforeEach(() => { /** * Both helpers make the same success/error split, so the cases are declared once and run * against each helper. `readConcurUploadBody` reads through the mocked - * `readResponseTextWithLimit`; `readConcurProxyBody` reads through `response.text()`. + * `readResponseTextWithLimit`; `readConcurApiBody` reads through `response.text()`. */ const helpers = [ { @@ -67,9 +66,9 @@ const helpers = [ }, }, { - name: 'readConcurProxyBody', + name: 'readConcurApiBody', read: (status: number, result: Promise) => - readConcurProxyBody(proxyResponse(status, () => result)), + readConcurApiBody(apiResponse(status, () => result)), }, ] as const diff --git a/apps/sim/lib/internal/sap-concur/schema.ts b/apps/sim/lib/internal/sap-concur/schema.ts new file mode 100644 index 00000000000..8e098b370b0 --- /dev/null +++ b/apps/sim/lib/internal/sap-concur/schema.ts @@ -0,0 +1,106 @@ +import { z } from 'zod' +import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' + +/** Published Concur token hosts, including server/client twins and implementation sandboxes. */ +export const SAP_CONCUR_ALLOWED_DATACENTERS = new Set([ + 'us.api.concursolutions.com', + 'www-us.api.concursolutions.com', + 'us2.api.concursolutions.com', + 'www-us2.api.concursolutions.com', + 'eu.api.concursolutions.com', + 'eu2.api.concursolutions.com', + 'www-eu2.api.concursolutions.com', + 'emea.api.concursolutions.com', + 'www-emea.api.concursolutions.com', + 'apj1.api.concursolutions.com', + 'www-apj1.api.concursolutions.com', + 'usg.api.concursolutions.com', + 'www-usg.api.concursolutions.com', + 'glz.api.concursolutions.com', + 'us-impl.api.concursolutions.com', + 'www-us-impl.api.concursolutions.com', + 'emea-impl.api.concursolutions.com', + 'www-emea-impl.api.concursolutions.com', +]) + +export const sapConcurDatacenterSchema = z + .string() + .min(1) + .refine((datacenter) => SAP_CONCUR_ALLOWED_DATACENTERS.has(datacenter), { + message: `datacenter must be one of: ${Array.from(SAP_CONCUR_ALLOWED_DATACENTERS).join(', ')}`, + }) + +export const sapConcurGrantTypeSchema = z.enum(['client_credentials', 'password']) + +export const sapConcurAuthSchema = z.object({ + datacenter: sapConcurDatacenterSchema.default('us.api.concursolutions.com'), + grantType: sapConcurGrantTypeSchema.default('client_credentials'), + clientId: z.string().min(1, 'clientId is required'), + clientSecret: z.string().min(1, 'clientSecret is required'), + username: z.string().optional(), + password: z.string().optional(), + companyUuid: z.string().optional(), + credtype: z.enum(['password', 'authtoken']).optional(), +}) + +export type SapConcurAuth = z.infer + +export const sapConcurHttpMethodSchema = z.enum(['GET', 'POST', 'PUT', 'PATCH', 'DELETE']) + +export const sapConcurApiPathSchema = z + .string() + .min(1, 'path is required') + .refine( + (path) => + !path.split(/[/\\]/).some((segment) => segment === '..' || segment === '.') && + !path.includes('#') && + !/%(?:2[eEfF]|5[cC]|23)/.test(path), + { + message: + 'path must not contain ".." or "." segments, "#", or percent-encoded path/fragment characters', + } + ) + +export const sapConcurApiInputSchema = sapConcurAuthSchema + .extend({ + path: sapConcurApiPathSchema, + method: sapConcurHttpMethodSchema.default('GET'), + query: z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])).optional(), + body: z.unknown().optional(), + contentType: z.string().optional(), + accept: z.string().optional(), + }) + .superRefine((input, context) => { + if (input.grantType !== 'password') return + if (!input.username && !input.companyUuid) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ['username'], + message: 'username is required for password grant (or companyUuid for company-level auth)', + }) + } + if (!input.password) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ['password'], + message: 'password is required for password grant', + }) + } + }) + +export type SapConcurApiInput = z.infer + +export const sapConcurUploadOperationSchema = z.enum([ + 'upload_receipt_image', + 'create_quick_expense_with_image', +]) + +export const sapConcurUploadInputSchema = sapConcurAuthSchema.extend({ + operation: sapConcurUploadOperationSchema, + userId: z.string().min(1, 'userId is required'), + contextType: z.string().optional(), + receipt: FileInputSchema, + body: z.union([z.record(z.string(), z.unknown()), z.string()]).optional(), +}) + +export type SapConcurUploadInput = z.infer diff --git a/apps/sim/lib/internal/sap-s4hana/client.test.ts b/apps/sim/lib/internal/sap-s4hana/client.test.ts new file mode 100644 index 00000000000..8a0883eea31 --- /dev/null +++ b/apps/sim/lib/internal/sap-s4hana/client.test.ts @@ -0,0 +1,132 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockSecureFetchWithValidation } = vi.hoisted(() => ({ + mockSecureFetchWithValidation: vi.fn(), +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + MAX_JSON_API_RESPONSE_BYTES: 10 * 1024 * 1024, + secureFetchWithValidation: mockSecureFetchWithValidation, +})) + +import { callSapOdata, fetchSapAccessToken, fetchSapCsrf } from '@/lib/internal/sap-s4hana/client' +import { sapS4HanaOperationInputSchema } from '@/lib/internal/sap-s4hana/schema' + +function response( + body: unknown, + status = 200, + headers: Record = {}, + setCookies: string[] = [] +) { + return { + ok: status >= 200 && status < 300, + status, + statusText: 'OK', + headers: { + get: vi.fn((name: string) => headers[name.toLowerCase()] ?? null), + getSetCookie: vi.fn().mockReturnValue(setCookies), + }, + text: vi.fn().mockResolvedValue(JSON.stringify(body)), + json: vi.fn().mockResolvedValue(body), + arrayBuffer: vi.fn(), + body: null, + } +} + +describe('SAP S/4HANA client', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('pins the response cap and forwards cancellation to the provider request', async () => { + mockSecureFetchWithValidation.mockResolvedValue(response({ d: { BusinessPartner: '100' } })) + const signal = new AbortController().signal + const input = sapS4HanaOperationInputSchema.parse({ + deploymentType: 'cloud_private', + authType: 'basic', + baseUrl: 'https://sap.example.com', + username: 'user', + password: 'password', + service: 'API_BUSINESS_PARTNER', + path: '/A_BusinessPartner', + method: 'GET', + query: { $format: 'json', $top: 10 }, + }) + + await expect(callSapOdata(input, null, null, signal)).resolves.toMatchObject({ status: 200 }) + + expect(mockSecureFetchWithValidation).toHaveBeenCalledWith( + 'https://sap.example.com/sap/opu/odata/sap/API_BUSINESS_PARTNER/A_BusinessPartner?$format=json&$top=10', + expect.objectContaining({ + method: 'GET', + maxResponseBytes: 10 * 1024 * 1024, + signal, + }), + 'baseUrl' + ) + }) + + it('forwards cancellation and the response cap to OAuth token requests', async () => { + mockSecureFetchWithValidation.mockResolvedValue( + response({ access_token: 'access-token', expires_in: 3600 }) + ) + const signal = new AbortController().signal + const input = sapS4HanaOperationInputSchema.parse({ + subdomain: 'token-test', + region: 'us30', + clientId: 'client-token-test', + clientSecret: 'secret-token-test', + service: 'API_BUSINESS_PARTNER', + path: '/A_BusinessPartner', + }) + + await expect(fetchSapAccessToken(input, signal)).resolves.toBe('access-token') + + expect(mockSecureFetchWithValidation).toHaveBeenCalledWith( + 'https://token-test.authentication.us30.hana.ondemand.com/oauth/token', + expect.objectContaining({ + method: 'POST', + maxResponseBytes: 10 * 1024 * 1024, + signal, + }), + 'tokenUrl' + ) + }) + + it('forwards cancellation and preserves CSRF cookies', async () => { + mockSecureFetchWithValidation.mockResolvedValue( + response({}, 200, { 'x-csrf-token': 'csrf-token' }, [ + 'sap-usercontext=sap-client=100; Path=/; Secure', + 'SAP_SESSIONID=session; Path=/; Secure', + ]) + ) + const signal = new AbortController().signal + const input = sapS4HanaOperationInputSchema.parse({ + deploymentType: 'cloud_private', + authType: 'basic', + baseUrl: 'https://sap.example.com', + username: 'user', + password: 'password', + service: 'API_BUSINESS_PARTNER', + path: '/A_BusinessPartner', + method: 'POST', + }) + + await expect(fetchSapCsrf(input, null, signal)).resolves.toEqual({ + token: 'csrf-token', + cookie: 'sap-usercontext=sap-client=100; SAP_SESSIONID=session', + }) + expect(mockSecureFetchWithValidation).toHaveBeenCalledWith( + 'https://sap.example.com/sap/opu/odata/sap/API_BUSINESS_PARTNER/$metadata', + expect.objectContaining({ + method: 'GET', + maxResponseBytes: 10 * 1024 * 1024, + signal, + }), + 'baseUrl' + ) + }) +}) diff --git a/apps/sim/lib/internal/sap-s4hana/client.ts b/apps/sim/lib/internal/sap-s4hana/client.ts new file mode 100644 index 00000000000..f4bd96fc4c3 --- /dev/null +++ b/apps/sim/lib/internal/sap-s4hana/client.ts @@ -0,0 +1,250 @@ +import { createHash } from 'node:crypto' +import { + MAX_JSON_API_RESPONSE_BYTES, + type SecureFetchResponse, + secureFetchWithValidation, +} from '@/lib/core/security/input-validation.server' +import { + assertSafeSapExternalUrl, + type SapS4HanaOperationInput, +} from '@/lib/internal/sap-s4hana/schema' + +interface CachedToken { + accessToken: string + expiresAt: number +} + +export interface SapCsrfBundle { + token: string + cookie: string +} + +export interface SapOdataInvocation { + status: number + body: unknown + csrfHeader: string +} + +const TOKEN_CACHE = new Map() +const TOKEN_CACHE_MAX_ENTRIES = 500 +const TOKEN_SAFETY_WINDOW_MS = 60_000 +const OUTBOUND_FETCH_TIMEOUT_MS = 30_000 + +function resolveTokenUrl(input: SapS4HanaOperationInput): string { + if (input.deploymentType === 'cloud_public') { + return `https://${input.subdomain}.authentication.${input.region}.hana.ondemand.com/oauth/token` + } + if (!input.tokenUrl) { + throw new Error('tokenUrl is required for OAuth on cloud_private/on_premise') + } + return input.tokenUrl +} + +function tokenCacheKey(input: SapS4HanaOperationInput): string { + const secretHash = input.clientSecret + ? createHash('sha256').update(input.clientSecret).digest('hex').slice(0, 16) + : '' + return `${resolveTokenUrl(input)}::${input.clientId ?? ''}::${secretHash}` +} + +function rememberToken(key: string, token: CachedToken): void { + if (TOKEN_CACHE.has(key)) TOKEN_CACHE.delete(key) + TOKEN_CACHE.set(key, token) + while (TOKEN_CACHE.size > TOKEN_CACHE_MAX_ENTRIES) { + const oldestKey = TOKEN_CACHE.keys().next().value + if (oldestKey === undefined) break + TOKEN_CACHE.delete(oldestKey) + } +} + +export async function fetchSapAccessToken( + input: SapS4HanaOperationInput, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const cacheKey = tokenCacheKey(input) + const cached = TOKEN_CACHE.get(cacheKey) + if (cached && cached.expiresAt - TOKEN_SAFETY_WINDOW_MS > Date.now()) { + return cached.accessToken + } + + const tokenUrl = assertSafeSapExternalUrl(resolveTokenUrl(input), 'tokenUrl').toString() + const basic = Buffer.from(`${input.clientId}:${input.clientSecret}`).toString('base64') + const response = await secureFetchWithValidation( + tokenUrl, + { + method: 'POST', + headers: { + Authorization: `Basic ${basic}`, + 'Content-Type': 'application/x-www-form-urlencoded', + Accept: 'application/json', + }, + body: 'grant_type=client_credentials', + timeout: OUTBOUND_FETCH_TIMEOUT_MS, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + signal, + }, + 'tokenUrl' + ) + signal?.throwIfAborted() + + if (!response.ok) { + await response.text().catch(() => '') + throw new Error(`SAP token request failed: HTTP ${response.status}`) + } + + const data = (await response.json()) as { access_token?: string; expires_in?: number } + signal?.throwIfAborted() + if (!data.access_token) { + throw new Error('SAP token response missing access_token') + } + + rememberToken(cacheKey, { + accessToken: data.access_token, + expiresAt: Date.now() + (data.expires_in ?? 3600) * 1000, + }) + return data.access_token +} + +function joinSetCookies(response: SecureFetchResponse): string { + return response.headers + .getSetCookie() + .map((cookie) => cookie.split(';')[0]?.trim()) + .filter(Boolean) + .join('; ') +} + +function buildAuthHeader(input: SapS4HanaOperationInput, accessToken: string | null): string { + if (input.authType === 'basic') { + return `Basic ${Buffer.from(`${input.username}:${input.password}`).toString('base64')}` + } + return `Bearer ${accessToken}` +} + +function resolveHost(input: SapS4HanaOperationInput): string { + if (input.deploymentType === 'cloud_public') { + const constructed = `https://${input.subdomain}-api.s4hana.ondemand.com` + return assertSafeSapExternalUrl(constructed, 'subdomain').toString().replace(/\/+$/, '') + } + if (!input.baseUrl) { + throw new Error('baseUrl is required for cloud_private and on_premise deployments') + } + return assertSafeSapExternalUrl(input.baseUrl.replace(/\/+$/, ''), 'baseUrl') + .toString() + .replace(/\/+$/, '') +} + +function buildOdataUrl(input: SapS4HanaOperationInput, pathOverride?: string): string { + const host = resolveHost(input) + const servicePath = `/sap/opu/odata/sap/${input.service}` + const subPath = pathOverride ?? input.path + const normalized = subPath.startsWith('/') ? subPath : `/${subPath}` + const base = `${host}${servicePath}${normalized}` + if (pathOverride !== undefined || !input.query || Object.keys(input.query).length === 0) { + return base + } + + const encode = (value: string) => encodeURIComponent(value).replace(/%24/g, '$') + const parts: string[] = [] + for (const [key, value] of Object.entries(input.query)) { + if (value === undefined || value === null) continue + parts.push(`${encode(key)}=${encode(String(value))}`) + } + const queryString = parts.join('&') + if (!queryString) return base + return base.includes('?') ? `${base}&${queryString}` : `${base}?${queryString}` +} + +export async function fetchSapCsrf( + input: SapS4HanaOperationInput, + accessToken: string | null, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const response = await secureFetchWithValidation( + buildOdataUrl(input, '/$metadata'), + { + method: 'GET', + headers: { + Authorization: buildAuthHeader(input, accessToken), + Accept: 'application/xml', + 'X-CSRF-Token': 'Fetch', + }, + timeout: OUTBOUND_FETCH_TIMEOUT_MS, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + signal, + }, + 'baseUrl' + ) + signal?.throwIfAborted() + if (!response.ok) { + await response.text().catch(() => '') + return null + } + + const token = response.headers.get('x-csrf-token') + if (!token) return null + return { token, cookie: joinSetCookies(response) } +} + +export async function callSapOdata( + input: SapS4HanaOperationInput, + accessToken: string | null, + csrf: SapCsrfBundle | null, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const headers: Record = { + Authorization: buildAuthHeader(input, accessToken), + Accept: 'application/json', + } + const isWrite = isSapWriteMethod(input.method) + const hasBody = input.body !== undefined && input.body !== null + if (hasBody) headers['Content-Type'] = 'application/json' + if (input.ifMatch) headers['If-Match'] = input.ifMatch + if (isWrite && csrf) { + headers['X-CSRF-Token'] = csrf.token + if (csrf.cookie) headers.Cookie = csrf.cookie + } + + const response = await secureFetchWithValidation( + buildOdataUrl(input), + { + method: input.method, + headers, + body: hasBody ? JSON.stringify(input.body) : undefined, + timeout: OUTBOUND_FETCH_TIMEOUT_MS, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + signal, + }, + 'baseUrl' + ) + signal?.throwIfAborted() + + const raw = await response.text() + signal?.throwIfAborted() + let body: unknown = null + if (raw.length > 0) { + try { + body = JSON.parse(raw) + } catch { + body = raw + } + } + + return { + status: response.status, + body, + csrfHeader: response.headers.get('x-csrf-token')?.toLowerCase() ?? '', + } +} + +export function isSapWriteMethod(method: SapS4HanaOperationInput['method']): boolean { + return ( + method === 'POST' || + method === 'PUT' || + method === 'PATCH' || + method === 'DELETE' || + method === 'MERGE' + ) +} diff --git a/apps/sim/lib/internal/sap-s4hana/execute-tool.test.ts b/apps/sim/lib/internal/sap-s4hana/execute-tool.test.ts new file mode 100644 index 00000000000..ad057faf11e --- /dev/null +++ b/apps/sim/lib/internal/sap-s4hana/execute-tool.test.ts @@ -0,0 +1,94 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const { mockExecuteOperation } = vi.hoisted(() => ({ + mockExecuteOperation: vi.fn(), +})) + +vi.mock('@/lib/internal/sap-s4hana/operations', () => { + class SapS4HanaProviderError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + } + } + return { + executeSapS4HanaOperation: mockExecuteOperation, + SapS4HanaProviderError, + } +}) + +import { executeSapS4HanaTool, SAP_S4HANA_TOOL_IDS } from '@/lib/internal/sap-s4hana/execute-tool' +import { SapS4HanaProviderError } from '@/lib/internal/sap-s4hana/operations' + +const VALID_INPUT = { + subdomain: 'example', + region: 'us30', + clientId: 'client', + clientSecret: 'secret', + service: 'API_BUSINESS_PARTNER', + path: '/A_BusinessPartner', + method: 'GET', +} + +function createCall(overrides: Partial = {}): InternalToolOperationCall { + return { + toolId: 'sap_s4hana_list_business_partners', + input: VALID_INPUT, + headers: new Headers(), + context: { userId: 'user-1', workspaceId: 'workspace-1' }, + requestId: 'request-1', + signal: new AbortController().signal, + ...overrides, + } +} + +describe('executeSapS4HanaTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mockExecuteOperation.mockResolvedValue({ status: 200, data: [] }) + }) + + it('dispatches every canonical SAP S/4HANA tool ID', async () => { + for (const toolId of SAP_S4HANA_TOOL_IDS) { + const response = await executeSapS4HanaTool(createCall({ toolId })) + expect(response.status).toBe(200) + } + expect(mockExecuteOperation).toHaveBeenCalledTimes(SAP_S4HANA_TOOL_IDS.length) + }) + + it('preserves the first validation error in the 400 envelope', async () => { + const response = await executeSapS4HanaTool(createCall({ input: {} })) + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + success: false, + error: 'Invalid input: expected string, received undefined', + }) + expect(mockExecuteOperation).not.toHaveBeenCalled() + }) + + it('preserves provider status and error envelopes', async () => { + mockExecuteOperation.mockRejectedValue(new SapS4HanaProviderError('Not found', 404)) + const response = await executeSapS4HanaTool(createCall()) + expect(response.status).toBe(404) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Not found', + status: 404, + }) + }) + + it('honors cancellation before dispatch', async () => { + const controller = new AbortController() + controller.abort() + await expect( + executeSapS4HanaTool(createCall({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mockExecuteOperation).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/sap-s4hana/execute-tool.ts b/apps/sim/lib/internal/sap-s4hana/execute-tool.ts new file mode 100644 index 00000000000..9a07420dd7a --- /dev/null +++ b/apps/sim/lib/internal/sap-s4hana/execute-tool.ts @@ -0,0 +1,86 @@ +import { toError } from '@sim/utils/errors' +import { + executeSapS4HanaOperation, + SapS4HanaProviderError, +} from '@/lib/internal/sap-s4hana/operations' +import { sapS4HanaOperationInputSchema } from '@/lib/internal/sap-s4hana/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +export const SAP_S4HANA_TOOL_IDS = [ + 'sap_s4hana_create_business_partner', + 'sap_s4hana_create_purchase_order', + 'sap_s4hana_create_purchase_requisition', + 'sap_s4hana_create_sales_order', + 'sap_s4hana_delete_sales_order', + 'sap_s4hana_get_billing_document', + 'sap_s4hana_get_business_partner', + 'sap_s4hana_get_customer', + 'sap_s4hana_get_inbound_delivery', + 'sap_s4hana_get_material_document', + 'sap_s4hana_get_outbound_delivery', + 'sap_s4hana_get_product', + 'sap_s4hana_get_purchase_order', + 'sap_s4hana_get_purchase_requisition', + 'sap_s4hana_get_sales_order', + 'sap_s4hana_get_supplier', + 'sap_s4hana_get_supplier_invoice', + 'sap_s4hana_list_billing_documents', + 'sap_s4hana_list_business_partners', + 'sap_s4hana_list_customers', + 'sap_s4hana_list_inbound_deliveries', + 'sap_s4hana_list_material_documents', + 'sap_s4hana_list_material_stock', + 'sap_s4hana_list_outbound_deliveries', + 'sap_s4hana_list_products', + 'sap_s4hana_list_purchase_orders', + 'sap_s4hana_list_purchase_requisitions', + 'sap_s4hana_list_sales_orders', + 'sap_s4hana_list_supplier_invoices', + 'sap_s4hana_list_suppliers', + 'sap_s4hana_odata_query', + 'sap_s4hana_update_business_partner', + 'sap_s4hana_update_customer', + 'sap_s4hana_update_product', + 'sap_s4hana_update_purchase_order', + 'sap_s4hana_update_purchase_requisition', + 'sap_s4hana_update_sales_order', + 'sap_s4hana_update_supplier', +] as const + +const SAP_S4HANA_TOOL_ID_SET = new Set(SAP_S4HANA_TOOL_IDS) + +export const executeSapS4HanaTool: InternalToolOperationHandler = async ({ + toolId, + input, + requestId, + signal, +}) => { + signal?.throwIfAborted() + if (!SAP_S4HANA_TOOL_ID_SET.has(toolId)) { + return Response.json({ error: `Unsupported SAP S/4HANA tool: ${toolId}` }, { status: 500 }) + } + + const parsed = sapS4HanaOperationInputSchema.safeParse(input) + if (!parsed.success) { + return Response.json( + { + success: false, + error: parsed.error.issues[0]?.message || 'Validation failed', + }, + { status: 400 } + ) + } + + try { + return Response.json(await executeSapS4HanaOperation(parsed.data, requestId, signal)) + } catch (error) { + signal?.throwIfAborted() + if (error instanceof SapS4HanaProviderError) { + return Response.json( + { success: false, error: error.message, status: error.status }, + { status: error.status } + ) + } + return Response.json({ success: false, error: toError(error).message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/sap-s4hana/operations.test.ts b/apps/sim/lib/internal/sap-s4hana/operations.test.ts new file mode 100644 index 00000000000..19f9f7a48d2 --- /dev/null +++ b/apps/sim/lib/internal/sap-s4hana/operations.test.ts @@ -0,0 +1,103 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockCallOdata, mockFetchAccessToken, mockFetchCsrf } = vi.hoisted(() => ({ + mockCallOdata: vi.fn(), + mockFetchAccessToken: vi.fn(), + mockFetchCsrf: vi.fn(), +})) + +vi.mock('@/lib/internal/sap-s4hana/client', () => ({ + callSapOdata: mockCallOdata, + fetchSapAccessToken: mockFetchAccessToken, + fetchSapCsrf: mockFetchCsrf, + isSapWriteMethod: (method: string) => + method === 'POST' || + method === 'PUT' || + method === 'PATCH' || + method === 'DELETE' || + method === 'MERGE', +})) + +import { + executeSapS4HanaOperation, + SapS4HanaProviderError, +} from '@/lib/internal/sap-s4hana/operations' +import { sapS4HanaOperationInputSchema } from '@/lib/internal/sap-s4hana/schema' + +const BASE_INPUT = sapS4HanaOperationInputSchema.parse({ + deploymentType: 'cloud_private', + authType: 'basic', + baseUrl: 'https://sap.example.com', + username: 'user', + password: 'password', + service: 'API_BUSINESS_PARTNER', + path: '/A_BusinessPartner', + method: 'GET', +}) + +describe('SAP S/4HANA operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mockFetchAccessToken.mockResolvedValue('token') + mockFetchCsrf.mockResolvedValue({ token: 'csrf', cookie: 'session=1' }) + }) + + it('preserves OData collection metadata while unwrapping the v2 envelope', async () => { + mockCallOdata.mockResolvedValue({ + status: 200, + body: { d: { results: [{ BusinessPartner: '100' }], __count: '1', __next: '/next' } }, + csrfHeader: '', + }) + + await expect(executeSapS4HanaOperation(BASE_INPUT, 'request-1')).resolves.toEqual({ + status: 200, + data: { + results: [{ BusinessPartner: '100' }], + __count: '1', + __next: '/next', + }, + }) + }) + + it('refreshes a rejected CSRF token once without workflow-level retries', async () => { + mockCallOdata + .mockResolvedValueOnce({ + status: 403, + body: { error: { message: { value: 'CSRF token validation failed' } } }, + csrfHeader: 'required', + }) + .mockResolvedValueOnce({ status: 204, body: null, csrfHeader: '' }) + + await expect( + executeSapS4HanaOperation( + { ...BASE_INPUT, method: 'MERGE', body: { Name: 'Updated' } }, + 'request-2' + ) + ).resolves.toEqual({ status: 204, data: null }) + expect(mockFetchCsrf).toHaveBeenCalledTimes(2) + expect(mockCallOdata).toHaveBeenCalledTimes(2) + }) + + it('preserves provider status and detailed OData errors', async () => { + mockCallOdata.mockResolvedValue({ + status: 400, + body: { + error: { + code: 'SAP/INVALID', + message: { value: 'Invalid request' }, + innererror: { + errordetails: [{ code: 'FIELD', message: 'Field is required', severity: 'error' }], + }, + }, + }, + csrfHeader: '', + }) + + await expect(executeSapS4HanaOperation(BASE_INPUT, 'request-3')).rejects.toEqual( + new SapS4HanaProviderError('[SAP/INVALID] Invalid request ([FIELD] Field is required)', 400) + ) + }) +}) diff --git a/apps/sim/lib/internal/sap-s4hana/operations.ts b/apps/sim/lib/internal/sap-s4hana/operations.ts new file mode 100644 index 00000000000..3b00205095d --- /dev/null +++ b/apps/sim/lib/internal/sap-s4hana/operations.ts @@ -0,0 +1,123 @@ +import { createLogger } from '@sim/logger' +import { + callSapOdata, + fetchSapAccessToken, + fetchSapCsrf, + isSapWriteMethod, + type SapOdataInvocation, +} from '@/lib/internal/sap-s4hana/client' +import type { SapS4HanaOperationInput } from '@/lib/internal/sap-s4hana/schema' + +const logger = createLogger('SapS4HanaOperations') + +export interface SapS4HanaOperationResult { + status: number + data: unknown +} + +export class SapS4HanaProviderError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + } +} + +function isCsrfRequired(invocation: SapOdataInvocation): boolean { + if (invocation.status !== 403) return false + if (invocation.csrfHeader === 'required') return true + if (typeof invocation.body !== 'object' || invocation.body === null) return false + const error = (invocation.body as { error?: { message?: { value?: string } | string } }).error + const messageField = error?.message + const message = typeof messageField === 'string' ? messageField : (messageField?.value ?? '') + return message.toLowerCase().includes('csrf') +} + +function extractOdataError(body: unknown, status: number): string { + if (body && typeof body === 'object') { + const error = ( + body as { + error?: { + message?: { value?: string } | string + code?: string + innererror?: { + errordetails?: Array<{ code?: string; message?: string; severity?: string }> + } + } + } + ).error + if (error) { + const messageField = error.message + const base = + typeof messageField === 'string' ? messageField : (messageField?.value ?? error.code ?? '') + const prefix = error.code ? `[${error.code}] ` : '' + const details = error.innererror?.errordetails + ?.filter((detail) => + Boolean(detail.message && (!detail.severity || detail.severity.toLowerCase() !== 'info')) + ) + .map((detail) => `${detail.code ? `[${detail.code}] ` : ''}${detail.message}`) + .filter((message): message is string => Boolean(message)) + if (details && details.length > 0) { + const extras = details.filter((detail) => !detail.endsWith(base)) + return extras.length > 0 ? `${prefix}${base} (${extras.join('; ')})` : `${prefix}${base}` + } + if (base) return `${prefix}${base}` + } + } + if (typeof body === 'string' && body.length > 0) return body + return `SAP request failed with HTTP ${status}` +} + +function unwrapOdata(body: unknown): unknown { + if (!body || typeof body !== 'object') return body + const root = (body as { d?: unknown }).d + if (root === undefined) return body + if (root && typeof root === 'object' && 'results' in (root as Record)) { + const result = root as { results: unknown; __count?: string; __next?: string } + if (result.__count !== undefined || result.__next !== undefined) { + return { + results: result.results, + ...(result.__count !== undefined && { __count: result.__count }), + ...(result.__next !== undefined && { __next: result.__next }), + } + } + return result.results + } + return root +} + +export async function executeSapS4HanaOperation( + input: SapS4HanaOperationInput, + requestId: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const isWrite = isSapWriteMethod(input.method) + const accessToken = + input.authType === 'oauth_client_credentials' ? await fetchSapAccessToken(input, signal) : null + const csrf = isWrite ? await fetchSapCsrf(input, accessToken, signal) : null + let invocation = await callSapOdata(input, accessToken, csrf, signal) + + if (isWrite && isCsrfRequired(invocation)) { + logger.info(`[${requestId}] CSRF token rejected, refetching once`) + const refreshed = await fetchSapCsrf(input, accessToken, signal) + if (refreshed) { + invocation = await callSapOdata(input, accessToken, refreshed, signal) + } + } + + signal?.throwIfAborted() + if (invocation.status >= 200 && invocation.status < 300) { + return { + status: invocation.status, + data: invocation.status === 204 ? null : unwrapOdata(invocation.body), + } + } + + const message = extractOdataError(invocation.body, invocation.status) + logger.warn( + `[${requestId}] SAP API error (${invocation.status}) ${input.service}${input.path}: ${message}` + ) + throw new SapS4HanaProviderError(message, invocation.status) +} diff --git a/apps/sim/lib/internal/sap-s4hana/schema.test.ts b/apps/sim/lib/internal/sap-s4hana/schema.test.ts new file mode 100644 index 00000000000..6b2f1014347 --- /dev/null +++ b/apps/sim/lib/internal/sap-s4hana/schema.test.ts @@ -0,0 +1,75 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + checkSapExternalUrlSafety, + sapS4HanaOperationInputSchema, +} from '@/lib/internal/sap-s4hana/schema' + +describe('SAP S/4HANA operation schema', () => { + it('applies public-cloud auth defaults', () => { + expect( + sapS4HanaOperationInputSchema.parse({ + subdomain: 'example', + region: 'us30', + clientId: 'client', + clientSecret: 'secret', + service: 'API_BUSINESS_PARTNER', + path: '/A_BusinessPartner', + }) + ).toMatchObject({ + deploymentType: 'cloud_public', + authType: 'oauth_client_credentials', + method: 'GET', + }) + }) + + it('accepts private-cloud Basic auth only with a public HTTPS base URL', () => { + expect( + sapS4HanaOperationInputSchema.safeParse({ + deploymentType: 'cloud_private', + authType: 'basic', + baseUrl: 'https://sap.example.com', + username: 'user', + password: 'password', + service: 'API_PRODUCT_SRV', + path: '/A_Product', + }).success + ).toBe(true) + + const rejected = sapS4HanaOperationInputSchema.safeParse({ + deploymentType: 'cloud_private', + authType: 'basic', + baseUrl: 'https://127.0.0.1', + username: 'user', + password: 'password', + service: 'API_PRODUCT_SRV', + path: '/A_Product', + }) + expect(rejected.success).toBe(false) + }) + + it('rejects path traversal and query injection in the service path', () => { + const result = sapS4HanaOperationInputSchema.safeParse({ + subdomain: 'example', + region: 'us30', + clientId: 'client', + clientSecret: 'secret', + service: 'API_BUSINESS_PARTNER', + path: '/../admin?$top=1', + }) + expect(result.success).toBe(false) + }) + + it('rejects non-HTTPS and private external URLs', () => { + expect(checkSapExternalUrlSafety('http://sap.example.com', 'baseUrl')).toEqual({ + ok: false, + message: 'baseUrl must use https://', + }) + expect(checkSapExternalUrlSafety('https://169.254.169.254', 'baseUrl')).toEqual({ + ok: false, + message: 'baseUrl host is not allowed', + }) + }) +}) diff --git a/apps/sim/lib/internal/sap-s4hana/schema.ts b/apps/sim/lib/internal/sap-s4hana/schema.ts new file mode 100644 index 00000000000..c8a13c8643c --- /dev/null +++ b/apps/sim/lib/internal/sap-s4hana/schema.ts @@ -0,0 +1,209 @@ +import { isPrivateIpHost } from '@sim/security/ssrf' +import { z } from 'zod' + +const sapHttpMethodSchema = z.enum(['GET', 'POST', 'PATCH', 'PUT', 'DELETE', 'MERGE']) +const sapDeploymentTypeSchema = z.enum(['cloud_public', 'cloud_private', 'on_premise']) +const sapAuthTypeSchema = z.enum(['oauth_client_credentials', 'basic']) + +const sapServiceNameSchema = z + .string() + .min(1, 'service is required') + .regex( + /^[A-Z][A-Z0-9_]*(;v=\d+)?$/, + 'service must be an uppercase OData service name optionally suffixed with ";v=NNNN" (e.g., API_BUSINESS_PARTNER, API_OUTBOUND_DELIVERY_SRV;v=0002)' + ) + +const sapServicePathSchema = z + .string() + .min(1, 'path is required') + .refine( + (path) => + !path.split(/[/\\]/).some((segment) => segment === '..' || segment === '.') && + !path.includes('?') && + !path.includes('#') && + !/%(?:2[eEfF]|5[cC]|3[fF]|23)/.test(path), + { + message: + 'path must not contain ".." or "." segments, "?", "#", or percent-encoded path/query/fragment characters', + } + ) + +const sapSubdomainSchema = z + .string() + .regex( + /^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$/i, + 'subdomain must contain only letters, digits, and hyphens (1-63 chars)' + ) + +const FORBIDDEN_SAP_HOSTS = new Set([ + 'localhost', + '0.0.0.0', + '127.0.0.1', + '169.254.169.254', + 'metadata.google.internal', + 'metadata', + '[::1]', + '[::]', + '[::ffff:127.0.0.1]', + '[fd00:ec2::254]', +]) + +export function checkSapExternalUrlSafety( + rawUrl: string, + label: string +): { ok: true; url: URL } | { ok: false; message: string } { + let parsed: URL + try { + parsed = new URL(rawUrl) + } catch { + return { ok: false, message: `${label} must be a valid URL` } + } + if (parsed.protocol !== 'https:') { + return { ok: false, message: `${label} must use https://` } + } + const host = parsed.hostname.toLowerCase() + if (FORBIDDEN_SAP_HOSTS.has(host) || FORBIDDEN_SAP_HOSTS.has(`[${host}]`)) { + return { ok: false, message: `${label} host is not allowed` } + } + if (isPrivateIpHost(host)) { + return { ok: false, message: `${label} host is not allowed (private/loopback range)` } + } + return { ok: true, url: parsed } +} + +export function assertSafeSapExternalUrl(rawUrl: string, label: string): URL { + const result = checkSapExternalUrlSafety(rawUrl, label) + if (!result.ok) throw new Error(result.message) + return result.url +} + +export const sapS4HanaOperationInputSchema = z + .object({ + deploymentType: sapDeploymentTypeSchema.default('cloud_public'), + authType: sapAuthTypeSchema.default('oauth_client_credentials'), + subdomain: sapSubdomainSchema.optional(), + region: z + .string() + .regex(/^[a-z]{2,4}\d{1,3}$/i, 'region must be an SAP BTP region code (e.g., eu10, us30)') + .optional(), + baseUrl: z.string().optional(), + tokenUrl: z.string().optional(), + clientId: z.string().optional(), + clientSecret: z.string().optional(), + username: z.string().optional(), + password: z.string().optional(), + service: sapServiceNameSchema, + path: sapServicePathSchema, + method: sapHttpMethodSchema.default('GET'), + query: z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])).optional(), + body: z.unknown().optional(), + ifMatch: z.string().optional(), + }) + .superRefine((input, context) => { + if (input.deploymentType === 'cloud_public') { + if (!input.subdomain) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ['subdomain'], + message: 'subdomain is required for cloud_public deployment', + }) + } + if (!input.region) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ['region'], + message: 'region is required for cloud_public deployment', + }) + } + if (input.authType !== 'oauth_client_credentials') { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ['authType'], + message: 'cloud_public deployment only supports oauth_client_credentials', + }) + } + if (!input.clientId) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ['clientId'], + message: 'clientId is required', + }) + } + if (!input.clientSecret) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ['clientSecret'], + message: 'clientSecret is required', + }) + } + return + } + + if (!input.baseUrl) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ['baseUrl'], + message: 'baseUrl is required for cloud_private and on_premise deployments', + }) + } else { + const baseUrlCheck = checkSapExternalUrlSafety(input.baseUrl, 'baseUrl') + if (!baseUrlCheck.ok) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ['baseUrl'], + message: baseUrlCheck.message, + }) + } + } + + if (input.authType === 'oauth_client_credentials') { + if (!input.tokenUrl) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ['tokenUrl'], + message: 'tokenUrl is required for OAuth on cloud_private/on_premise', + }) + } else { + const tokenUrlCheck = checkSapExternalUrlSafety(input.tokenUrl, 'tokenUrl') + if (!tokenUrlCheck.ok) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ['tokenUrl'], + message: tokenUrlCheck.message, + }) + } + } + if (!input.clientId) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ['clientId'], + message: 'clientId is required for OAuth', + }) + } + if (!input.clientSecret) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ['clientSecret'], + message: 'clientSecret is required for OAuth', + }) + } + return + } + + if (!input.username) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ['username'], + message: 'username is required for Basic auth', + }) + } + if (!input.password) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ['password'], + message: 'password is required for Basic auth', + }) + } + }) + +export type SapS4HanaOperationInput = z.output diff --git a/apps/sim/lib/internal/search/execute-tool.ts b/apps/sim/lib/internal/search/execute-tool.ts new file mode 100644 index 00000000000..63e4d3ef696 --- /dev/null +++ b/apps/sim/lib/internal/search/execute-tool.ts @@ -0,0 +1,37 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { z } from 'zod' +import { executeSearchOperation } from '@/lib/internal/search/operations' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const searchInputSchema = z.object({ query: z.string().min(1) }) + +export const executeSearchTool: InternalToolOperationHandler = async ({ + toolId, + input, + context, + signal, +}) => { + signal?.throwIfAborted() + if (toolId !== 'search_tool') { + return Response.json({ error: `Unsupported Search tool: ${toolId}` }, { status: 500 }) + } + if (!context.userId) { + return Response.json({ success: false, error: 'Unauthorized' }, { status: 401 }) + } + + const parsed = searchInputSchema.safeParse(input) + if (!parsed.success) { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + + try { + return Response.json(await executeSearchOperation(parsed.data, signal)) + } catch (error) { + signal?.throwIfAborted() + const message = getErrorMessage(error, 'Search failed') + return Response.json( + { success: false, error: message }, + { status: message === 'Search service not configured' ? 503 : 500 } + ) + } +} diff --git a/apps/sim/lib/internal/search/operations.test.ts b/apps/sim/lib/internal/search/operations.test.ts new file mode 100644 index 00000000000..69067abbf8d --- /dev/null +++ b/apps/sim/lib/internal/search/operations.test.ts @@ -0,0 +1,60 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockExecuteTool } = vi.hoisted(() => ({ mockExecuteTool: vi.fn() })) + +vi.mock('@/lib/core/config/env', () => ({ env: { EXA_API_KEY: 'exa-key' } })) +vi.mock('@/tools', () => ({ executeTool: mockExecuteTool })) + +import { executeSearchOperation } from '@/lib/internal/search/operations' + +describe('executeSearchOperation', () => { + beforeEach(() => vi.clearAllMocks()) + + it('projects the external Exa result into the canonical search output', async () => { + mockExecuteTool.mockResolvedValue({ + success: true, + output: { + results: [ + { + title: 'Result', + url: 'https://example.com', + highlights: ['First', 'Second'], + publishedDate: '2026-08-27', + }, + ], + }, + }) + + const output = await executeSearchOperation({ query: 'test query' }) + + expect(mockExecuteTool).toHaveBeenCalledWith( + 'exa_search', + expect.objectContaining({ query: 'test query', apiKey: 'exa-key' }), + { signal: undefined } + ) + expect(output).toMatchObject({ + query: 'test query', + totalResults: 1, + source: 'exa', + results: [ + { + title: 'Result', + link: 'https://example.com', + snippet: 'First ... Second', + date: '2026-08-27', + position: 1, + }, + ], + }) + }) + + it('propagates external search failures without retrying the composite operation', async () => { + mockExecuteTool.mockResolvedValue({ success: false, error: 'provider failed' }) + + await expect(executeSearchOperation({ query: 'test query' })).rejects.toThrow('provider failed') + expect(mockExecuteTool).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/sim/lib/internal/search/operations.ts b/apps/sim/lib/internal/search/operations.ts new file mode 100644 index 00000000000..a3612526f5c --- /dev/null +++ b/apps/sim/lib/internal/search/operations.ts @@ -0,0 +1,74 @@ +import { isRecordLike } from '@sim/utils/object' +import { SEARCH_TOOL_COST } from '@/lib/billing/constants' +import { env } from '@/lib/core/config/env' +import type { SearchResponse } from '@/tools/search/types' + +interface SearchOperationInput { + query: string +} + +function optionalString(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined +} + +export async function executeSearchOperation( + input: SearchOperationInput, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const exaApiKey = env.EXA_API_KEY + if (!exaApiKey) throw new Error('Search service not configured') + + const { executeTool } = await import('@/tools') + const result = await executeTool( + 'exa_search', + { + query: input.query, + type: 'auto', + useAutoprompt: true, + highlights: true, + apiKey: exaApiKey, + }, + { signal } + ) + signal?.throwIfAborted() + if (!result.success) throw new Error(result.error || 'Search failed') + + const rawResults = + isRecordLike(result.output) && Array.isArray(result.output.results) ? result.output.results : [] + const results = rawResults.map((rawResult, index) => { + const resultRecord = isRecordLike(rawResult) ? rawResult : {} + const highlights = Array.isArray(resultRecord.highlights) + ? resultRecord.highlights.filter((value): value is string => typeof value === 'string') + : [] + return { + title: optionalString(resultRecord.title) ?? '', + link: optionalString(resultRecord.url) ?? '', + snippet: highlights.join(' ... '), + ...(optionalString(resultRecord.publishedDate) + ? { date: optionalString(resultRecord.publishedDate) } + : {}), + position: index + 1, + } + }) + + return { + results, + query: input.query, + totalResults: results.length, + source: 'exa', + cost: { + input: 0, + output: 0, + total: SEARCH_TOOL_COST, + tokens: { input: 0, output: 0, total: 0 }, + model: 'search-exa', + pricing: { + input: 0, + cachedInput: 0, + output: 0, + updatedAt: new Date().toISOString(), + }, + }, + } +} diff --git a/apps/sim/lib/internal/secrets-manager/client.ts b/apps/sim/lib/internal/secrets-manager/client.ts new file mode 100644 index 00000000000..7e3808dc776 --- /dev/null +++ b/apps/sim/lib/internal/secrets-manager/client.ts @@ -0,0 +1,293 @@ +import type { RotationRulesType, SecretListEntry, Tag } from '@aws-sdk/client-secrets-manager' +import { + CreateSecretCommand, + DeleteSecretCommand, + DescribeSecretCommand, + GetSecretValueCommand, + ListSecretsCommand, + RestoreSecretCommand, + RotateSecretCommand, + SecretsManagerClient, + TagResourceCommand, + UntagResourceCommand, + UpdateSecretCommand, +} from '@aws-sdk/client-secrets-manager' + +interface SecretsManagerConnectionConfig { + region: string + accessKeyId: string + secretAccessKey: string +} + +function mapRotationRules(rules: RotationRulesType | undefined) { + if (!rules) return null + return { + automaticallyAfterDays: rules.AutomaticallyAfterDays ?? null, + duration: rules.Duration ?? null, + scheduleExpression: rules.ScheduleExpression ?? null, + } +} + +export function createSecretsManagerClient( + config: SecretsManagerConnectionConfig +): SecretsManagerClient { + return new SecretsManagerClient({ + region: config.region, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + }) +} + +export async function getSecretValue( + client: SecretsManagerClient, + secretId: string, + versionId?: string | null, + versionStage?: string | null, + signal?: AbortSignal +) { + const command = new GetSecretValueCommand({ + SecretId: secretId, + ...(versionId ? { VersionId: versionId } : {}), + ...(versionStage ? { VersionStage: versionStage } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + + if (!response.SecretString && response.SecretBinary) { + throw new Error( + 'Secret is stored as binary (SecretBinary). This integration only supports string secrets.' + ) + } + + return { + name: response.Name ?? '', + secretValue: response.SecretString ?? '', + arn: response.ARN ?? '', + versionId: response.VersionId ?? '', + versionStages: response.VersionStages ?? [], + createdDate: response.CreatedDate?.toISOString() ?? null, + } +} + +export async function listSecrets( + client: SecretsManagerClient, + maxResults?: number | null, + nextToken?: string | null, + signal?: AbortSignal +) { + const command = new ListSecretsCommand({ + ...(maxResults ? { MaxResults: maxResults } : {}), + ...(nextToken ? { NextToken: nextToken } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + const secrets = (response.SecretList ?? []).map((secret: SecretListEntry) => ({ + name: secret.Name ?? '', + arn: secret.ARN ?? '', + description: secret.Description ?? null, + createdDate: secret.CreatedDate?.toISOString() ?? null, + lastChangedDate: secret.LastChangedDate?.toISOString() ?? null, + lastAccessedDate: secret.LastAccessedDate?.toISOString() ?? null, + rotationEnabled: secret.RotationEnabled ?? false, + tags: secret.Tags?.map((tag: Tag) => ({ key: tag.Key ?? '', value: tag.Value ?? '' })) ?? [], + rotationRules: mapRotationRules(secret.RotationRules), + lastRotatedDate: secret.LastRotatedDate?.toISOString() ?? null, + nextRotationDate: secret.NextRotationDate?.toISOString() ?? null, + deletedDate: secret.DeletedDate?.toISOString() ?? null, + secretVersionsToStages: secret.SecretVersionsToStages ?? null, + })) + + return { + secrets, + nextToken: response.NextToken ?? null, + count: secrets.length, + } +} + +export async function createSecret( + client: SecretsManagerClient, + name: string, + secretValue: string, + description?: string | null, + signal?: AbortSignal +) { + const command = new CreateSecretCommand({ + Name: name, + SecretString: secretValue, + ...(description ? { Description: description } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + return { + name: response.Name ?? '', + arn: response.ARN ?? '', + versionId: response.VersionId ?? '', + } +} + +export async function updateSecretValue( + client: SecretsManagerClient, + secretId: string, + secretValue: string, + description?: string | null, + signal?: AbortSignal +) { + const command = new UpdateSecretCommand({ + SecretId: secretId, + SecretString: secretValue, + ...(description ? { Description: description } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + return { + name: response.Name ?? '', + arn: response.ARN ?? '', + versionId: response.VersionId ?? '', + } +} + +export async function deleteSecret( + client: SecretsManagerClient, + secretId: string, + recoveryWindowInDays?: number | null, + forceDelete?: boolean | null, + signal?: AbortSignal +) { + const command = new DeleteSecretCommand({ + SecretId: secretId, + ...(forceDelete ? { ForceDeleteWithoutRecovery: true } : {}), + ...(!forceDelete && recoveryWindowInDays ? { RecoveryWindowInDays: recoveryWindowInDays } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + return { + name: response.Name ?? '', + arn: response.ARN ?? '', + deletionDate: response.DeletionDate?.toISOString() ?? null, + } +} + +export async function describeSecret( + client: SecretsManagerClient, + secretId: string, + signal?: AbortSignal +) { + const command = new DescribeSecretCommand({ SecretId: secretId }) + const response = await client.send(command, { abortSignal: signal }) + + return { + name: response.Name ?? '', + arn: response.ARN ?? '', + description: response.Description ?? null, + kmsKeyId: response.KmsKeyId ?? null, + rotationEnabled: response.RotationEnabled ?? false, + rotationLambdaARN: response.RotationLambdaARN ?? null, + rotationRules: mapRotationRules(response.RotationRules), + lastRotatedDate: response.LastRotatedDate?.toISOString() ?? null, + lastChangedDate: response.LastChangedDate?.toISOString() ?? null, + lastAccessedDate: response.LastAccessedDate?.toISOString() ?? null, + deletedDate: response.DeletedDate?.toISOString() ?? null, + nextRotationDate: response.NextRotationDate?.toISOString() ?? null, + tags: response.Tags?.map((tag: Tag) => ({ key: tag.Key ?? '', value: tag.Value ?? '' })) ?? [], + versionIdsToStages: response.VersionIdsToStages ?? null, + owningService: response.OwningService ?? null, + createdDate: response.CreatedDate?.toISOString() ?? null, + primaryRegion: response.PrimaryRegion ?? null, + replicationStatus: + response.ReplicationStatus?.map((replication) => ({ + region: replication.Region ?? '', + kmsKeyId: replication.KmsKeyId ?? null, + status: replication.Status ?? null, + statusMessage: replication.StatusMessage ?? null, + lastAccessedDate: replication.LastAccessedDate?.toISOString() ?? null, + })) ?? [], + } +} + +export async function tagResource( + client: SecretsManagerClient, + secretId: string, + tags: Tag[], + signal?: AbortSignal +) { + const command = new TagResourceCommand({ SecretId: secretId, Tags: tags }) + await client.send(command, { abortSignal: signal }) + return { name: secretId } +} + +export async function untagResource( + client: SecretsManagerClient, + secretId: string, + tagKeys: string[], + signal?: AbortSignal +) { + const command = new UntagResourceCommand({ SecretId: secretId, TagKeys: tagKeys }) + await client.send(command, { abortSignal: signal }) + return { name: secretId } +} + +export async function restoreSecret( + client: SecretsManagerClient, + secretId: string, + signal?: AbortSignal +) { + const command = new RestoreSecretCommand({ SecretId: secretId }) + const response = await client.send(command, { abortSignal: signal }) + return { + name: response.Name ?? '', + arn: response.ARN ?? '', + } +} + +interface RotationRulesInput { + automaticallyAfterDays?: number | null + duration?: string | null + scheduleExpression?: string | null +} + +export async function rotateSecret( + client: SecretsManagerClient, + secretId: string, + clientRequestToken?: string | null, + rotationLambdaARN?: string | null, + rotationRules?: RotationRulesInput | null, + rotateImmediately?: boolean | null, + signal?: AbortSignal +) { + const hasRotationRules = Boolean( + rotationRules?.automaticallyAfterDays || + rotationRules?.duration || + rotationRules?.scheduleExpression + ) + + const command = new RotateSecretCommand({ + SecretId: secretId, + ...(clientRequestToken ? { ClientRequestToken: clientRequestToken } : {}), + ...(rotationLambdaARN ? { RotationLambdaARN: rotationLambdaARN } : {}), + ...(hasRotationRules + ? { + RotationRules: { + ...(rotationRules?.automaticallyAfterDays + ? { AutomaticallyAfterDays: rotationRules.automaticallyAfterDays } + : {}), + ...(rotationRules?.duration ? { Duration: rotationRules.duration } : {}), + ...(rotationRules?.scheduleExpression + ? { ScheduleExpression: rotationRules.scheduleExpression } + : {}), + }, + } + : {}), + ...(rotateImmediately === undefined || rotateImmediately === null + ? {} + : { RotateImmediately: rotateImmediately }), + }) + + const response = await client.send(command, { abortSignal: signal }) + return { + name: response.Name ?? '', + arn: response.ARN ?? '', + versionId: response.VersionId ?? '', + } +} diff --git a/apps/sim/lib/internal/secrets-manager/execute-tool.test.ts b/apps/sim/lib/internal/secrets-manager/execute-tool.test.ts new file mode 100644 index 00000000000..fcac06d0ac7 --- /dev/null +++ b/apps/sim/lib/internal/secrets-manager/execute-tool.test.ts @@ -0,0 +1,152 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mockOperations = vi.hoisted(() => ({ + executeSecretsManagerGetSecret: vi.fn(), + executeSecretsManagerListSecrets: vi.fn(), + executeSecretsManagerCreateSecret: vi.fn(), + executeSecretsManagerUpdateSecret: vi.fn(), + executeSecretsManagerDeleteSecret: vi.fn(), + executeSecretsManagerDescribeSecret: vi.fn(), + executeSecretsManagerTagResource: vi.fn(), + executeSecretsManagerUntagResource: vi.fn(), + executeSecretsManagerRestoreSecret: vi.fn(), + executeSecretsManagerRotateSecret: vi.fn(), +})) + +vi.mock('@/lib/internal/secrets-manager/operations', () => mockOperations) + +import { executeSecretsManagerTool } from '@/lib/internal/secrets-manager/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const CONNECTION = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', +} + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'secrets_manager_list_secrets', + input: CONNECTION, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + userId: 'user-1', + metadata: {}, + }, + requestId: 'request-1', + ...overrides, + } +} + +const TOOL_CASES = [ + { + toolId: 'secrets_manager_get_secret', + input: { ...CONNECTION, secretId: 'secret-1' }, + operation: mockOperations.executeSecretsManagerGetSecret, + }, + { + toolId: 'secrets_manager_list_secrets', + input: CONNECTION, + operation: mockOperations.executeSecretsManagerListSecrets, + }, + { + toolId: 'secrets_manager_create_secret', + input: { ...CONNECTION, name: 'secret-1', secretValue: 'value' }, + operation: mockOperations.executeSecretsManagerCreateSecret, + }, + { + toolId: 'secrets_manager_update_secret', + input: { ...CONNECTION, secretId: 'secret-1', secretValue: 'value' }, + operation: mockOperations.executeSecretsManagerUpdateSecret, + }, + { + toolId: 'secrets_manager_delete_secret', + input: { ...CONNECTION, secretId: 'secret-1' }, + operation: mockOperations.executeSecretsManagerDeleteSecret, + }, + { + toolId: 'secrets_manager_describe_secret', + input: { ...CONNECTION, secretId: 'secret-1' }, + operation: mockOperations.executeSecretsManagerDescribeSecret, + }, + { + toolId: 'secrets_manager_tag_resource', + input: { ...CONNECTION, secretId: 'secret-1', tags: [{ key: 'team', value: 'platform' }] }, + operation: mockOperations.executeSecretsManagerTagResource, + }, + { + toolId: 'secrets_manager_untag_resource', + input: { ...CONNECTION, secretId: 'secret-1', tagKeys: ['team'] }, + operation: mockOperations.executeSecretsManagerUntagResource, + }, + { + toolId: 'secrets_manager_restore_secret', + input: { ...CONNECTION, secretId: 'secret-1' }, + operation: mockOperations.executeSecretsManagerRestoreSecret, + }, + { + toolId: 'secrets_manager_rotate_secret', + input: { ...CONNECTION, secretId: 'secret-1', automaticallyAfterDays: 30 }, + operation: mockOperations.executeSecretsManagerRotateSecret, + }, +] as const + +describe('executeSecretsManagerTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it.each(TOOL_CASES)('validates and dispatches $toolId', async ({ toolId, input, operation }) => { + const controller = new AbortController() + operation.mockResolvedValue({ toolId }) + + const response = await executeSecretsManagerTool( + createRequest({ toolId, input, signal: controller.signal }) + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ toolId }) + expect(operation).toHaveBeenCalledWith(input, controller.signal) + }) + + it('returns the route-compatible validation envelope before provider work', async () => { + const response = await executeSecretsManagerTool(createRequest({ input: { region: '' } })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(mockOperations.executeSecretsManagerListSecrets).not.toHaveBeenCalled() + }) + + it('preserves the provider error envelope', async () => { + mockOperations.executeSecretsManagerListSecrets.mockRejectedValue( + new Error('AWS rejected credentials') + ) + + const response = await executeSecretsManagerTool(createRequest()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'Failed to list secrets: AWS rejected credentials', + }) + }) + + it('propagates cancellation without starting provider work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeSecretsManagerTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mockOperations.executeSecretsManagerListSecrets).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/secrets-manager/execute-tool.ts b/apps/sim/lib/internal/secrets-manager/execute-tool.ts new file mode 100644 index 00000000000..4fc606b3608 --- /dev/null +++ b/apps/sim/lib/internal/secrets-manager/execute-tool.ts @@ -0,0 +1,145 @@ +import { getErrorMessage } from '@sim/utils/errors' +import type { AnyApiRouteContract, ContractBody } from '@/lib/api/contracts' +import { awsSecretsManagerCreateSecretContract } from '@/lib/api/contracts/tools/aws/secrets-manager-create-secret' +import { awsSecretsManagerDeleteSecretContract } from '@/lib/api/contracts/tools/aws/secrets-manager-delete-secret' +import { awsSecretsManagerDescribeSecretContract } from '@/lib/api/contracts/tools/aws/secrets-manager-describe-secret' +import { awsSecretsManagerGetSecretContract } from '@/lib/api/contracts/tools/aws/secrets-manager-get-secret' +import { awsSecretsManagerListSecretsContract } from '@/lib/api/contracts/tools/aws/secrets-manager-list-secrets' +import { awsSecretsManagerRestoreSecretContract } from '@/lib/api/contracts/tools/aws/secrets-manager-restore-secret' +import { awsSecretsManagerRotateSecretContract } from '@/lib/api/contracts/tools/aws/secrets-manager-rotate-secret' +import { awsSecretsManagerTagResourceContract } from '@/lib/api/contracts/tools/aws/secrets-manager-tag-resource' +import { awsSecretsManagerUntagResourceContract } from '@/lib/api/contracts/tools/aws/secrets-manager-untag-resource' +import { awsSecretsManagerUpdateSecretContract } from '@/lib/api/contracts/tools/aws/secrets-manager-update-secret' +import { + executeSecretsManagerCreateSecret, + executeSecretsManagerDeleteSecret, + executeSecretsManagerDescribeSecret, + executeSecretsManagerGetSecret, + executeSecretsManagerListSecrets, + executeSecretsManagerRestoreSecret, + executeSecretsManagerRotateSecret, + executeSecretsManagerTagResource, + executeSecretsManagerUntagResource, + executeSecretsManagerUpdateSecret, +} from '@/lib/internal/secrets-manager/operations' +import { parseInternalToolInput } from '@/lib/internal/tool-operations/parse-input' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +async function executeOperation( + contract: C, + input: unknown, + execute: (input: ContractBody, signal?: AbortSignal) => Promise, + errorMessage: string, + signal?: AbortSignal +): Promise { + const parsed = parseInternalToolInput(contract, input) + if (!parsed.success) return parsed.response + + try { + const result = await execute(parsed.data, signal) + signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + signal?.throwIfAborted() + return Response.json( + { error: `${errorMessage}: ${getErrorMessage(error, 'Unknown error occurred')}` }, + { status: 500 } + ) + } +} + +export const executeSecretsManagerTool: InternalToolOperationHandler = async ({ + toolId, + input, + signal, +}) => { + signal?.throwIfAborted() + + switch (toolId) { + case 'secrets_manager_get_secret': + return executeOperation( + awsSecretsManagerGetSecretContract, + input, + executeSecretsManagerGetSecret, + 'Failed to retrieve secret', + signal + ) + case 'secrets_manager_list_secrets': + return executeOperation( + awsSecretsManagerListSecretsContract, + input, + executeSecretsManagerListSecrets, + 'Failed to list secrets', + signal + ) + case 'secrets_manager_create_secret': + return executeOperation( + awsSecretsManagerCreateSecretContract, + input, + executeSecretsManagerCreateSecret, + 'Failed to create secret', + signal + ) + case 'secrets_manager_update_secret': + return executeOperation( + awsSecretsManagerUpdateSecretContract, + input, + executeSecretsManagerUpdateSecret, + 'Failed to update secret', + signal + ) + case 'secrets_manager_delete_secret': + return executeOperation( + awsSecretsManagerDeleteSecretContract, + input, + executeSecretsManagerDeleteSecret, + 'Failed to delete secret', + signal + ) + case 'secrets_manager_describe_secret': + return executeOperation( + awsSecretsManagerDescribeSecretContract, + input, + executeSecretsManagerDescribeSecret, + 'Failed to describe secret', + signal + ) + case 'secrets_manager_tag_resource': + return executeOperation( + awsSecretsManagerTagResourceContract, + input, + executeSecretsManagerTagResource, + 'Failed to tag secret', + signal + ) + case 'secrets_manager_untag_resource': + return executeOperation( + awsSecretsManagerUntagResourceContract, + input, + executeSecretsManagerUntagResource, + 'Failed to untag secret', + signal + ) + case 'secrets_manager_restore_secret': + return executeOperation( + awsSecretsManagerRestoreSecretContract, + input, + executeSecretsManagerRestoreSecret, + 'Failed to restore secret', + signal + ) + case 'secrets_manager_rotate_secret': + return executeOperation( + awsSecretsManagerRotateSecretContract, + input, + executeSecretsManagerRotateSecret, + 'Failed to rotate secret', + signal + ) + default: + return Response.json( + { error: `Unsupported Secrets Manager tool: ${toolId}` }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/secrets-manager/operations.test.ts b/apps/sim/lib/internal/secrets-manager/operations.test.ts new file mode 100644 index 00000000000..9d9c7f7bdb9 --- /dev/null +++ b/apps/sim/lib/internal/secrets-manager/operations.test.ts @@ -0,0 +1,63 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockCreateSecretsManagerClient, mockDestroy, mockListSecrets } = vi.hoisted(() => ({ + mockCreateSecretsManagerClient: vi.fn(), + mockDestroy: vi.fn(), + mockListSecrets: vi.fn(), +})) + +vi.mock('@/lib/internal/secrets-manager/client', () => ({ + createSecret: vi.fn(), + createSecretsManagerClient: mockCreateSecretsManagerClient, + deleteSecret: vi.fn(), + describeSecret: vi.fn(), + getSecretValue: vi.fn(), + listSecrets: mockListSecrets, + restoreSecret: vi.fn(), + rotateSecret: vi.fn(), + tagResource: vi.fn(), + untagResource: vi.fn(), + updateSecretValue: vi.fn(), +})) + +import { executeSecretsManagerListSecrets } from '@/lib/internal/secrets-manager/operations' + +const INPUT = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', + maxResults: 10, + nextToken: 'next-token', +} + +describe('Secrets Manager operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCreateSecretsManagerClient.mockReturnValue({ destroy: mockDestroy }) + }) + + it('forwards cancellation and destroys the AWS client after success', async () => { + const controller = new AbortController() + const result = { secrets: [], nextToken: null, count: 0 } + mockListSecrets.mockResolvedValue(result) + + await expect(executeSecretsManagerListSecrets(INPUT, controller.signal)).resolves.toBe(result) + expect(mockListSecrets).toHaveBeenCalledWith( + { destroy: mockDestroy }, + 10, + 'next-token', + controller.signal + ) + expect(mockDestroy).toHaveBeenCalledOnce() + }) + + it('destroys the AWS client when provider execution fails', async () => { + mockListSecrets.mockRejectedValue(new Error('provider failure')) + + await expect(executeSecretsManagerListSecrets(INPUT)).rejects.toThrow('provider failure') + expect(mockDestroy).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/internal/secrets-manager/operations.ts b/apps/sim/lib/internal/secrets-manager/operations.ts new file mode 100644 index 00000000000..fc428db3d0f --- /dev/null +++ b/apps/sim/lib/internal/secrets-manager/operations.ts @@ -0,0 +1,186 @@ +import type { AwsSecretsManagerCreateSecretBody } from '@/lib/api/contracts/tools/aws/secrets-manager-create-secret' +import type { AwsSecretsManagerDeleteSecretBody } from '@/lib/api/contracts/tools/aws/secrets-manager-delete-secret' +import type { AwsSecretsManagerDescribeSecretBody } from '@/lib/api/contracts/tools/aws/secrets-manager-describe-secret' +import type { AwsSecretsManagerGetSecretBody } from '@/lib/api/contracts/tools/aws/secrets-manager-get-secret' +import type { AwsSecretsManagerListSecretsBody } from '@/lib/api/contracts/tools/aws/secrets-manager-list-secrets' +import type { AwsSecretsManagerRestoreSecretBody } from '@/lib/api/contracts/tools/aws/secrets-manager-restore-secret' +import type { AwsSecretsManagerRotateSecretBody } from '@/lib/api/contracts/tools/aws/secrets-manager-rotate-secret' +import type { AwsSecretsManagerTagResourceBody } from '@/lib/api/contracts/tools/aws/secrets-manager-tag-resource' +import type { AwsSecretsManagerUntagResourceBody } from '@/lib/api/contracts/tools/aws/secrets-manager-untag-resource' +import type { AwsSecretsManagerUpdateSecretBody } from '@/lib/api/contracts/tools/aws/secrets-manager-update-secret' +import { + createSecret, + createSecretsManagerClient, + deleteSecret, + describeSecret, + getSecretValue, + listSecrets, + restoreSecret, + rotateSecret, + tagResource, + untagResource, + updateSecretValue, +} from '@/lib/internal/secrets-manager/client' + +export async function executeSecretsManagerGetSecret( + input: AwsSecretsManagerGetSecretBody, + signal?: AbortSignal +) { + const client = createSecretsManagerClient(input) + try { + return await getSecretValue(client, input.secretId, input.versionId, input.versionStage, signal) + } finally { + client.destroy() + } +} + +export async function executeSecretsManagerListSecrets( + input: AwsSecretsManagerListSecretsBody, + signal?: AbortSignal +) { + const client = createSecretsManagerClient(input) + try { + return await listSecrets(client, input.maxResults, input.nextToken, signal) + } finally { + client.destroy() + } +} + +export async function executeSecretsManagerCreateSecret( + input: AwsSecretsManagerCreateSecretBody, + signal?: AbortSignal +) { + const client = createSecretsManagerClient(input) + try { + const result = await createSecret( + client, + input.name, + input.secretValue, + input.description, + signal + ) + return { message: `Secret "${result.name}" created successfully`, ...result } + } finally { + client.destroy() + } +} + +export async function executeSecretsManagerUpdateSecret( + input: AwsSecretsManagerUpdateSecretBody, + signal?: AbortSignal +) { + const client = createSecretsManagerClient(input) + try { + const result = await updateSecretValue( + client, + input.secretId, + input.secretValue, + input.description, + signal + ) + return { message: `Secret "${result.name}" updated successfully`, ...result } + } finally { + client.destroy() + } +} + +export async function executeSecretsManagerDeleteSecret( + input: AwsSecretsManagerDeleteSecretBody, + signal?: AbortSignal +) { + const client = createSecretsManagerClient(input) + try { + const result = await deleteSecret( + client, + input.secretId, + input.recoveryWindowInDays, + input.forceDelete, + signal + ) + const action = input.forceDelete ? 'permanently deleted' : 'scheduled for deletion' + return { message: `Secret "${result.name}" ${action}`, ...result } + } finally { + client.destroy() + } +} + +export async function executeSecretsManagerDescribeSecret( + input: AwsSecretsManagerDescribeSecretBody, + signal?: AbortSignal +) { + const client = createSecretsManagerClient(input) + try { + return await describeSecret(client, input.secretId, signal) + } finally { + client.destroy() + } +} + +export async function executeSecretsManagerTagResource( + input: AwsSecretsManagerTagResourceBody, + signal?: AbortSignal +) { + const client = createSecretsManagerClient(input) + try { + const result = await tagResource( + client, + input.secretId, + input.tags.map((tag) => ({ Key: tag.key, Value: tag.value })), + signal + ) + return { message: `Secret "${result.name}" tagged successfully`, ...result } + } finally { + client.destroy() + } +} + +export async function executeSecretsManagerUntagResource( + input: AwsSecretsManagerUntagResourceBody, + signal?: AbortSignal +) { + const client = createSecretsManagerClient(input) + try { + const result = await untagResource(client, input.secretId, input.tagKeys, signal) + return { message: `Secret "${result.name}" untagged successfully`, ...result } + } finally { + client.destroy() + } +} + +export async function executeSecretsManagerRestoreSecret( + input: AwsSecretsManagerRestoreSecretBody, + signal?: AbortSignal +) { + const client = createSecretsManagerClient(input) + try { + const result = await restoreSecret(client, input.secretId, signal) + return { message: `Secret "${result.name}" restored successfully`, ...result } + } finally { + client.destroy() + } +} + +export async function executeSecretsManagerRotateSecret( + input: AwsSecretsManagerRotateSecretBody, + signal?: AbortSignal +) { + const client = createSecretsManagerClient(input) + try { + const result = await rotateSecret( + client, + input.secretId, + input.clientRequestToken, + input.rotationLambdaARN, + { + automaticallyAfterDays: input.automaticallyAfterDays, + duration: input.duration, + scheduleExpression: input.scheduleExpression, + }, + input.rotateImmediately, + signal + ) + return { message: `Rotation started for secret "${result.name}"`, ...result } + } finally { + client.destroy() + } +} diff --git a/apps/sim/lib/internal/sendgrid/client.ts b/apps/sim/lib/internal/sendgrid/client.ts new file mode 100644 index 00000000000..75610b27c0b --- /dev/null +++ b/apps/sim/lib/internal/sendgrid/client.ts @@ -0,0 +1,51 @@ +import { isRecordLike } from '@sim/utils/object' +import { consumeOrCancelBody, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' +import { SendGridOperationError } from '@/lib/internal/sendgrid/errors' + +const MAX_SENDGRID_ERROR_BYTES = 64 * 1024 + +function record(value: unknown): Record { + return isRecordLike(value) ? value : {} +} + +function errorMessage(value: unknown): string { + const root = record(value) + const errors = Array.isArray(root.errors) ? root.errors : [] + const first = record(errors[0]) + if (typeof first.message === 'string') return first.message + if (typeof root.message === 'string') return root.message + return 'Failed to send email' +} + +export async function sendSendGridMail( + apiKey: string, + body: Record, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const response = await fetch('https://api.sendgrid.com/v3/mail/send', { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(body), + signal, + }) + signal?.throwIfAborted() + if (!response.ok) { + const error = await readResponseJsonWithLimit(response, { + maxBytes: MAX_SENDGRID_ERROR_BYTES, + label: 'SendGrid error response', + signal, + }).catch(() => { + signal?.throwIfAborted() + return {} + }) + signal?.throwIfAborted() + throw new SendGridOperationError(errorMessage(error), response.status) + } + await consumeOrCancelBody(response) + signal?.throwIfAborted() + return response.headers.get('X-Message-Id') || undefined +} diff --git a/apps/sim/lib/internal/sendgrid/errors.ts b/apps/sim/lib/internal/sendgrid/errors.ts new file mode 100644 index 00000000000..ac07bf9b230 --- /dev/null +++ b/apps/sim/lib/internal/sendgrid/errors.ts @@ -0,0 +1,10 @@ +export class SendGridOperationError extends Error { + constructor( + message: string, + readonly status: number, + readonly body: Record = { success: false, error: message } + ) { + super(message) + this.name = 'SendGridOperationError' + } +} diff --git a/apps/sim/lib/internal/sendgrid/execute-tool.ts b/apps/sim/lib/internal/sendgrid/execute-tool.ts new file mode 100644 index 00000000000..87d88f7f95e --- /dev/null +++ b/apps/sim/lib/internal/sendgrid/execute-tool.ts @@ -0,0 +1,62 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { SendGridOperationError } from '@/lib/internal/sendgrid/errors' +import { executeSendGridSend } from '@/lib/internal/sendgrid/operations' +import { sendGridSendInputSchema } from '@/lib/internal/sendgrid/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const logger = createLogger('SendGridToolExecution') + +export const executeSendGridTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'sendgrid_send_mail') { + return Response.json( + { success: false, error: `Unsupported SendGrid tool: ${request.toolId}` }, + { status: 500 } + ) + } + if (!request.context.userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + let serializedInput: string + try { + serializedInput = JSON.stringify(request.input) ?? '' + } catch { + return Response.json({ error: 'Validation error', details: [] }, { status: 400 }) + } + if (Buffer.byteLength(serializedInput, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + const parsed = sendGridSendInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { error: 'Validation error', details: parsed.error.issues }, + { status: 400 } + ) + } + + try { + const result = await executeSendGridSend(parsed.data, { + headers: request.headers, + requestId: request.requestId, + signal: request.signal, + userId: request.context.userId, + }) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof SendGridOperationError) { + return Response.json(error.body, { status: error.status }) + } + const message = getErrorMessage(error, 'Unknown error') + logger.error('SendGrid send failed', { error: message, requestId: request.requestId }) + return Response.json({ success: false, error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/sendgrid/operations.test.ts b/apps/sim/lib/internal/sendgrid/operations.test.ts new file mode 100644 index 00000000000..25cfdd2b5cc --- /dev/null +++ b/apps/sim/lib/internal/sendgrid/operations.test.ts @@ -0,0 +1,98 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { PRIVATE_MODEL_INPUT_PROVENANCE_HEADER } from '@/lib/execution/model-input-provenance' +import { + RESOLVED_SECRET_PROVENANCE_FIELD, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, +} from '@/lib/execution/private-tool-metadata' + +const mocks = vi.hoisted(() => ({ materialize: vi.fn(), send: vi.fn() })) +vi.mock('@/lib/internal/mail/attachment-materialization', async () => ({ + MailAttachmentMaterializationError: class extends Error {}, + materializeAuthorizedMailAttachments: mocks.materialize, +})) +vi.mock('@/lib/internal/sendgrid/client', () => ({ sendSendGridMail: mocks.send })) + +import { executeSendGridSend } from '@/lib/internal/sendgrid/operations' + +const context = { headers: new Headers(), requestId: 'request-1', userId: 'user-1' } + +describe('SendGrid operation', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.send.mockResolvedValue('message-1') + mocks.materialize.mockResolvedValue([ + { name: 'a.txt', contentType: 'text/plain', buffer: Buffer.from('abc') }, + ]) + }) + + it('builds template personalizations and authorized attachments exactly', async () => { + const attachment = { key: 'workspace/ws-1/a.txt', name: 'a.txt', size: 3 } + await expect( + executeSendGridSend( + { + apiKey: 'secret', + from: 'from@example.com', + fromName: 'From', + to: 'to@example.com', + toName: 'To', + subject: null, + templateId: 'template-1', + dynamicTemplateData: '{"name":"Ada"}', + cc: 'cc@example.com', + attachments: [attachment], + }, + context + ) + ).resolves.toMatchObject({ output: { messageId: 'message-1', to: 'to@example.com' } }) + expect(mocks.materialize).toHaveBeenCalledWith([attachment], context, { + label: 'Total attachment size', + maxTotalBytes: 30 * 1024 * 1024, + }) + expect(mocks.send).toHaveBeenCalledWith( + 'secret', + expect.objectContaining({ + personalizations: [ + expect.objectContaining({ + to: [{ email: 'to@example.com', name: 'To' }], + cc: [{ email: 'cc@example.com' }], + dynamic_template_data: { name: 'Ada' }, + }), + ], + template_id: 'template-1', + attachments: [ + { + content: 'YWJj', + filename: 'a.txt', + type: 'text/plain', + disposition: 'attachment', + }, + ], + }), + undefined + ) + }) + + it('fails closed on incomplete attachment provenance', async () => { + const headers = new Headers({ + [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1, + }) + await expect( + executeSendGridSend( + { + apiKey: 'secret', + from: 'from@example.com', + to: 'to@example.com', + [RESOLVED_SECRET_PROVENANCE_FIELD]: { version: 1, complete: false, entries: [] }, + }, + { ...context, headers } + ) + ).rejects.toMatchObject({ + status: 400, + body: { success: false, error: 'Model input provenance is unavailable' }, + }) + expect(mocks.send).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/sendgrid/operations.ts b/apps/sim/lib/internal/sendgrid/operations.ts new file mode 100644 index 00000000000..2db21669dd6 --- /dev/null +++ b/apps/sim/lib/internal/sendgrid/operations.ts @@ -0,0 +1,108 @@ +import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' +import { + MailAttachmentMaterializationError, + materializeAuthorizedMailAttachments, +} from '@/lib/internal/mail/attachment-materialization' +import { sendSendGridMail } from '@/lib/internal/sendgrid/client' +import { SendGridOperationError } from '@/lib/internal/sendgrid/errors' +import type { SendGridSendInput } from '@/lib/internal/sendgrid/schema' + +const MAX_ATTACHMENT_TOTAL_BYTES = 30 * 1024 * 1024 + +export interface SendGridOperationContext { + headers: Headers + requestId: string + signal?: AbortSignal + userId: string +} + +function sizeError(observedBytes: number): SendGridOperationError { + const sizeMB = (observedBytes / (1024 * 1024)).toFixed(2) + return new SendGridOperationError( + `Total attachment size (${sizeMB}MB) exceeds SendGrid's limit of 30MB`, + 400 + ) +} + +function attachmentError(error: MailAttachmentMaterializationError): SendGridOperationError { + if (error.kind === 'size') { + return sizeError(error.observedBytes ?? MAX_ATTACHMENT_TOTAL_BYTES) + } + return new SendGridOperationError(error.message, error.status, error.body) +} + +export async function executeSendGridSend( + input: SendGridSendInput, + context: SendGridOperationContext +) { + context.signal?.throwIfAborted() + const provenance = validateOpaqueModelInputProvenance({ + headers: context.headers, + payload: input, + isInternalRequest: true, + }) + if (!provenance.success) { + throw new SendGridOperationError(provenance.error, provenance.status) + } + + const personalization: Record = { + to: [{ email: input.to, ...(input.toName && { name: input.toName }) }], + } + if (input.cc) personalization.cc = [{ email: input.cc }] + if (input.bcc) personalization.bcc = [{ email: input.bcc }] + if (input.templateId && input.dynamicTemplateData) { + personalization.dynamic_template_data = + typeof input.dynamicTemplateData === 'string' + ? JSON.parse(input.dynamicTemplateData) + : input.dynamicTemplateData + } + + const body: Record = { + personalizations: [personalization], + from: { email: input.from, ...(input.fromName && { name: input.fromName }) }, + subject: input.subject, + } + if (input.templateId) { + body.template_id = input.templateId + } else { + body.content = [{ type: input.contentType || 'text/plain', value: input.content }] + } + if (input.replyTo) { + body.reply_to = { + email: input.replyTo, + ...(input.replyToName && { name: input.replyToName }), + } + } + + if (input.attachments?.length) { + try { + const attachments = await materializeAuthorizedMailAttachments(input.attachments, context, { + label: 'Total attachment size', + maxTotalBytes: MAX_ATTACHMENT_TOTAL_BYTES, + }) + if (attachments.length > 0) { + body.attachments = attachments.map((file) => ({ + content: file.buffer.toString('base64'), + filename: file.name, + type: file.contentType, + disposition: 'attachment', + })) + } + } catch (error) { + context.signal?.throwIfAborted() + if (error instanceof MailAttachmentMaterializationError) throw attachmentError(error) + throw error + } + } + + const messageId = await sendSendGridMail(input.apiKey, body, context.signal) + return { + success: true, + output: { + success: true, + messageId, + to: input.to, + subject: input.subject || '', + }, + } +} diff --git a/apps/sim/lib/internal/sendgrid/schema.ts b/apps/sim/lib/internal/sendgrid/schema.ts new file mode 100644 index 00000000000..ff42e97a84c --- /dev/null +++ b/apps/sim/lib/internal/sendgrid/schema.ts @@ -0,0 +1,25 @@ +import { z } from 'zod' +import { resolvedSecretTraceProvenanceSchema } from '@/lib/api/contracts/primitives' +import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' + +export const sendGridSendInputSchema = z.object({ + apiKey: z.string().min(1, 'API key is required'), + from: z.string().min(1, 'From email is required'), + fromName: z.string().optional().nullable(), + to: z.string().min(1, 'To email is required'), + toName: z.string().optional().nullable(), + subject: z.string().optional().nullable(), + content: z.string().optional().nullable(), + contentType: z.string().optional().nullable(), + cc: z.string().optional().nullable(), + bcc: z.string().optional().nullable(), + replyTo: z.string().optional().nullable(), + replyToName: z.string().optional().nullable(), + templateId: z.string().optional().nullable(), + dynamicTemplateData: z.unknown().optional().nullable(), + attachments: RawFileInputArraySchema.optional().nullable(), + [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), +}) + +export type SendGridSendInput = z.output diff --git a/apps/sim/lib/internal/servicenow/client.ts b/apps/sim/lib/internal/servicenow/client.ts new file mode 100644 index 00000000000..e880dc9ee7a --- /dev/null +++ b/apps/sim/lib/internal/servicenow/client.ts @@ -0,0 +1,56 @@ +import { isRecordLike } from '@sim/utils/object' +import { + MAX_JSON_API_RESPONSE_BYTES, + secureFetchWithValidation, +} from '@/lib/core/security/input-validation.server' +import { ServiceNowOperationError } from '@/lib/internal/servicenow/errors' +import type { ServiceNowAttachment } from '@/tools/servicenow/types' +import { createBasicAuthHeader } from '@/tools/servicenow/utils' + +export async function uploadServiceNowAttachment( + input: { + contentType: string + fileName: string + instanceUrl: string + password: string + recordSysId: string + tableName: string + username: string + }, + buffer: Buffer, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const baseUrl = input.instanceUrl.trim().replace(/\/$/, '') + const params = new URLSearchParams({ + table_name: input.tableName.trim(), + table_sys_id: input.recordSysId.trim(), + file_name: input.fileName, + }) + const response = await secureFetchWithValidation( + `${baseUrl}/api/now/attachment/file?${params.toString()}`, + { + method: 'POST', + headers: { + Authorization: createBasicAuthHeader(input.username, input.password), + 'Content-Type': input.contentType, + Accept: 'application/json', + }, + body: buffer, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + signal, + }, + 'instanceUrl' + ) + const data = await response.json().catch(() => null) + if (!response.ok) { + const error = isRecordLike(data) && isRecordLike(data.error) ? data.error : null + const message = + error && typeof error.message === 'string' + ? error.message + : `ServiceNow API error: ${response.status} ${response.statusText}` + throw new ServiceNowOperationError(message, response.status) + } + if (!isRecordLike(data) || !isRecordLike(data.result)) return null + return data.result as ServiceNowAttachment +} diff --git a/apps/sim/lib/internal/servicenow/errors.ts b/apps/sim/lib/internal/servicenow/errors.ts new file mode 100644 index 00000000000..a8b10f0a53f --- /dev/null +++ b/apps/sim/lib/internal/servicenow/errors.ts @@ -0,0 +1,10 @@ +export class ServiceNowOperationError extends Error { + constructor( + message: string, + readonly status: number, + readonly body: Record = { success: false, error: message } + ) { + super(message) + this.name = 'ServiceNowOperationError' + } +} diff --git a/apps/sim/lib/internal/servicenow/execute-tool.ts b/apps/sim/lib/internal/servicenow/execute-tool.ts new file mode 100644 index 00000000000..0dc21a5c6e9 --- /dev/null +++ b/apps/sim/lib/internal/servicenow/execute-tool.ts @@ -0,0 +1,70 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { ServiceNowOperationError } from '@/lib/internal/servicenow/errors' +import { executeServiceNowUploadAttachment } from '@/lib/internal/servicenow/operations' +import { serviceNowUploadAttachmentInputSchema } from '@/lib/internal/servicenow/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const logger = createLogger('ServiceNowToolExecution') + +function inputSizeError(input: unknown): Response | null { + let serialized: string + try { + serialized = JSON.stringify(input) ?? '' + } catch { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + return Buffer.byteLength(serialized) > DEFAULT_MAX_JSON_BODY_BYTES + ? Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + : null +} + +export const executeServiceNowTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'servicenow_upload_attachment') { + return Response.json( + { success: false, error: `Unsupported ServiceNow tool: ${request.toolId}` }, + { status: 500 } + ) + } + const userId = request.context.userId + if (!userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + const sizeError = inputSizeError(request.input) + if (sizeError) return sizeError + const parsed = serviceNowUploadAttachmentInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { success: false, error: getValidationErrorMessage(parsed.error, 'Invalid request data') }, + { status: 400 } + ) + } + try { + const result = await executeServiceNowUploadAttachment(parsed.data, { + requestId: request.requestId, + signal: request.signal, + userId, + }) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof ServiceNowOperationError) { + return Response.json(error.body, { status: error.status }) + } + const message = getErrorMessage(error, 'Internal server error') + logger.error('ServiceNow attachment upload failed', { + error: message, + requestId: request.requestId, + }) + return Response.json({ success: false, error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/servicenow/operations.ts b/apps/sim/lib/internal/servicenow/operations.ts new file mode 100644 index 00000000000..98bd57c83f8 --- /dev/null +++ b/apps/sim/lib/internal/servicenow/operations.ts @@ -0,0 +1,93 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { isRecordLike } from '@sim/utils/object' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { uploadServiceNowAttachment } from '@/lib/internal/servicenow/client' +import { ServiceNowOperationError } from '@/lib/internal/servicenow/errors' +import type { ServiceNowUploadAttachmentInput } from '@/lib/internal/servicenow/schema' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/doc-not-ready' +import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +const logger = createLogger('ServiceNowOperations') + +export interface ServiceNowOperationContext { + requestId: string + signal?: AbortSignal + userId: string +} + +async function deniedBody(response: Response): Promise> { + const body: unknown = await response.json().catch(() => null) + return isRecordLike(body) ? body : { success: false, error: 'File not found' } +} + +export async function executeServiceNowUploadAttachment( + input: ServiceNowUploadAttachmentInput, + context: ServiceNowOperationContext +) { + context.signal?.throwIfAborted() + if (!input.file) throw new ServiceNowOperationError('A file is required', 400) + let userFile + try { + userFile = processSingleFileToUserFile(input.file, context.requestId, logger) + } catch (error) { + throw new ServiceNowOperationError(getErrorMessage(error, 'Failed to process file'), 400) + } + const denied = await assertToolFileAccess(userFile.key, context.userId, context.requestId, logger) + context.signal?.throwIfAborted() + if (denied) { + throw new ServiceNowOperationError('File not found', denied.status, await deniedBody(denied)) + } + + let buffer: Buffer + let resolvedContentType: string + try { + const downloaded = await downloadServableFileFromStorage(userFile, context.requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + signal: context.signal, + }) + buffer = downloaded.buffer + resolvedContentType = downloaded.contentType + } catch (error) { + context.signal?.throwIfAborted() + if (isDocNotReadyError(error)) throw new ServiceNowOperationError(docNotReadyMessage(), 409) + throw new ServiceNowOperationError( + getErrorMessage(error, 'Failed to download file'), + isPayloadSizeLimitError(error) ? 413 : 500 + ) + } + const result = await uploadServiceNowAttachment( + { + contentType: resolvedContentType || userFile.type || 'application/octet-stream', + fileName: input.fileName, + instanceUrl: input.instanceUrl, + password: input.password, + recordSysId: input.recordSysId, + tableName: input.tableName, + username: input.username, + }, + buffer, + context.signal + ) + context.signal?.throwIfAborted() + return { + success: true, + output: { + attachment: result + ? { + sys_id: result.sys_id ?? null, + file_name: result.file_name ?? null, + content_type: result.content_type ?? null, + size_bytes: result.size_bytes ?? null, + table_name: result.table_name ?? null, + table_sys_id: result.table_sys_id ?? null, + download_link: result.download_link ?? null, + } + : null, + metadata: { recordCount: 1 }, + }, + } +} diff --git a/apps/sim/lib/internal/servicenow/schema.ts b/apps/sim/lib/internal/servicenow/schema.ts new file mode 100644 index 00000000000..16b6a555ae4 --- /dev/null +++ b/apps/sim/lib/internal/servicenow/schema.ts @@ -0,0 +1,14 @@ +import { z } from 'zod' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' + +export const serviceNowUploadAttachmentInputSchema = z.object({ + instanceUrl: z.string().min(1, 'Instance URL is required'), + username: z.string().min(1, 'Username is required'), + password: z.string().min(1, 'Password is required'), + tableName: z.string().min(1, 'Table name is required'), + recordSysId: z.string().min(1, 'Record sys_id is required'), + fileName: z.string().min(1, 'File name is required'), + file: RawFileInputSchema.optional().nullable(), +}) + +export type ServiceNowUploadAttachmentInput = z.output diff --git a/apps/sim/lib/internal/ses/client.ts b/apps/sim/lib/internal/ses/client.ts new file mode 100644 index 00000000000..8ab4c75bdfa --- /dev/null +++ b/apps/sim/lib/internal/ses/client.ts @@ -0,0 +1,595 @@ +import { + CreateConfigurationSetCommand, + CreateEmailIdentityCommand, + CreateEmailTemplateCommand, + DeleteEmailIdentityCommand, + DeleteEmailTemplateCommand, + DeleteSuppressedDestinationCommand, + GetAccountCommand, + GetEmailIdentityCommand, + GetEmailTemplateCommand, + GetSuppressedDestinationCommand, + ListEmailIdentitiesCommand, + ListEmailTemplatesCommand, + ListSuppressedDestinationsCommand, + PutSuppressedDestinationCommand, + SESv2Client, + SendBulkEmailCommand, + SendCustomVerificationEmailCommand, + SendEmailCommand, + type SuppressionListReason, + type TlsPolicy, + UpdateEmailTemplateCommand, +} from '@aws-sdk/client-sesv2' +import { z } from 'zod' +import type { SESConnectionConfig } from '@/tools/ses/types' + +const SesBulkEmailDestinationSchema = z.object({ + toAddresses: z.array(z.string().email()), + templateData: z.string().optional(), +}) + +type SesBulkEmailDestination = z.infer + +export function createSESClient(config: SESConnectionConfig): SESv2Client { + return new SESv2Client({ + region: config.region, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + }) +} + +export async function sendEmail( + client: SESv2Client, + params: { + fromAddress: string + toAddresses: string[] + subject: string + bodyText?: string | null + bodyHtml?: string | null + ccAddresses?: string[] | null + bccAddresses?: string[] | null + replyToAddresses?: string[] | null + configurationSetName?: string | null + }, + signal?: AbortSignal +) { + const command = new SendEmailCommand({ + FromEmailAddress: params.fromAddress, + Destination: { + ToAddresses: params.toAddresses, + ...(params.ccAddresses?.length ? { CcAddresses: params.ccAddresses } : {}), + ...(params.bccAddresses?.length ? { BccAddresses: params.bccAddresses } : {}), + }, + Content: { + Simple: { + Subject: { Data: params.subject }, + Body: { + ...(params.bodyText ? { Text: { Data: params.bodyText } } : {}), + ...(params.bodyHtml ? { Html: { Data: params.bodyHtml } } : {}), + }, + }, + }, + ...(params.replyToAddresses?.length ? { ReplyToAddresses: params.replyToAddresses } : {}), + ...(params.configurationSetName ? { ConfigurationSetName: params.configurationSetName } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + + return { + messageId: response.MessageId ?? '', + } +} + +export async function sendTemplatedEmail( + client: SESv2Client, + params: { + fromAddress: string + toAddresses: string[] + templateName: string + templateData: string + ccAddresses?: string[] | null + bccAddresses?: string[] | null + configurationSetName?: string | null + }, + signal?: AbortSignal +) { + const command = new SendEmailCommand({ + FromEmailAddress: params.fromAddress, + Destination: { + ToAddresses: params.toAddresses, + ...(params.ccAddresses?.length ? { CcAddresses: params.ccAddresses } : {}), + ...(params.bccAddresses?.length ? { BccAddresses: params.bccAddresses } : {}), + }, + Content: { + Template: { + TemplateName: params.templateName, + TemplateData: params.templateData, + }, + }, + ...(params.configurationSetName ? { ConfigurationSetName: params.configurationSetName } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + + return { + messageId: response.MessageId ?? '', + } +} + +export function parseBulkEmailDestinations(destinationsJson: string): SesBulkEmailDestination[] { + const destinations = JSON.parse(destinationsJson) + return z.array(SesBulkEmailDestinationSchema).parse(destinations) +} + +export async function sendBulkEmail( + client: SESv2Client, + params: { + fromAddress: string + templateName: string + destinations: SesBulkEmailDestination[] + defaultTemplateData?: string | null + configurationSetName?: string | null + }, + signal?: AbortSignal +) { + const command = new SendBulkEmailCommand({ + FromEmailAddress: params.fromAddress, + DefaultContent: { + Template: { + TemplateName: params.templateName, + ...(params.defaultTemplateData ? { TemplateData: params.defaultTemplateData } : {}), + }, + }, + BulkEmailEntries: params.destinations.map((dest) => ({ + Destination: { ToAddresses: dest.toAddresses }, + ...(dest.templateData + ? { + ReplacementEmailContent: { + ReplacementTemplate: { + ReplacementTemplateData: dest.templateData, + }, + }, + } + : {}), + })), + ...(params.configurationSetName ? { ConfigurationSetName: params.configurationSetName } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + + const results = (response.BulkEmailEntryResults ?? []).map((r) => ({ + messageId: r.MessageId ?? null, + status: r.Status ?? 'UNKNOWN', + error: r.Error ?? null, + })) + + const successCount = results.filter((r) => r.status === 'SUCCESS').length + const failureCount = results.length - successCount + + return { results, successCount, failureCount } +} + +export async function listIdentities( + client: SESv2Client, + params: { + pageSize?: number | null + nextToken?: string | null + }, + signal?: AbortSignal +) { + const command = new ListEmailIdentitiesCommand({ + ...(params.pageSize != null ? { PageSize: params.pageSize } : {}), + ...(params.nextToken ? { NextToken: params.nextToken } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + + const identities = (response.EmailIdentities ?? []).map((identity) => ({ + identityName: identity.IdentityName ?? '', + identityType: identity.IdentityType ?? '', + sendingEnabled: identity.SendingEnabled ?? false, + verificationStatus: identity.VerificationStatus ?? '', + })) + + return { + identities, + nextToken: response.NextToken ?? null, + count: identities.length, + } +} + +export async function getAccount(client: SESv2Client, signal?: AbortSignal) { + const command = new GetAccountCommand({}) + const response = await client.send(command, { abortSignal: signal }) + + return { + sendingEnabled: response.SendingEnabled ?? false, + max24HourSend: response.SendQuota?.Max24HourSend ?? 0, + maxSendRate: response.SendQuota?.MaxSendRate ?? 0, + sentLast24Hours: response.SendQuota?.SentLast24Hours ?? 0, + } +} + +export async function createTemplate( + client: SESv2Client, + params: { + templateName: string + subjectPart: string + textPart?: string | null + htmlPart?: string | null + }, + signal?: AbortSignal +) { + const command = new CreateEmailTemplateCommand({ + TemplateName: params.templateName, + TemplateContent: { + Subject: params.subjectPart, + ...(params.textPart ? { Text: params.textPart } : {}), + ...(params.htmlPart ? { Html: params.htmlPart } : {}), + }, + }) + + await client.send(command, { abortSignal: signal }) + + return { + message: `Template '${params.templateName}' created successfully`, + } +} + +export async function getTemplate(client: SESv2Client, templateName: string, signal?: AbortSignal) { + const command = new GetEmailTemplateCommand({ TemplateName: templateName }) + const response = await client.send(command, { abortSignal: signal }) + + return { + templateName: response.TemplateName ?? '', + subjectPart: response.TemplateContent?.Subject ?? '', + textPart: response.TemplateContent?.Text ?? null, + htmlPart: response.TemplateContent?.Html ?? null, + } +} + +export async function listTemplates( + client: SESv2Client, + params: { + pageSize?: number | null + nextToken?: string | null + }, + signal?: AbortSignal +) { + const command = new ListEmailTemplatesCommand({ + ...(params.pageSize != null ? { PageSize: params.pageSize } : {}), + ...(params.nextToken ? { NextToken: params.nextToken } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + + const templates = (response.TemplatesMetadata ?? []).map((t) => ({ + templateName: t.TemplateName ?? '', + createdTimestamp: t.CreatedTimestamp?.toISOString() ?? null, + })) + + return { + templates, + nextToken: response.NextToken ?? null, + count: templates.length, + } +} + +export async function deleteTemplate( + client: SESv2Client, + templateName: string, + signal?: AbortSignal +) { + const command = new DeleteEmailTemplateCommand({ TemplateName: templateName }) + await client.send(command, { abortSignal: signal }) + + return { + message: `Template '${templateName}' deleted successfully`, + } +} + +export async function updateTemplate( + client: SESv2Client, + params: { + templateName: string + subjectPart: string + textPart?: string | null + htmlPart?: string | null + }, + signal?: AbortSignal +) { + const command = new UpdateEmailTemplateCommand({ + TemplateName: params.templateName, + TemplateContent: { + Subject: params.subjectPart, + ...(params.textPart ? { Text: params.textPart } : {}), + ...(params.htmlPart ? { Html: params.htmlPart } : {}), + }, + }) + + await client.send(command, { abortSignal: signal }) + + return { + message: `Template '${params.templateName}' updated successfully`, + } +} + +export async function putSuppressedDestination( + client: SESv2Client, + params: { emailAddress: string; reason: SuppressionListReason }, + signal?: AbortSignal +) { + const command = new PutSuppressedDestinationCommand({ + EmailAddress: params.emailAddress, + Reason: params.reason, + }) + + await client.send(command, { abortSignal: signal }) + + return { + message: `Email address '${params.emailAddress}' added to the suppression list`, + } +} + +export async function deleteSuppressedDestination( + client: SESv2Client, + emailAddress: string, + signal?: AbortSignal +) { + const command = new DeleteSuppressedDestinationCommand({ EmailAddress: emailAddress }) + await client.send(command, { abortSignal: signal }) + + return { + message: `Email address '${emailAddress}' removed from the suppression list`, + } +} + +export async function getSuppressedDestination( + client: SESv2Client, + emailAddress: string, + signal?: AbortSignal +) { + const command = new GetSuppressedDestinationCommand({ EmailAddress: emailAddress }) + const response = await client.send(command, { abortSignal: signal }) + const destination = response.SuppressedDestination + + return { + emailAddress: destination?.EmailAddress ?? emailAddress, + reason: destination?.Reason ?? '', + lastUpdateTime: destination?.LastUpdateTime?.toISOString() ?? null, + messageId: destination?.Attributes?.MessageId ?? null, + feedbackId: destination?.Attributes?.FeedbackId ?? null, + } +} + +export async function listSuppressedDestinations( + client: SESv2Client, + params: { + reasons?: SuppressionListReason[] | null + startDate?: Date | null + endDate?: Date | null + pageSize?: number | null + nextToken?: string | null + }, + signal?: AbortSignal +) { + const command = new ListSuppressedDestinationsCommand({ + ...(params.reasons?.length ? { Reasons: params.reasons } : {}), + ...(params.startDate ? { StartDate: params.startDate } : {}), + ...(params.endDate ? { EndDate: params.endDate } : {}), + ...(params.pageSize != null ? { PageSize: params.pageSize } : {}), + ...(params.nextToken ? { NextToken: params.nextToken } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + + const destinations = (response.SuppressedDestinationSummaries ?? []).map((d) => ({ + emailAddress: d.EmailAddress ?? '', + reason: d.Reason ?? '', + lastUpdateTime: d.LastUpdateTime?.toISOString() ?? null, + })) + + return { + destinations, + nextToken: response.NextToken ?? null, + count: destinations.length, + } +} + +export async function createEmailIdentity( + client: SESv2Client, + params: { + emailIdentity: string + dkimSigningAttributes?: { + domainSigningSelector?: string + domainSigningPrivateKey?: string + nextSigningKeyLength?: 'RSA_1024_BIT' | 'RSA_2048_BIT' + } | null + tags?: Array<{ key: string; value: string }> | null + configurationSetName?: string | null + }, + signal?: AbortSignal +) { + const command = new CreateEmailIdentityCommand({ + EmailIdentity: params.emailIdentity, + ...(params.dkimSigningAttributes + ? { + DkimSigningAttributes: { + ...(params.dkimSigningAttributes.domainSigningSelector + ? { DomainSigningSelector: params.dkimSigningAttributes.domainSigningSelector } + : {}), + ...(params.dkimSigningAttributes.domainSigningPrivateKey + ? { DomainSigningPrivateKey: params.dkimSigningAttributes.domainSigningPrivateKey } + : {}), + ...(params.dkimSigningAttributes.nextSigningKeyLength + ? { NextSigningKeyLength: params.dkimSigningAttributes.nextSigningKeyLength } + : {}), + }, + } + : {}), + ...(params.tags?.length + ? { Tags: params.tags.map((t) => ({ Key: t.key, Value: t.value })) } + : {}), + ...(params.configurationSetName ? { ConfigurationSetName: params.configurationSetName } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + + return { + identityType: response.IdentityType ?? '', + verifiedForSendingStatus: response.VerifiedForSendingStatus ?? false, + dkimAttributes: response.DkimAttributes + ? { + signingEnabled: response.DkimAttributes.SigningEnabled ?? null, + status: response.DkimAttributes.Status ?? null, + tokens: response.DkimAttributes.Tokens ?? [], + signingAttributesOrigin: response.DkimAttributes.SigningAttributesOrigin ?? null, + nextSigningKeyLength: response.DkimAttributes.NextSigningKeyLength ?? null, + currentSigningKeyLength: response.DkimAttributes.CurrentSigningKeyLength ?? null, + lastKeyGenerationTimestamp: + response.DkimAttributes.LastKeyGenerationTimestamp?.toISOString() ?? null, + signingHostedZone: response.DkimAttributes.SigningHostedZone ?? null, + } + : null, + } +} + +export async function deleteEmailIdentity( + client: SESv2Client, + emailIdentity: string, + signal?: AbortSignal +) { + const command = new DeleteEmailIdentityCommand({ EmailIdentity: emailIdentity }) + await client.send(command, { abortSignal: signal }) + + return { + message: `Email identity '${emailIdentity}' deleted successfully`, + } +} + +export async function getEmailIdentity( + client: SESv2Client, + emailIdentity: string, + signal?: AbortSignal +) { + const command = new GetEmailIdentityCommand({ EmailIdentity: emailIdentity }) + const response = await client.send(command, { abortSignal: signal }) + + return { + identityType: response.IdentityType ?? '', + verifiedForSendingStatus: response.VerifiedForSendingStatus ?? false, + verificationStatus: response.VerificationStatus ?? null, + feedbackForwardingStatus: response.FeedbackForwardingStatus ?? null, + configurationSetName: response.ConfigurationSetName ?? null, + dkimAttributes: response.DkimAttributes + ? { + signingEnabled: response.DkimAttributes.SigningEnabled ?? null, + status: response.DkimAttributes.Status ?? null, + tokens: response.DkimAttributes.Tokens ?? [], + signingAttributesOrigin: response.DkimAttributes.SigningAttributesOrigin ?? null, + nextSigningKeyLength: response.DkimAttributes.NextSigningKeyLength ?? null, + currentSigningKeyLength: response.DkimAttributes.CurrentSigningKeyLength ?? null, + lastKeyGenerationTimestamp: + response.DkimAttributes.LastKeyGenerationTimestamp?.toISOString() ?? null, + signingHostedZone: response.DkimAttributes.SigningHostedZone ?? null, + } + : null, + mailFromAttributes: response.MailFromAttributes + ? { + mailFromDomain: response.MailFromAttributes.MailFromDomain ?? null, + mailFromDomainStatus: response.MailFromAttributes.MailFromDomainStatus ?? null, + behaviorOnMxFailure: response.MailFromAttributes.BehaviorOnMxFailure ?? null, + } + : null, + policies: response.Policies ?? null, + tags: (response.Tags ?? []).map((t) => ({ key: t.Key ?? '', value: t.Value ?? '' })), + verificationInfo: response.VerificationInfo + ? { + errorType: response.VerificationInfo.ErrorType ?? null, + lastCheckedTimestamp: + response.VerificationInfo.LastCheckedTimestamp?.toISOString() ?? null, + lastSuccessTimestamp: + response.VerificationInfo.LastSuccessTimestamp?.toISOString() ?? null, + } + : null, + } +} + +export async function createConfigurationSet( + client: SESv2Client, + params: { + configurationSetName: string + customRedirectDomain?: string | null + httpsPolicy?: 'REQUIRE' | 'REQUIRE_OPEN_ONLY' | 'OPTIONAL' | null + tlsPolicy?: TlsPolicy | null + sendingPoolName?: string | null + reputationMetricsEnabled?: boolean | null + sendingEnabled?: boolean | null + suppressedReasons?: SuppressionListReason[] | null + tags?: Array<{ key: string; value: string }> | null + }, + signal?: AbortSignal +) { + const command = new CreateConfigurationSetCommand({ + ConfigurationSetName: params.configurationSetName, + ...(params.customRedirectDomain + ? { + TrackingOptions: { + CustomRedirectDomain: params.customRedirectDomain, + ...(params.httpsPolicy ? { HttpsPolicy: params.httpsPolicy } : {}), + }, + } + : {}), + ...(params.tlsPolicy || params.sendingPoolName + ? { + DeliveryOptions: { + ...(params.tlsPolicy ? { TlsPolicy: params.tlsPolicy } : {}), + ...(params.sendingPoolName ? { SendingPoolName: params.sendingPoolName } : {}), + }, + } + : {}), + ...(params.reputationMetricsEnabled != null + ? { ReputationOptions: { ReputationMetricsEnabled: params.reputationMetricsEnabled } } + : {}), + ...(params.sendingEnabled != null + ? { SendingOptions: { SendingEnabled: params.sendingEnabled } } + : {}), + ...(params.suppressedReasons?.length + ? { SuppressionOptions: { SuppressedReasons: params.suppressedReasons } } + : {}), + ...(params.tags?.length + ? { Tags: params.tags.map((t) => ({ Key: t.key, Value: t.value })) } + : {}), + }) + + await client.send(command, { abortSignal: signal }) + + return { + message: `Configuration set '${params.configurationSetName}' created successfully`, + } +} + +export async function sendCustomVerificationEmail( + client: SESv2Client, + params: { + emailAddress: string + templateName: string + configurationSetName?: string | null + }, + signal?: AbortSignal +) { + const command = new SendCustomVerificationEmailCommand({ + EmailAddress: params.emailAddress, + TemplateName: params.templateName, + ...(params.configurationSetName ? { ConfigurationSetName: params.configurationSetName } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + + return { + messageId: response.MessageId ?? '', + } +} diff --git a/apps/sim/lib/internal/ses/execute-tool.test.ts b/apps/sim/lib/internal/ses/execute-tool.test.ts new file mode 100644 index 00000000000..04c3dd115de --- /dev/null +++ b/apps/sim/lib/internal/ses/execute-tool.test.ts @@ -0,0 +1,173 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operationMocks = vi.hoisted(() => { + class SesOperationInputError extends Error {} + + return { + SesOperationInputError, + executeSesCreateConfigurationSet: vi.fn(), + executeSesCreateEmailIdentity: vi.fn(), + executeSesCreateTemplate: vi.fn(), + executeSesDeleteEmailIdentity: vi.fn(), + executeSesDeleteSuppressedDestination: vi.fn(), + executeSesDeleteTemplate: vi.fn(), + executeSesGetAccount: vi.fn(), + executeSesGetEmailIdentity: vi.fn(), + executeSesGetSuppressedDestination: vi.fn(), + executeSesGetTemplate: vi.fn(), + executeSesListIdentities: vi.fn(), + executeSesListSuppressedDestinations: vi.fn(), + executeSesListTemplates: vi.fn(), + executeSesPutSuppressedDestination: vi.fn(), + executeSesSendBulkEmail: vi.fn(), + executeSesSendCustomVerificationEmail: vi.fn(), + executeSesSendEmail: vi.fn(), + executeSesSendTemplatedEmail: vi.fn(), + executeSesUpdateTemplate: vi.fn(), + } +}) + +vi.mock('@/lib/internal/ses/operations', () => operationMocks) + +import { executeSesTool } from '@/lib/internal/ses/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const VALID_BODY = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', +} as const + +const SUPPORTED_TOOL_IDS = [ + 'ses_create_configuration_set', + 'ses_create_email_identity', + 'ses_create_template', + 'ses_delete_email_identity', + 'ses_delete_suppressed_destination', + 'ses_delete_template', + 'ses_get_account', + 'ses_get_email_identity', + 'ses_get_suppressed_destination', + 'ses_get_template', + 'ses_list_identities', + 'ses_list_suppressed_destinations', + 'ses_list_templates', + 'ses_put_suppressed_destination', + 'ses_send_bulk_email', + 'ses_send_custom_verification_email', + 'ses_send_email', + 'ses_send_templated_email', + 'ses_update_template', +] as const + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'ses_get_account', + input: VALID_BODY, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeSesTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('validates and executes the matching SES operation with cancellation', async () => { + const controller = new AbortController() + operationMocks.executeSesGetAccount.mockResolvedValue({ + sendingEnabled: true, + max24HourSend: 1000, + maxSendRate: 10, + sentLast24Hours: 25, + }) + + const response = await executeSesTool(createRequest({ signal: controller.signal })) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + sendingEnabled: true, + max24HourSend: 1000, + maxSendRate: 10, + sentLast24Hours: 25, + }) + expect(operationMocks.executeSesGetAccount).toHaveBeenCalledWith(VALID_BODY, controller.signal) + }) + + it('returns the route-compatible contract validation envelope before provider work', async () => { + const response = await executeSesTool(createRequest({ input: { region: 'us-east-1' } })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(operationMocks.executeSesGetAccount).not.toHaveBeenCalled() + }) + + it.each(SUPPORTED_TOOL_IDS)('recognizes the canonical tool ID %s', async (toolId) => { + const response = await executeSesTool(createRequest({ toolId, input: {} })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ error: 'Invalid request data' }) + }) + + it('preserves the provider error envelope', async () => { + operationMocks.executeSesGetAccount.mockRejectedValue(new Error('AWS rejected credentials')) + + const response = await executeSesTool(createRequest()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'Failed to get account information: AWS rejected credentials', + }) + }) + + it('preserves operation-level 400 errors', async () => { + operationMocks.executeSesSendBulkEmail.mockRejectedValue( + new operationMocks.SesOperationInputError( + 'destinations must be a valid JSON array of destination objects' + ) + ) + + const response = await executeSesTool( + createRequest({ + toolId: 'ses_send_bulk_email', + input: { + ...VALID_BODY, + fromAddress: 'sender@example.com', + templateName: 'template', + destinations: '[]', + }, + }) + ) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ + error: 'destinations must be a valid JSON array of destination objects', + }) + }) + + it('propagates cancellation without converting it into a provider failure', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeSesTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(operationMocks.executeSesGetAccount).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/ses/execute-tool.ts b/apps/sim/lib/internal/ses/execute-tool.ts new file mode 100644 index 00000000000..bc4e491561d --- /dev/null +++ b/apps/sim/lib/internal/ses/execute-tool.ts @@ -0,0 +1,229 @@ +import { toError } from '@sim/utils/errors' +import type { AnyApiRouteContract, ContractBody } from '@/lib/api/contracts' +import { awsSesCreateConfigurationSetContract } from '@/lib/api/contracts/tools/aws/ses-create-configuration-set' +import { awsSesCreateEmailIdentityContract } from '@/lib/api/contracts/tools/aws/ses-create-email-identity' +import { awsSesCreateTemplateContract } from '@/lib/api/contracts/tools/aws/ses-create-template' +import { awsSesDeleteEmailIdentityContract } from '@/lib/api/contracts/tools/aws/ses-delete-email-identity' +import { awsSesDeleteSuppressedDestinationContract } from '@/lib/api/contracts/tools/aws/ses-delete-suppressed-destination' +import { awsSesDeleteTemplateContract } from '@/lib/api/contracts/tools/aws/ses-delete-template' +import { awsSesGetAccountContract } from '@/lib/api/contracts/tools/aws/ses-get-account' +import { awsSesGetEmailIdentityContract } from '@/lib/api/contracts/tools/aws/ses-get-email-identity' +import { awsSesGetSuppressedDestinationContract } from '@/lib/api/contracts/tools/aws/ses-get-suppressed-destination' +import { awsSesGetTemplateContract } from '@/lib/api/contracts/tools/aws/ses-get-template' +import { awsSesListIdentitiesContract } from '@/lib/api/contracts/tools/aws/ses-list-identities' +import { awsSesListSuppressedDestinationsContract } from '@/lib/api/contracts/tools/aws/ses-list-suppressed-destinations' +import { awsSesListTemplatesContract } from '@/lib/api/contracts/tools/aws/ses-list-templates' +import { awsSesPutSuppressedDestinationContract } from '@/lib/api/contracts/tools/aws/ses-put-suppressed-destination' +import { awsSesSendBulkEmailContract } from '@/lib/api/contracts/tools/aws/ses-send-bulk-email' +import { awsSesSendCustomVerificationEmailContract } from '@/lib/api/contracts/tools/aws/ses-send-custom-verification-email' +import { awsSesSendEmailContract } from '@/lib/api/contracts/tools/aws/ses-send-email' +import { awsSesSendTemplatedEmailContract } from '@/lib/api/contracts/tools/aws/ses-send-templated-email' +import { awsSesUpdateTemplateContract } from '@/lib/api/contracts/tools/aws/ses-update-template' +import { + executeSesCreateConfigurationSet, + executeSesCreateEmailIdentity, + executeSesCreateTemplate, + executeSesDeleteEmailIdentity, + executeSesDeleteSuppressedDestination, + executeSesDeleteTemplate, + executeSesGetAccount, + executeSesGetEmailIdentity, + executeSesGetSuppressedDestination, + executeSesGetTemplate, + executeSesListIdentities, + executeSesListSuppressedDestinations, + executeSesListTemplates, + executeSesPutSuppressedDestination, + executeSesSendBulkEmail, + executeSesSendCustomVerificationEmail, + executeSesSendEmail, + executeSesSendTemplatedEmail, + executeSesUpdateTemplate, + SesOperationInputError, +} from '@/lib/internal/ses/operations' +import { parseInternalToolInput } from '@/lib/internal/tool-operations/parse-input' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +async function executeOperation( + contract: C, + input: unknown, + execute: (input: ContractBody, signal?: AbortSignal) => Promise, + errorMessage: string, + signal?: AbortSignal +): Promise { + const parsed = parseInternalToolInput(contract, input) + if (!parsed.success) return parsed.response + + try { + const result = await execute(parsed.data, signal) + signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + signal?.throwIfAborted() + if (error instanceof SesOperationInputError) { + return Response.json({ error: error.message }, { status: 400 }) + } + return Response.json({ error: `${errorMessage}: ${toError(error).message}` }, { status: 500 }) + } +} + +export const executeSesTool: InternalToolOperationHandler = async ({ toolId, input, signal }) => { + signal?.throwIfAborted() + + switch (toolId) { + case 'ses_create_configuration_set': + return executeOperation( + awsSesCreateConfigurationSetContract, + input, + executeSesCreateConfigurationSet, + 'Failed to create configuration set', + signal + ) + case 'ses_create_email_identity': + return executeOperation( + awsSesCreateEmailIdentityContract, + input, + executeSesCreateEmailIdentity, + 'Failed to create email identity', + signal + ) + case 'ses_create_template': + return executeOperation( + awsSesCreateTemplateContract, + input, + executeSesCreateTemplate, + 'Failed to create template', + signal + ) + case 'ses_delete_email_identity': + return executeOperation( + awsSesDeleteEmailIdentityContract, + input, + executeSesDeleteEmailIdentity, + 'Failed to delete email identity', + signal + ) + case 'ses_delete_suppressed_destination': + return executeOperation( + awsSesDeleteSuppressedDestinationContract, + input, + executeSesDeleteSuppressedDestination, + 'Failed to remove suppressed destination', + signal + ) + case 'ses_delete_template': + return executeOperation( + awsSesDeleteTemplateContract, + input, + executeSesDeleteTemplate, + 'Failed to delete template', + signal + ) + case 'ses_get_account': + return executeOperation( + awsSesGetAccountContract, + input, + executeSesGetAccount, + 'Failed to get account information', + signal + ) + case 'ses_get_email_identity': + return executeOperation( + awsSesGetEmailIdentityContract, + input, + executeSesGetEmailIdentity, + 'Failed to get email identity', + signal + ) + case 'ses_get_suppressed_destination': + return executeOperation( + awsSesGetSuppressedDestinationContract, + input, + executeSesGetSuppressedDestination, + 'Failed to get suppressed destination', + signal + ) + case 'ses_get_template': + return executeOperation( + awsSesGetTemplateContract, + input, + executeSesGetTemplate, + 'Failed to get template', + signal + ) + case 'ses_list_identities': + return executeOperation( + awsSesListIdentitiesContract, + input, + executeSesListIdentities, + 'Failed to list identities', + signal + ) + case 'ses_list_suppressed_destinations': + return executeOperation( + awsSesListSuppressedDestinationsContract, + input, + executeSesListSuppressedDestinations, + 'Failed to list suppressed destinations', + signal + ) + case 'ses_list_templates': + return executeOperation( + awsSesListTemplatesContract, + input, + executeSesListTemplates, + 'Failed to list templates', + signal + ) + case 'ses_put_suppressed_destination': + return executeOperation( + awsSesPutSuppressedDestinationContract, + input, + executeSesPutSuppressedDestination, + 'Failed to add suppressed destination', + signal + ) + case 'ses_send_bulk_email': + return executeOperation( + awsSesSendBulkEmailContract, + input, + executeSesSendBulkEmail, + 'Failed to send bulk email', + signal + ) + case 'ses_send_custom_verification_email': + return executeOperation( + awsSesSendCustomVerificationEmailContract, + input, + executeSesSendCustomVerificationEmail, + 'Failed to send custom verification email', + signal + ) + case 'ses_send_email': + return executeOperation( + awsSesSendEmailContract, + input, + executeSesSendEmail, + 'Failed to send email', + signal + ) + case 'ses_send_templated_email': + return executeOperation( + awsSesSendTemplatedEmailContract, + input, + executeSesSendTemplatedEmail, + 'Failed to send templated email', + signal + ) + case 'ses_update_template': + return executeOperation( + awsSesUpdateTemplateContract, + input, + executeSesUpdateTemplate, + 'Failed to update template', + signal + ) + default: + return Response.json({ error: `Unsupported SES tool: ${toolId}` }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/ses/operations.test.ts b/apps/sim/lib/internal/ses/operations.test.ts new file mode 100644 index 00000000000..d896d5226b2 --- /dev/null +++ b/apps/sim/lib/internal/ses/operations.test.ts @@ -0,0 +1,107 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const clientMocks = vi.hoisted(() => ({ + createConfigurationSet: vi.fn(), + createEmailIdentity: vi.fn(), + createSESClient: vi.fn(), + createTemplate: vi.fn(), + deleteEmailIdentity: vi.fn(), + deleteSuppressedDestination: vi.fn(), + deleteTemplate: vi.fn(), + getAccount: vi.fn(), + getEmailIdentity: vi.fn(), + getSuppressedDestination: vi.fn(), + getTemplate: vi.fn(), + listIdentities: vi.fn(), + listSuppressedDestinations: vi.fn(), + listTemplates: vi.fn(), + parseBulkEmailDestinations: vi.fn(), + putSuppressedDestination: vi.fn(), + sendBulkEmail: vi.fn(), + sendCustomVerificationEmail: vi.fn(), + sendEmail: vi.fn(), + sendTemplatedEmail: vi.fn(), + updateTemplate: vi.fn(), +})) + +vi.mock('@/lib/internal/ses/client', () => clientMocks) + +import { + executeSesGetAccount, + executeSesListSuppressedDestinations, + executeSesSendBulkEmail, + SesOperationInputError, +} from '@/lib/internal/ses/operations' + +const CONNECTION = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', +} as const + +describe('SES operations', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('passes the abort signal to SES and destroys the client after success', async () => { + const controller = new AbortController() + const client = { destroy: vi.fn() } + clientMocks.createSESClient.mockReturnValue(client) + clientMocks.getAccount.mockResolvedValue({ + sendingEnabled: true, + max24HourSend: 1000, + maxSendRate: 10, + sentLast24Hours: 25, + }) + + await expect(executeSesGetAccount(CONNECTION, controller.signal)).resolves.toMatchObject({ + sendingEnabled: true, + }) + + expect(clientMocks.createSESClient).toHaveBeenCalledWith(CONNECTION) + expect(clientMocks.getAccount).toHaveBeenCalledWith(client, controller.signal) + expect(client.destroy).toHaveBeenCalledOnce() + }) + + it('destroys the SES client when the provider rejects', async () => { + const client = { destroy: vi.fn() } + clientMocks.createSESClient.mockReturnValue(client) + clientMocks.getAccount.mockRejectedValue(new Error('provider failed')) + + await expect(executeSesGetAccount(CONNECTION)).rejects.toThrow('provider failed') + expect(client.destroy).toHaveBeenCalledOnce() + }) + + it('rejects malformed bulk destinations before creating an SES client', async () => { + clientMocks.parseBulkEmailDestinations.mockImplementation(() => { + throw new Error('invalid JSON') + }) + + expect(() => + executeSesSendBulkEmail({ + ...CONNECTION, + fromAddress: 'sender@example.com', + templateName: 'template', + destinations: 'not-json', + }) + ).toThrow( + new SesOperationInputError('destinations must be a valid JSON array of destination objects') + ) + expect(clientMocks.createSESClient).not.toHaveBeenCalled() + }) + + it('rejects invalid suppression reasons before creating an SES client', async () => { + expect(() => + executeSesListSuppressedDestinations({ ...CONNECTION, reasons: 'BOUNCE,INVALID' }) + ).toThrow( + new SesOperationInputError( + 'Invalid suppression reason(s): INVALID. Must be one of: BOUNCE, COMPLAINT' + ) + ) + expect(clientMocks.createSESClient).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/ses/operations.ts b/apps/sim/lib/internal/ses/operations.ts new file mode 100644 index 00000000000..b12955770ac --- /dev/null +++ b/apps/sim/lib/internal/ses/operations.ts @@ -0,0 +1,334 @@ +import type { SESv2Client, SuppressionListReason } from '@aws-sdk/client-sesv2' +import type { AwsSesCreateConfigurationSetBody } from '@/lib/api/contracts/tools/aws/ses-create-configuration-set' +import type { AwsSesCreateEmailIdentityBody } from '@/lib/api/contracts/tools/aws/ses-create-email-identity' +import type { AwsSesCreateTemplateBody } from '@/lib/api/contracts/tools/aws/ses-create-template' +import type { AwsSesDeleteEmailIdentityBody } from '@/lib/api/contracts/tools/aws/ses-delete-email-identity' +import type { AwsSesDeleteSuppressedDestinationBody } from '@/lib/api/contracts/tools/aws/ses-delete-suppressed-destination' +import type { AwsSesDeleteTemplateBody } from '@/lib/api/contracts/tools/aws/ses-delete-template' +import type { AwsSesGetAccountBody } from '@/lib/api/contracts/tools/aws/ses-get-account' +import type { AwsSesGetEmailIdentityBody } from '@/lib/api/contracts/tools/aws/ses-get-email-identity' +import type { AwsSesGetSuppressedDestinationBody } from '@/lib/api/contracts/tools/aws/ses-get-suppressed-destination' +import type { AwsSesGetTemplateBody } from '@/lib/api/contracts/tools/aws/ses-get-template' +import type { AwsSesListIdentitiesBody } from '@/lib/api/contracts/tools/aws/ses-list-identities' +import type { AwsSesListSuppressedDestinationsBody } from '@/lib/api/contracts/tools/aws/ses-list-suppressed-destinations' +import type { AwsSesListTemplatesBody } from '@/lib/api/contracts/tools/aws/ses-list-templates' +import type { AwsSesPutSuppressedDestinationBody } from '@/lib/api/contracts/tools/aws/ses-put-suppressed-destination' +import type { AwsSesSendBulkEmailBody } from '@/lib/api/contracts/tools/aws/ses-send-bulk-email' +import type { AwsSesSendCustomVerificationEmailBody } from '@/lib/api/contracts/tools/aws/ses-send-custom-verification-email' +import type { AwsSesSendEmailBody } from '@/lib/api/contracts/tools/aws/ses-send-email' +import type { AwsSesSendTemplatedEmailBody } from '@/lib/api/contracts/tools/aws/ses-send-templated-email' +import type { AwsSesUpdateTemplateBody } from '@/lib/api/contracts/tools/aws/ses-update-template' +import { + createConfigurationSet, + createEmailIdentity, + createSESClient, + createTemplate, + deleteEmailIdentity, + deleteSuppressedDestination, + deleteTemplate, + getAccount, + getEmailIdentity, + getSuppressedDestination, + getTemplate, + listIdentities, + listSuppressedDestinations, + listTemplates, + parseBulkEmailDestinations, + putSuppressedDestination, + sendBulkEmail, + sendCustomVerificationEmail, + sendEmail, + sendTemplatedEmail, + updateTemplate, +} from '@/lib/internal/ses/client' +import type { SESConnectionConfig } from '@/tools/ses/types' + +const VALID_SUPPRESSION_REASONS: SuppressionListReason[] = ['BOUNCE', 'COMPLAINT'] + +export class SesOperationInputError extends Error {} + +async function withSesClient( + input: SESConnectionConfig, + execute: (client: SESv2Client) => Promise +): Promise { + const client = createSESClient(input) + try { + return await execute(client) + } finally { + client.destroy() + } +} + +function splitEmailAddresses(value?: string | null): string[] | null { + if (!value) return null + return value + .split(',') + .map((address) => address.trim()) + .filter(Boolean) +} + +export function executeSesSendEmail(input: AwsSesSendEmailBody, signal?: AbortSignal) { + return withSesClient(input, (client) => + sendEmail( + client, + { + fromAddress: input.fromAddress, + toAddresses: splitEmailAddresses(input.toAddresses) ?? [], + subject: input.subject, + bodyText: input.bodyText, + bodyHtml: input.bodyHtml, + ccAddresses: splitEmailAddresses(input.ccAddresses), + bccAddresses: splitEmailAddresses(input.bccAddresses), + replyToAddresses: splitEmailAddresses(input.replyToAddresses), + configurationSetName: input.configurationSetName, + }, + signal + ) + ) +} + +export function executeSesSendTemplatedEmail( + input: AwsSesSendTemplatedEmailBody, + signal?: AbortSignal +) { + return withSesClient(input, (client) => + sendTemplatedEmail( + client, + { + fromAddress: input.fromAddress, + toAddresses: splitEmailAddresses(input.toAddresses) ?? [], + templateName: input.templateName, + templateData: input.templateData, + ccAddresses: splitEmailAddresses(input.ccAddresses), + bccAddresses: splitEmailAddresses(input.bccAddresses), + configurationSetName: input.configurationSetName, + }, + signal + ) + ) +} + +export function executeSesSendBulkEmail(input: AwsSesSendBulkEmailBody, signal?: AbortSignal) { + let destinations: ReturnType + try { + destinations = parseBulkEmailDestinations(input.destinations) + } catch { + throw new SesOperationInputError( + 'destinations must be a valid JSON array of destination objects' + ) + } + + return withSesClient(input, (client) => + sendBulkEmail( + client, + { + fromAddress: input.fromAddress, + templateName: input.templateName, + destinations, + defaultTemplateData: input.defaultTemplateData, + configurationSetName: input.configurationSetName, + }, + signal + ) + ) +} + +export function executeSesListIdentities(input: AwsSesListIdentitiesBody, signal?: AbortSignal) { + return withSesClient(input, (client) => + listIdentities(client, { pageSize: input.pageSize, nextToken: input.nextToken }, signal) + ) +} + +export function executeSesGetAccount(input: AwsSesGetAccountBody, signal?: AbortSignal) { + return withSesClient(input, (client) => getAccount(client, signal)) +} + +export function executeSesCreateTemplate(input: AwsSesCreateTemplateBody, signal?: AbortSignal) { + return withSesClient(input, (client) => + createTemplate( + client, + { + templateName: input.templateName, + subjectPart: input.subjectPart, + textPart: input.textPart, + htmlPart: input.htmlPart, + }, + signal + ) + ) +} + +export function executeSesGetTemplate(input: AwsSesGetTemplateBody, signal?: AbortSignal) { + return withSesClient(input, (client) => getTemplate(client, input.templateName, signal)) +} + +export function executeSesListTemplates(input: AwsSesListTemplatesBody, signal?: AbortSignal) { + return withSesClient(input, (client) => + listTemplates(client, { pageSize: input.pageSize, nextToken: input.nextToken }, signal) + ) +} + +export function executeSesDeleteTemplate(input: AwsSesDeleteTemplateBody, signal?: AbortSignal) { + return withSesClient(input, (client) => deleteTemplate(client, input.templateName, signal)) +} + +export function executeSesUpdateTemplate(input: AwsSesUpdateTemplateBody, signal?: AbortSignal) { + return withSesClient(input, (client) => + updateTemplate( + client, + { + templateName: input.templateName, + subjectPart: input.subjectPart, + textPart: input.textPart, + htmlPart: input.htmlPart, + }, + signal + ) + ) +} + +export function executeSesPutSuppressedDestination( + input: AwsSesPutSuppressedDestinationBody, + signal?: AbortSignal +) { + return withSesClient(input, (client) => + putSuppressedDestination( + client, + { emailAddress: input.emailAddress, reason: input.reason }, + signal + ) + ) +} + +export function executeSesDeleteSuppressedDestination( + input: AwsSesDeleteSuppressedDestinationBody, + signal?: AbortSignal +) { + return withSesClient(input, (client) => + deleteSuppressedDestination(client, input.emailAddress, signal) + ) +} + +export function executeSesGetSuppressedDestination( + input: AwsSesGetSuppressedDestinationBody, + signal?: AbortSignal +) { + return withSesClient(input, (client) => + getSuppressedDestination(client, input.emailAddress, signal) + ) +} + +export function executeSesListSuppressedDestinations( + input: AwsSesListSuppressedDestinationsBody, + signal?: AbortSignal +) { + let reasons: SuppressionListReason[] | null = null + if (input.reasons) { + const candidates = input.reasons + .split(',') + .map((reason) => reason.trim()) + .filter(Boolean) + const invalid = candidates.filter( + (reason) => !VALID_SUPPRESSION_REASONS.includes(reason as SuppressionListReason) + ) + if (invalid.length > 0) { + throw new SesOperationInputError( + `Invalid suppression reason(s): ${invalid.join(', ')}. Must be one of: ${VALID_SUPPRESSION_REASONS.join(', ')}` + ) + } + reasons = candidates as SuppressionListReason[] + } + + return withSesClient(input, (client) => + listSuppressedDestinations( + client, + { + reasons, + startDate: input.startDate ? new Date(input.startDate) : null, + endDate: input.endDate ? new Date(input.endDate) : null, + pageSize: input.pageSize, + nextToken: input.nextToken, + }, + signal + ) + ) +} + +export function executeSesCreateEmailIdentity( + input: AwsSesCreateEmailIdentityBody, + signal?: AbortSignal +) { + return withSesClient(input, (client) => + createEmailIdentity( + client, + { + emailIdentity: input.emailIdentity, + dkimSigningAttributes: input.dkimSigningAttributes, + tags: input.tags, + configurationSetName: input.configurationSetName, + }, + signal + ) + ) +} + +export function executeSesDeleteEmailIdentity( + input: AwsSesDeleteEmailIdentityBody, + signal?: AbortSignal +) { + return withSesClient(input, (client) => deleteEmailIdentity(client, input.emailIdentity, signal)) +} + +export function executeSesGetEmailIdentity( + input: AwsSesGetEmailIdentityBody, + signal?: AbortSignal +) { + return withSesClient(input, (client) => getEmailIdentity(client, input.emailIdentity, signal)) +} + +export function executeSesCreateConfigurationSet( + input: AwsSesCreateConfigurationSetBody, + signal?: AbortSignal +) { + const suppressedReasons = input.suppressedReasons + ? (input.suppressedReasons + .split(',') + .map((reason) => reason.trim()) + .filter(Boolean) as SuppressionListReason[]) + : null + + return withSesClient(input, (client) => + createConfigurationSet( + client, + { + configurationSetName: input.configurationSetName, + customRedirectDomain: input.customRedirectDomain, + httpsPolicy: input.httpsPolicy, + tlsPolicy: input.tlsPolicy, + sendingPoolName: input.sendingPoolName, + reputationMetricsEnabled: input.reputationMetricsEnabled, + sendingEnabled: input.sendingEnabled, + suppressedReasons, + tags: input.tags, + }, + signal + ) + ) +} + +export function executeSesSendCustomVerificationEmail( + input: AwsSesSendCustomVerificationEmailBody, + signal?: AbortSignal +) { + return withSesClient(input, (client) => + sendCustomVerificationEmail( + client, + { + emailAddress: input.emailAddress, + templateName: input.templateName, + configurationSetName: input.configurationSetName, + }, + signal + ) + ) +} diff --git a/apps/sim/lib/internal/sftp/client.test.ts b/apps/sim/lib/internal/sftp/client.test.ts new file mode 100644 index 00000000000..83fc3c7f048 --- /dev/null +++ b/apps/sim/lib/internal/sftp/client.test.ts @@ -0,0 +1,187 @@ +/** + * @vitest-environment node + */ +import { createHash } from 'node:crypto' +import { Readable } from 'node:stream' +import type { ConnectConfig, SFTPWrapper } from 'ssh2' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' + +const mocks = vi.hoisted(() => ({ + connectConfigs: [] as ConnectConfig[], + clients: [] as Array<{ + emit: (event: string, value?: unknown) => void + destroy: ReturnType + }>, + emitReady: true, + validateHost: vi.fn(), +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + validateDatabaseHost: mocks.validateHost, +})) + +vi.mock('ssh2', () => ({ + Client: class { + private listeners = new Map void>>() + destroy = vi.fn(() => this.emit('close')) + end = vi.fn(() => this.emit('close')) + + constructor() { + mocks.clients.push(this) + } + + on(event: string, listener: (value?: unknown) => void) { + const listeners = this.listeners.get(event) ?? new Set() + listeners.add(listener) + this.listeners.set(event, listeners) + return this + } + + once(event: string, listener: (value?: unknown) => void) { + const onceListener = (value?: unknown) => { + this.off(event, onceListener) + listener(value) + } + return this.on(event, onceListener) + } + + off(event: string, listener: (value?: unknown) => void) { + this.listeners.get(event)?.delete(listener) + return this + } + + emit(event: string, value?: unknown) { + for (const listener of [...(this.listeners.get(event) ?? [])]) listener(value) + } + + connect(config: ConnectConfig) { + mocks.connectConfigs.push(config) + if (mocks.emitReady) this.emit('ready') + } + }, +})) + +import { + createSftpConnection, + MAX_SFTP_READ_BYTES, + readSftpFileCapped, + sanitizeFileName, +} from '@/lib/internal/sftp/client' + +function fakeSftp(chunkSize: number, chunkCount: number) { + let emitted = 0 + const stream = new Readable({ + read() { + if (emitted >= chunkCount) { + this.push(null) + return + } + emitted++ + this.push(Buffer.alloc(chunkSize, 0x41)) + }, + }) + const createReadStream = vi.fn(() => stream) + return { sftp: { createReadStream } as unknown as SFTPWrapper, stream, createReadStream } +} + +describe('SFTP client boundary', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.clients.length = 0 + mocks.connectConfigs.length = 0 + mocks.emitReady = true + mocks.validateHost.mockResolvedValue({ isValid: true, resolvedIP: '203.0.113.10' }) + }) + + it('pins the connection to the validated IP while preserving credentials', async () => { + await createSftpConnection({ + host: 'sftp.example.com', + port: 22, + username: 'user', + password: 'secret', + }) + + expect(mocks.connectConfigs[0]).toMatchObject({ + host: '203.0.113.10', + port: 22, + username: 'user', + password: 'secret', + }) + }) + + it('installs a SHA-256 host-key verifier and accepts the pinned key', async () => { + const hostKey = Buffer.from('trusted-host-key') + const fingerprint = createHash('sha256').update(hostKey).digest('base64').replace(/=+$/, '') + + await createSftpConnection({ + host: 'sftp.example.com', + port: 22, + username: 'user', + password: 'secret', + hostFingerprint: `SHA256:${fingerprint}`, + }) + + expect(mocks.connectConfigs[0].hostVerifier?.(hostKey)).toBe(true) + expect(mocks.connectConfigs[0].hostVerifier?.(Buffer.from('other-key'))).toBe(false) + }) + + it('destroys a connection and rejects when canceled during connect', async () => { + mocks.emitReady = false + const controller = new AbortController() + const connection = createSftpConnection({ + host: 'sftp.example.com', + port: 22, + username: 'user', + password: 'secret', + signal: controller.signal, + }) + await vi.waitFor(() => expect(mocks.clients).toHaveLength(1)) + + controller.abort(new Error('execution canceled')) + + await expect(connection).rejects.toThrow('execution canceled') + expect(mocks.clients[0].destroy).toHaveBeenCalledOnce() + }) + + it('resolves with the full contents when under the byte cap', async () => { + const { sftp, createReadStream } = fakeSftp(4, 3) + const buffer = await readSftpFileCapped(sftp, '/file', 1024, 'file') + expect(buffer.toString()).toBe('A'.repeat(12)) + expect(createReadStream).toHaveBeenCalledWith('/file') + }) + + it('destroys a remote stream when actual bytes exceed the cap', async () => { + const { sftp, stream } = fakeSftp(8, 1_000_000) + await expect(readSftpFileCapped(sftp, '/bomb', 16, 'file')).rejects.toSatisfy( + isPayloadSizeLimitError + ) + expect(stream.destroyed).toBe(true) + }) + + it('destroys a remote stream when the execution is canceled', async () => { + const { sftp, stream } = fakeSftp(8, 1_000_000) + const controller = new AbortController() + const read = readSftpFileCapped(sftp, '/file', 1024 * 1024, 'file', controller.signal) + controller.abort(new Error('execution canceled')) + await expect(read).rejects.toThrow('execution canceled') + expect(stream.destroyed).toBe(true) + }) + + it('caps SFTP downloads at 50MB', () => { + expect(MAX_SFTP_READ_BYTES).toBe(50 * 1024 * 1024) + }) + + it.each([ + ['../../secret.txt', '_.._secret.txt'], + ['....//secret.txt', '_secret.txt'], + ['..\\..\\secret.txt', '_.._secret.txt'], + ['%2e%2e%2f%2e%2e%2fsecret.txt', '_.._secret.txt'], + ['folder///nested\\file.txt', 'folder_nested_file.txt'], + ])('keeps an untrusted upload name in one path segment', (input, expected) => { + const sanitized = sanitizeFileName(input) + + expect(sanitized).toBe(expected) + expect(sanitized).not.toMatch(/[/\\]/) + }) +}) diff --git a/apps/sim/lib/internal/sftp/client.ts b/apps/sim/lib/internal/sftp/client.ts new file mode 100644 index 00000000000..abf7f1ee9ff --- /dev/null +++ b/apps/sim/lib/internal/sftp/client.ts @@ -0,0 +1,346 @@ +import { createHash } from 'node:crypto' +import { createLogger } from '@sim/logger' +import { safeCompare } from '@sim/security/compare' +import { toError } from '@sim/utils/errors' +import { type Attributes, Client, type ConnectConfig, type SFTPWrapper } from 'ssh2' +import { validateDatabaseHost } from '@/lib/core/security/input-validation.server' +import { readNodeStreamToBufferWithLimit } from '@/lib/core/utils/stream-limits' + +const logger = createLogger('SftpClient') +const S_IFMT = 0o170000 +const S_IFDIR = 0o040000 +const S_IFREG = 0o100000 +const S_IFLNK = 0o120000 + +export interface SftpConnectionConfig { + host: string + port: number + username: string + password?: string | null + privateKey?: string | null + passphrase?: string | null + timeout?: number + keepaliveInterval?: number + readyTimeout?: number + hostFingerprint?: string | null + signal?: AbortSignal +} + +function normalizeSha256Fingerprint(value: string): string { + return value + .trim() + .replace(/^sha256:/i, '') + .replace(/=+$/, '') + .trim() +} + +function computeHostKeyFingerprint(hostKey: Buffer): string { + return createHash('sha256').update(hostKey).digest('base64').replace(/=+$/, '') +} + +function formatSftpError(err: Error, config: { host: string; port: number }): Error { + const errorMessage = err.message.toLowerCase() + const { host, port } = config + + if (errorMessage.includes('econnrefused') || errorMessage.includes('connection refused')) { + return new Error( + `Connection refused to ${host}:${port}. Please verify: (1) SSH/SFTP server is running, (2) Port ${port} is correct, (3) Firewall allows connections.` + ) + } + if (errorMessage.includes('econnreset') || errorMessage.includes('connection reset')) { + return new Error( + `Connection reset by ${host}:${port}. This usually means: (1) Wrong port number, (2) Server rejected the connection, (3) Network/firewall interrupted the connection.` + ) + } + if (errorMessage.includes('etimedout') || errorMessage.includes('timeout')) { + return new Error( + `Connection timed out to ${host}:${port}. Please verify: (1) Host is reachable, (2) No firewall is blocking the connection, (3) The SFTP server is responding.` + ) + } + if (errorMessage.includes('enotfound') || errorMessage.includes('getaddrinfo')) { + return new Error( + `Could not resolve hostname "${host}". Please verify the hostname or IP address is correct.` + ) + } + if (errorMessage.includes('authentication') || errorMessage.includes('auth')) { + return new Error( + `Authentication failed on ${host}:${port}. Please verify: (1) Username is correct, (2) Password or private key is valid, (3) User has SFTP access on the server.` + ) + } + if ( + errorMessage.includes('key') && + (errorMessage.includes('parse') || errorMessage.includes('invalid')) + ) { + return new Error( + 'Invalid private key format. Please ensure you\'re using a valid OpenSSH private key (starts with "-----BEGIN" and ends with "-----END").' + ) + } + if (errorMessage.includes('host key') || errorMessage.includes('hostkey')) { + return new Error( + `Host key verification issue for ${host}. This may be the first connection or the server's key has changed.` + ) + } + return new Error(`SFTP connection to ${host}:${port} failed: ${err.message}`) +} + +export async function createSftpConnection(config: SftpConnectionConfig): Promise { + const host = config.host + if (!host || host.trim() === '') { + throw new Error('Host is required. Please provide a valid hostname or IP address.') + } + config.signal?.throwIfAborted() + + const hostValidation = await validateDatabaseHost(host, 'host') + config.signal?.throwIfAborted() + if (!hostValidation.isValid) throw new Error(hostValidation.error) + + const resolvedHost = hostValidation.resolvedIP ?? host.trim() + + return new Promise((resolve, reject) => { + const client = new Client() + const port = config.port || 22 + const hasPassword = Boolean(config.password?.trim()) + const hasPrivateKey = Boolean(config.privateKey?.trim()) + let ready = false + let settled = false + + const cleanupBeforeReady = () => { + client.off('ready', onReady) + client.off('timeout', onTimeout) + } + const fail = (error: Error) => { + if (ready || settled) return + settled = true + cleanupBeforeReady() + config.signal?.removeEventListener('abort', onAbort) + reject(error) + } + const onAbort = () => { + const error = toError(config.signal?.reason ?? new Error('Aborted')) + if (!ready) fail(error) + client.destroy() + } + const onReady = () => { + if (settled) return + settled = true + ready = true + client.off('ready', onReady) + resolve(client) + } + let hostKeyRejection: Error | undefined + const onError = (error: Error) => { + fail(hostKeyRejection ?? formatSftpError(error, { host, port })) + } + const onTimeout = () => { + client.destroy() + fail( + new Error( + `Connection to ${host}:${port} timed out after ${config.timeout}ms of inactivity.` + ) + ) + } + + if (!hasPassword && !hasPrivateKey) { + fail(new Error('Authentication required. Please provide either a password or private key.')) + return + } + + const connectConfig: ConnectConfig = { host: resolvedHost, port, username: config.username } + if (config.readyTimeout !== undefined) connectConfig.readyTimeout = config.readyTimeout + if (config.keepaliveInterval !== undefined) { + connectConfig.keepaliveInterval = config.keepaliveInterval + } + if (config.timeout !== undefined) connectConfig.timeout = config.timeout + + const suppliedFingerprint = config.hostFingerprint?.trim() + const expectedFingerprint = suppliedFingerprint + ? normalizeSha256Fingerprint(suppliedFingerprint) + : undefined + if (suppliedFingerprint && !expectedFingerprint) { + fail( + new Error( + 'Host key fingerprint is not a valid SHA-256 fingerprint. Expected the base64 form printed by `ssh-keyscan | ssh-keygen -lf -`.' + ) + ) + return + } + if (expectedFingerprint) { + connectConfig.hostVerifier = (hostKey: Buffer): boolean => { + const actualFingerprint = computeHostKeyFingerprint(hostKey) + if (safeCompare(actualFingerprint, expectedFingerprint)) return true + hostKeyRejection = new Error( + `Host key verification failed for ${host}:${port}. Expected SHA256:${expectedFingerprint} but the server presented SHA256:${actualFingerprint}. Either the server's host key changed, or the connection was intercepted. Re-run "ssh-keyscan -t rsa,ecdsa,ed25519 ${host}" to confirm the current key before updating the fingerprint.` + ) + logger.warn('SFTP host key fingerprint mismatch', { host, port }) + return false + } + } + + if (hasPrivateKey) { + connectConfig.privateKey = config.privateKey ?? undefined + if (config.passphrase?.trim()) connectConfig.passphrase = config.passphrase + } else { + connectConfig.password = config.password ?? undefined + } + + client.on('ready', onReady) + client.on('error', onError) + client.on('timeout', onTimeout) + client.once('close', () => config.signal?.removeEventListener('abort', onAbort)) + config.signal?.addEventListener('abort', onAbort, { once: true }) + + if (config.signal?.aborted) { + onAbort() + return + } + try { + client.connect(connectConfig) + } catch (error) { + fail(formatSftpError(toError(error), { host, port })) + } + }) +} + +export function getSftp(client: Client, signal?: AbortSignal): Promise { + signal?.throwIfAborted() + return new Promise((resolve, reject) => { + let settled = false + const finish = (callback: () => void) => { + if (settled) return + settled = true + signal?.removeEventListener('abort', onAbort) + callback() + } + const onAbort = () => { + finish(() => reject(toError(signal?.reason ?? new Error('Aborted')))) + } + signal?.addEventListener('abort', onAbort, { once: true }) + if (signal?.aborted) { + onAbort() + return + } + client.sftp((error, sftp) => { + if (signal?.aborted) { + onAbort() + } else if (error) { + finish(() => reject(new Error(`Failed to start SFTP session: ${error.message}`))) + } else { + finish(() => resolve(sftp)) + } + }) + }) +} + +export const MAX_SFTP_READ_BYTES = 50 * 1024 * 1024 + +export function readSftpFileCapped( + sftp: SFTPWrapper, + remotePath: string, + maxBytes: number, + label: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const stream = sftp.createReadStream(remotePath) + stream.on('error', () => {}) + return readNodeStreamToBufferWithLimit(stream, { maxBytes, label, signal }) +} + +export function sanitizePath(path: string): string { + return decodeURIComponent(path.replace(/\0/g, '')).replace(/\\/g, '/').replace(/\/+/g, '/').trim() +} + +export function sanitizeFileName(fileName: string): string { + let sanitized = fileName.replace(/\0/g, '') + try { + sanitized = decodeURIComponent(sanitized) + } catch {} + sanitized = sanitized + .replace(/[/\\]+/g, '_') + .replace(/^\.+/, '') + .replace(/[\x00-\x1f\x7f]/g, '') + .trim() + return sanitized || 'unnamed_file' +} + +export function isPathSafe(path: string): boolean { + const normalizedPath = path.replace(/\\/g, '/') + if (normalizedPath.includes('../') || normalizedPath.includes('..\\')) return false + try { + const decoded = decodeURIComponent(normalizedPath) + if (decoded.includes('../') || decoded.includes('..\\')) return false + } catch { + return false + } + return !normalizedPath.includes('\0') +} + +export function parsePermissions(mode: number): string { + return `0${(mode & 0o777).toString(8)}` +} + +export function getFileType(attrs: Attributes): 'file' | 'directory' | 'symlink' | 'other' { + const fileType = attrs.mode & S_IFMT + if (fileType === S_IFDIR) return 'directory' + if (fileType === S_IFREG) return 'file' + if (fileType === S_IFLNK) return 'symlink' + return 'other' +} + +export function sftpExists( + sftp: SFTPWrapper, + path: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + return new Promise((resolve, reject) => { + let settled = false + const finish = (callback: () => void) => { + if (settled) return + settled = true + signal?.removeEventListener('abort', onAbort) + callback() + } + const onAbort = () => { + finish(() => reject(toError(signal?.reason ?? new Error('Aborted')))) + } + signal?.addEventListener('abort', onAbort, { once: true }) + if (signal?.aborted) { + onAbort() + return + } + sftp.stat(path, (error) => { + if (signal?.aborted) onAbort() + else finish(() => resolve(!error)) + }) + }) +} + +export function sftpIsDirectory( + sftp: SFTPWrapper, + path: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + return new Promise((resolve, reject) => { + let settled = false + const finish = (callback: () => void) => { + if (settled) return + settled = true + signal?.removeEventListener('abort', onAbort) + callback() + } + const onAbort = () => { + finish(() => reject(toError(signal?.reason ?? new Error('Aborted')))) + } + signal?.addEventListener('abort', onAbort, { once: true }) + if (signal?.aborted) { + onAbort() + return + } + sftp.stat(path, (error, stats) => { + if (signal?.aborted) onAbort() + else finish(() => resolve(!error && getFileType(stats) === 'directory')) + }) + }) +} diff --git a/apps/sim/lib/internal/sftp/execute-tool.test.ts b/apps/sim/lib/internal/sftp/execute-tool.test.ts new file mode 100644 index 00000000000..901baba7b09 --- /dev/null +++ b/apps/sim/lib/internal/sftp/execute-tool.test.ts @@ -0,0 +1,92 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + executeDelete: vi.fn(), + executeDownload: vi.fn(), + executeList: vi.fn(), + executeMkdir: vi.fn(), + executeUpload: vi.fn(), +})) + +vi.mock('@/lib/internal/sftp/operations', () => ({ + executeSftpDelete: mocks.executeDelete, + executeSftpDownload: mocks.executeDownload, + executeSftpList: mocks.executeList, + executeSftpMkdir: mocks.executeMkdir, + executeSftpUpload: mocks.executeUpload, +})) + +import { executeSftpTool } from '@/lib/internal/sftp/execute-tool' +import { sftpDeleteTool } from '@/tools/sftp/delete' +import { sftpDownloadTool } from '@/tools/sftp/download' +import { sftpListTool } from '@/tools/sftp/list' +import { sftpMkdirTool } from '@/tools/sftp/mkdir' +import { sftpUploadTool } from '@/tools/sftp/upload' + +const baseInput = { + host: 'sftp.example.com', + port: 22, + username: 'user', + password: 'secret', + remotePath: '/files', +} + +describe('SFTP tool execution', () => { + beforeEach(() => { + vi.clearAllMocks() + for (const execute of Object.values(mocks)) { + execute.mockResolvedValue(Response.json({ success: true })) + } + }) + + it.each([ + ['sftp_delete', mocks.executeDelete], + ['sftp_download', mocks.executeDownload], + ['sftp_list', mocks.executeList], + ['sftp_mkdir', mocks.executeMkdir], + ['sftp_upload', mocks.executeUpload], + ])('dispatches %s through the typed operation', async (toolId, execute) => { + await executeSftpTool({ + toolId, + input: + toolId === 'sftp_upload' + ? { ...baseInput, fileName: 'note.txt', fileContent: 'hello' } + : baseInput, + headers: new Headers(), + context: { userId: 'user-1' }, + requestId: 'request-1', + }) + + expect(execute).toHaveBeenCalledOnce() + expect(execute.mock.calls[0][1]).toMatchObject({ userId: 'user-1', requestId: 'request-1' }) + }) + + it('rejects missing credentials before any operation runs', async () => { + const response = await executeSftpTool({ + toolId: 'sftp_list', + input: { ...baseInput, password: undefined }, + headers: new Headers(), + context: { userId: 'user-1' }, + requestId: 'request-1', + }) + + expect(response.status).toBe(400) + expect(mocks.executeList).not.toHaveBeenCalled() + }) + + it('uses operation-only declarations with no HTTP-shaped request metadata', () => { + for (const tool of [ + sftpDeleteTool, + sftpDownloadTool, + sftpListTool, + sftpMkdirTool, + sftpUploadTool, + ]) { + expect(tool.operation).toBeDefined() + expect('request' in tool).toBe(false) + } + }) +}) diff --git a/apps/sim/lib/internal/sftp/execute-tool.ts b/apps/sim/lib/internal/sftp/execute-tool.ts new file mode 100644 index 00000000000..70d5b1d947b --- /dev/null +++ b/apps/sim/lib/internal/sftp/execute-tool.ts @@ -0,0 +1,99 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { z } from 'zod' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { + executeSftpDelete, + executeSftpDownload, + executeSftpList, + executeSftpMkdir, + executeSftpUpload, + type SftpOperationContext, +} from '@/lib/internal/sftp/operations' +import { + sftpDeleteInputSchema, + sftpDownloadInputSchema, + sftpListInputSchema, + sftpMkdirInputSchema, + sftpUploadInputSchema, +} from '@/lib/internal/sftp/schema' +import type { + InternalToolOperationCall, + InternalToolOperationHandler, +} from '@/lib/internal/tool-operations/types' + +const logger = createLogger('SftpToolExecution') + +async function executeParsed( + request: InternalToolOperationCall, + schema: S, + execute: (input: z.output, context: SftpOperationContext) => Promise +): Promise { + const parsed = schema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { + error: getValidationErrorMessage(parsed.error, 'Invalid request data'), + details: parsed.error.issues, + }, + { status: 400 } + ) + } + const userId = request.context.userId + if (!userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + return execute(parsed.data, { + userId, + requestId: request.requestId, + signal: request.signal, + }) +} + +export const executeSftpTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + let serializedInput: string + try { + serializedInput = JSON.stringify(request.input) ?? '' + } catch { + return Response.json({ error: 'Invalid request data' }, { status: 400 }) + } + if (Buffer.byteLength(serializedInput, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + + try { + switch (request.toolId) { + case 'sftp_delete': + return executeParsed(request, sftpDeleteInputSchema, executeSftpDelete) + case 'sftp_download': + return executeParsed(request, sftpDownloadInputSchema, executeSftpDownload) + case 'sftp_list': + return executeParsed(request, sftpListInputSchema, executeSftpList) + case 'sftp_mkdir': + return executeParsed(request, sftpMkdirInputSchema, executeSftpMkdir) + case 'sftp_upload': + return executeParsed(request, sftpUploadInputSchema, executeSftpUpload) + default: + return Response.json( + { success: false, error: `Unsupported SFTP tool: ${request.toolId}` }, + { status: 500 } + ) + } + } catch (error) { + request.signal?.throwIfAborted() + const message = getErrorMessage(error, 'Unknown error') + logger.error('SFTP operation dispatch failed', { + error: message, + requestId: request.requestId, + toolId: request.toolId, + }) + return Response.json({ success: false, error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/sftp/operations.test.ts b/apps/sim/lib/internal/sftp/operations.test.ts new file mode 100644 index 00000000000..eaac26e5b46 --- /dev/null +++ b/apps/sim/lib/internal/sftp/operations.test.ts @@ -0,0 +1,166 @@ +/** + * @vitest-environment node + */ +import type { Attributes, SFTPWrapper } from 'ssh2' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + clientEnd: vi.fn(), + createConnection: vi.fn(), + getSftp: vi.fn(), + readFile: vi.fn(), + exists: vi.fn(), + isDirectory: vi.fn(), + processFiles: vi.fn(), + downloadFile: vi.fn(), + assertFileAccess: vi.fn(), + docNotReadyResponse: vi.fn(), +})) + +vi.mock('@/lib/internal/sftp/client', () => ({ + createSftpConnection: mocks.createConnection, + getSftp: mocks.getSftp, + isPathSafe: (value: string) => !value.includes('../'), + sanitizePath: (value: string) => value.trim(), + sanitizeFileName: (value: string) => value.replaceAll('/', '_'), + getFileType: (attributes: Attributes) => + (attributes.mode & 0o170000) === 0o040000 ? 'directory' : 'file', + parsePermissions: (mode: number) => `0${(mode & 0o777).toString(8)}`, + MAX_SFTP_READ_BYTES: 50 * 1024 * 1024, + readSftpFileCapped: mocks.readFile, + sftpExists: mocks.exists, + sftpIsDirectory: mocks.isDirectory, +})) + +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + getFileExtension: (name: string) => name.split('.').pop() ?? '', + getMimeTypeFromExtension: () => 'text/plain', + processFilesToUserFiles: mocks.processFiles, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mocks.downloadFile, +})) + +vi.mock('@/lib/uploads/utils/servable-file-response', () => ({ + docNotReadyResponse: mocks.docNotReadyResponse, +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertFileAccess, +})) + +import { + executeSftpDownload, + executeSftpList, + executeSftpUpload, +} from '@/lib/internal/sftp/operations' + +const connectionInput = { + host: 'sftp.example.com', + port: 22, + username: 'user', + password: 'secret', + privateKey: null, + passphrase: null, +} +const context = { userId: 'user-1', requestId: 'request-1' } + +describe('SFTP operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.createConnection.mockResolvedValue({ end: mocks.clientEnd }) + mocks.assertFileAccess.mockResolvedValue(null) + mocks.docNotReadyResponse.mockReturnValue(null) + }) + + it('returns the historical list shape and sorts directories first', async () => { + const sftp = { + readdir: vi.fn((_path, callback) => + callback(null, [ + { filename: 'z.txt', attrs: { mode: 0o100644, size: 4, mtime: 1 } }, + { filename: 'folder', attrs: { mode: 0o040755, size: 0, mtime: 2 } }, + { filename: '.', attrs: { mode: 0o040755, size: 0, mtime: 2 } }, + ]) + ), + } as unknown as SFTPWrapper + mocks.getSftp.mockResolvedValue(sftp) + + const response = await executeSftpList( + { ...connectionInput, remotePath: '/files', detailed: true }, + context + ) + + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ + success: true, + path: '/files', + entries: [ + { + name: 'folder', + type: 'directory', + size: 0, + permissions: '0755', + modifiedAt: new Date(2000).toISOString(), + }, + { + name: 'z.txt', + type: 'file', + size: 4, + permissions: '0644', + modifiedAt: new Date(1000).toISOString(), + }, + ], + count: 2, + message: 'Found 2 entries in /files', + }) + expect(mocks.clientEnd).toHaveBeenCalledOnce() + }) + + it('rejects declared downloads over 50MB before opening a read stream', async () => { + const sftp = { + stat: vi.fn((_path, callback) => callback(null, { size: 50 * 1024 * 1024 + 1 })), + } as unknown as SFTPWrapper + mocks.getSftp.mockResolvedValue(sftp) + + const response = await executeSftpDownload( + { ...connectionInput, remotePath: '/huge.bin', encoding: 'base64' }, + context + ) + + expect(response.status).toBe(413) + expect(mocks.readFile).not.toHaveBeenCalled() + expect(mocks.clientEnd).toHaveBeenCalledOnce() + }) + + it('authorizes every referenced Sim file before reading or uploading it', async () => { + const denied = Response.json({ success: false, error: 'File not found' }, { status: 404 }) + const file = { key: 'workspace/file', name: 'private.txt', size: 4 } + mocks.getSftp.mockResolvedValue({} as SFTPWrapper) + mocks.processFiles.mockReturnValue([file]) + mocks.assertFileAccess.mockResolvedValue(denied) + + const response = await executeSftpUpload( + { + ...connectionInput, + remotePath: '/files', + files: [file], + fileContent: null, + fileName: null, + overwrite: true, + permissions: null, + }, + context + ) + + expect(response.status).toBe(404) + expect(mocks.assertFileAccess).toHaveBeenCalledWith( + 'workspace/file', + 'user-1', + 'request-1', + expect.anything() + ) + expect(mocks.downloadFile).not.toHaveBeenCalled() + expect(mocks.clientEnd).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/internal/sftp/operations.ts b/apps/sim/lib/internal/sftp/operations.ts new file mode 100644 index 00000000000..dc82c211ba4 --- /dev/null +++ b/apps/sim/lib/internal/sftp/operations.ts @@ -0,0 +1,505 @@ +import path from 'node:path' +import { createLogger } from '@sim/logger' +import { getErrorMessage, toError } from '@sim/utils/errors' +import type { FileEntry, SFTPWrapper } from 'ssh2' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { + createSftpConnection, + getFileType, + getSftp, + isPathSafe, + MAX_SFTP_READ_BYTES, + parsePermissions, + readSftpFileCapped, + sanitizeFileName, + sanitizePath, + sftpExists, + sftpIsDirectory, +} from '@/lib/internal/sftp/client' +import type { + SftpDeleteInput, + SftpDownloadInput, + SftpListInput, + SftpMkdirInput, + SftpUploadInput, +} from '@/lib/internal/sftp/schema' +import { + getFileExtension, + getMimeTypeFromExtension, + processFilesToUserFiles, +} from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +const logger = createLogger('SftpOperations') +const MAX_SFTP_UPLOAD_BYTES = 100 * 1024 * 1024 + +export interface SftpOperationContext { + userId: string + requestId: string + signal?: AbortSignal +} + +type SftpConnectionInput = Pick< + SftpDeleteInput, + 'host' | 'port' | 'username' | 'password' | 'privateKey' | 'passphrase' +> + +function operationFailure(operation: string, error: unknown): Response { + return Response.json( + { error: `SFTP ${operation} failed: ${getErrorMessage(error, 'Unknown error occurred')}` }, + { status: 500 } + ) +} + +function unsafePathResponse(): Response { + return Response.json( + { error: 'Invalid remote path: path traversal sequences are not allowed' }, + { status: 400 } + ) +} + +async function withSftp( + input: SftpConnectionInput, + context: SftpOperationContext, + execute: (sftp: SFTPWrapper) => Promise +): Promise { + context.signal?.throwIfAborted() + const client = await createSftpConnection({ ...input, signal: context.signal }) + try { + const sftp = await getSftp(client, context.signal) + return await execute(sftp) + } finally { + client.end() + } +} + +function sftpCall( + signal: AbortSignal | undefined, + start: (callback: (error: Error | undefined, value?: T) => void) => void +): Promise { + signal?.throwIfAborted() + return new Promise((resolve, reject) => { + let settled = false + const finish = (callback: () => void) => { + if (settled) return + settled = true + signal?.removeEventListener('abort', onAbort) + callback() + } + const onAbort = () => { + finish(() => reject(toError(signal?.reason ?? new Error('Aborted')))) + } + signal?.addEventListener('abort', onAbort, { once: true }) + if (signal?.aborted) { + onAbort() + return + } + start((error, value) => { + if (error) finish(() => reject(error)) + else finish(() => resolve(value as T)) + }) + }) +} + +async function deleteRecursive( + sftp: SFTPWrapper, + directoryPath: string, + signal?: AbortSignal +): Promise { + const entries = await sftpCall(signal, (callback) => + sftp.readdir(directoryPath, (error, list) => callback(error ?? undefined, list)) + ) + for (const entry of entries) { + signal?.throwIfAborted() + if (entry.filename === '.' || entry.filename === '..') continue + const entryPath = `${directoryPath}/${entry.filename}` + if (getFileType(entry.attrs) === 'directory') { + await deleteRecursive(sftp, entryPath, signal) + } else { + await sftpCall(signal, (callback) => + sftp.unlink(entryPath, (error) => callback(error ?? undefined)) + ) + } + } + await sftpCall(signal, (callback) => + sftp.rmdir(directoryPath, (error) => callback(error ?? undefined)) + ) +} + +async function mkdirRecursive( + sftp: SFTPWrapper, + directoryPath: string, + signal?: AbortSignal +): Promise { + const parts = directoryPath.split('/').filter(Boolean) + let currentPath = '' + for (const part of parts) { + signal?.throwIfAborted() + currentPath = currentPath + ? `${currentPath}/${part}` + : directoryPath.startsWith('/') + ? `/${part}` + : part + if (!(await sftpExists(sftp, currentPath, signal))) { + await sftpCall(signal, (callback) => + sftp.mkdir(currentPath, (error) => + callback(error && !error.message.includes('already exists') ? error : undefined) + ) + ) + } + } +} + +function writeSftpFile( + sftp: SFTPWrapper, + remotePath: string, + content: Buffer, + permissions: string | null | undefined, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + return new Promise((resolve, reject) => { + const stream = sftp.createWriteStream(remotePath, { + mode: permissions ? Number.parseInt(permissions, 8) : 0o644, + }) + let settled = false + const cleanup = () => { + stream.off('error', onError) + stream.off('close', onClose) + signal?.removeEventListener('abort', onAbort) + } + const finish = (callback: () => void) => { + if (settled) return + settled = true + cleanup() + callback() + } + const onError = (error: Error) => finish(() => reject(error)) + const onClose = () => finish(resolve) + const onAbort = () => { + const error = toError(signal?.reason ?? new Error('Aborted')) + stream.destroy() + finish(() => reject(error)) + } + stream.on('error', onError) + stream.on('close', onClose) + signal?.addEventListener('abort', onAbort, { once: true }) + if (signal?.aborted) { + onAbort() + return + } + stream.end(content) + }) +} + +export async function executeSftpDelete( + input: SftpDeleteInput, + context: SftpOperationContext +): Promise { + if (!isPathSafe(input.remotePath)) return unsafePathResponse() + try { + return await withSftp(input, context, async (sftp) => { + const remotePath = sanitizePath(input.remotePath) + const isDirectory = await sftpIsDirectory(sftp, remotePath, context.signal) + if (isDirectory) { + if (input.recursive) { + await deleteRecursive(sftp, remotePath, context.signal) + } else { + await sftpCall(context.signal, (callback) => + sftp.rmdir(remotePath, (error) => { + if (error?.message.includes('not empty')) { + callback( + new Error( + 'Directory is not empty. Use recursive: true to delete non-empty directories.' + ) + ) + } else callback(error ?? undefined) + }) + ) + } + } else { + await sftpCall(context.signal, (callback) => + sftp.unlink(remotePath, (error) => + callback( + error?.message.includes('No such file') + ? new Error(`File not found: ${remotePath}`) + : (error ?? undefined) + ) + ) + ) + } + return Response.json({ + success: true, + deletedPath: remotePath, + message: `Successfully deleted ${remotePath}`, + }) + }) + } catch (error) { + context.signal?.throwIfAborted() + return operationFailure('delete', error) + } +} + +export async function executeSftpMkdir( + input: SftpMkdirInput, + context: SftpOperationContext +): Promise { + if (!isPathSafe(input.remotePath)) return unsafePathResponse() + try { + return await withSftp(input, context, async (sftp) => { + const remotePath = sanitizePath(input.remotePath) + if (input.recursive) { + await mkdirRecursive(sftp, remotePath, context.signal) + } else { + if (await sftpExists(sftp, remotePath, context.signal)) { + return Response.json( + { error: `Directory already exists: ${remotePath}` }, + { status: 409 } + ) + } + await sftpCall(context.signal, (callback) => + sftp.mkdir(remotePath, (error) => + callback( + error?.message.includes('No such file') + ? new Error( + 'Parent directory does not exist. Use recursive: true to create parent directories.' + ) + : (error ?? undefined) + ) + ) + ) + } + return Response.json({ + success: true, + createdPath: remotePath, + message: `Successfully created directory ${remotePath}`, + }) + }) + } catch (error) { + context.signal?.throwIfAborted() + return operationFailure('mkdir', error) + } +} + +export async function executeSftpList( + input: SftpListInput, + context: SftpOperationContext +): Promise { + if (!isPathSafe(input.remotePath)) return unsafePathResponse() + try { + return await withSftp(input, context, async (sftp) => { + const remotePath = sanitizePath(input.remotePath) + const files = await sftpCall(context.signal, (callback) => + sftp.readdir(remotePath, (error, list) => + callback( + error?.message.includes('No such file') + ? new Error(`Directory not found: ${remotePath}`) + : (error ?? undefined), + list + ) + ) + ) + const entries = files + .filter((item) => item.filename !== '.' && item.filename !== '..') + .map((item) => ({ + name: item.filename, + type: getFileType(item.attrs), + ...(input.detailed + ? { + size: item.attrs.size, + permissions: parsePermissions(item.attrs.mode), + ...(item.attrs.mtime + ? { modifiedAt: new Date(item.attrs.mtime * 1000).toISOString() } + : {}), + } + : {}), + })) + entries.sort((a, b) => { + if (a.type === 'directory' && b.type !== 'directory') return -1 + if (a.type !== 'directory' && b.type === 'directory') return 1 + return a.name.localeCompare(b.name) + }) + return Response.json({ + success: true, + path: remotePath, + entries, + count: entries.length, + message: `Found ${entries.length} entries in ${remotePath}`, + }) + }) + } catch (error) { + context.signal?.throwIfAborted() + return operationFailure('list', error) + } +} + +export async function executeSftpDownload( + input: SftpDownloadInput, + context: SftpOperationContext +): Promise { + if (!isPathSafe(input.remotePath)) return unsafePathResponse() + try { + return await withSftp(input, context, async (sftp) => { + const remotePath = sanitizePath(input.remotePath) + const stats = await sftpCall<{ size: number }>(context.signal, (callback) => + sftp.stat(remotePath, (error, attributes) => + callback( + error?.message.includes('No such file') + ? new Error(`File not found: ${remotePath}`) + : (error ?? undefined), + attributes + ) + ) + ) + if (stats.size > MAX_SFTP_READ_BYTES) { + const sizeMB = (stats.size / (1024 * 1024)).toFixed(2) + return Response.json( + { success: false, error: `File size (${sizeMB}MB) exceeds download limit of 50MB` }, + { status: 413 } + ) + } + const buffer = await readSftpFileCapped( + sftp, + remotePath, + MAX_SFTP_READ_BYTES, + 'SFTP download', + context.signal + ) + const fileName = path.basename(remotePath) + const extension = getFileExtension(fileName) + const mimeType = getMimeTypeFromExtension(extension) + return Response.json({ + success: true, + fileName, + file: { + name: fileName, + mimeType, + data: buffer.toString('base64'), + size: buffer.length, + }, + content: buffer.toString(input.encoding === 'base64' ? 'base64' : 'utf-8'), + size: buffer.length, + encoding: input.encoding, + message: `Successfully downloaded ${fileName}`, + }) + }) + } catch (error) { + context.signal?.throwIfAborted() + if (isPayloadSizeLimitError(error)) { + return Response.json( + { success: false, error: getErrorMessage(error, 'Unknown error occurred') }, + { status: 413 } + ) + } + return operationFailure('download', error) + } +} + +export async function executeSftpUpload( + input: SftpUploadInput, + context: SftpOperationContext +): Promise { + const hasFiles = Boolean(input.files?.length) + const hasDirectContent = Boolean(input.fileContent && input.fileName) + if (!hasFiles && !hasDirectContent) { + return Response.json( + { error: 'Either files or fileContent with fileName must be provided' }, + { status: 400 } + ) + } + if (!isPathSafe(input.remotePath)) return unsafePathResponse() + + try { + return await withSftp(input, context, async (sftp) => { + const remotePath = sanitizePath(input.remotePath) + const uploadedFiles: Array<{ name: string; remotePath: string; size: number }> = [] + if (hasFiles) { + const userFiles = processFilesToUserFiles(input.files ?? [], context.requestId, logger) + const totalSize = userFiles.reduce((sum, file) => sum + file.size, 0) + if (totalSize > MAX_SFTP_UPLOAD_BYTES) { + return Response.json( + { + success: false, + error: `Total file size (${(totalSize / (1024 * 1024)).toFixed(2)}MB) exceeds limit of 100MB`, + }, + { status: 400 } + ) + } + + let resolvedTotal = 0 + for (const file of userFiles) { + context.signal?.throwIfAborted() + try { + const denied = await assertToolFileAccess( + file.key, + context.userId, + context.requestId, + logger + ) + if (denied) return denied + const { buffer } = await downloadServableFileFromStorage( + file, + context.requestId, + logger, + { maxBytes: MAX_SFTP_UPLOAD_BYTES - resolvedTotal, signal: context.signal } + ) + resolvedTotal += buffer.length + const safeName = sanitizeFileName(file.name) + const destination = sanitizePath( + remotePath.endsWith('/') ? `${remotePath}${safeName}` : `${remotePath}/${safeName}` + ) + if (!input.overwrite && (await sftpExists(sftp, destination, context.signal))) continue + await writeSftpFile(sftp, destination, buffer, input.permissions, context.signal) + uploadedFiles.push({ name: safeName, remotePath: destination, size: buffer.length }) + } catch (error) { + context.signal?.throwIfAborted() + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + const observed = resolvedTotal + (error.observedBytes ?? file.size) + return Response.json( + { + success: false, + error: `Total file size (${(observed / (1024 * 1024)).toFixed(2)}MB) exceeds limit of 100MB`, + }, + { status: 400 } + ) + } + throw new Error( + `Failed to upload file "${file.name}": ${getErrorMessage(error, 'Unknown error')}` + ) + } + } + } + + if (hasDirectContent) { + const safeName = sanitizeFileName(input.fileName ?? '') + const destination = sanitizePath( + remotePath.endsWith('/') ? `${remotePath}${safeName}` : `${remotePath}/${safeName}` + ) + if (!input.overwrite && (await sftpExists(sftp, destination, context.signal))) { + return Response.json( + { error: 'File already exists and overwrite is disabled' }, + { status: 409 } + ) + } + const rawContent = input.fileContent ?? '' + let content = Buffer.from(rawContent, 'base64') + if (content.toString('base64') !== rawContent) content = Buffer.from(rawContent, 'utf-8') + await writeSftpFile(sftp, destination, content, input.permissions, context.signal) + uploadedFiles.push({ name: safeName, remotePath: destination, size: content.length }) + } + + return Response.json({ + success: true, + uploadedFiles, + message: `Successfully uploaded ${uploadedFiles.length} file(s)`, + }) + }) + } catch (error) { + context.signal?.throwIfAborted() + return operationFailure('upload', error) + } +} diff --git a/apps/sim/lib/internal/sftp/schema.ts b/apps/sim/lib/internal/sftp/schema.ts new file mode 100644 index 00000000000..6a3ab54e14d --- /dev/null +++ b/apps/sim/lib/internal/sftp/schema.ts @@ -0,0 +1,71 @@ +import { z } from 'zod' +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' + +const connectionFields = { + host: z.string().min(1, 'Host is required'), + port: z.coerce.number().int().positive().default(22), + username: z.string().min(1, 'Username is required'), + password: z.string().nullish(), + privateKey: z.string().nullish(), + passphrase: z.string().nullish(), +} + +function requireCredentials(schema: S): S { + return schema.refine( + (value) => { + const connection = value as { password?: string | null; privateKey?: string | null } + return Boolean(connection.password || connection.privateKey) + }, + { message: 'Either password or privateKey must be provided' } + ) as S +} + +export const sftpListInputSchema = requireCredentials( + z.object({ + ...connectionFields, + remotePath: z.string().min(1, 'Remote path is required'), + detailed: z.boolean().default(false), + }) +) + +export const sftpDeleteInputSchema = requireCredentials( + z.object({ + ...connectionFields, + remotePath: z.string().min(1, 'Remote path is required'), + recursive: z.boolean().default(false), + }) +) + +export const sftpMkdirInputSchema = requireCredentials( + z.object({ + ...connectionFields, + remotePath: z.string().min(1, 'Remote path is required'), + recursive: z.boolean().default(false), + }) +) + +export const sftpDownloadInputSchema = requireCredentials( + z.object({ + ...connectionFields, + remotePath: z.string().min(1, 'Remote path is required'), + encoding: z.enum(['utf-8', 'base64']).default('utf-8'), + }) +) + +export const sftpUploadInputSchema = requireCredentials( + z.object({ + ...connectionFields, + remotePath: z.string().min(1, 'Remote path is required'), + files: RawFileInputArraySchema.optional().nullable(), + fileContent: z.string().nullish(), + fileName: z.string().nullish(), + overwrite: z.boolean().default(true), + permissions: z.string().nullish(), + }) +) + +export type SftpListInput = z.output +export type SftpDeleteInput = z.output +export type SftpMkdirInput = z.output +export type SftpDownloadInput = z.output +export type SftpUploadInput = z.output diff --git a/apps/sim/lib/internal/sharepoint/client.test.ts b/apps/sim/lib/internal/sharepoint/client.test.ts new file mode 100644 index 00000000000..98f255b1481 --- /dev/null +++ b/apps/sim/lib/internal/sharepoint/client.test.ts @@ -0,0 +1,77 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' + +const mocks = vi.hoisted(() => ({ + validateUrl: vi.fn(), + pinnedFetch: vi.fn(), + validatedFetch: vi.fn(), +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + validateUrlWithDNS: mocks.validateUrl, + secureFetchWithPinnedIP: mocks.pinnedFetch, + secureFetchWithValidation: mocks.validatedFetch, +})) + +import { SharePointClient } from '@/lib/internal/sharepoint/client' + +describe('SharePointClient', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.validateUrl.mockResolvedValue({ isValid: true, resolvedIP: '20.190.128.1' }) + }) + + it('pins Graph downloads, strips authorization on redirect, and enforces the file cap', async () => { + const controller = new AbortController() + mocks.pinnedFetch.mockResolvedValue( + new Response(Buffer.from('content'), { status: 200, headers: { 'content-length': '7' } }) + ) + + const result = await new SharePointClient('token', controller.signal).download( + 'drive/id', + 'item id' + ) + + expect(result).toEqual(Buffer.from('content')) + expect(mocks.pinnedFetch).toHaveBeenCalledWith( + 'https://graph.microsoft.com/v1.0/drives/drive%2Fid/items/item%20id/content', + '20.190.128.1', + { + headers: { Authorization: 'Bearer token' }, + stripAuthOnRedirect: true, + maxResponseBytes: MAX_FILE_SIZE, + signal: controller.signal, + } + ) + }) + + it('passes cancellation into validated Graph uploads', async () => { + const controller = new AbortController() + mocks.validatedFetch.mockResolvedValue( + Response.json({ id: 'item', name: 'file', webUrl: 'https://example.com', size: 4 }) + ) + const buffer = Buffer.from('file') + await new SharePointClient('token', controller.signal).upload( + 'https://graph.microsoft.com/upload', + buffer, + 'application/pdf' + ) + + expect(mocks.validatedFetch).toHaveBeenCalledWith( + 'https://graph.microsoft.com/upload', + { + method: 'PUT', + headers: { + Authorization: 'Bearer token', + 'Content-Type': 'application/pdf', + }, + body: buffer, + signal: controller.signal, + }, + 'uploadUrl' + ) + }) +}) diff --git a/apps/sim/lib/internal/sharepoint/client.ts b/apps/sim/lib/internal/sharepoint/client.ts new file mode 100644 index 00000000000..2697463f462 --- /dev/null +++ b/apps/sim/lib/internal/sharepoint/client.ts @@ -0,0 +1,193 @@ +import { + type SecureFetchOptions, + type SecureFetchResponse, + secureFetchWithPinnedIP, + secureFetchWithValidation, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + readResponseJsonWithLimit, + readResponseToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' +import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' + +interface GraphErrorBody { + error?: { message?: string } +} + +export interface SharePointDriveItemMetadata { + id?: string + name?: string + folder?: Record + file?: { mimeType?: string } +} + +export interface SharePointUploadedItem { + id: string + name: string + webUrl: string + size: number + createdDateTime?: string + lastModifiedDateTime?: string +} + +export interface SharePointUploadResult { + ok: boolean + status: number + data: SharePointUploadedItem | GraphErrorBody +} + +export class SharePointGraphError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'SharePointGraphError' + } +} + +async function readJsonOrEmpty( + response: SecureFetchResponse, + label: string, + signal?: AbortSignal +): Promise { + try { + return await readResponseJsonWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label, + signal, + }) + } catch { + signal?.throwIfAborted() + return {} as T + } +} + +function readJson( + response: SecureFetchResponse, + label: string, + signal?: AbortSignal +): Promise { + return readResponseJsonWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label, + signal, + }) +} + +function graphErrorMessage( + data: SharePointUploadedItem | GraphErrorBody, + fallback: string +): string { + return 'error' in data && data.error?.message ? data.error.message : fallback +} + +export class SharePointClient { + constructor( + private readonly accessToken: string, + private readonly signal?: AbortSignal + ) {} + + private get authorization(): string { + return `Bearer ${this.accessToken}` + } + + private async pinnedFetch( + url: string, + paramName: string, + options: SecureFetchOptions + ): Promise { + this.signal?.throwIfAborted() + const validation = await validateUrlWithDNS(url, paramName) + this.signal?.throwIfAborted() + if (!validation.isValid || !validation.resolvedIP) { + throw new SharePointGraphError(validation.error || `Invalid ${paramName}`, 400) + } + return secureFetchWithPinnedIP(url, validation.resolvedIP, { + ...options, + signal: this.signal, + }) + } + + async getMetadata(driveId: string, itemId: string): Promise { + const url = `https://graph.microsoft.com/v1.0/drives/${encodeURIComponent(driveId)}/items/${encodeURIComponent(itemId)}` + const response = await this.pinnedFetch(url, 'metadataUrl', { + headers: { Authorization: this.authorization }, + }) + if (!response.ok) { + const data = await readJsonOrEmpty( + response, + 'SharePoint metadata error response', + this.signal + ) + throw new SharePointGraphError(data.error?.message || 'Failed to get file metadata', 400) + } + return readJson( + response, + 'SharePoint metadata response', + this.signal + ) + } + + async download(driveId: string, itemId: string): Promise { + const url = `https://graph.microsoft.com/v1.0/drives/${encodeURIComponent(driveId)}/items/${encodeURIComponent(itemId)}/content` + const response = await this.pinnedFetch(url, 'downloadUrl', { + headers: { Authorization: this.authorization }, + stripAuthOnRedirect: true, + maxResponseBytes: MAX_FILE_SIZE, + }) + if (!response.ok) { + const data = await readJsonOrEmpty( + response, + 'SharePoint download error response', + this.signal + ) + throw new SharePointGraphError(data.error?.message || 'Failed to download file', 400) + } + return readResponseToBufferWithLimit(response, { + maxBytes: MAX_FILE_SIZE, + label: 'SharePoint file download', + signal: this.signal, + }) + } + + async upload( + url: string, + buffer: Buffer, + contentType: string, + label = 'uploadUrl' + ): Promise { + this.signal?.throwIfAborted() + const response = await secureFetchWithValidation( + url, + { + method: 'PUT', + headers: { + Authorization: this.authorization, + 'Content-Type': contentType, + }, + body: buffer, + signal: this.signal, + }, + label + ) + const data = response.ok + ? await readJson( + response, + `SharePoint ${label} response`, + this.signal + ) + : await readJsonOrEmpty( + response, + `SharePoint ${label} error response`, + this.signal + ) + return { ok: response.ok, status: response.status, data } + } + + static errorMessage(result: SharePointUploadResult, fallback: string): string { + return graphErrorMessage(result.data, fallback) + } +} diff --git a/apps/sim/lib/internal/sharepoint/execute-tool.test.ts b/apps/sim/lib/internal/sharepoint/execute-tool.test.ts new file mode 100644 index 00000000000..d268dc14ce3 --- /dev/null +++ b/apps/sim/lib/internal/sharepoint/execute-tool.test.ts @@ -0,0 +1,99 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + download: vi.fn(), + upload: vi.fn(), +})) + +vi.mock('@/lib/internal/sharepoint/operations', () => ({ + executeSharePointDownloadFile: mocks.download, + executeSharePointUploadFile: mocks.upload, +})) + +import { executeSharePointTool } from '@/lib/internal/sharepoint/execute-tool' +import { downloadFileTool } from '@/tools/sharepoint/download_file' +import { uploadFileTool } from '@/tools/sharepoint/upload_file' + +const context = { + userId: 'user-1', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', +} + +describe('executeSharePointTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.download.mockResolvedValue(Response.json({ success: true, output: { file: {} } })) + mocks.upload.mockResolvedValue(Response.json({ success: true, output: { uploadedFiles: [] } })) + }) + + it.each([ + [ + 'sharepoint_download_file', + mocks.download, + { accessToken: 'token', driveId: 'd', itemId: 'i' }, + ], + ['sharepoint_upload_file', mocks.upload, { accessToken: 'token', siteId: 'root', files: [] }], + ])('dispatches %s through its typed operation', async (toolId, operation, input) => { + await executeSharePointTool({ + toolId, + input, + headers: new Headers(), + context, + requestId: 'request-1', + }) + + expect(operation).toHaveBeenCalledWith( + expect.objectContaining(input), + expect.objectContaining({ userId: 'user-1', requestId: 'request-1' }) + ) + }) + + it('requires trusted execution identity before parsing tool input', async () => { + const response = await executeSharePointTool({ + toolId: 'sharepoint_download_file', + input: {}, + headers: new Headers(), + context: {}, + requestId: 'request-1', + }) + + expect(response.status).toBe(401) + expect(await response.json()).toEqual({ success: false, error: 'Authentication required' }) + expect(mocks.download).not.toHaveBeenCalled() + }) +}) + +describe('SharePoint internal tool declarations', () => { + it('contain operation metadata only and keep private upload data out of model input', () => { + expect(downloadFileTool).not.toHaveProperty('request') + expect(uploadFileTool).not.toHaveProperty('request') + + const file = { key: 'workspace/file.pdf', name: 'file.pdf', size: 10 } + const params = { + accessToken: 'private-token', + siteId: 'private-site', + driveId: 'drive', + folderPath: '/Reports', + fileName: 'report.pdf', + files: [file], + } + expect(uploadFileTool.operation.modelInput?.select?.(params)).toEqual({ + driveId: 'drive', + folderPath: '/Reports', + fileName: 'report.pdf', + }) + expect(uploadFileTool.operation.input(params)).toEqual({ + accessToken: 'private-token', + siteId: 'private-site', + driveId: 'drive', + folderPath: '/Reports', + fileName: 'report.pdf', + files: [file], + }) + }) +}) diff --git a/apps/sim/lib/internal/sharepoint/execute-tool.ts b/apps/sim/lib/internal/sharepoint/execute-tool.ts new file mode 100644 index 00000000000..8d331672e52 --- /dev/null +++ b/apps/sim/lib/internal/sharepoint/execute-tool.ts @@ -0,0 +1,88 @@ +import { getErrorMessage } from '@sim/utils/errors' +import type { z } from 'zod' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { + executeSharePointDownloadFile, + executeSharePointUploadFile, + type SharePointOperationContext, +} from '@/lib/internal/sharepoint/operations' +import { + sharePointDownloadFileInputSchema, + sharePointUploadFileInputSchema, +} from '@/lib/internal/sharepoint/schema' +import type { + InternalToolOperationCall, + InternalToolOperationHandler, +} from '@/lib/internal/tool-operations/types' + +async function executeParsed( + request: InternalToolOperationCall, + schema: S, + execute: (input: z.output, context: SharePointOperationContext) => Promise +): Promise { + let serializedInput: string + try { + serializedInput = JSON.stringify(request.input) ?? '' + } catch { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + if (Buffer.byteLength(serializedInput, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + success: false, + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + const parsed = schema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { + success: false, + error: getValidationErrorMessage(parsed.error, 'Invalid request data'), + }, + { status: 400 } + ) + } + const userId = request.context.userId + if (!userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + return execute(parsed.data, { + userId, + requestId: request.requestId, + signal: request.signal, + }) +} + +export const executeSharePointTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!request.context.userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + try { + switch (request.toolId) { + case 'sharepoint_download_file': + return executeParsed( + request, + sharePointDownloadFileInputSchema, + executeSharePointDownloadFile + ) + case 'sharepoint_upload_file': + return executeParsed(request, sharePointUploadFileInputSchema, executeSharePointUploadFile) + default: + return Response.json( + { success: false, error: `Unsupported SharePoint tool: ${request.toolId}` }, + { status: 500 } + ) + } + } catch (error) { + request.signal?.throwIfAborted() + return Response.json( + { success: false, error: getErrorMessage(error, 'Unknown error occurred') }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/sharepoint/operations.test.ts b/apps/sim/lib/internal/sharepoint/operations.test.ts new file mode 100644 index 00000000000..863cb6a0f6a --- /dev/null +++ b/apps/sim/lib/internal/sharepoint/operations.test.ts @@ -0,0 +1,176 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + clientConstructed: vi.fn(), + getMetadata: vi.fn(), + downloadGraph: vi.fn(), + uploadGraph: vi.fn(), + processFiles: vi.fn(), + downloadStorage: vi.fn(), + assertAccess: vi.fn(), +})) + +vi.mock('@/lib/internal/sharepoint/client', () => { + class SharePointGraphError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + } + } + class SharePointClient { + static errorMessage(result: { data?: { error?: { message?: string } } }, fallback: string) { + return result.data?.error?.message || fallback + } + + constructor(accessToken: string, signal?: AbortSignal) { + mocks.clientConstructed(accessToken, signal) + } + + getMetadata = mocks.getMetadata + download = mocks.downloadGraph + upload = mocks.uploadGraph + } + return { SharePointClient, SharePointGraphError } +}) + +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + processFilesToUserFiles: mocks.processFiles, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mocks.downloadStorage, +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertAccess, +})) + +import { + executeSharePointDownloadFile, + executeSharePointUploadFile, + MAX_SHAREPOINT_UPLOAD_BYTES, +} from '@/lib/internal/sharepoint/operations' + +const userFile = { + key: 'workspace/file.pdf', + name: 'file.pdf', + size: 4, + type: 'application/pdf', +} + +describe('SharePoint operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.processFiles.mockReturnValue([userFile]) + mocks.assertAccess.mockResolvedValue(null) + mocks.downloadStorage.mockResolvedValue({ + buffer: Buffer.from('file'), + contentType: 'application/pdf', + }) + mocks.uploadGraph.mockResolvedValue({ + ok: true, + status: 201, + data: { + id: 'item-1', + name: 'file.pdf', + webUrl: 'https://example.com/file.pdf', + size: 4, + }, + }) + }) + + it('authorizes input provenance and carries cancellation through storage and Graph upload', async () => { + const controller = new AbortController() + const response = await executeSharePointUploadFile( + { + accessToken: 'token', + siteId: 'root', + driveId: 'drive/id', + folderPath: '/Shared Documents/Reports/', + fileName: null, + files: [userFile], + }, + { userId: 'user-1', requestId: 'request-1', signal: controller.signal } + ) + + expect(mocks.assertAccess).toHaveBeenCalledWith( + userFile.key, + 'user-1', + 'request-1', + expect.anything() + ) + expect(mocks.downloadStorage).toHaveBeenCalledWith(userFile, 'request-1', expect.anything(), { + maxBytes: MAX_SHAREPOINT_UPLOAD_BYTES, + signal: controller.signal, + }) + expect(mocks.clientConstructed).toHaveBeenCalledWith('token', controller.signal) + expect(mocks.uploadGraph).toHaveBeenCalledWith( + 'https://graph.microsoft.com/v1.0/drives/drive%2Fid/root:/Shared%20Documents/Reports/file.pdf:/content', + Buffer.from('file'), + 'application/pdf' + ) + expect(await response.json()).toEqual({ + success: true, + output: { + uploadedFiles: [ + { + id: 'item-1', + name: 'file.pdf', + webUrl: 'https://example.com/file.pdf', + size: 4, + }, + ], + fileCount: 1, + skippedFiles: [], + skippedCount: 0, + errors: [], + }, + }) + }) + + it('does not materialize a file when its provenance check fails', async () => { + mocks.assertAccess.mockResolvedValue( + Response.json({ success: false, error: 'File not found' }, { status: 404 }) + ) + const response = await executeSharePointUploadFile( + { accessToken: 'token', siteId: 'root', files: [userFile] }, + { userId: 'user-1', requestId: 'request-1' } + ) + + expect(response.status).toBe(404) + expect(mocks.downloadStorage).not.toHaveBeenCalled() + expect(mocks.uploadGraph).not.toHaveBeenCalled() + }) + + it('preserves the inline download output contract and cancellation signal', async () => { + const controller = new AbortController() + mocks.getMetadata.mockResolvedValue({ + name: 'source.txt', + file: { mimeType: 'text/plain' }, + }) + mocks.downloadGraph.mockResolvedValue(Buffer.from('hello')) + + const response = await executeSharePointDownloadFile( + { accessToken: 'token', driveId: 'drive', itemId: 'item', fileName: 'renamed.txt' }, + { userId: 'user-1', requestId: 'request-1', signal: controller.signal } + ) + + expect(mocks.clientConstructed).toHaveBeenCalledWith('token', controller.signal) + expect(await response.json()).toEqual({ + success: true, + output: { + file: { + name: 'renamed.txt', + mimeType: 'text/plain', + data: Buffer.from('hello').toString('base64'), + size: 5, + }, + }, + }) + }) +}) diff --git a/apps/sim/lib/internal/sharepoint/operations.ts b/apps/sim/lib/internal/sharepoint/operations.ts new file mode 100644 index 00000000000..392185a2ee9 --- /dev/null +++ b/apps/sim/lib/internal/sharepoint/operations.ts @@ -0,0 +1,215 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { + SharePointClient, + SharePointGraphError, + type SharePointUploadedItem, +} from '@/lib/internal/sharepoint/client' +import type { + SharePointDownloadFileInput, + SharePointUploadFileInput, +} from '@/lib/internal/sharepoint/schema' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' +import { assertToolFileAccess } from '@/app/api/files/authorization' +import type { SharepointSkippedFile, SharepointUploadError } from '@/tools/sharepoint/types' + +const logger = createLogger('SharePointOperations') +export const MAX_SHAREPOINT_UPLOAD_BYTES = 250 * 1024 * 1024 + +export interface SharePointOperationContext { + userId: string + requestId: string + signal?: AbortSignal +} + +function failureResponse( + error: string, + status: number, + output?: Record +): Response { + return Response.json({ success: false, error, ...(output ? { output } : {}) }, { status }) +} + +function uploadedFile(data: SharePointUploadedItem): SharePointUploadedItem { + return { + id: data.id, + name: data.name, + webUrl: data.webUrl, + size: data.size, + createdDateTime: data.createdDateTime, + lastModifiedDateTime: data.lastModifiedDateTime, + } +} + +export async function executeSharePointDownloadFile( + input: SharePointDownloadFileInput, + context: SharePointOperationContext +): Promise { + context.signal?.throwIfAborted() + try { + const client = new SharePointClient(input.accessToken, context.signal) + const metadata = await client.getMetadata(input.driveId, input.itemId) + if (metadata.folder && !metadata.file) { + return failureResponse( + `Cannot download folder "${metadata.name}". Please select a file instead.`, + 400 + ) + } + const mimeType = metadata.file?.mimeType || 'application/octet-stream' + const buffer = await client.download(input.driveId, input.itemId) + context.signal?.throwIfAborted() + return Response.json({ + success: true, + output: { + file: { + name: input.fileName || metadata.name || 'download', + mimeType, + data: buffer.toString('base64'), + size: buffer.length, + }, + }, + }) + } catch (error) { + context.signal?.throwIfAborted() + if (error instanceof SharePointGraphError) { + return failureResponse(error.message, error.status) + } + logger.error('Error downloading SharePoint file', { + error: getErrorMessage(error), + requestId: context.requestId, + }) + return failureResponse(getErrorMessage(error, 'Unknown error occurred'), 500) + } +} + +function buildUploadUrl( + input: SharePointUploadFileInput, + fileName: string +): { url: string; replaceUrl: string } { + const folderPath = input.folderPath?.trim() || '' + const normalizedPath = folderPath.startsWith('/') ? folderPath : `/${folderPath}` + const cleanPath = normalizedPath.endsWith('/') ? normalizedPath.slice(0, -1) : normalizedPath + const uploadPath = folderPath ? `${cleanPath}/${fileName}` : `/${fileName}` + const encodedPath = uploadPath + .split('/') + .map((segment) => (segment ? encodeURIComponent(segment) : '')) + .join('/') + const driveId = input.driveId?.trim() + const siteId = input.siteId.trim() || 'root' + const url = driveId + ? `https://graph.microsoft.com/v1.0/drives/${encodeURIComponent(driveId)}/root:${encodedPath}:/content` + : `https://graph.microsoft.com/v1.0/sites/${encodeURIComponent(siteId)}/drive/root:${encodedPath}:/content` + return { url, replaceUrl: `${url}?@microsoft.graph.conflictBehavior=replace` } +} + +export async function executeSharePointUploadFile( + input: SharePointUploadFileInput, + context: SharePointOperationContext +): Promise { + context.signal?.throwIfAborted() + if (!input.files?.length) { + return failureResponse('At least one file is required for upload', 400) + } + const userFiles = processFilesToUserFiles(input.files, context.requestId, logger) + if (userFiles.length === 0) return failureResponse('No valid files to upload', 400) + + const client = new SharePointClient(input.accessToken, context.signal) + const uploadedFiles: SharePointUploadedItem[] = [] + const skippedFiles: SharepointSkippedFile[] = [] + const errors: SharepointUploadError[] = [] + + try { + for (const userFile of userFiles) { + context.signal?.throwIfAborted() + const denied = await assertToolFileAccess( + userFile.key, + context.userId, + context.requestId, + logger + ) + context.signal?.throwIfAborted() + if (denied) return denied + + const fileName = input.fileName || userFile.name + const skipOversized = (size: number) => { + skippedFiles.push({ + name: fileName, + size, + limit: MAX_SHAREPOINT_UPLOAD_BYTES, + reason: 'File exceeds the 250 MB Microsoft Graph small upload limit', + }) + } + + let buffer: Buffer + let downloadedContentType = '' + try { + const result = await downloadServableFileFromStorage(userFile, context.requestId, logger, { + maxBytes: MAX_SHAREPOINT_UPLOAD_BYTES, + signal: context.signal, + }) + buffer = result.buffer + downloadedContentType = result.contentType + } catch (error) { + context.signal?.throwIfAborted() + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + skipOversized(error.observedBytes ?? userFile.size) + continue + } + throw error + } + + const { url, replaceUrl } = buildUploadUrl(input, fileName) + const contentType = downloadedContentType || userFile.type || 'application/octet-stream' + const uploadResult = await client.upload(url, buffer, contentType) + if (!uploadResult.ok) { + if (uploadResult.status === 409) { + const replaceResult = await client.upload(replaceUrl, buffer, contentType, 'replaceUrl') + if (!replaceResult.ok) { + errors.push({ + name: fileName, + status: replaceResult.status, + error: SharePointClient.errorMessage( + replaceResult, + `Failed to replace file: ${fileName}` + ), + }) + continue + } + uploadedFiles.push(uploadedFile(replaceResult.data as SharePointUploadedItem)) + continue + } + errors.push({ + name: fileName, + status: uploadResult.status, + error: SharePointClient.errorMessage(uploadResult, `Failed to upload file: ${fileName}`), + }) + continue + } + uploadedFiles.push(uploadedFile(uploadResult.data as SharePointUploadedItem)) + } + + const output = { + uploadedFiles, + fileCount: uploadedFiles.length, + skippedFiles, + skippedCount: skippedFiles.length, + errors, + } + if (uploadedFiles.length === 0) { + return failureResponse('No files were uploaded successfully', 200, output) + } + return Response.json({ success: true, output }) + } catch (error) { + context.signal?.throwIfAborted() + logger.error('Error uploading files to SharePoint', { + error: getErrorMessage(error), + requestId: context.requestId, + }) + return failureResponse(getErrorMessage(error, 'Unknown error occurred'), 500) + } +} diff --git a/apps/sim/lib/internal/sharepoint/schema.ts b/apps/sim/lib/internal/sharepoint/schema.ts new file mode 100644 index 00000000000..e6dcb73806c --- /dev/null +++ b/apps/sim/lib/internal/sharepoint/schema.ts @@ -0,0 +1,23 @@ +import { z } from 'zod' +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' + +const accessTokenSchema = z.string().min(1, 'Access token is required') + +export const sharePointDownloadFileInputSchema = z.object({ + accessToken: accessTokenSchema, + driveId: z.string().min(1, 'Drive ID is required'), + itemId: z.string().min(1, 'Item ID is required'), + fileName: z.string().optional().nullable(), +}) + +export const sharePointUploadFileInputSchema = z.object({ + accessToken: accessTokenSchema, + siteId: z.string().default('root'), + driveId: z.string().optional().nullable(), + folderPath: z.string().optional().nullable(), + fileName: z.string().optional().nullable(), + files: RawFileInputArraySchema.optional().nullable(), +}) + +export type SharePointDownloadFileInput = z.output +export type SharePointUploadFileInput = z.output diff --git a/apps/sim/lib/internal/slack/client.ts b/apps/sim/lib/internal/slack/client.ts new file mode 100644 index 00000000000..4a957b6a8d1 --- /dev/null +++ b/apps/sim/lib/internal/slack/client.ts @@ -0,0 +1,113 @@ +import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' + +const MAX_SLACK_JSON_BYTES = 2 * 1024 * 1024 + +export type SlackJsonObject = Record + +export interface SlackApiResult { + data: SlackJsonObject + status: number + statusText: string +} + +export interface SlackApiRequest { + accessToken: string + method: string + httpMethod?: 'GET' | 'POST' + body?: SlackJsonObject | URLSearchParams + query?: Record + signal?: AbortSignal + tolerateInvalidErrorJson?: boolean +} + +function isSlackJsonObject(value: unknown): value is SlackJsonObject { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +export function slackString(data: SlackJsonObject, key: string): string | undefined { + const value = data[key] + return typeof value === 'string' ? value : undefined +} + +export function slackObject(data: SlackJsonObject, key: string): SlackJsonObject | undefined { + const value = data[key] + return isSlackJsonObject(value) ? value : undefined +} + +export function slackArray(data: SlackJsonObject, key: string): unknown[] | undefined { + const value = data[key] + return Array.isArray(value) ? value : undefined +} + +export function slackOk(data: SlackJsonObject): boolean { + return data.ok === true +} + +/** Calls one fixed Slack Web API method with bounded JSON parsing and caller cancellation. */ +export async function requestSlackApi({ + accessToken, + method, + httpMethod = 'POST', + body, + query, + signal, + tolerateInvalidErrorJson, +}: SlackApiRequest): Promise { + signal?.throwIfAborted() + const url = new URL(`https://slack.com/api/${method}`) + for (const [key, value] of Object.entries(query ?? {})) { + if (value !== undefined) url.searchParams.set(key, String(value)) + } + + const isForm = body instanceof URLSearchParams + const response = await fetch(url, { + method: httpMethod, + headers: { + Authorization: `Bearer ${accessToken}`, + ...(httpMethod === 'POST' + ? { + 'Content-Type': isForm ? 'application/x-www-form-urlencoded' : 'application/json', + } + : {}), + }, + ...(body ? { body: isForm ? body.toString() : JSON.stringify(body) } : {}), + signal, + }) + signal?.throwIfAborted() + let parsed: unknown + try { + parsed = await readResponseJsonWithLimit(response, { + maxBytes: MAX_SLACK_JSON_BYTES, + label: 'Slack API response', + }) + } catch (error) { + if (!response.ok && tolerateInvalidErrorJson && !isPayloadSizeLimitError(error)) { + parsed = {} + } else { + throw error + } + } + signal?.throwIfAborted() + if (!isSlackJsonObject(parsed)) throw new Error('Slack API returned an invalid response') + return { data: parsed, status: response.status, statusText: response.statusText } +} + +/** Opens a Slack direct-message conversation while retaining the legacy thrown-error behavior. */ +export async function openSlackDm( + accessToken: string, + userId: string, + signal?: AbortSignal +): Promise { + const { data } = await requestSlackApi({ + accessToken, + method: 'conversations.open', + body: { users: userId }, + signal, + }) + if (!slackOk(data)) { + throw new Error(slackString(data, 'error') || 'Failed to open DM channel with user') + } + const channelId = slackString(slackObject(data, 'channel') ?? {}, 'id') + if (!channelId) throw new Error('Failed to open DM channel with user') + return channelId +} diff --git a/apps/sim/lib/internal/slack/errors.ts b/apps/sim/lib/internal/slack/errors.ts new file mode 100644 index 00000000000..7efc8486749 --- /dev/null +++ b/apps/sim/lib/internal/slack/errors.ts @@ -0,0 +1,13 @@ +export class SlackOperationError extends Error { + constructor( + readonly status: number, + readonly body: unknown + ) { + super( + typeof body === 'object' && body !== null && 'error' in body + ? String(body.error) + : 'Slack operation failed' + ) + this.name = 'SlackOperationError' + } +} diff --git a/apps/sim/lib/internal/slack/execute-tool.test.ts b/apps/sim/lib/internal/slack/execute-tool.test.ts new file mode 100644 index 00000000000..ac8b523a615 --- /dev/null +++ b/apps/sim/lib/internal/slack/execute-tool.test.ts @@ -0,0 +1,190 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + addReaction: vi.fn(), + deleteMessage: vi.fn(), + download: vi.fn(), + readMessages: vi.fn(), + removeReaction: vi.fn(), + sendEphemeral: vi.fn(), + sendMessage: vi.fn(), + updateMessage: vi.fn(), +})) + +vi.mock('@/lib/internal/slack/operations', () => ({ + executeSlackAddReaction: mocks.addReaction, + executeSlackDeleteMessage: mocks.deleteMessage, + executeSlackDownload: mocks.download, + executeSlackReadMessages: mocks.readMessages, + executeSlackRemoveReaction: mocks.removeReaction, + executeSlackSendEphemeral: mocks.sendEphemeral, + executeSlackSendMessage: mocks.sendMessage, + executeSlackUpdateMessage: mocks.updateMessage, +})) + +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { SlackOperationError } from '@/lib/internal/slack/errors' +import { executeSlackTool } from '@/lib/internal/slack/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const INPUTS = { + slack_add_reaction: { + accessToken: 'token', + channel: 'C1', + timestamp: '1.0', + name: 'eyes', + }, + slack_delete_message: { accessToken: 'token', channel: 'C1', timestamp: '1.0' }, + slack_download: { accessToken: 'token', fileId: 'F1', fileName: 'report.pdf' }, + slack_ephemeral_message: { + accessToken: 'token', + channel: 'C1', + user: 'U1', + text: 'hello', + }, + slack_message: { accessToken: 'token', channel: 'C1', text: 'hello' }, + slack_message_reader: { accessToken: 'token', channel: 'C1', limit: 2 }, + slack_remove_reaction: { + accessToken: 'token', + channel: 'C1', + timestamp: '1.0', + name: 'eyes', + }, + slack_update_message: { + accessToken: 'token', + channel: 'C1', + timestamp: '1.0', + text: 'updated', + }, +} as const + +const DISPATCH = { + slack_add_reaction: mocks.addReaction, + slack_delete_message: mocks.deleteMessage, + slack_download: mocks.download, + slack_ephemeral_message: mocks.sendEphemeral, + slack_message: mocks.sendMessage, + slack_message_reader: mocks.readMessages, + slack_remove_reaction: mocks.removeReaction, + slack_update_message: mocks.updateMessage, +} as const + +function request( + toolId: keyof typeof INPUTS, + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId, + input: INPUTS[toolId], + headers: new Headers(), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + userId: 'user-1', + workspaceId: 'workspace-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeSlackTool', () => { + beforeEach(() => { + vi.clearAllMocks() + for (const operation of Object.values(DISPATCH)) { + operation.mockResolvedValue({ success: true, output: { ok: true } }) + } + }) + + it.each(Object.keys(INPUTS) as Array)( + 'validates and dispatches %s from typed input', + async (toolId) => { + const controller = new AbortController() + const response = await executeSlackTool(request(toolId, { signal: controller.signal })) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ success: true, output: { ok: true } }) + expect(DISPATCH[toolId]).toHaveBeenCalledOnce() + expect(DISPATCH[toolId].mock.calls[0]?.[0]).toEqual(INPUTS[toolId]) + if (toolId === 'slack_message') { + expect(DISPATCH[toolId].mock.calls[0]?.[1]).toEqual({ + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + }) + } else { + expect(DISPATCH[toolId].mock.calls[0]?.[1]).toBe(controller.signal) + } + } + ) + + it('returns the canonical validation envelope before provider work', async () => { + const response = await executeSlackTool( + request('slack_add_reaction', { input: { accessToken: '', channel: '' } }) + ) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(mocks.addReaction).not.toHaveBeenCalled() + }) + + it('keeps message file authority tied to the trusted execution context', async () => { + const response = await executeSlackTool( + request('slack_message', { + context: { ...createExecutionContext({ workflowId: 'workflow-1' }), userId: undefined }, + }) + ) + + expect(response.status).toBe(401) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Authentication required', + }) + expect(mocks.sendMessage).not.toHaveBeenCalled() + }) + + it('preserves Slack logical-error status and envelope', async () => { + mocks.addReaction.mockRejectedValue( + new SlackOperationError(200, { success: false, error: 'already_reacted' }) + ) + + const response = await executeSlackTool(request('slack_add_reaction')) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'already_reacted', + }) + }) + + it('propagates cancellation before validation or provider work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeSlackTool(request('slack_download', { signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mocks.download).not.toHaveBeenCalled() + }) + + it('retains the clean 413 projection for oversized Slack downloads', async () => { + mocks.download.mockRejectedValue( + new PayloadSizeLimitError({ + label: 'response body', + maxBytes: 10, + observedBytes: 11, + }) + ) + + const response = await executeSlackTool(request('slack_download')) + + expect(response.status).toBe(413) + await expect(response.json()).resolves.toMatchObject({ success: false }) + }) +}) diff --git a/apps/sim/lib/internal/slack/execute-tool.ts b/apps/sim/lib/internal/slack/execute-tool.ts new file mode 100644 index 00000000000..fe98dd27b29 --- /dev/null +++ b/apps/sim/lib/internal/slack/execute-tool.ts @@ -0,0 +1,119 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { AnyApiRouteContract, ContractBody } from '@/lib/api/contracts' +import { + slackAddReactionContract, + slackDeleteMessageContract, + slackDownloadContract, + slackReadMessagesContract, + slackRemoveReactionContract, + slackSendEphemeralContract, + slackSendMessageContract, + slackUpdateMessageContract, +} from '@/lib/api/contracts/tools/communication/slack' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { SlackOperationError } from '@/lib/internal/slack/errors' +import { + executeSlackAddReaction, + executeSlackDeleteMessage, + executeSlackDownload, + executeSlackReadMessages, + executeSlackRemoveReaction, + executeSlackSendEphemeral, + executeSlackSendMessage, + executeSlackUpdateMessage, + type SlackOperationContext, +} from '@/lib/internal/slack/operations' +import { parseInternalToolInput } from '@/lib/internal/tool-operations/parse-input' +import type { + InternalToolOperationCall, + InternalToolOperationHandler, +} from '@/lib/internal/tool-operations/types' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' + +const logger = createLogger('SlackToolExecution') + +async function executeOperation( + contract: C, + request: InternalToolOperationCall, + execute: (input: ContractBody) => Promise +): Promise { + request.signal?.throwIfAborted() + const parsed = parseInternalToolInput(contract, request.input) + if (!parsed.success) return parsed.response + try { + const result = await execute(parsed.data) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof SlackOperationError) { + return Response.json(error.body, { status: error.status }) + } + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + const message = getErrorMessage(error, 'Unknown error occurred') + logger.error('Slack operation failed', { + error: message, + requestId: request.requestId, + toolId: request.toolId, + }) + return Response.json( + { success: false, error: message }, + { + status: isPayloadSizeLimitError(error) && request.toolId === 'slack_download' ? 413 : 500, + } + ) + } +} + +export const executeSlackTool: InternalToolOperationHandler = async (request) => { + const context: SlackOperationContext = { + requestId: request.requestId, + signal: request.signal, + userId: request.context.userId, + } + + switch (request.toolId) { + case 'slack_add_reaction': + return executeOperation(slackAddReactionContract, request, (input) => + executeSlackAddReaction(input, request.signal) + ) + case 'slack_delete_message': + return executeOperation(slackDeleteMessageContract, request, (input) => + executeSlackDeleteMessage(input, request.signal) + ) + case 'slack_download': + return executeOperation(slackDownloadContract, request, (input) => + executeSlackDownload(input, request.signal) + ) + case 'slack_ephemeral_message': + return executeOperation(slackSendEphemeralContract, request, (input) => + executeSlackSendEphemeral(input, request.signal) + ) + case 'slack_message': + if (!request.context.userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + return executeOperation(slackSendMessageContract, request, (input) => + executeSlackSendMessage(input, context) + ) + case 'slack_message_reader': + return executeOperation(slackReadMessagesContract, request, (input) => + executeSlackReadMessages(input, request.signal) + ) + case 'slack_remove_reaction': + return executeOperation(slackRemoveReactionContract, request, (input) => + executeSlackRemoveReaction(input, request.signal) + ) + case 'slack_update_message': + return executeOperation(slackUpdateMessageContract, request, (input) => + executeSlackUpdateMessage(input, request.signal) + ) + default: + return Response.json( + { success: false, error: `Unsupported Slack tool: ${request.toolId}` }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/slack/file-input.test.ts b/apps/sim/lib/internal/slack/file-input.test.ts new file mode 100644 index 00000000000..e86e0048a52 --- /dev/null +++ b/apps/sim/lib/internal/slack/file-input.test.ts @@ -0,0 +1,125 @@ +/** + * @vitest-environment node + */ +import { createLogger } from '@sim/logger' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' + +const mocks = vi.hoisted(() => ({ + assertAccess: vi.fn(), + download: vi.fn(), +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertAccess, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mocks.download, +})) + +import type { SlackOperationError } from '@/lib/internal/slack/errors' +import { forEachSlackAttachmentFile } from '@/lib/internal/slack/file-input' + +const logger = createLogger('SlackFileInputTest') +const FILES = [ + { id: 'file-1', key: 'workspace/file-1', name: 'one.txt', size: 3, type: 'text/plain' }, + { id: 'file-2', key: 'execution/file-2', name: 'two.txt', size: 2, type: 'text/plain' }, +] + +describe('resolveSlackAttachmentFiles', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.assertAccess.mockResolvedValue(null) + mocks.download + .mockResolvedValueOnce({ buffer: Buffer.from('one'), contentType: 'text/plain' }) + .mockResolvedValueOnce({ buffer: Buffer.from('22'), contentType: 'text/plain' }) + }) + + it('authorizes every supported storage reference and applies one aggregate byte budget', async () => { + const controller = new AbortController() + const contents: string[] = [] + await forEachSlackAttachmentFile( + FILES, + { + logger, + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + }, + async (file) => { + contents.push(file.buffer.toString()) + } + ) + + expect(contents).toEqual(['one', '22']) + expect(mocks.assertAccess).toHaveBeenNthCalledWith( + 1, + 'workspace/file-1', + 'user-1', + 'request-1', + logger + ) + expect(mocks.assertAccess).toHaveBeenNthCalledWith( + 2, + 'execution/file-2', + 'user-1', + 'request-1', + logger + ) + expect(mocks.download.mock.calls[0]?.[3]).toEqual({ + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + signal: controller.signal, + }) + expect(mocks.download.mock.calls[1]?.[3]).toEqual({ + maxBytes: MAX_BUFFERED_TRANSFER_BYTES - 3, + signal: controller.signal, + }) + }) + + it('fails closed without trusted executor identity', async () => { + await expect( + forEachSlackAttachmentFile(FILES, { logger, requestId: 'request-1' }, async () => {}) + ).rejects.toMatchObject>({ status: 401 }) + expect(mocks.assertAccess).not.toHaveBeenCalled() + }) + + it('conceals denied files as not found and never reads their bytes', async () => { + mocks.assertAccess.mockResolvedValueOnce(new Response(null, { status: 404 })) + + await expect( + forEachSlackAttachmentFile( + FILES, + { + logger, + requestId: 'request-1', + userId: 'user-1', + }, + async () => {} + ) + ).rejects.toMatchObject>({ status: 404 }) + expect(mocks.download).not.toHaveBeenCalled() + }) + + it('stops before the next authorization when cancellation arrives', async () => { + const controller = new AbortController() + mocks.download.mockReset().mockImplementationOnce(async () => { + controller.abort(new DOMException('cancelled', 'AbortError')) + return { buffer: Buffer.from('one'), contentType: 'text/plain' } + }) + + await expect( + forEachSlackAttachmentFile( + FILES, + { + logger, + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + }, + async () => {} + ) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mocks.assertAccess).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/internal/slack/file-input.ts b/apps/sim/lib/internal/slack/file-input.ts new file mode 100644 index 00000000000..de5895943cd --- /dev/null +++ b/apps/sim/lib/internal/slack/file-input.ts @@ -0,0 +1,66 @@ +import type { Logger } from '@sim/logger' +import { SlackOperationError } from '@/lib/internal/slack/errors' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +export interface SlackResolvedFile { + buffer: Buffer + contentType: string + name: string + type?: string +} + +export interface SlackFileInputContext { + logger: Logger + requestId: string + signal?: AbortSignal + userId?: string +} + +/** Resolves and consumes one Slack attachment at a time under one aggregate byte cap. */ +export async function forEachSlackAttachmentFile( + files: RawFileInput[], + context: SlackFileInputContext, + consume: (file: SlackResolvedFile) => Promise +): Promise { + context.signal?.throwIfAborted() + if (!context.userId) { + throw new SlackOperationError(401, { success: false, error: 'Authentication required' }) + } + + const userFiles = processFilesToUserFiles(files, context.requestId, context.logger) + let remainingBytes = MAX_BUFFERED_TRANSFER_BYTES + + for (const file of userFiles) { + context.signal?.throwIfAborted() + const denied = await assertToolFileAccess( + file.key, + context.userId, + context.requestId, + context.logger + ) + context.signal?.throwIfAborted() + if (denied) { + throw new SlackOperationError(404, { success: false, error: 'File not found' }) + } + + const downloaded = await downloadServableFileFromStorage( + file, + context.requestId, + context.logger, + { maxBytes: remainingBytes, signal: context.signal } + ) + context.signal?.throwIfAborted() + remainingBytes -= downloaded.buffer.length + await consume({ + buffer: downloaded.buffer, + contentType: downloaded.contentType, + name: file.name, + type: file.type, + }) + context.signal?.throwIfAborted() + } +} diff --git a/apps/sim/lib/internal/slack/operations.test.ts b/apps/sim/lib/internal/slack/operations.test.ts new file mode 100644 index 00000000000..4eca2326476 --- /dev/null +++ b/apps/sim/lib/internal/slack/operations.test.ts @@ -0,0 +1,241 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + resolveFiles: vi.fn(), + secureFetchWithPinnedIP: vi.fn(), + secureFetchWithValidation: vi.fn(), + validateUrlWithDNS: vi.fn(), +})) + +vi.mock('@/lib/internal/slack/file-input', () => ({ + forEachSlackAttachmentFile: mocks.resolveFiles, +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + secureFetchWithPinnedIP: mocks.secureFetchWithPinnedIP, + secureFetchWithValidation: mocks.secureFetchWithValidation, + validateUrlWithDNS: mocks.validateUrlWithDNS, +})) + +import type { SlackOperationError } from '@/lib/internal/slack/errors' +import { + executeSlackAddReaction, + executeSlackDownload, + executeSlackReadMessages, + executeSlackSendMessage, + executeSlackUpdateMessage, +} from '@/lib/internal/slack/operations' +import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' + +const originalFetch = global.fetch + +function slackResponse(body: unknown, status = 200): Response { + return Response.json(body, { status }) +} + +describe('Slack operations', () => { + beforeEach(() => { + vi.clearAllMocks() + global.fetch = vi.fn() as unknown as typeof fetch + mocks.validateUrlWithDNS.mockResolvedValue({ + isValid: true, + resolvedIP: '93.184.216.34', + originalHostname: 'files.slack.com', + }) + mocks.secureFetchWithValidation.mockResolvedValue(new Response(null, { status: 200 })) + }) + + afterEach(() => { + global.fetch = originalFetch + }) + + it('passes cancellation through Slack Web API calls and preserves logical-error status', async () => { + const controller = new AbortController() + vi.mocked(global.fetch).mockResolvedValueOnce( + slackResponse({ ok: false, error: 'already_reacted' }) + ) + + await expect( + executeSlackAddReaction( + { accessToken: 'token', channel: 'C1', timestamp: '1.0', name: 'eyes' }, + controller.signal + ) + ).rejects.toMatchObject>({ + status: 200, + body: { success: false, error: 'already_reacted' }, + }) + expect(vi.mocked(global.fetch).mock.calls[0]?.[1]).toMatchObject({ + signal: controller.signal, + }) + }) + + it('retains the update fallback message and exact metadata projection', async () => { + vi.mocked(global.fetch).mockResolvedValueOnce( + slackResponse({ ok: true, channel: 'C1', ts: '2.0', text: 'normalized' }) + ) + + const result = await executeSlackUpdateMessage({ + accessToken: 'token', + channel: 'C1', + timestamp: '1.0', + text: 'updated', + }) + + expect(result).toEqual({ + success: true, + output: { + message: { + type: 'message', + ts: '2.0', + text: 'normalized', + channel: 'C1', + }, + content: 'Message updated successfully', + metadata: { channel: 'C1', timestamp: '2.0', text: 'normalized' }, + }, + }) + }) + + it('opens a DM, keeps the read limit bounded by the contract, and maps legacy fields', async () => { + vi.mocked(global.fetch) + .mockResolvedValueOnce(slackResponse({ ok: true, channel: { id: 'D1' } })) + .mockResolvedValueOnce( + slackResponse({ + ok: true, + messages: [ + { + ts: '2.0', + text: 'hello', + reactions: [{ name: 'eyes', count: 1 }], + files: [{ id: 'F1', name: 'report.pdf' }], + edited: { user: 'U2', ts: '2.1' }, + }, + ], + }) + ) + + const result = await executeSlackReadMessages({ + accessToken: 'token', + userId: 'U1', + limit: 15, + }) + + expect(result.output.messages).toHaveLength(1) + expect(result.output.messages[0]).toMatchObject({ + type: 'message', + ts: '2.0', + reactions: [{ name: 'eyes', count: 1, users: [] }], + files: [{ id: 'F1', name: 'report.pdf' }], + edited: { user: 'U2', ts: '2.1' }, + }) + const historyUrl = String(vi.mocked(global.fetch).mock.calls[1]?.[0]) + expect(historyUrl).toContain('channel=D1') + expect(historyUrl).toContain('limit=15') + }) + + it('uses the protected file resolver, validated upload URL, and completes sharing', async () => { + const controller = new AbortController() + mocks.resolveFiles.mockImplementation(async (_files, _context, consume) => { + await consume({ + buffer: Buffer.from('hello'), + contentType: 'text/plain', + name: 'hello.txt', + type: 'text/plain', + }) + }) + vi.mocked(global.fetch) + .mockResolvedValueOnce( + slackResponse({ + ok: true, + upload_url: 'https://files.slack.com/upload/signed', + file_id: 'F1', + }) + ) + .mockResolvedValueOnce( + slackResponse({ + ok: true, + files: [{ id: 'F1', name: 'hello.txt', created: 10, mimetype: 'text/plain' }], + }) + ) + + const result = await executeSlackSendMessage( + { + accessToken: 'token', + channel: 'C1', + text: 'hello', + files: [{ key: 'workspace/file-1', name: 'hello.txt', size: 5 }], + }, + { + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + } + ) + + expect(mocks.resolveFiles).toHaveBeenCalledWith( + [{ key: 'workspace/file-1', name: 'hello.txt', size: 5 }], + expect.objectContaining({ + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + }), + expect.any(Function) + ) + expect(mocks.secureFetchWithValidation).toHaveBeenCalledWith( + 'https://files.slack.com/upload/signed', + expect.objectContaining({ + method: 'POST', + body: Buffer.from('hello'), + signal: controller.signal, + }), + 'uploadUrl' + ) + expect(result.output).toMatchObject({ + channel: 'C1', + fileCount: 1, + ts: '10', + files: [{ name: 'hello.txt', size: 5 }], + }) + }) + + it('keeps private Slack downloads DNS-pinned, bounded, and cancellable', async () => { + const controller = new AbortController() + vi.mocked(global.fetch).mockResolvedValueOnce( + slackResponse({ + ok: true, + file: { + name: 'report.pdf', + mimetype: 'application/pdf', + url_private: 'https://files.slack.com/report.pdf', + }, + }) + ) + mocks.secureFetchWithPinnedIP.mockResolvedValue( + new Response(Buffer.from('pdf'), { status: 200 }) + ) + + const result = await executeSlackDownload( + { accessToken: 'token', fileId: 'F1' }, + controller.signal + ) + + expect(mocks.secureFetchWithPinnedIP).toHaveBeenCalledWith( + 'https://files.slack.com/report.pdf', + '93.184.216.34', + { + headers: { Authorization: 'Bearer token' }, + maxResponseBytes: MAX_FILE_SIZE, + signal: controller.signal, + } + ) + expect(result.output.file).toEqual({ + name: 'report.pdf', + mimeType: 'application/pdf', + data: Buffer.from('pdf').toString('base64'), + size: 3, + }) + }) +}) diff --git a/apps/sim/lib/internal/slack/operations.ts b/apps/sim/lib/internal/slack/operations.ts new file mode 100644 index 00000000000..b3649cdfdc4 --- /dev/null +++ b/apps/sim/lib/internal/slack/operations.ts @@ -0,0 +1,465 @@ +import { createLogger } from '@sim/logger' +import type { + SlackDeleteMessageBody, + SlackDownloadBody, + SlackReactionBody, + SlackReadMessagesBody, + SlackSendEphemeralBody, + SlackSendMessageBody, + SlackUpdateMessageBody, +} from '@/lib/api/contracts/tools/communication/slack' +import { + secureFetchWithPinnedIP, + secureFetchWithValidation, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { + openSlackDm, + requestSlackApi, + type SlackJsonObject, + slackArray, + slackObject, + slackOk, + slackString, +} from '@/lib/internal/slack/client' +import { SlackOperationError } from '@/lib/internal/slack/errors' +import { forEachSlackAttachmentFile } from '@/lib/internal/slack/file-input' +import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' +import type { ToolFileData } from '@/tools/types' + +const logger = createLogger('SlackOperations') + +export interface SlackOperationContext { + requestId: string + signal?: AbortSignal + userId?: string +} + +function failure(status: number, error: string): never { + throw new SlackOperationError(status, { success: false, error }) +} + +function providerError(data: SlackJsonObject, status: number, fallback: string): never { + return failure(status, slackString(data, 'error') || fallback) +} + +function record(value: unknown): SlackJsonObject { + return typeof value === 'object' && value !== null && !Array.isArray(value) + ? (value as SlackJsonObject) + : {} +} + +function mapReaction(value: unknown) { + const reaction = record(value) + return { + name: reaction.name, + count: reaction.count, + users: Array.isArray(reaction.users) ? reaction.users : [], + } +} + +function mapFile(value: unknown) { + const file = record(value) + return { + id: file.id, + name: file.name, + mimetype: file.mimetype, + size: file.size, + url_private: file.url_private, + permalink: file.permalink, + mode: file.mode, + } +} + +function mapReaderMessage(value: unknown) { + const message = record(value) + const edited = record(message.edited) + return { + type: message.type || 'message', + ts: message.ts, + text: message.text || '', + user: message.user, + bot_id: message.bot_id, + username: message.username, + channel: message.channel, + team: message.team, + thread_ts: message.thread_ts, + parent_user_id: message.parent_user_id, + reply_count: message.reply_count, + reply_users_count: message.reply_users_count, + latest_reply: message.latest_reply, + subscribed: message.subscribed, + last_read: message.last_read, + unread_count: message.unread_count, + subtype: message.subtype, + reactions: Array.isArray(message.reactions) ? message.reactions.map(mapReaction) : undefined, + is_starred: message.is_starred, + pinned_to: message.pinned_to, + files: Array.isArray(message.files) ? message.files.map(mapFile) : undefined, + attachments: message.attachments, + blocks: message.blocks, + edited: + message.edited && typeof message.edited === 'object' + ? { user: edited.user, ts: edited.ts } + : undefined, + permalink: message.permalink, + } +} + +export async function executeSlackAddReaction(input: SlackReactionBody, signal?: AbortSignal) { + const { data, status } = await requestSlackApi({ + accessToken: input.accessToken, + method: 'reactions.add', + body: { channel: input.channel, timestamp: input.timestamp, name: input.name }, + signal, + }) + if (!slackOk(data)) providerError(data, status, 'Failed to add reaction') + return { + success: true as const, + output: { + content: `Successfully added :${input.name}: reaction`, + metadata: { channel: input.channel, timestamp: input.timestamp, reaction: input.name }, + }, + } +} + +export async function executeSlackRemoveReaction(input: SlackReactionBody, signal?: AbortSignal) { + const { data, status } = await requestSlackApi({ + accessToken: input.accessToken, + method: 'reactions.remove', + body: { channel: input.channel, timestamp: input.timestamp, name: input.name }, + signal, + }) + if (!slackOk(data)) providerError(data, status, 'Failed to remove reaction') + return { + success: true as const, + output: { + content: `Successfully removed :${input.name}: reaction`, + metadata: { channel: input.channel, timestamp: input.timestamp, reaction: input.name }, + }, + } +} + +export async function executeSlackDeleteMessage( + input: SlackDeleteMessageBody, + signal?: AbortSignal +) { + const { data, status } = await requestSlackApi({ + accessToken: input.accessToken, + method: 'chat.delete', + body: { channel: input.channel, ts: input.timestamp }, + signal, + }) + if (!slackOk(data)) providerError(data, status, 'Failed to delete message') + return { + success: true as const, + output: { + content: 'Message deleted successfully', + metadata: { channel: data.channel, timestamp: data.ts }, + }, + } +} + +export async function executeSlackUpdateMessage( + input: SlackUpdateMessageBody, + signal?: AbortSignal +) { + const { data, status } = await requestSlackApi({ + accessToken: input.accessToken, + method: 'chat.update', + body: { + channel: input.channel, + ts: input.timestamp, + text: input.text, + ...(input.blocks?.length ? { blocks: input.blocks } : {}), + }, + signal, + }) + if (!slackOk(data)) providerError(data, status, 'Failed to update message') + const message = data.message ?? { + type: 'message', + ts: data.ts, + text: data.text || input.text, + channel: data.channel, + } + return { + success: true as const, + output: { + message, + content: 'Message updated successfully', + metadata: { + channel: data.channel, + timestamp: data.ts, + text: data.text || input.text, + }, + }, + } +} + +export async function executeSlackSendEphemeral( + input: SlackSendEphemeralBody, + signal?: AbortSignal +) { + const { data } = await requestSlackApi({ + accessToken: input.accessToken, + method: 'chat.postEphemeral', + body: { + channel: input.channel, + user: input.user, + text: input.text, + ...(input.thread_ts ? { thread_ts: input.thread_ts } : {}), + ...(input.blocks?.length ? { blocks: input.blocks } : {}), + }, + signal, + }) + if (!slackOk(data)) providerError(data, 400, 'Failed to send ephemeral message') + return { + success: true as const, + output: { messageTs: data.message_ts, channel: input.channel }, + } +} + +export async function executeSlackReadMessages(input: SlackReadMessagesBody, signal?: AbortSignal) { + let channel = input.channel ?? undefined + if (!channel && input.userId) channel = await openSlackDm(input.accessToken, input.userId, signal) + if (!channel) failure(400, 'Either channel or userId is required') + const { data } = await requestSlackApi({ + accessToken: input.accessToken, + method: 'conversations.history', + httpMethod: 'GET', + query: { + channel, + limit: typeof input.limit === 'number' ? input.limit : 10, + oldest: input.oldest ?? undefined, + latest: input.latest ?? undefined, + }, + signal, + }) + if (!slackOk(data)) { + const error = slackString(data, 'error') + if (error === 'not_in_channel') { + failure( + 400, + 'Bot is not in the channel. Please invite the Sim bot to your Slack channel by typing: /invite @Sim Studio' + ) + } + if (error === 'channel_not_found') { + failure(400, 'Channel not found. Please check the channel ID and try again.') + } + if (error === 'missing_scope') { + failure( + 400, + 'Missing required permissions. Reconnect your Slack account to grant channel history access (channels:history, groups:history). Reading direct message history is not supported with the Sim bot.' + ) + } + failure(400, error || 'Failed to fetch messages') + } + return { + success: true as const, + output: { messages: (slackArray(data, 'messages') ?? []).map(mapReaderMessage) }, + } +} + +function defaultMessage(ts: unknown, text: string, channel: unknown) { + return { type: 'message', ts, text, channel } +} + +function sentMessageOutput(data: SlackJsonObject, text: string) { + return { + message: data.message ?? defaultMessage(data.ts, text, data.channel), + ts: data.ts, + channel: data.channel, + } +} + +async function postSlackMessage( + input: SlackSendMessageBody, + channel: string, + signal?: AbortSignal +) { + return requestSlackApi({ + accessToken: input.accessToken, + method: 'chat.postMessage', + body: { + channel, + text: input.text, + ...(input.thread_ts ? { thread_ts: input.thread_ts } : {}), + ...(input.blocks?.length ? { blocks: input.blocks } : {}), + }, + signal, + }) +} + +async function uploadSlackFiles( + input: SlackSendMessageBody, + channel: string, + context: SlackOperationContext +): Promise<{ fileIds: string[]; files: ToolFileData[]; message?: unknown }> { + const fileIds: string[] = [] + const files: ToolFileData[] = [] + + await forEachSlackAttachmentFile( + input.files ?? [], + { + logger, + requestId: context.requestId, + signal: context.signal, + userId: context.userId, + }, + async (file) => { + context.signal?.throwIfAborted() + const { data } = await requestSlackApi({ + accessToken: input.accessToken, + method: 'files.getUploadURLExternal', + body: new URLSearchParams({ filename: file.name, length: String(file.buffer.length) }), + signal: context.signal, + }) + const uploadUrl = slackString(data, 'upload_url') + const fileId = slackString(data, 'file_id') + if (!slackOk(data) || !uploadUrl || !fileId) { + logger.error(`[${context.requestId}] Failed to get Slack upload URL`, { + error: slackString(data, 'error'), + }) + return + } + const uploaded = await secureFetchWithValidation( + uploadUrl, + { + method: 'POST', + body: file.buffer, + maxResponseBytes: 64 * 1024, + signal: context.signal, + }, + 'uploadUrl' + ) + context.signal?.throwIfAborted() + if (!uploaded.ok) { + logger.error(`[${context.requestId}] Failed to upload Slack file data`, { + status: uploaded.status, + }) + return + } + fileIds.push(fileId) + files.push({ + name: file.name, + mimeType: file.contentType || file.type || 'application/octet-stream', + data: file.buffer.toString('base64'), + size: file.buffer.length, + }) + } + ) + + if (fileIds.length === 0) return { fileIds, files } + const { data } = await requestSlackApi({ + accessToken: input.accessToken, + method: 'files.completeUploadExternal', + body: { + files: fileIds.map((id) => ({ id })), + channel_id: channel, + ...(input.blocks?.length ? { blocks: input.blocks } : { initial_comment: input.text }), + ...(input.thread_ts ? { thread_ts: input.thread_ts } : {}), + }, + signal: context.signal, + }) + if (!slackOk(data)) providerError(data, 400, 'Failed to complete file upload') + const slackFiles = slackArray(data, 'files') ?? [] + const first = record(slackFiles[0]) + const message = { + type: 'message', + ts: + first.created !== undefined && first.created !== null + ? String(first.created) + : String(Date.now() / 1000), + text: input.text, + channel, + files: slackFiles.map((value) => { + const slackFile = record(value) + return { + id: slackFile.id, + name: slackFile.name, + mimetype: slackFile.mimetype, + size: slackFile.size, + url_private: slackFile.url_private, + permalink: slackFile.permalink, + } + }), + } + return { fileIds, files, message } +} + +export async function executeSlackSendMessage( + input: SlackSendMessageBody, + context: SlackOperationContext +) { + context.signal?.throwIfAborted() + if (!context.userId) failure(401, 'Authentication required') + let channel = input.channel ?? undefined + if (!channel && input.userId) { + channel = await openSlackDm(input.accessToken, input.userId, context.signal) + } + if (!channel) failure(400, 'Either channel or userId is required') + + if (!input.files?.length) { + const { data } = await postSlackMessage(input, channel, context.signal) + if (!slackOk(data)) providerError(data, 400, 'Failed to send message') + return { success: true as const, output: sentMessageOutput(data, input.text) } + } + + const uploaded = await uploadSlackFiles(input, channel, context) + if (uploaded.fileIds.length === 0) { + const { data } = await postSlackMessage(input, channel, context.signal) + if (!slackOk(data)) providerError(data, 400, 'Failed to send message') + return { success: true as const, output: sentMessageOutput(data, input.text) } + } + + return { + success: true as const, + output: { + message: uploaded.message, + ts: record(uploaded.message).ts, + channel, + fileCount: uploaded.fileIds.length, + files: uploaded.files, + }, + } +} + +export async function executeSlackDownload(input: SlackDownloadBody, signal?: AbortSignal) { + const { data, status, statusText } = await requestSlackApi({ + accessToken: input.accessToken, + method: 'files.info', + httpMethod: 'GET', + query: { file: input.fileId }, + signal, + tolerateInvalidErrorJson: true, + }) + if (status < 200 || status >= 300) { + logger.error('Failed to get Slack file info', { status, statusText }) + failure(400, slackString(data, 'error') || 'Failed to get file info') + } + if (!slackOk(data)) providerError(data, 400, 'Slack API error') + const file = slackObject(data, 'file') ?? {} + const name = input.fileName || slackString(file, 'name') || 'download' + const mimeType = slackString(file, 'mimetype') || 'application/octet-stream' + const urlPrivate = slackString(file, 'url_private') + if (!urlPrivate) failure(400, 'File does not have a download URL') + const downloadUrl = urlPrivate + const validation = await validateUrlWithDNS(downloadUrl, 'urlPrivate') + signal?.throwIfAborted() + if (!validation.isValid) failure(400, validation.error || 'Invalid Slack file URL') + const response = await secureFetchWithPinnedIP(downloadUrl, validation.resolvedIP!, { + headers: { Authorization: `Bearer ${input.accessToken}` }, + maxResponseBytes: MAX_FILE_SIZE, + signal, + }) + signal?.throwIfAborted() + if (!response.ok) failure(400, 'Failed to download file content') + const buffer = Buffer.from(await response.arrayBuffer()) + signal?.throwIfAborted() + return { + success: true as const, + output: { + file: { name, mimeType, data: buffer.toString('base64'), size: buffer.length }, + }, + } +} diff --git a/apps/sim/lib/internal/sms/execute-tool.test.ts b/apps/sim/lib/internal/sms/execute-tool.test.ts new file mode 100644 index 00000000000..27880819c86 --- /dev/null +++ b/apps/sim/lib/internal/sms/execute-tool.test.ts @@ -0,0 +1,50 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockSendSms } = vi.hoisted(() => ({ mockSendSms: vi.fn() })) + +vi.mock('@/lib/core/config/env', () => ({ env: { TWILIO_PHONE_NUMBER: '+15555550100' } })) +vi.mock('@/lib/messaging/sms/service', () => ({ sendSMS: mockSendSms })) + +import { executeSmsTool } from '@/lib/internal/sms/execute-tool' + +const context = { workflowId: 'workflow-1', workspaceId: 'workspace-1', userId: 'user-1' } + +describe('executeSmsTool', () => { + beforeEach(() => vi.clearAllMocks()) + + it('sends validated operation input through the SMS service once', async () => { + mockSendSms.mockResolvedValue({ success: true, message: 'sent' }) + + const response = await executeSmsTool({ + toolId: 'sms_send', + input: { to: '+15555550101', body: 'hello' }, + headers: new Headers(), + context, + requestId: 'request-1', + }) + + expect(response.status).toBe(200) + expect(mockSendSms).toHaveBeenCalledWith({ + to: '+15555550101', + body: 'hello', + from: '+15555550100', + }) + expect(mockSendSms).toHaveBeenCalledTimes(1) + }) + + it('rejects invalid input before calling the provider', async () => { + const response = await executeSmsTool({ + toolId: 'sms_send', + input: { to: '', body: 'hello' }, + headers: new Headers(), + context, + requestId: 'request-1', + }) + + expect(response.status).toBe(400) + expect(mockSendSms).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/sms/execute-tool.ts b/apps/sim/lib/internal/sms/execute-tool.ts new file mode 100644 index 00000000000..c7d96b405a4 --- /dev/null +++ b/apps/sim/lib/internal/sms/execute-tool.ts @@ -0,0 +1,39 @@ +import { smsSendBodySchema } from '@/lib/api/contracts/tools/communication/messaging' +import { env } from '@/lib/core/config/env' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { sendSMS } from '@/lib/messaging/sms/service' + +export const executeSmsTool: InternalToolOperationHandler = async ({ + toolId, + input, + context, + signal, +}) => { + signal?.throwIfAborted() + if (toolId !== 'sms_send') { + return Response.json({ error: `Unsupported SMS tool: ${toolId}` }, { status: 500 }) + } + if (!context.userId) { + return Response.json({ success: false, message: 'Authentication required' }, { status: 401 }) + } + + const parsed = smsSendBodySchema.safeParse(input) + if (!parsed.success) { + return Response.json( + { success: false, message: parsed.error.issues[0]?.message ?? 'Invalid request data' }, + { status: 400 } + ) + } + + const fromNumber = env.TWILIO_PHONE_NUMBER + if (!fromNumber) { + return Response.json( + { success: false, message: 'SMS sending failed: No phone number configured.' }, + { status: 500 } + ) + } + + const result = await sendSMS({ ...parsed.data, from: fromNumber }) + signal?.throwIfAborted() + return Response.json(result) +} diff --git a/apps/sim/lib/internal/smtp/client.test.ts b/apps/sim/lib/internal/smtp/client.test.ts new file mode 100644 index 00000000000..4fdbd95d51f --- /dev/null +++ b/apps/sim/lib/internal/smtp/client.test.ts @@ -0,0 +1,47 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ close: vi.fn(), create: vi.fn(), send: vi.fn() })) +vi.mock('nodemailer', () => ({ + default: { createTransport: mocks.create }, +})) + +import { sendSmtpMessage } from '@/lib/internal/smtp/client' + +const config = { + host: '203.0.113.10', + port: 465, + secure: true, + auth: { user: 'user', pass: 'password' }, + name: 'sim.example.com', + tls: { rejectUnauthorized: true, servername: 'smtp.example.com' }, +} + +describe('SMTP client lifecycle', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.create.mockReturnValue({ close: mocks.close, sendMail: mocks.send }) + mocks.send.mockResolvedValue({ messageId: 'message-1' }) + }) + + it('closes the transporter after successful delivery', async () => { + await expect(sendSmtpMessage(config, { to: 'to@example.com' })).resolves.toEqual({ + messageId: 'message-1', + }) + expect(mocks.close).toHaveBeenCalledOnce() + }) + + it('closes the transporter when execution aborts', async () => { + const controller = new AbortController() + mocks.send.mockImplementation(async () => { + controller.abort(new Error('Execution aborted')) + throw new Error('connection closed') + }) + await expect( + sendSmtpMessage(config, { to: 'to@example.com' }, controller.signal) + ).rejects.toThrow() + expect(mocks.close).toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/smtp/client.ts b/apps/sim/lib/internal/smtp/client.ts new file mode 100644 index 00000000000..e2ea25c3007 --- /dev/null +++ b/apps/sim/lib/internal/smtp/client.ts @@ -0,0 +1,46 @@ +import { createLogger } from '@sim/logger' +import nodemailer from 'nodemailer' + +const logger = createLogger('SmtpClient') + +export interface SmtpClientConfig { + host: string + port: number + secure: boolean + auth: { user: string; pass: string } + name?: string + tls: { rejectUnauthorized: boolean; servername: string } +} + +function closeTransporter(transporter: nodemailer.Transporter): void { + try { + transporter.close() + } catch (error) { + logger.warn('Failed to close SMTP transporter', { error }) + } +} + +export async function sendSmtpMessage( + config: SmtpClientConfig, + message: nodemailer.SendMailOptions, + signal?: AbortSignal +): Promise<{ messageId?: string }> { + signal?.throwIfAborted() + const transporter = nodemailer.createTransport(config) + let closed = false + const close = () => { + if (closed) return + closed = true + closeTransporter(transporter) + } + const abort = () => close() + signal?.addEventListener('abort', abort, { once: true }) + try { + const result = await transporter.sendMail(message) + signal?.throwIfAborted() + return { messageId: result.messageId } + } finally { + signal?.removeEventListener('abort', abort) + close() + } +} diff --git a/apps/sim/lib/internal/smtp/errors.ts b/apps/sim/lib/internal/smtp/errors.ts new file mode 100644 index 00000000000..c6b3a10e4d7 --- /dev/null +++ b/apps/sim/lib/internal/smtp/errors.ts @@ -0,0 +1,10 @@ +export class SmtpOperationError extends Error { + constructor( + message: string, + readonly status: number, + readonly body: Record = { success: false, error: message } + ) { + super(message) + this.name = 'SmtpOperationError' + } +} diff --git a/apps/sim/lib/internal/smtp/execute-tool.ts b/apps/sim/lib/internal/smtp/execute-tool.ts new file mode 100644 index 00000000000..2d8cd660cd8 --- /dev/null +++ b/apps/sim/lib/internal/smtp/execute-tool.ts @@ -0,0 +1,57 @@ +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { SmtpOperationError } from '@/lib/internal/smtp/errors' +import { executeSmtpSend } from '@/lib/internal/smtp/operations' +import { smtpSendInputSchema } from '@/lib/internal/smtp/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +export const executeSmtpTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'smtp_send_mail') { + return Response.json( + { success: false, error: `Unsupported SMTP tool: ${request.toolId}` }, + { status: 500 } + ) + } + if (!request.context.userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + let serializedInput: string + try { + serializedInput = JSON.stringify(request.input) ?? '' + } catch { + return Response.json({ error: 'Validation error', details: [] }, { status: 400 }) + } + if (Buffer.byteLength(serializedInput, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + const parsed = smtpSendInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { error: 'Validation error', details: parsed.error.issues }, + { status: 400 } + ) + } + try { + const result = await executeSmtpSend(parsed.data, { + requestId: request.requestId, + signal: request.signal, + userId: request.context.userId, + }) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof SmtpOperationError) { + return Response.json(error.body, { status: error.status }) + } + return Response.json( + { success: false, error: 'Failed to send email via SMTP' }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/smtp/operations.test.ts b/apps/sim/lib/internal/smtp/operations.test.ts new file mode 100644 index 00000000000..ec55e4d4972 --- /dev/null +++ b/apps/sim/lib/internal/smtp/operations.test.ts @@ -0,0 +1,102 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ host: vi.fn(), materialize: vi.fn(), send: vi.fn() })) +vi.mock('@/lib/core/security/input-validation.server', () => ({ + validateDatabaseHost: mocks.host, +})) +vi.mock('@/lib/internal/mail/attachment-materialization', async () => ({ + MailAttachmentMaterializationError: class extends Error {}, + materializeAuthorizedMailAttachments: mocks.materialize, +})) +vi.mock('@/lib/internal/smtp/client', () => ({ sendSmtpMessage: mocks.send })) +vi.mock('@/lib/messaging/email/ehlo', () => ({ getSmtpEhloName: () => 'sim.example.com' })) + +import { executeSmtpSend } from '@/lib/internal/smtp/operations' + +const base = { + smtpHost: 'smtp.example.com', + smtpPort: 465, + smtpUsername: 'user', + smtpPassword: 'password', + smtpSecure: 'SSL' as const, + from: 'from@example.com', + to: 'to@example.com', + subject: 'Hello', + body: 'Hello', + contentType: 'html' as const, +} +const context = { requestId: 'request-1', userId: 'user-1', signal: new AbortController().signal } + +describe('SMTP operation', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.host.mockResolvedValue({ isValid: true, resolvedIP: '203.0.113.10' }) + mocks.materialize.mockResolvedValue([]) + mocks.send.mockResolvedValue({ messageId: 'message-1' }) + }) + + it('pins DNS while preserving TLS SNI, SSL, EHLO, and message behavior', async () => { + await expect(executeSmtpSend({ ...base, fromName: 'Sender' }, context)).resolves.toEqual({ + success: true, + messageId: 'message-1', + to: 'to@example.com', + subject: 'Hello', + }) + expect(mocks.send).toHaveBeenCalledWith( + { + host: '203.0.113.10', + port: 465, + secure: true, + auth: { user: 'user', pass: 'password' }, + name: 'sim.example.com', + tls: { rejectUnauthorized: true, servername: 'smtp.example.com' }, + }, + expect.objectContaining({ + from: '"Sender" ', + to: 'to@example.com', + subject: 'Hello', + html: 'Hello', + }), + context.signal + ) + }) + + it('preserves None mode TLS behavior and the 25MB attachment policy', async () => { + const attachment = { key: 'workspace/ws-1/a.txt', name: 'a.txt', size: 3 } + await executeSmtpSend({ ...base, smtpSecure: 'None', attachments: [attachment] }, context) + expect(mocks.materialize).toHaveBeenCalledWith([attachment], context, { + label: 'Total attachment size', + maxTotalBytes: 25 * 1024 * 1024, + preflightDeclaredSize: true, + }) + expect(mocks.send.mock.calls[0][0]).toMatchObject({ + secure: false, + tls: { rejectUnauthorized: false, servername: 'smtp.example.com' }, + }) + }) + + it('rejects unsafe SMTP destinations before creating a transport', async () => { + mocks.host.mockResolvedValue({ isValid: false, error: 'Private address' }) + await expect(executeSmtpSend(base, context)).rejects.toMatchObject({ + status: 400, + body: { success: false, error: 'Private address' }, + }) + expect(mocks.send).not.toHaveBeenCalled() + }) + + it('preserves SMTP authentication error semantics', async () => { + const error = new Error('bad auth') as NodeJS.ErrnoException + error.code = 'EAUTH' + mocks.send.mockRejectedValue(error) + await expect(executeSmtpSend(base, context)).rejects.toMatchObject({ + status: 500, + body: { + success: false, + error: 'SMTP authentication failed - check username and password', + }, + }) + }) +}) diff --git a/apps/sim/lib/internal/smtp/operations.ts b/apps/sim/lib/internal/smtp/operations.ts new file mode 100644 index 00000000000..54eabe86286 --- /dev/null +++ b/apps/sim/lib/internal/smtp/operations.ts @@ -0,0 +1,142 @@ +import { toError } from '@sim/utils/errors' +import type nodemailer from 'nodemailer' +import { validateDatabaseHost } from '@/lib/core/security/input-validation.server' +import { + MailAttachmentMaterializationError, + materializeAuthorizedMailAttachments, +} from '@/lib/internal/mail/attachment-materialization' +import { sendSmtpMessage } from '@/lib/internal/smtp/client' +import { SmtpOperationError } from '@/lib/internal/smtp/errors' +import type { SmtpSendInput } from '@/lib/internal/smtp/schema' +import { getSmtpEhloName } from '@/lib/messaging/email/ehlo' + +const MAX_ATTACHMENT_TOTAL_BYTES = 25 * 1024 * 1024 + +export interface SmtpOperationContext { + requestId: string + signal?: AbortSignal + userId: string +} + +function sizeError(observedBytes: number): SmtpOperationError { + const sizeMB = (observedBytes / (1024 * 1024)).toFixed(2) + return new SmtpOperationError( + `Total attachment size (${sizeMB}MB) exceeds SMTP limit of 25MB`, + 400 + ) +} + +function attachmentError(error: MailAttachmentMaterializationError): SmtpOperationError { + if (error.kind === 'size') { + return sizeError(error.observedBytes ?? MAX_ATTACHMENT_TOTAL_BYTES) + } + return new SmtpOperationError(error.message, error.status, error.body) +} + +function hasResponseCode(error: unknown): error is { responseCode: number } { + return ( + typeof error === 'object' && + error !== null && + 'responseCode' in error && + typeof error.responseCode === 'number' + ) +} + +function isNodeError(error: unknown): error is NodeJS.ErrnoException { + return error instanceof Error && 'code' in error +} + +function smtpError(error: unknown): SmtpOperationError { + let message = 'Failed to send email via SMTP' + if (isNodeError(error)) { + if (error.code === 'EAUTH') { + message = 'SMTP authentication failed - check username and password' + } else if ( + error.code === 'ECONNECTION' || + error.code === 'ECONNREFUSED' || + error.code === 'ECONNRESET' || + error.code === 'ETIMEDOUT' + ) { + message = 'Could not connect to SMTP server - check host and port' + } + } + if (hasResponseCode(error)) { + if (error.responseCode >= 500) { + message = 'SMTP server error - please try again later' + } else if (error.responseCode >= 400) { + message = 'Email rejected by SMTP server - check recipient addresses' + } + } + return new SmtpOperationError(message, 500) +} + +export async function executeSmtpSend(input: SmtpSendInput, context: SmtpOperationContext) { + context.signal?.throwIfAborted() + const hostValidation = await validateDatabaseHost(input.smtpHost, 'smtpHost') + context.signal?.throwIfAborted() + if (!hostValidation.isValid) { + throw new SmtpOperationError(hostValidation.error || 'Invalid SMTP host', 400) + } + + const from = input.fromName ? `"${input.fromName}" <${input.from}>` : input.from + const contentType = input.contentType || 'text' + const message: nodemailer.SendMailOptions = { + from, + to: input.to, + subject: input.subject, + [contentType === 'html' ? 'html' : 'text']: input.body, + } + if (input.cc) message.cc = input.cc + if (input.bcc) message.bcc = input.bcc + if (input.replyTo) message.replyTo = input.replyTo + + if (input.attachments?.length) { + try { + const attachments = await materializeAuthorizedMailAttachments(input.attachments, context, { + label: 'Total attachment size', + maxTotalBytes: MAX_ATTACHMENT_TOTAL_BYTES, + preflightDeclaredSize: true, + }) + if (attachments.length > 0) { + message.attachments = attachments.map((file) => ({ + filename: file.name, + content: file.buffer, + contentType: file.contentType, + })) + } + } catch (error) { + context.signal?.throwIfAborted() + if (error instanceof MailAttachmentMaterializationError) throw attachmentError(error) + throw error + } + } + + try { + const result = await sendSmtpMessage( + { + host: hostValidation.resolvedIP ?? input.smtpHost, + port: input.smtpPort, + secure: input.smtpSecure === 'SSL', + auth: { user: input.smtpUsername, pass: input.smtpPassword }, + name: getSmtpEhloName(), + tls: + input.smtpSecure === 'None' + ? { rejectUnauthorized: false, servername: input.smtpHost } + : { rejectUnauthorized: true, servername: input.smtpHost }, + }, + message, + context.signal + ) + return { + success: true, + messageId: result.messageId, + to: input.to, + subject: input.subject, + } + } catch (error) { + context.signal?.throwIfAborted() + if (error instanceof SmtpOperationError) throw error + const normalized = toError(error) + throw smtpError(normalized) + } +} diff --git a/apps/sim/lib/internal/smtp/schema.ts b/apps/sim/lib/internal/smtp/schema.ts new file mode 100644 index 00000000000..2cfa3caa91f --- /dev/null +++ b/apps/sim/lib/internal/smtp/schema.ts @@ -0,0 +1,22 @@ +import { z } from 'zod' +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' + +export const smtpSendInputSchema = z.object({ + smtpHost: z.string().min(1, 'SMTP host is required'), + smtpPort: z.number().min(1).max(65535, 'Port must be between 1 and 65535'), + smtpUsername: z.string().min(1, 'SMTP username is required'), + smtpPassword: z.string().min(1, 'SMTP password is required'), + smtpSecure: z.enum(['TLS', 'SSL', 'None']), + from: z.string().email('Invalid from email address').min(1, 'From address is required'), + to: z.string().min(1, 'To email is required'), + subject: z.string().min(1, 'Subject is required'), + body: z.string().min(1, 'Email body is required'), + contentType: z.enum(['text', 'html']).optional().nullable(), + fromName: z.string().optional().nullable(), + cc: z.string().optional().nullable(), + bcc: z.string().optional().nullable(), + replyTo: z.string().optional().nullable(), + attachments: RawFileInputArraySchema.optional().nullable(), +}) + +export type SmtpSendInput = z.output diff --git a/apps/sim/lib/internal/sqs/client.ts b/apps/sim/lib/internal/sqs/client.ts new file mode 100644 index 00000000000..ec0f3875cf0 --- /dev/null +++ b/apps/sim/lib/internal/sqs/client.ts @@ -0,0 +1,41 @@ +import { SendMessageCommand, type SendMessageCommandOutput, SQSClient } from '@aws-sdk/client-sqs' +import type { SqsConnectionConfig } from '@/tools/sqs/types' + +export function createSqsClient(config: SqsConnectionConfig): SQSClient { + return new SQSClient({ + region: config.region, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + }) +} + +export async function sendMessage( + client: SQSClient, + queueUrl: string, + data: Record, + messageGroupId?: string | null, + messageDeduplicationId?: string | null, + signal?: AbortSignal +): Promise | null> { + const command = new SendMessageCommand({ + QueueUrl: queueUrl, + MessageBody: JSON.stringify(data), + MessageGroupId: messageGroupId ?? undefined, + ...(messageDeduplicationId ? { MessageDeduplicationId: messageDeduplicationId } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + return parseSendMessageResponse(response) +} + +function parseSendMessageResponse( + response: SendMessageCommandOutput +): Record | null { + if (!response) { + return null + } + + return { id: response.MessageId } +} diff --git a/apps/sim/lib/internal/sqs/execute-tool.test.ts b/apps/sim/lib/internal/sqs/execute-tool.test.ts new file mode 100644 index 00000000000..86c798b1206 --- /dev/null +++ b/apps/sim/lib/internal/sqs/execute-tool.test.ts @@ -0,0 +1,93 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockExecuteSqsSend } = vi.hoisted(() => ({ + mockExecuteSqsSend: vi.fn(), +})) + +vi.mock('@/lib/internal/sqs/operations', () => ({ + executeSqsSend: mockExecuteSqsSend, +})) + +import { executeSqsTool } from '@/lib/internal/sqs/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const BODY = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', + queueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/test-queue', + data: { action: 'process' }, + messageGroupId: 'group-1', + messageDeduplicationId: 'message-1', +} + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'sqs_send', + input: BODY, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + userId: 'user-1', + metadata: {}, + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeSqsTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('validates and executes the SQS send operation', async () => { + const controller = new AbortController() + const result = { message: `Message sent to SQS queue ${BODY.queueUrl}`, id: 'message-id' } + mockExecuteSqsSend.mockResolvedValue(result) + + const response = await executeSqsTool(createRequest({ signal: controller.signal })) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual(result) + expect(mockExecuteSqsSend).toHaveBeenCalledWith(BODY, controller.signal) + }) + + it('returns the route-compatible validation envelope before provider work', async () => { + const response = await executeSqsTool(createRequest({ input: { ...BODY, data: {} } })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(mockExecuteSqsSend).not.toHaveBeenCalled() + }) + + it('preserves the provider error envelope', async () => { + mockExecuteSqsSend.mockRejectedValue(new Error('AWS rejected credentials')) + + const response = await executeSqsTool(createRequest()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'SQS send message failed: AWS rejected credentials', + }) + }) + + it('propagates cancellation without starting provider work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeSqsTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mockExecuteSqsSend).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/sqs/execute-tool.ts b/apps/sim/lib/internal/sqs/execute-tool.ts new file mode 100644 index 00000000000..ddddcb6d6d9 --- /dev/null +++ b/apps/sim/lib/internal/sqs/execute-tool.ts @@ -0,0 +1,30 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { executeSqsSend } from '@/lib/internal/sqs/operations' +import { sqsSendInputSchema } from '@/lib/internal/sqs/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +export const executeSqsTool: InternalToolOperationHandler = async ({ toolId, input, signal }) => { + signal?.throwIfAborted() + + if (toolId !== 'sqs_send') { + return Response.json({ error: `Unsupported SQS tool: ${toolId}` }, { status: 500 }) + } + + const parsed = sqsSendInputSchema.safeParse(input) + if (!parsed.success) { + return Response.json( + { error: 'Invalid request data', details: parsed.error.issues }, + { status: 400 } + ) + } + + try { + return Response.json(await executeSqsSend(parsed.data, signal)) + } catch (error) { + signal?.throwIfAborted() + return Response.json( + { error: `SQS send message failed: ${getErrorMessage(error, 'Unknown error occurred')}` }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/sqs/operations.test.ts b/apps/sim/lib/internal/sqs/operations.test.ts new file mode 100644 index 00000000000..f1447e6bd4c --- /dev/null +++ b/apps/sim/lib/internal/sqs/operations.test.ts @@ -0,0 +1,60 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockCreateSqsClient, mockDestroy, mockSendMessage } = vi.hoisted(() => ({ + mockCreateSqsClient: vi.fn(), + mockDestroy: vi.fn(), + mockSendMessage: vi.fn(), +})) + +vi.mock('@/lib/internal/sqs/client', () => ({ + createSqsClient: mockCreateSqsClient, + sendMessage: mockSendMessage, +})) + +import { executeSqsSend } from '@/lib/internal/sqs/operations' + +const INPUT = { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', + queueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/test-queue', + data: { action: 'process' }, + messageGroupId: 'group-1', + messageDeduplicationId: 'message-1', +} + +describe('SQS operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCreateSqsClient.mockReturnValue({ destroy: mockDestroy }) + }) + + it('forwards cancellation and destroys the AWS client after success', async () => { + const controller = new AbortController() + mockSendMessage.mockResolvedValue({ id: 'message-id' }) + + await expect(executeSqsSend(INPUT, controller.signal)).resolves.toEqual({ + message: `Message sent to SQS queue ${INPUT.queueUrl}`, + id: 'message-id', + }) + expect(mockSendMessage).toHaveBeenCalledWith( + { destroy: mockDestroy }, + INPUT.queueUrl, + INPUT.data, + INPUT.messageGroupId, + INPUT.messageDeduplicationId, + controller.signal + ) + expect(mockDestroy).toHaveBeenCalledOnce() + }) + + it('destroys the AWS client when provider execution fails', async () => { + mockSendMessage.mockRejectedValue(new Error('provider failure')) + + await expect(executeSqsSend(INPUT)).rejects.toThrow('provider failure') + expect(mockDestroy).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/internal/sqs/operations.ts b/apps/sim/lib/internal/sqs/operations.ts new file mode 100644 index 00000000000..2ad44d2e41f --- /dev/null +++ b/apps/sim/lib/internal/sqs/operations.ts @@ -0,0 +1,24 @@ +import { createSqsClient, sendMessage } from '@/lib/internal/sqs/client' +import type { SqsSendInput } from '@/lib/internal/sqs/schema' + +export async function executeSqsSend(input: SqsSendInput, signal?: AbortSignal) { + signal?.throwIfAborted() + const client = createSqsClient(input) + try { + const result = await sendMessage( + client, + input.queueUrl, + input.data, + input.messageGroupId, + input.messageDeduplicationId, + signal + ) + signal?.throwIfAborted() + return { + message: `Message sent to SQS queue ${input.queueUrl}`, + id: result?.id, + } + } finally { + client.destroy() + } +} diff --git a/apps/sim/lib/internal/sqs/schema.ts b/apps/sim/lib/internal/sqs/schema.ts new file mode 100644 index 00000000000..6d56c0a69e6 --- /dev/null +++ b/apps/sim/lib/internal/sqs/schema.ts @@ -0,0 +1,15 @@ +import { z } from 'zod' + +export const sqsSendInputSchema = z.object({ + region: z.string().min(1, 'AWS region is required'), + accessKeyId: z.string().min(1, 'AWS access key ID is required'), + secretAccessKey: z.string().min(1, 'AWS secret access key is required'), + queueUrl: z.string().min(1, 'Queue URL is required'), + messageGroupId: z.string().nullish(), + messageDeduplicationId: z.string().nullish(), + data: z.record(z.string(), z.unknown()).refine((obj) => Object.keys(obj).length > 0, { + message: 'Data object must have at least one field', + }), +}) + +export type SqsSendInput = z.output diff --git a/apps/sim/lib/internal/square/execute-tool.test.ts b/apps/sim/lib/internal/square/execute-tool.test.ts new file mode 100644 index 00000000000..fee90a999c2 --- /dev/null +++ b/apps/sim/lib/internal/square/execute-tool.test.ts @@ -0,0 +1,79 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockExecuteSquareCreateCatalogImage } = vi.hoisted(() => ({ + mockExecuteSquareCreateCatalogImage: vi.fn(), +})) + +vi.mock('@/lib/internal/square/operations', () => ({ + executeSquareCreateCatalogImage: mockExecuteSquareCreateCatalogImage, +})) + +import { executeSquareTool } from '@/lib/internal/square/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const FILE = { + id: 'file-1', + key: 'workspace/workspace-1/image.png', + name: 'image.png', + size: 4, + type: 'image/png', + url: '/api/files/serve?key=image.png', +} + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'square_create_catalog_image', + input: { accessToken: 'square-token', file: FILE, idempotencyKey: 'stable-key' }, + headers: new Headers(), + context: { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + userId: 'user-1', + metadata: {}, + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeSquareTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mockExecuteSquareCreateCatalogImage.mockResolvedValue(Response.json({ success: true })) + }) + + it('dispatches typed input with trusted identity', async () => { + const response = await executeSquareTool(createRequest()) + + expect(response.status).toBe(200) + expect(mockExecuteSquareCreateCatalogImage).toHaveBeenCalledWith( + expect.objectContaining({ accessToken: 'square-token', file: FILE }), + { userId: 'user-1', requestId: 'request-1', signal: undefined } + ) + }) + + it('rejects missing files before provider work', async () => { + const response = await executeSquareTool( + createRequest({ input: { accessToken: 'square-token' } }) + ) + + expect(response.status).toBe(400) + expect(mockExecuteSquareCreateCatalogImage).not.toHaveBeenCalled() + }) + + it('requires trusted execution identity', async () => { + const response = await executeSquareTool( + createRequest({ + context: { workflowId: 'workflow-1', workspaceId: 'workspace-1', metadata: {} }, + }) + ) + + expect(response.status).toBe(401) + expect(mockExecuteSquareCreateCatalogImage).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/square/execute-tool.ts b/apps/sim/lib/internal/square/execute-tool.ts new file mode 100644 index 00000000000..329a623c1b9 --- /dev/null +++ b/apps/sim/lib/internal/square/execute-tool.ts @@ -0,0 +1,28 @@ +import { getValidationErrorMessage } from '@/lib/api/server' +import { executeSquareCreateCatalogImage } from '@/lib/internal/square/operations' +import { squareCatalogImageInputSchema } from '@/lib/internal/square/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +export const executeSquareTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'square_create_catalog_image') { + return Response.json({ error: `Unsupported Square tool: ${request.toolId}` }, { status: 500 }) + } + if (!request.context.userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + + const parsed = squareCatalogImageInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { success: false, error: getValidationErrorMessage(parsed.error, 'Invalid request data') }, + { status: 400 } + ) + } + + return executeSquareCreateCatalogImage(parsed.data, { + userId: request.context.userId, + requestId: request.requestId, + signal: request.signal, + }) +} diff --git a/apps/sim/lib/internal/square/operations.test.ts b/apps/sim/lib/internal/square/operations.test.ts new file mode 100644 index 00000000000..acef2374ce9 --- /dev/null +++ b/apps/sim/lib/internal/square/operations.test.ts @@ -0,0 +1,116 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + assertToolFileAccess: vi.fn(), + downloadFileFromStorage: vi.fn(), +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertToolFileAccess, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadFileFromStorage: mocks.downloadFileFromStorage, +})) + +import { executeSquareCreateCatalogImage } from '@/lib/internal/square/operations' + +const FILE = { + id: 'file-1', + key: 'workspace/workspace-1/image.png', + name: 'image.png', + size: 4, + type: 'image/png', + url: '/api/files/serve?key=image.png', +} + +describe('executeSquareCreateCatalogImage', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.assertToolFileAccess.mockResolvedValue(null) + mocks.downloadFileFromStorage.mockResolvedValue(Buffer.from('image')) + vi.stubGlobal( + 'fetch', + vi + .fn() + .mockResolvedValue(Response.json({ image: { id: 'image-1', type: 'IMAGE', version: 1 } })) + ) + }) + + it('authorizes, loads, and uploads the file once with the supplied idempotency key', async () => { + const response = await executeSquareCreateCatalogImage( + { + accessToken: 'square-token', + file: FILE, + fileName: null, + objectId: 'item-1', + caption: 'Product', + idempotencyKey: 'stable-key', + }, + { userId: 'user-1', requestId: 'request-1' } + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + output: { metadata: { id: 'image-1', type: 'IMAGE', version: 1 } }, + }) + expect(mocks.assertToolFileAccess).toHaveBeenCalledOnce() + expect(mocks.downloadFileFromStorage).toHaveBeenCalledOnce() + expect(fetch).toHaveBeenCalledOnce() + const formData = vi.mocked(fetch).mock.calls[0][1]?.body as FormData + expect(JSON.parse(String(formData.get('request')))).toMatchObject({ + idempotency_key: 'stable-key', + object_id: 'item-1', + }) + }) + + it('preserves Square provider error status and detail', async () => { + vi.mocked(fetch).mockResolvedValueOnce( + Response.json({ errors: [{ detail: 'Invalid image' }] }, { status: 400 }) + ) + + const response = await executeSquareCreateCatalogImage( + { + accessToken: 'square-token', + file: FILE, + fileName: null, + objectId: null, + caption: null, + idempotencyKey: 'stable-key', + }, + { userId: 'user-1', requestId: 'request-1' } + ) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Invalid image', + }) + }) + + it('forwards cancellation to Square', async () => { + const controller = new AbortController() + vi.mocked(fetch).mockImplementationOnce(async (_url, init) => { + controller.abort(new DOMException('cancelled', 'AbortError')) + throw init?.signal?.reason + }) + + await expect( + executeSquareCreateCatalogImage( + { + accessToken: 'square-token', + file: FILE, + fileName: null, + objectId: null, + caption: null, + idempotencyKey: 'stable-key', + }, + { userId: 'user-1', requestId: 'request-1', signal: controller.signal } + ) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) +}) diff --git a/apps/sim/lib/internal/square/operations.ts b/apps/sim/lib/internal/square/operations.ts new file mode 100644 index 00000000000..3fb8ffec558 --- /dev/null +++ b/apps/sim/lib/internal/square/operations.ts @@ -0,0 +1,127 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { + readResponseJsonWithLimit, + readResponseTextWithLimit, +} from '@/lib/core/utils/stream-limits' +import type { SquareCatalogImageInput } from '@/lib/internal/square/schema' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' +import { SQUARE_API_VERSION, SQUARE_BASE_URL } from '@/tools/square/types' + +const logger = createLogger('SquareCatalogImage') +const MAX_SQUARE_RESPONSE_BYTES = 10 * 1024 * 1024 + +export interface SquareOperationContext { + userId: string + requestId: string + signal?: AbortSignal +} + +function failureResponse(error: string, status: number): Response { + return Response.json({ success: false, error }, { status }) +} + +export async function executeSquareCreateCatalogImage( + input: SquareCatalogImageInput, + context: SquareOperationContext +): Promise { + try { + context.signal?.throwIfAborted() + if (!input.file) return failureResponse('File is required', 400) + if (typeof input.file === 'string') return failureResponse('Invalid file input', 400) + + const userFile = processFilesToUserFiles([input.file], context.requestId, logger)[0] + if (!userFile) return failureResponse('Invalid file input', 400) + + const denied = await assertToolFileAccess( + userFile.key, + context.userId, + context.requestId, + logger + ) + context.signal?.throwIfAborted() + if (denied) return denied + + const fileBuffer = await downloadFileFromStorage(userFile, context.requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) + context.signal?.throwIfAborted() + + const imageRequest: Record = { + idempotency_key: input.idempotencyKey || generateId(), + image: { + type: 'IMAGE', + id: '#square_catalog_image', + image_data: input.caption ? { caption: input.caption } : {}, + }, + } + if (input.objectId) imageRequest.object_id = input.objectId + + const formData = new FormData() + formData.append('request', JSON.stringify(imageRequest)) + formData.append( + 'file', + new Blob([new Uint8Array(fileBuffer)], { + type: userFile.type || 'application/octet-stream', + }), + input.fileName || userFile.name + ) + + const response = await fetch(`${SQUARE_BASE_URL}/v2/catalog/images`, { + method: 'POST', + headers: { + Authorization: `Bearer ${input.accessToken}`, + 'Square-Version': SQUARE_API_VERSION, + }, + body: formData, + signal: context.signal, + }) + + if (!response.ok) { + const errorText = await readResponseTextWithLimit(response, { + maxBytes: MAX_SQUARE_RESPONSE_BYTES, + label: 'Square error response', + signal: context.signal, + }) + let detail: string | undefined + try { + const parsed = JSON.parse(errorText) as { errors?: Array<{ detail?: string }> } + detail = parsed.errors?.[0]?.detail + } catch { + detail = undefined + } + return failureResponse( + detail || `Failed to upload catalog image (HTTP ${response.status})`, + response.status + ) + } + + const data = await readResponseJsonWithLimit<{ image?: Record }>(response, { + maxBytes: MAX_SQUARE_RESPONSE_BYTES, + label: 'Square catalog image response', + signal: context.signal, + }) + const object = data.image ?? {} + return Response.json({ + success: true, + output: { + object, + metadata: { + id: typeof object.id === 'string' ? object.id : '', + type: typeof object.type === 'string' ? object.type : null, + version: typeof object.version === 'number' ? object.version : null, + }, + }, + }) + } catch (error) { + context.signal?.throwIfAborted() + logger.error(`[${context.requestId}] Square catalog image upload failed`, { + error: getErrorMessage(error), + }) + return failureResponse(getErrorMessage(error, 'Unknown error'), 500) + } +} diff --git a/apps/sim/lib/internal/square/schema.ts b/apps/sim/lib/internal/square/schema.ts new file mode 100644 index 00000000000..7601289c158 --- /dev/null +++ b/apps/sim/lib/internal/square/schema.ts @@ -0,0 +1,13 @@ +import { z } from 'zod' +import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' + +export const squareCatalogImageInputSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + file: FileInputSchema, + fileName: z.string().optional().nullable(), + objectId: z.string().optional().nullable(), + caption: z.string().optional().nullable(), + idempotencyKey: z.string().optional().nullable(), +}) + +export type SquareCatalogImageInput = z.output diff --git a/apps/sim/lib/internal/ssh/client.ts b/apps/sim/lib/internal/ssh/client.ts new file mode 100644 index 00000000000..f315dad3a77 --- /dev/null +++ b/apps/sim/lib/internal/ssh/client.ts @@ -0,0 +1,375 @@ +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { type Attributes, Client, type ClientChannel, type ConnectConfig } from 'ssh2' +import { validateDatabaseHost } from '@/lib/core/security/input-validation.server' + +const logger = createLogger('SSHClient') + +const S_IFMT = 0o170000 +const S_IFDIR = 0o040000 +const S_IFREG = 0o100000 +const S_IFLNK = 0o120000 + +export interface SSHConnectionConfig { + host: string + port: number + username: string + password?: string | null + privateKey?: string | null + passphrase?: string | null + timeout?: number + keepaliveInterval?: number + readyTimeout?: number +} + +export interface SSHCommandResult { + stdout: string + stderr: string + exitCode: number +} + +/** Formats SSH connection errors with actionable provider context. */ +function formatSSHError(err: Error, config: { host: string; port: number }): Error { + const errorMessage = err.message.toLowerCase() + const host = config.host + const port = config.port + + if (errorMessage.includes('econnrefused') || errorMessage.includes('connection refused')) { + return new Error( + `Connection refused to ${host}:${port}. ` + + `Please verify: (1) SSH server is running on the target machine, ` + + `(2) Port ${port} is correct (default SSH port is 22), ` + + `(3) Firewall allows connections to port ${port}.` + ) + } + + if (errorMessage.includes('econnreset') || errorMessage.includes('connection reset')) { + return new Error( + `Connection reset by ${host}:${port}. ` + + `This usually means: (1) Wrong port number (SSH default is 22), ` + + `(2) Server rejected the connection, ` + + `(3) Network/firewall interrupted the connection. ` + + `Verify your SSH server configuration and port number.` + ) + } + + if (errorMessage.includes('etimedout') || errorMessage.includes('timeout')) { + return new Error( + `Connection timed out to ${host}:${port}. ` + + `Please verify: (1) Host "${host}" is reachable, ` + + `(2) No firewall is blocking the connection, ` + + `(3) The SSH server is responding.` + ) + } + + if (errorMessage.includes('enotfound') || errorMessage.includes('getaddrinfo')) { + return new Error( + `Could not resolve hostname "${host}". ` + + `Please verify the hostname or IP address is correct.` + ) + } + + if (errorMessage.includes('authentication') || errorMessage.includes('auth')) { + return new Error( + `Authentication failed for user on ${host}:${port}. ` + + `Please verify: (1) Username is correct, ` + + `(2) Password or private key is valid, ` + + `(3) User has SSH access on the server.` + ) + } + + if ( + errorMessage.includes('key') && + (errorMessage.includes('parse') || errorMessage.includes('invalid')) + ) { + return new Error( + `Invalid private key format. ` + + `Please ensure you're using a valid OpenSSH private key. ` + + `The key should start with "-----BEGIN" and end with "-----END".` + ) + } + + if (errorMessage.includes('host key') || errorMessage.includes('hostkey')) { + return new Error( + `Host key verification issue for ${host}. ` + + `This may be the first connection to this server or the server's key has changed.` + ) + } + + return new Error(`SSH connection to ${host}:${port} failed: ${err.message}`) +} + +/** Opens an SSRF-validated SSH connection and cancels the handshake with the caller. */ +export async function createSSHConnection( + config: SSHConnectionConfig, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const host = config.host + + if (!host || host.trim() === '') { + throw new Error('Host is required. Please provide a valid hostname or IP address.') + } + + const hostValidation = await validateDatabaseHost(host, 'host') + signal?.throwIfAborted() + if (!hostValidation.isValid) { + throw new Error(hostValidation.error) + } + + const resolvedHost = hostValidation.resolvedIP ?? host.trim() + + return new Promise((resolve, reject) => { + const client = new Client() + const port = config.port || 22 + + const hasPassword = config.password && config.password.trim() !== '' + const hasPrivateKey = config.privateKey && config.privateKey.trim() !== '' + + if (!hasPassword && !hasPrivateKey) { + reject(new Error('Authentication required. Please provide either a password or private key.')) + return + } + + const connectConfig: ConnectConfig = { + host: resolvedHost, + port, + username: config.username, + } + + if (config.readyTimeout !== undefined) { + connectConfig.readyTimeout = config.readyTimeout + } + if (config.keepaliveInterval !== undefined) { + connectConfig.keepaliveInterval = config.keepaliveInterval + } + if (config.timeout !== undefined) { + connectConfig.timeout = config.timeout + } + + if (hasPrivateKey) { + connectConfig.privateKey = config.privateKey! + if (config.passphrase && config.passphrase.trim() !== '') { + connectConfig.passphrase = config.passphrase + } + } else if (hasPassword) { + connectConfig.password = config.password! + } + + let settled = false + const cleanup = () => signal?.removeEventListener('abort', onAbort) + const finish = (callback: () => void) => { + if (settled) return + settled = true + cleanup() + callback() + } + const onAbort = () => { + const reason = signal?.reason ?? new DOMException('Aborted', 'AbortError') + finish(() => reject(reason)) + client.destroy() + } + + signal?.addEventListener('abort', onAbort, { once: true }) + + client.on('ready', () => finish(() => resolve(client))) + + client.on('error', (err) => { + finish(() => reject(formatSSHError(err, { host, port }))) + }) + + try { + client.connect(connectConfig) + } catch (err) { + finish(() => reject(formatSSHError(toError(err), { host, port }))) + } + }) +} + +const MAX_OUTPUT_BYTES = 16 * 1024 * 1024 + +/** Executes a command while bounding stdout and stderr independently to 16 MiB. */ +export function executeSSHCommand( + client: Client, + command: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + return new Promise((resolve, reject) => { + let stream: ClientChannel | undefined + let settled = false + const cleanup = () => signal?.removeEventListener('abort', onAbort) + const finish = (callback: () => void) => { + if (settled) return + settled = true + cleanup() + callback() + } + const onAbort = () => { + stream?.close() + finish(() => reject(signal?.reason ?? new DOMException('Aborted', 'AbortError'))) + } + + signal?.addEventListener('abort', onAbort, { once: true }) + + client.exec(command, (err, channel) => { + if (err) { + finish(() => reject(err)) + return + } + + stream = channel + if (signal?.aborted) { + onAbort() + return + } + + let stdout = '' + let stderr = '' + let stdoutBytes = 0 + let stderrBytes = 0 + let stdoutTruncated = false + let stderrTruncated = false + + channel.on('error', (streamError: Error) => finish(() => reject(streamError))) + channel.on('close', (code: number) => { + finish(() => + resolve({ + stdout: stdoutTruncated + ? `${stdout.trim()}\n[output truncated: exceeded 16MB limit]` + : stdout.trim(), + stderr: stderrTruncated + ? `${stderr.trim()}\n[stderr truncated: exceeded 16MB limit]` + : stderr.trim(), + exitCode: code ?? -1, + }) + ) + }) + + channel.on('data', (data: Buffer) => { + const remaining = MAX_OUTPUT_BYTES - stdoutBytes + if (remaining <= 0) { + stdoutTruncated = true + return + } + const chunk = data.subarray(0, remaining) + stdout += chunk.toString() + stdoutBytes += chunk.length + if (data.length > remaining) stdoutTruncated = true + }) + + channel.stderr.on('data', (data: Buffer) => { + const remaining = MAX_OUTPUT_BYTES - stderrBytes + if (remaining <= 0) { + stderrTruncated = true + return + } + const chunk = data.subarray(0, remaining) + stderr += chunk.toString() + stderrBytes += chunk.length + if (data.length > remaining) stderrTruncated = true + }) + }) + }) +} + +/** Removes unsafe control bytes while preserving intentional shell syntax. */ +export function sanitizeCommand(command: string): string { + let sanitized = command.replace(/\0/g, '') + + sanitized = sanitized.replace(/[\x0B\x0C]/g, '') + + sanitized = sanitized.trim() + + const dangerousPatterns = [ + { pattern: /\$\(.*\)/, name: 'command substitution $()' }, + { pattern: /`.*`/, name: 'backtick command substitution' }, + { pattern: /;\s*rm\s+-rf/i, name: 'destructive rm -rf command' }, + { pattern: /;\s*dd\s+/i, name: 'dd command (disk operations)' }, + { pattern: /mkfs/i, name: 'filesystem formatting command' }, + { pattern: />\s*\/dev\/sd[a-z]/i, name: 'direct disk write' }, + ] + + for (const { pattern, name } of dangerousPatterns) { + if (pattern.test(sanitized)) { + logger.warn(`Command contains ${name}`, { + command: sanitized.substring(0, 100) + (sanitized.length > 100 ? '...' : ''), + }) + } + } + + return sanitized +} + +/** Removes invalid control bytes and rejects encoded or literal path traversal. */ +export function sanitizePath(path: string): string { + let sanitized = path.replace(/\0/g, '') + sanitized = sanitized.trim() + + if (sanitized.includes('%00')) { + logger.warn('Path contains URL-encoded null bytes', { + path: path.substring(0, 100), + }) + throw new Error('Path contains invalid characters') + } + + const pathTraversalPatterns = [ + '../', + '..\\', + '/../', + '\\..\\', + '%2e%2e%2f', + '%2e%2e/', + '%2e%2e%5c', + '%2e%2e\\', + '..%2f', + '..%5c', + '%252e%252e', + '..%252f', + '..%255c', + ] + + const lowerPath = sanitized.toLowerCase() + for (const pattern of pathTraversalPatterns) { + if (lowerPath.includes(pattern.toLowerCase())) { + logger.warn('Path traversal attempt detected', { + pattern, + path: path.substring(0, 100), + }) + throw new Error('Path contains invalid path traversal sequences') + } + } + + const segments = sanitized.split(/[/\\]/) + for (const segment of segments) { + if (segment === '..') { + logger.warn('Path traversal attempt detected (.. as path segment)', { + path: path.substring(0, 100), + }) + throw new Error('Path contains invalid path traversal sequences') + } + } + + return sanitized +} + +/** Escapes a value for interpolation inside a single-quoted shell argument. */ +export function escapeShellArg(arg: string): string { + return arg.replace(/'/g, "'\\''") +} + +/** Formats POSIX permission bits as an octal string. */ +export function parsePermissions(mode: number): string { + return `0${(mode & 0o777).toString(8)}` +} + +/** Maps POSIX mode bits to the tool's stable file-type vocabulary. */ +export function getFileType(attrs: Attributes): 'file' | 'directory' | 'symlink' | 'other' { + const mode = attrs.mode + const fileType = mode & S_IFMT + + if (fileType === S_IFDIR) return 'directory' + if (fileType === S_IFREG) return 'file' + if (fileType === S_IFLNK) return 'symlink' + return 'other' +} diff --git a/apps/sim/lib/internal/ssh/errors.ts b/apps/sim/lib/internal/ssh/errors.ts new file mode 100644 index 00000000000..9ff6e78520b --- /dev/null +++ b/apps/sim/lib/internal/ssh/errors.ts @@ -0,0 +1,9 @@ +export class SshOperationError extends Error { + constructor( + readonly status: number, + readonly body: { error: string } + ) { + super(body.error) + this.name = 'SshOperationError' + } +} diff --git a/apps/sim/lib/internal/ssh/execute-tool.test.ts b/apps/sim/lib/internal/ssh/execute-tool.test.ts new file mode 100644 index 00000000000..7fe223caf1b --- /dev/null +++ b/apps/sim/lib/internal/ssh/execute-tool.test.ts @@ -0,0 +1,141 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operationMocks = vi.hoisted(() => ({ + executeSshCheckCommandExists: vi.fn(), + executeSshCheckFileExists: vi.fn(), + executeSshCreateDirectory: vi.fn(), + executeSshDeleteFile: vi.fn(), + executeSshDownloadFile: vi.fn(), + executeSshExecuteCommand: vi.fn(), + executeSshExecuteScript: vi.fn(), + executeSshGetSystemInfo: vi.fn(), + executeSshListDirectory: vi.fn(), + executeSshMoveRename: vi.fn(), + executeSshReadFileContent: vi.fn(), + executeSshUploadFile: vi.fn(), + executeSshWriteFileContent: vi.fn(), +})) + +vi.mock('@/lib/internal/ssh/operations', () => operationMocks) + +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { SshOperationError } from '@/lib/internal/ssh/errors' +import { executeSshTool } from '@/lib/internal/ssh/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const CONNECTION = { + host: 'ssh.example.com', + port: 22, + username: 'deploy', + password: 'not-a-real-password', +} + +const TOOL_IDS = [ + 'ssh_check_command_exists', + 'ssh_check_file_exists', + 'ssh_create_directory', + 'ssh_delete_file', + 'ssh_download_file', + 'ssh_execute_command', + 'ssh_execute_script', + 'ssh_get_system_info', + 'ssh_list_directory', + 'ssh_move_rename', + 'ssh_read_file_content', + 'ssh_upload_file', + 'ssh_write_file_content', +] as const + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'ssh_execute_command', + input: { ...CONNECTION, command: 'pwd' }, + headers: new Headers(), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeSshTool', () => { + beforeEach(() => { + vi.clearAllMocks() + for (const operation of Object.values(operationMocks)) { + operation.mockResolvedValue({ handled: true }) + } + }) + + it('validates typed operation input and dispatches without reading a serialized body', async () => { + const controller = new AbortController() + const input = { ...CONNECTION, command: 'pwd' } + + const response = await executeSshTool(createRequest({ input, signal: controller.signal })) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ handled: true }) + expect(operationMocks.executeSshExecuteCommand).toHaveBeenCalledWith(input, { + signal: controller.signal, + }) + }) + + it.each(TOOL_IDS)('recognizes canonical tool ID %s', async (toolId) => { + const response = await executeSshTool(createRequest({ toolId, input: {} })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + }) + + it('preserves expected statuses and generic error prefixes', async () => { + operationMocks.executeSshExecuteCommand.mockRejectedValueOnce( + new SshOperationError(409, { error: 'conflict' }) + ) + const conflict = await executeSshTool(createRequest()) + expect(conflict.status).toBe(409) + await expect(conflict.json()).resolves.toEqual({ error: 'conflict' }) + + operationMocks.executeSshExecuteCommand.mockRejectedValueOnce( + new PayloadSizeLimitError({ label: 'SSH file', maxBytes: 1, observedBytes: 2 }) + ) + const oversized = await executeSshTool(createRequest()) + expect(oversized.status).toBe(413) + await expect(oversized.json()).resolves.toEqual({ + error: 'SSH file exceeds maximum size of 1 bytes (2 bytes received)', + }) + + operationMocks.executeSshExecuteCommand.mockRejectedValueOnce(new Error('connection reset')) + const generic = await executeSshTool(createRequest()) + expect(generic.status).toBe(500) + await expect(generic.json()).resolves.toEqual({ + error: 'SSH command execution failed: connection reset', + }) + }) + + it('propagates cancellation before provider work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeSshTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(operationMocks.executeSshExecuteCommand).not.toHaveBeenCalled() + }) + + it('rejects unsupported SSH IDs without provider work', async () => { + const response = await executeSshTool(createRequest({ toolId: 'ssh_unknown' })) + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ error: 'Unsupported SSH tool: ssh_unknown' }) + }) +}) diff --git a/apps/sim/lib/internal/ssh/execute-tool.ts b/apps/sim/lib/internal/ssh/execute-tool.ts new file mode 100644 index 00000000000..58010742226 --- /dev/null +++ b/apps/sim/lib/internal/ssh/execute-tool.ts @@ -0,0 +1,178 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { AnyApiRouteContract, ContractBody } from '@/lib/api/contracts' +import { + sshCheckCommandExistsContract, + sshCheckFileExistsContract, + sshCreateDirectoryContract, + sshDeleteFileContract, + sshDownloadFileContract, + sshExecuteCommandContract, + sshExecuteScriptContract, + sshGetSystemInfoContract, + sshListDirectoryContract, + sshMoveRenameContract, + sshReadFileContentContract, + sshUploadFileContract, + sshWriteFileContentContract, +} from '@/lib/api/contracts/storage-transfer' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { SshOperationError } from '@/lib/internal/ssh/errors' +import { + executeSshCheckCommandExists, + executeSshCheckFileExists, + executeSshCreateDirectory, + executeSshDeleteFile, + executeSshDownloadFile, + executeSshExecuteCommand, + executeSshExecuteScript, + executeSshGetSystemInfo, + executeSshListDirectory, + executeSshMoveRename, + executeSshReadFileContent, + executeSshUploadFile, + executeSshWriteFileContent, + type SshOperationContext, +} from '@/lib/internal/ssh/operations' +import type { + InternalToolOperationCall, + InternalToolOperationHandler, +} from '@/lib/internal/tool-operations/types' + +const logger = createLogger('SshToolExecution') + +async function executeOperation( + contract: C, + request: InternalToolOperationCall, + operation: (input: ContractBody, context: SshOperationContext) => Promise, + failureMessage: string +): Promise { + request.signal?.throwIfAborted() + if (!contract.body) throw new Error(`SSH contract ${contract.path} has no operation input`) + const parsed = contract.body.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { error: 'Invalid request data', details: parsed.error.issues }, + { status: 400 } + ) + } + + try { + const result = await operation(parsed.data as ContractBody, { signal: request.signal }) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof SshOperationError) { + return Response.json(error.body, { status: error.status }) + } + if (isPayloadSizeLimitError(error)) { + return Response.json({ error: error.message }, { status: 413 }) + } + const message = getErrorMessage(error, 'Unknown error occurred') + logger.error('SSH operation failed', { + error: message, + requestId: request.requestId, + toolId: request.toolId, + }) + return Response.json({ error: `${failureMessage}: ${message}` }, { status: 500 }) + } +} + +export const executeSshTool: InternalToolOperationHandler = async (request) => { + switch (request.toolId) { + case 'ssh_check_command_exists': + return executeOperation( + sshCheckCommandExistsContract, + request, + executeSshCheckCommandExists, + 'SSH check command exists failed' + ) + case 'ssh_check_file_exists': + return executeOperation( + sshCheckFileExistsContract, + request, + executeSshCheckFileExists, + 'SSH check file exists failed' + ) + case 'ssh_create_directory': + return executeOperation( + sshCreateDirectoryContract, + request, + executeSshCreateDirectory, + 'SSH create directory failed' + ) + case 'ssh_delete_file': + return executeOperation( + sshDeleteFileContract, + request, + executeSshDeleteFile, + 'SSH delete file failed' + ) + case 'ssh_download_file': + return executeOperation( + sshDownloadFileContract, + request, + executeSshDownloadFile, + 'SSH file download failed' + ) + case 'ssh_execute_command': + return executeOperation( + sshExecuteCommandContract, + request, + executeSshExecuteCommand, + 'SSH command execution failed' + ) + case 'ssh_execute_script': + return executeOperation( + sshExecuteScriptContract, + request, + executeSshExecuteScript, + 'SSH script execution failed' + ) + case 'ssh_get_system_info': + return executeOperation( + sshGetSystemInfoContract, + request, + executeSshGetSystemInfo, + 'SSH get system info failed' + ) + case 'ssh_list_directory': + return executeOperation( + sshListDirectoryContract, + request, + executeSshListDirectory, + 'SSH list directory failed' + ) + case 'ssh_move_rename': + return executeOperation( + sshMoveRenameContract, + request, + executeSshMoveRename, + 'SSH move/rename failed' + ) + case 'ssh_read_file_content': + return executeOperation( + sshReadFileContentContract, + request, + executeSshReadFileContent, + 'SSH read file content failed' + ) + case 'ssh_upload_file': + return executeOperation( + sshUploadFileContract, + request, + executeSshUploadFile, + 'SSH file upload failed' + ) + case 'ssh_write_file_content': + return executeOperation( + sshWriteFileContentContract, + request, + executeSshWriteFileContent, + 'SSH write file content failed' + ) + default: + return Response.json({ error: `Unsupported SSH tool: ${request.toolId}` }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/ssh/operations.test.ts b/apps/sim/lib/internal/ssh/operations.test.ts new file mode 100644 index 00000000000..5927e232ed4 --- /dev/null +++ b/apps/sim/lib/internal/ssh/operations.test.ts @@ -0,0 +1,74 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + client: { destroy: vi.fn(), end: vi.fn() }, + createSSHConnection: vi.fn(), + executeSSHCommand: vi.fn(), +})) + +vi.mock('@/lib/internal/ssh/client', () => ({ + createSSHConnection: mocks.createSSHConnection, + escapeShellArg: (value: string) => value.replace(/'/g, "'\\''"), + executeSSHCommand: mocks.executeSSHCommand, + getFileType: vi.fn(), + parsePermissions: vi.fn(), + sanitizeCommand: (value: string) => value.trim(), + sanitizePath: (value: string) => value.trim(), +})) + +import { executeSshExecuteCommand } from '@/lib/internal/ssh/operations' + +const INPUT = { + host: 'ssh.example.com', + port: 22, + username: 'deploy', + password: 'not-a-real-password', + command: ' pwd ', + workingDirectory: "/srv/app's", +} + +describe('SSH operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.createSSHConnection.mockResolvedValue(mocks.client) + mocks.executeSSHCommand.mockResolvedValue({ stdout: '/srv/app', stderr: '', exitCode: 0 }) + }) + + it('threads cancellation to connection and command while preserving command semantics', async () => { + const controller = new AbortController() + + const result = await executeSshExecuteCommand(INPUT, { signal: controller.signal }) + + expect(result).toEqual({ + stdout: '/srv/app', + stderr: '', + exitCode: 0, + success: true, + message: 'Command executed with exit code 0', + }) + expect(mocks.createSSHConnection).toHaveBeenCalledWith(INPUT, controller.signal) + expect(mocks.executeSSHCommand).toHaveBeenCalledWith( + mocks.client, + "cd '/srv/app'\\''s' && pwd", + controller.signal + ) + expect(mocks.client.end).toHaveBeenCalledOnce() + }) + + it('destroys and closes the client when cancellation wins during provider work', async () => { + const controller = new AbortController() + mocks.executeSSHCommand.mockImplementationOnce(async () => { + controller.abort(new DOMException('cancelled', 'AbortError')) + return { stdout: '', stderr: '', exitCode: 0 } + }) + + await expect( + executeSshExecuteCommand(INPUT, { signal: controller.signal }) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mocks.client.destroy).toHaveBeenCalledOnce() + expect(mocks.client.end).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/internal/ssh/operations.ts b/apps/sim/lib/internal/ssh/operations.ts new file mode 100644 index 00000000000..0072c49f4aa --- /dev/null +++ b/apps/sim/lib/internal/ssh/operations.ts @@ -0,0 +1,623 @@ +import path from 'node:path' +import { generateId } from '@sim/utils/id' +import type { Client, FileEntry, SFTPWrapper, Stats } from 'ssh2' +import type { ContractBody } from '@/lib/api/contracts' +import type { + sshCheckCommandExistsContract, + sshCheckFileExistsContract, + sshCreateDirectoryContract, + sshDeleteFileContract, + sshDownloadFileContract, + sshExecuteCommandContract, + sshExecuteScriptContract, + sshGetSystemInfoContract, + sshListDirectoryContract, + sshMoveRenameContract, + sshReadFileContentContract, + sshUploadFileContract, + sshWriteFileContentContract, +} from '@/lib/api/contracts/storage-transfer' +import { + assertKnownSizeWithinLimit, + isPayloadSizeLimitError, + readNodeStreamToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' +import { + createSSHConnection, + escapeShellArg, + executeSSHCommand, + getFileType, + parsePermissions, + type SSHConnectionConfig, + sanitizeCommand, + sanitizePath, +} from '@/lib/internal/ssh/client' +import { SshOperationError } from '@/lib/internal/ssh/errors' +import { getFileExtension, getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils' + +export interface SshOperationContext { + signal?: AbortSignal +} + +type CheckCommandExistsInput = ContractBody +type CheckFileExistsInput = ContractBody +type CreateDirectoryInput = ContractBody +type DeleteFileInput = ContractBody +type DownloadFileInput = ContractBody +type ExecuteCommandInput = ContractBody +type ExecuteScriptInput = ContractBody +type GetSystemInfoInput = ContractBody +type ListDirectoryInput = ContractBody +type MoveRenameInput = ContractBody +type ReadFileContentInput = ContractBody +type UploadFileInput = ContractBody +type WriteFileContentInput = ContractBody + +export const MAX_SSH_FILE_BYTES = 50 * 1024 * 1024 +const MAX_SSH_UPLOAD_INPUT_BYTES = Math.ceil((MAX_SSH_FILE_BYTES * 4) / 3) + 4 + +async function withClient( + input: SSHConnectionConfig, + context: SshOperationContext, + operation: (client: Client) => Promise +): Promise { + const client = await createSSHConnection(input, context.signal) + const abort = () => client.destroy() + context.signal?.addEventListener('abort', abort, { once: true }) + try { + context.signal?.throwIfAborted() + const result = await operation(client) + context.signal?.throwIfAborted() + return result + } catch (error) { + context.signal?.throwIfAborted() + throw error + } finally { + context.signal?.removeEventListener('abort', abort) + client.end() + } +} + +async function getSftp(client: Client, signal?: AbortSignal): Promise { + signal?.throwIfAborted() + const sftp = await new Promise((resolve, reject) => { + client.sftp((error, channel) => (error ? reject(error) : resolve(channel))) + }) + signal?.throwIfAborted() + return sftp +} + +function stat(sftp: SFTPWrapper, filePath: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted() + return new Promise((resolve, reject) => { + sftp.stat(filePath, (error, stats) => { + if (signal?.aborted) reject(signal.reason ?? new DOMException('Aborted', 'AbortError')) + else if (error) reject(error) + else resolve(stats) + }) + }) +} + +async function pathExists( + sftp: SFTPWrapper, + filePath: string, + signal?: AbortSignal +): Promise { + try { + await stat(sftp, filePath, signal) + return true + } catch { + signal?.throwIfAborted() + return false + } +} + +async function readFile( + sftp: SFTPWrapper, + filePath: string, + maxBytes: number, + label: string, + signal?: AbortSignal +): Promise { + const stream = sftp.createReadStream(filePath) + stream.on('error', () => {}) + return readNodeStreamToBufferWithLimit(stream, { maxBytes, label, signal }) +} + +async function writeFile( + sftp: SFTPWrapper, + filePath: string, + content: Buffer, + mode: number, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + await new Promise((resolve, reject) => { + const stream = sftp.createWriteStream(filePath, { mode }) + const abort = () => stream.destroy() + const cleanup = () => signal?.removeEventListener('abort', abort) + signal?.addEventListener('abort', abort, { once: true }) + stream.on('error', (error: Error) => { + cleanup() + reject(signal?.aborted ? signal.reason : error) + }) + stream.on('close', () => { + cleanup() + if (signal?.aborted) reject(signal.reason) + else resolve() + }) + stream.end(content) + }) + signal?.throwIfAborted() +} + +export async function executeSshCheckCommandExists( + input: CheckCommandExistsInput, + context: SshOperationContext +): Promise { + return withClient(input, context, async (client) => { + const escapedCommand = escapeShellArg(input.commandName) + const result = await executeSSHCommand( + client, + `command -v '${escapedCommand}' 2>/dev/null || which '${escapedCommand}' 2>/dev/null`, + context.signal + ) + const exists = result.exitCode === 0 && result.stdout.trim().length > 0 + const commandPath = exists ? result.stdout.trim() : undefined + let version: string | undefined + if (exists) { + try { + const versionResult = await executeSSHCommand( + client, + `'${escapedCommand}' --version 2>&1 | head -1 || '${escapedCommand}' -v 2>&1 | head -1`, + context.signal + ) + if (versionResult.exitCode === 0 && versionResult.stdout.trim()) { + version = versionResult.stdout.trim() + } + } catch { + context.signal?.throwIfAborted() + } + } + return { + exists, + path: commandPath, + version, + message: exists + ? `Command '${input.commandName}' found at ${commandPath}` + : `Command '${input.commandName}' not found`, + } + }) +} + +export async function executeSshCheckFileExists( + input: CheckFileExistsInput, + context: SshOperationContext +): Promise { + return withClient(input, context, async (client) => { + const sftp = await getSftp(client, context.signal) + const filePath = sanitizePath(input.path) + let stats: Stats + try { + stats = await stat(sftp, filePath, context.signal) + } catch { + context.signal?.throwIfAborted() + return { exists: false, type: 'not_found', message: `Path does not exist: ${filePath}` } + } + const fileType = getFileType(stats) + const metadata = { + type: fileType, + size: stats.size, + permissions: parsePermissions(stats.mode), + modified: new Date((stats.mtime || 0) * 1000).toISOString(), + } + if (input.type !== 'any' && fileType !== input.type) { + return { + exists: false, + ...metadata, + message: `Path exists but is a ${fileType}, not a ${input.type}`, + } + } + return { exists: true, ...metadata, message: `Path exists: ${filePath} (${fileType})` } + }) +} + +export async function executeSshCreateDirectory( + input: CreateDirectoryInput, + context: SshOperationContext +): Promise { + return withClient(input, context, async (client) => { + const dirPath = sanitizePath(input.path) + const escapedPath = escapeShellArg(dirPath) + const check = await executeSSHCommand( + client, + `test -d '${escapedPath}' && echo "exists"`, + context.signal + ) + if (check.stdout.trim() === 'exists') { + return { + created: false, + path: dirPath, + alreadyExists: true, + message: `Directory already exists: ${dirPath}`, + } + } + const result = await executeSSHCommand( + client, + `mkdir ${input.recursive ? '-p' : ''} -m ${input.permissions} '${escapedPath}'`, + context.signal + ) + if (result.exitCode !== 0) throw new Error(result.stderr || 'Failed to create directory') + return { + created: true, + path: dirPath, + alreadyExists: false, + message: `Directory created successfully: ${dirPath}`, + } + }) +} + +export async function executeSshDeleteFile( + input: DeleteFileInput, + context: SshOperationContext +): Promise { + return withClient(input, context, async (client) => { + const filePath = sanitizePath(input.path) + const escapedPath = escapeShellArg(filePath) + const check = await executeSSHCommand( + client, + `test -e '${escapedPath}' && echo "exists"`, + context.signal + ) + if (check.stdout.trim() !== 'exists') { + throw new SshOperationError(404, { error: `Path does not exist: ${filePath}` }) + } + const flag = input.recursive ? (input.force ? '-rf' : '-r') : input.force ? '-f' : '' + const result = await executeSSHCommand(client, `rm ${flag} '${escapedPath}'`, context.signal) + if (result.exitCode !== 0) throw new Error(result.stderr || 'Failed to delete path') + return { deleted: true, path: filePath, message: `Successfully deleted: ${filePath}` } + }) +} + +export async function executeSshDownloadFile( + input: DownloadFileInput, + context: SshOperationContext +): Promise { + return withClient(input, context, async (client) => { + const sftp = await getSftp(client, context.signal) + const remotePath = sanitizePath(input.remotePath) + let stats: Stats + try { + stats = await stat(sftp, remotePath, context.signal) + } catch { + context.signal?.throwIfAborted() + throw new Error(`File not found: ${remotePath}`) + } + if (stats.size > MAX_SSH_FILE_BYTES) { + const sizeMB = (stats.size / (1024 * 1024)).toFixed(2) + throw new SshOperationError(413, { + error: `File size (${sizeMB}MB) exceeds download limit of 50MB`, + }) + } + const content = await readFile( + sftp, + remotePath, + MAX_SSH_FILE_BYTES, + 'SSH file download', + context.signal + ) + const fileName = path.basename(remotePath) + const base64Content = content.toString('base64') + return { + downloaded: true, + file: { + name: fileName, + mimeType: getMimeTypeFromExtension(getFileExtension(fileName)), + data: base64Content, + size: content.length, + }, + content: base64Content, + fileName, + remotePath, + size: content.length, + message: `File downloaded successfully from ${remotePath}`, + } + }) +} + +export async function executeSshExecuteCommand( + input: ExecuteCommandInput, + context: SshOperationContext +): Promise { + return withClient(input, context, async (client) => { + let command = sanitizeCommand(input.command) + if (input.workingDirectory) { + command = `cd '${escapeShellArg(input.workingDirectory)}' && ${command}` + } + const result = await executeSSHCommand(client, command, context.signal) + return { + ...result, + success: result.exitCode === 0, + message: `Command executed with exit code ${result.exitCode}`, + } + }) +} + +export async function executeSshExecuteScript( + input: ExecuteScriptInput, + context: SshOperationContext +): Promise { + return withClient(input, context, async (client) => { + const scriptPath = `/tmp/sim_script_${generateId().slice(0, 8)}.sh` + const escapedScriptPath = escapeShellArg(scriptPath) + const heredocDelimiter = `SIMEOF_${generateId().replace(/-/g, '')}` + let command = `cat > '${escapedScriptPath}' << '${heredocDelimiter}' +${input.script} +${heredocDelimiter} +chmod +x '${escapedScriptPath}'` + if (input.workingDirectory) command += `\ncd '${escapeShellArg(input.workingDirectory)}'` + command += `\ntrap "rm -f ${scriptPath}" EXIT +'${escapeShellArg(input.interpreter)}' '${escapedScriptPath}' +exit_code=$? +exit $exit_code` + const result = await executeSSHCommand(client, command, context.signal) + return { + ...result, + success: result.exitCode === 0, + scriptPath, + message: `Script executed with exit code ${result.exitCode}`, + } + }) +} + +export async function executeSshGetSystemInfo( + input: GetSystemInfoInput, + context: SshOperationContext +): Promise { + return withClient(input, context, async (client) => { + const run = (command: string) => executeSSHCommand(client, command, context.signal) + const hostname = (await run('hostname')).stdout.trim() + const os = (await run('uname -s')).stdout.trim() + const architecture = (await run('uname -m')).stdout.trim() + const uptime = + Number.parseInt( + ( + await run( + "cat /proc/uptime 2>/dev/null | awk '{print int($1)}' || sysctl -n kern.boottime 2>/dev/null | awk '{print int(($(date +%s)) - $4)}'" + ) + ).stdout.trim() + ) || 0 + const memoryParts = ( + await run( + "free -b 2>/dev/null | awk '/Mem:/ {print $2, $7, $3}' || vm_stat 2>/dev/null | awk '/Pages free|Pages active|Pages speculative|Pages wired|page size/ {gsub(/[^0-9]/, \"\"); print}'" + ) + ).stdout + .trim() + .split(/\s+/) + const diskParts = ( + await run( + "df -B1 / 2>/dev/null | awk 'NR==2 {print $2, $4, $3}' || df -k / 2>/dev/null | awk 'NR==2 {print $2*1024, $4*1024, $3*1024}'" + ) + ).stdout + .trim() + .split(/\s+/) + const parseMetrics = (parts: string[]) => + parts.length >= 3 + ? { + total: Number.parseInt(parts[0]) || 0, + free: Number.parseInt(parts[1]) || 0, + used: Number.parseInt(parts[2]) || 0, + } + : { total: 0, free: 0, used: 0 } + return { + hostname, + os, + architecture, + uptime, + memory: parseMetrics(memoryParts), + diskSpace: parseMetrics(diskParts), + message: `System info retrieved for ${hostname}`, + } + }) +} + +export async function executeSshListDirectory( + input: ListDirectoryInput, + context: SshOperationContext +): Promise { + return withClient(input, context, async (client) => { + const sftp = await getSftp(client, context.signal) + const dirPath = sanitizePath(input.path) + const list = await new Promise((resolve, reject) => { + sftp.readdir(dirPath, (error, entries) => (error ? reject(error) : resolve(entries))) + }) + context.signal?.throwIfAborted() + const entries = list.map((entry) => ({ + name: entry.filename, + type: getFileType(entry.attrs), + size: entry.attrs.size, + permissions: parsePermissions(entry.attrs.mode), + modified: new Date((entry.attrs.mtime || 0) * 1000).toISOString(), + })) + const totalFiles = entries.filter((entry) => entry.type === 'file').length + const totalDirectories = entries.filter((entry) => entry.type === 'directory').length + return { + entries, + totalFiles, + totalDirectories, + message: `Found ${totalFiles} files and ${totalDirectories} directories`, + } + }) +} + +export async function executeSshMoveRename( + input: MoveRenameInput, + context: SshOperationContext +): Promise { + return withClient(input, context, async (client) => { + const sourcePath = sanitizePath(input.sourcePath) + const destinationPath = sanitizePath(input.destinationPath) + const escapedSource = escapeShellArg(sourcePath) + const escapedDestination = escapeShellArg(destinationPath) + const source = await executeSSHCommand( + client, + `test -e '${escapedSource}' && echo "exists"`, + context.signal + ) + if (source.stdout.trim() !== 'exists') { + throw new SshOperationError(404, { error: `Source path does not exist: ${sourcePath}` }) + } + if (!input.overwrite) { + const destination = await executeSSHCommand( + client, + `test -e '${escapedDestination}' && echo "exists"`, + context.signal + ) + if (destination.stdout.trim() === 'exists') { + throw new SshOperationError(409, { + error: `Destination already exists and overwrite is disabled: ${destinationPath}`, + }) + } + } + const result = await executeSSHCommand( + client, + `mv${input.overwrite ? ' -f' : ''} '${escapedSource}' '${escapedDestination}'`, + context.signal + ) + if (result.exitCode !== 0) throw new Error(result.stderr || 'Failed to move/rename') + return { + success: true, + sourcePath, + destinationPath, + message: `Successfully moved ${sourcePath} to ${destinationPath}`, + } + }) +} + +export async function executeSshReadFileContent( + input: ReadFileContentInput, + context: SshOperationContext +): Promise { + return withClient(input, context, async (client) => { + const sftp = await getSftp(client, context.signal) + const filePath = sanitizePath(input.path) + const maxBytes = input.maxSize * 1024 * 1024 + let stats: Stats + try { + stats = await stat(sftp, filePath, context.signal) + } catch { + context.signal?.throwIfAborted() + throw new Error(`File not found: ${filePath}`) + } + if (stats.size > maxBytes) { + throw new SshOperationError(413, { + error: `File size (${stats.size} bytes) exceeds maximum allowed (${maxBytes} bytes)`, + }) + } + const buffer = await readFile(sftp, filePath, maxBytes, `File '${filePath}'`, context.signal) + const content = buffer.toString(input.encoding as BufferEncoding) + const lines = content.split('\n').length + return { + content, + size: buffer.length, + lines, + path: filePath, + message: `File read successfully: ${buffer.length} bytes, ${lines} lines`, + } + }) +} + +function decodeUploadContent(fileContent: string): Buffer { + try { + const content = Buffer.from(fileContent, 'base64') + return content.toString('base64') === fileContent ? content : Buffer.from(fileContent, 'utf-8') + } catch { + return Buffer.from(fileContent, 'utf-8') + } +} + +export async function executeSshUploadFile( + input: UploadFileInput, + context: SshOperationContext +): Promise { + return withClient(input, context, async (client) => { + const sftp = await getSftp(client, context.signal) + const remotePath = sanitizePath(input.remotePath) + if (!input.overwrite && (await pathExists(sftp, remotePath, context.signal))) { + throw new SshOperationError(409, { error: 'File already exists and overwrite is disabled' }) + } + assertKnownSizeWithinLimit( + Buffer.byteLength(input.fileContent, 'utf8'), + MAX_SSH_UPLOAD_INPUT_BYTES, + 'SSH upload input' + ) + const content = decodeUploadContent(input.fileContent) + assertKnownSizeWithinLimit(content.length, MAX_SSH_FILE_BYTES, 'SSH upload file') + await writeFile( + sftp, + remotePath, + content, + input.permissions ? Number.parseInt(input.permissions, 8) : 0o644, + context.signal + ) + return { + uploaded: true, + remotePath, + size: content.length, + message: `File uploaded successfully to ${remotePath}`, + } + }) +} + +export async function executeSshWriteFileContent( + input: WriteFileContentInput, + context: SshOperationContext +): Promise { + return withClient(input, context, async (client) => { + const sftp = await getSftp(client, context.signal) + const filePath = sanitizePath(input.path) + if (input.mode === 'create' && (await pathExists(sftp, filePath, context.signal))) { + throw new SshOperationError(409, { + error: `File already exists and mode is 'create': ${filePath}`, + }) + } + const inputBytes = Buffer.byteLength(input.content, 'utf8') + assertKnownSizeWithinLimit(inputBytes, MAX_SSH_FILE_BYTES, `File '${filePath}'`) + let content = Buffer.from(input.content, 'utf-8') + if (input.mode === 'append') { + try { + const existing = await readFile( + sftp, + filePath, + MAX_SSH_FILE_BYTES, + `Existing file '${filePath}'`, + context.signal + ) + assertKnownSizeWithinLimit( + existing.length + inputBytes, + MAX_SSH_FILE_BYTES, + `File '${filePath}'` + ) + content = Buffer.concat([existing, content], existing.length + inputBytes) + } catch (error) { + context.signal?.throwIfAborted() + if (isPayloadSizeLimitError(error)) throw error + } + } + assertKnownSizeWithinLimit(content.length, MAX_SSH_FILE_BYTES, `File '${filePath}'`) + await writeFile( + sftp, + filePath, + content, + input.permissions ? Number.parseInt(input.permissions, 8) : 0o644, + context.signal + ) + const stats = await stat(sftp, filePath, context.signal) + return { + written: true, + path: filePath, + size: stats.size, + message: `File written successfully: ${stats.size} bytes`, + } + }) +} diff --git a/apps/sim/lib/internal/stagehand/client.test.ts b/apps/sim/lib/internal/stagehand/client.test.ts new file mode 100644 index 00000000000..d799d97950a --- /dev/null +++ b/apps/sim/lib/internal/stagehand/client.test.ts @@ -0,0 +1,73 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + close: vi.fn(), + init: vi.fn(), + instances: [] as Array<{ options: Record }>, +})) + +vi.mock('@/lib/core/config/env', () => ({ + env: { BROWSERBASE_API_KEY: 'browserbase-key', BROWSERBASE_PROJECT_ID: 'project-id' }, +})) + +vi.mock('@browserbasehq/stagehand', () => ({ + Stagehand: class { + options: Record + close = mocks.close + init = mocks.init + + constructor(options: Record) { + this.options = options + mocks.instances.push(this) + } + }, +})) + +import { createStagehandSession } from '@/lib/internal/stagehand/client' + +describe('Stagehand session', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.instances.length = 0 + mocks.init.mockResolvedValue(undefined) + mocks.close.mockResolvedValue(undefined) + }) + + it('preserves Browserbase and provider configuration', async () => { + const session = await createStagehandSession({ + provider: 'anthropic', + apiKey: 'sk-ant-test', + disableApi: true, + }) + + expect(mocks.instances[0].options).toMatchObject({ + env: 'BROWSERBASE', + apiKey: 'browserbase-key', + projectId: 'project-id', + disableAPI: true, + model: { modelName: 'anthropic/claude-sonnet-4-6', apiKey: 'sk-ant-test' }, + }) + await session.close() + expect(mocks.close).toHaveBeenCalledOnce() + }) + + it('closes the browser and rejects an in-flight operation on cancellation', async () => { + const controller = new AbortController() + const session = await createStagehandSession({ + provider: 'openai', + apiKey: 'sk-test', + disableApi: false, + signal: controller.signal, + }) + const pending = new Promise(() => {}) + const operation = session.run(pending) + + controller.abort(new Error('execution canceled')) + + await expect(operation).rejects.toThrow('execution canceled') + await vi.waitFor(() => expect(mocks.close).toHaveBeenCalledOnce()) + }) +}) diff --git a/apps/sim/lib/internal/stagehand/client.ts b/apps/sim/lib/internal/stagehand/client.ts new file mode 100644 index 00000000000..36a176f886e --- /dev/null +++ b/apps/sim/lib/internal/stagehand/client.ts @@ -0,0 +1,104 @@ +import type { Stagehand as StagehandType } from '@browserbasehq/stagehand' +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { env } from '@/lib/core/config/env' + +const logger = createLogger('StagehandClient') + +export interface StagehandSession { + stagehand: StagehandType + run(operation: Promise): Promise + close(): Promise +} + +export function hasBrowserbaseConfiguration(): boolean { + return Boolean(env.BROWSERBASE_API_KEY && env.BROWSERBASE_PROJECT_ID) +} + +export function getBrowserbaseApiKey(): string | undefined { + return env.BROWSERBASE_API_KEY +} + +function abortable(operation: Promise, signal?: AbortSignal): Promise { + signal?.throwIfAborted() + if (!signal) return operation + return new Promise((resolve, reject) => { + let settled = false + const finish = (callback: () => void) => { + if (settled) return + settled = true + signal.removeEventListener('abort', onAbort) + callback() + } + const onAbort = () => { + finish(() => reject(toError(signal.reason ?? new Error('Aborted')))) + } + signal.addEventListener('abort', onAbort, { once: true }) + if (signal.aborted) { + onAbort() + return + } + operation.then( + (value) => finish(() => resolve(value)), + (error) => finish(() => reject(error)) + ) + }) +} + +export async function createStagehandSession({ + provider, + apiKey, + disableApi, + signal, +}: { + provider: 'openai' | 'anthropic' + apiKey: string + disableApi: boolean + signal?: AbortSignal +}): Promise { + signal?.throwIfAborted() + const browserbaseApiKey = env.BROWSERBASE_API_KEY + const projectId = env.BROWSERBASE_PROJECT_ID + if (!browserbaseApiKey || !projectId) { + throw new Error('Server configuration error: Missing required environment variables') + } + const modelName = provider === 'anthropic' ? 'anthropic/claude-sonnet-4-6' : 'openai/gpt-5' + const { Stagehand } = await import('@browserbasehq/stagehand') + signal?.throwIfAborted() + const stagehand = new Stagehand({ + env: 'BROWSERBASE', + apiKey: browserbaseApiKey, + projectId, + verbose: 1, + ...(disableApi ? { disableAPI: true } : {}), + logger: (message) => + logger.info(typeof message === 'string' ? message : JSON.stringify(message)), + model: { modelName, apiKey }, + }) + let closePromise: Promise | undefined + const closeInstance = () => { + closePromise ??= stagehand.close().catch((error) => { + logger.error('Error closing Stagehand instance', { error }) + }) + return closePromise + } + const onAbort = () => void closeInstance() + signal?.addEventListener('abort', onAbort, { once: true }) + + try { + await abortable(stagehand.init(), signal) + } catch (error) { + signal?.removeEventListener('abort', onAbort) + await closeInstance() + throw error + } + + return { + stagehand, + run: (operation: Promise) => abortable(operation, signal), + close: async () => { + signal?.removeEventListener('abort', onAbort) + await closeInstance() + }, + } +} diff --git a/apps/sim/lib/internal/stagehand/execute-tool.test.ts b/apps/sim/lib/internal/stagehand/execute-tool.test.ts new file mode 100644 index 00000000000..152f4089ca3 --- /dev/null +++ b/apps/sim/lib/internal/stagehand/execute-tool.test.ts @@ -0,0 +1,103 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + agent: vi.fn(), + extract: vi.fn(), +})) + +vi.mock('@/lib/internal/stagehand/operations', () => ({ + executeStagehandAgent: mocks.agent, + executeStagehandExtract: mocks.extract, +})) + +import { executeStagehandTool } from '@/lib/internal/stagehand/execute-tool' +import { agentTool } from '@/tools/stagehand/agent' +import { extractTool } from '@/tools/stagehand/extract' + +describe('Stagehand internal tool execution', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.agent.mockResolvedValue(Response.json({ agentResult: {} })) + mocks.extract.mockResolvedValue(Response.json({ data: {} })) + }) + + it.each([ + [ + 'stagehand_agent', + { + task: 'Complete the task', + startUrl: 'https://example.com', + outputSchema: {}, + variables: {}, + provider: 'openai', + apiKey: 'sk-test', + mode: 'dom', + maxSteps: 20, + }, + mocks.agent, + ], + [ + 'stagehand_extract', + { + instruction: 'Extract the title', + schema: { type: 'object', properties: { title: { type: 'string' } } }, + provider: 'anthropic', + apiKey: 'sk-ant-test', + url: 'https://example.com', + }, + mocks.extract, + ], + ])('dispatches %s through its typed operation', async (toolId, input, execute) => { + await executeStagehandTool({ + toolId, + input, + headers: new Headers(), + context: { userId: 'user-1' }, + requestId: 'request-1', + }) + + expect(execute).toHaveBeenCalledOnce() + }) + + it('keeps provider keys and browser configuration out of model input', () => { + const agentParams = { + task: 'Use %account%', + startUrl: 'example.com', + variables: { account: 'private' }, + outputSchema: { type: 'object' }, + provider: 'openai' as const, + apiKey: 'sk-private', + } + expect(agentTool.operation.modelInput?.select(agentParams)).toEqual({ + task: 'Use %account%', + variables: { account: 'private' }, + outputSchema: { type: 'object' }, + }) + expect(agentTool.operation.input(agentParams)).toMatchObject({ + startUrl: 'https://example.com', + apiKey: 'sk-private', + }) + + const extractParams = { + instruction: 'Extract', + schema: { type: 'object' }, + provider: 'anthropic' as const, + apiKey: 'sk-ant-private', + url: 'https://example.com', + } + expect(extractTool.operation.modelInput?.select(extractParams)).toEqual({ + instruction: 'Extract', + schema: { type: 'object' }, + }) + }) + + it('uses operation-only declarations', () => { + for (const tool of [agentTool, extractTool]) { + expect(tool.operation).toBeDefined() + expect('request' in tool).toBe(false) + } + }) +}) diff --git a/apps/sim/lib/internal/stagehand/execute-tool.ts b/apps/sim/lib/internal/stagehand/execute-tool.ts new file mode 100644 index 00000000000..6fdcb969745 --- /dev/null +++ b/apps/sim/lib/internal/stagehand/execute-tool.ts @@ -0,0 +1,82 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { z } from 'zod' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { + executeStagehandAgent, + executeStagehandExtract, + type StagehandOperationContext, +} from '@/lib/internal/stagehand/operations' +import { + stagehandAgentInputSchema, + stagehandExtractInputSchema, +} from '@/lib/internal/stagehand/schema' +import type { + InternalToolOperationCall, + InternalToolOperationHandler, +} from '@/lib/internal/tool-operations/types' + +const logger = createLogger('StagehandToolExecution') + +async function executeParsed( + request: InternalToolOperationCall, + schema: S, + execute: (input: z.output, context: StagehandOperationContext) => Promise +): Promise { + const parsed = schema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { + error: getValidationErrorMessage(parsed.error, 'Invalid request parameters'), + details: parsed.error.issues, + }, + { status: 400 } + ) + } + if (!request.context.userId) { + return Response.json({ error: 'Unauthorized' }, { status: 401 }) + } + return execute(parsed.data, { signal: request.signal }) +} + +export const executeStagehandTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + let serializedInput: string + try { + serializedInput = JSON.stringify(request.input) ?? '' + } catch { + return Response.json({ error: 'Invalid request parameters' }, { status: 400 }) + } + if (Buffer.byteLength(serializedInput, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + + try { + switch (request.toolId) { + case 'stagehand_agent': + return executeParsed(request, stagehandAgentInputSchema, executeStagehandAgent) + case 'stagehand_extract': + return executeParsed(request, stagehandExtractInputSchema, executeStagehandExtract) + default: + return Response.json( + { error: `Unsupported Stagehand tool: ${request.toolId}` }, + { status: 500 } + ) + } + } catch (error) { + request.signal?.throwIfAborted() + const message = getErrorMessage(error, 'Unknown error') + logger.error('Stagehand operation dispatch failed', { + error: message, + requestId: request.requestId, + toolId: request.toolId, + }) + return Response.json({ error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/stagehand/operations.ts b/apps/sim/lib/internal/stagehand/operations.ts new file mode 100644 index 00000000000..e12f9f5f66d --- /dev/null +++ b/apps/sim/lib/internal/stagehand/operations.ts @@ -0,0 +1,315 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { isRecordLike } from '@sim/utils/object' +import { validateUrlWithDNS } from '@/lib/core/security/input-validation.server' +import { isSensitiveKey, REDACTED_MARKER } from '@/lib/core/security/redaction' +import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' +import { + createStagehandSession, + getBrowserbaseApiKey, + hasBrowserbaseConfiguration, + type StagehandSession, +} from '@/lib/internal/stagehand/client' +import type { StagehandAgentInput, StagehandExtractInput } from '@/lib/internal/stagehand/schema' +import { ensureZodObject, normalizeStagehandUrl } from '@/lib/internal/stagehand/schema-conversion' + +const logger = createLogger('StagehandOperations') +const MAX_BROWSERBASE_DEBUG_RESPONSE_BYTES = 256 * 1024 + +export interface StagehandOperationContext { + signal?: AbortSignal +} + +function getSchemaObject(outputSchema: unknown): Record | undefined { + if (!isRecordLike(outputSchema)) return undefined + return isRecordLike(outputSchema.schema) ? outputSchema.schema : outputSchema +} + +function formatSchemaForInstructions(schema: Record): string { + try { + return JSON.stringify(schema, null, 2) + } catch (error) { + logger.error('Error formatting schema for instructions', { error }) + return JSON.stringify(schema) + } +} + +function processVariables(variables: unknown): Record | undefined { + if (!variables) return undefined + let parsed: unknown = variables + if (typeof variables === 'string') { + try { + parsed = JSON.parse(variables) as unknown + } catch { + logger.warn('Failed to parse variables string as JSON') + return undefined + } + } + + const result: Record = {} + if (Array.isArray(parsed)) { + for (const item of parsed) { + if (!isRecordLike(item) || !isRecordLike(item.cells) || typeof item.cells.Key !== 'string') { + continue + } + result[item.cells.Key] = String(item.cells.Value ?? '') + } + } else if (isRecordLike(parsed)) { + for (const [key, value] of Object.entries(parsed)) result[key] = String(value ?? '') + } + return Object.keys(result).length > 0 ? result : undefined +} + +function substituteVariables(text: string, variables?: Record): string { + if (!variables) return text + let result = text + for (const [key, value] of Object.entries(variables)) { + result = result.split(`%${key}%`).join(value) + } + return result +} + +function validateProviderApiKey( + provider: 'openai' | 'anthropic', + apiKey: string +): Response | undefined { + if (!apiKey) return Response.json({ error: 'API key is required' }, { status: 400 }) + if (provider === 'openai' && !apiKey.startsWith('sk-')) { + return Response.json({ error: 'Invalid OpenAI API key format' }, { status: 400 }) + } + if (provider === 'anthropic' && !apiKey.startsWith('sk-ant-')) { + return Response.json({ error: 'Invalid Anthropic API key format' }, { status: 400 }) + } + return undefined +} + +function errorDetails(error: unknown): Record { + if (!(error instanceof Error)) return {} + const details: Record = { name: error.name, stack: error.stack } + if (isRecordLike(error)) { + if (error.code !== undefined) details.code = error.code + if (error.statusCode !== undefined) details.statusCode = error.statusCode + if (error.response !== undefined) details.response = error.response + } + return details +} + +async function getLiveViewUrl(sessionId: string, signal?: AbortSignal): Promise { + const browserbaseApiKey = getBrowserbaseApiKey() + if (!browserbaseApiKey) return null + try { + const response = await fetch(`https://api.browserbase.com/v1/sessions/${sessionId}/debug`, { + method: 'GET', + headers: { 'X-BB-API-Key': browserbaseApiKey }, + signal, + }) + if (!response.ok) return null + const data = await readResponseJsonWithLimit(response, { + maxBytes: MAX_BROWSERBASE_DEBUG_RESPONSE_BYTES, + label: 'Browserbase debug response', + signal, + }) + if (!isRecordLike(data)) return null + if (typeof data.debuggerFullscreenUrl === 'string') return data.debuggerFullscreenUrl + return typeof data.debuggerUrl === 'string' ? data.debuggerUrl : null + } catch (error) { + signal?.throwIfAborted() + logger.warn('Error fetching Browserbase debug URL', { error }) + return null + } +} + +export async function executeStagehandAgent( + input: StagehandAgentInput, + context: StagehandOperationContext +): Promise { + context.signal?.throwIfAborted() + let session: StagehandSession | undefined + let sessionId: string | null = null + let liveViewUrl: string | null = null + + try { + const startUrl = normalizeStagehandUrl(input.startUrl) + const urlValidation = await validateUrlWithDNS(startUrl, 'startUrl') + context.signal?.throwIfAborted() + if (!urlValidation.isValid) { + return Response.json({ error: urlValidation.error }, { status: 400 }) + } + if (!hasBrowserbaseConfiguration()) { + return Response.json( + { error: 'Server configuration error: Missing required environment variables' }, + { status: 500 } + ) + } + const apiKeyError = validateProviderApiKey(input.provider, input.apiKey) + if (apiKeyError) return apiKeyError + + try { + session = await createStagehandSession({ + provider: input.provider, + apiKey: input.apiKey, + disableApi: true, + signal: context.signal, + }) + const { stagehand } = session + sessionId = stagehand.browserbaseSessionID ?? null + if (sessionId) liveViewUrl = await getLiveViewUrl(sessionId, context.signal) + + const page = stagehand.context.pages()[0] + await session.run(page.goto(startUrl, { waitUntil: 'networkidle' })) + + const variables = processVariables(input.variables) + const task = substituteVariables(input.task, variables) + let instructions = `You are a helpful web browsing assistant. Complete the following task: ${task}` + if (variables) { + const safeKeys = Object.keys(variables).map((key) => + isSensitiveKey(key) ? `${key}: ${REDACTED_MARKER}` : key + ) + logger.info('Variables available for task', { variables: safeKeys }) + } + const schemaObject = getSchemaObject(input.outputSchema) + if (schemaObject) { + instructions += `\n\nIMPORTANT: You MUST return your final result in the following JSON format exactly:\n${formatSchemaForInstructions(schemaObject)}\n\nYour response should consist of valid JSON only, with no additional text.` + } + + const modelName = + input.provider === 'anthropic' ? 'anthropic/claude-sonnet-4-6' : 'openai/gpt-5' + const agent = stagehand.agent({ + model: { modelName, apiKey: input.apiKey }, + executionModel: { modelName, apiKey: input.apiKey }, + systemPrompt: instructions, + mode: input.mode, + }) + const execution = await session.run( + agent.execute({ instruction: task, maxSteps: input.maxSteps }) + ) + const agentResult = { + success: execution.success, + completed: execution.completed, + message: execution.message, + actions: execution.actions, + } + + let structuredOutput: unknown = null + if (agentResult.message) { + try { + let jsonContent = agentResult.message + const jsonBlockMatch = jsonContent.match(/```(?:json)?\s*([\s\S]*?)\s*```/) + if (jsonBlockMatch?.[1]) jsonContent = jsonBlockMatch[1] + structuredOutput = JSON.parse(jsonContent) as unknown + } catch (error) { + if (schemaObject) { + logger.warn('Failed to parse JSON from agent message, attempting fallback extraction', { + error, + }) + try { + const zodSchema = ensureZodObject(logger, schemaObject) + structuredOutput = await session.run( + stagehand.extract( + 'Extract the requested information from this page according to the schema', + zodSchema + ) + ) + } catch (extractError) { + context.signal?.throwIfAborted() + logger.error('Fallback extraction also failed', { error: extractError }) + } + } + } + } + + return Response.json({ agentResult, structuredOutput, liveViewUrl, sessionId }) + } catch (error) { + context.signal?.throwIfAborted() + return Response.json( + { + error: getErrorMessage(error, 'Unknown error during agent execution'), + details: errorDetails(error), + liveViewUrl, + sessionId, + }, + { status: 500 } + ) + } + } catch (error) { + context.signal?.throwIfAborted() + return Response.json( + { error: 'Internal server error', details: getErrorMessage(error, 'Unknown error') }, + { status: 500 } + ) + } finally { + await session?.close() + } +} + +export async function executeStagehandExtract( + input: StagehandExtractInput, + context: StagehandOperationContext +): Promise { + context.signal?.throwIfAborted() + let session: StagehandSession | undefined + + try { + const url = normalizeStagehandUrl(input.url) + const urlValidation = await validateUrlWithDNS(url, 'url') + context.signal?.throwIfAborted() + if (!urlValidation.isValid) { + return Response.json({ error: urlValidation.error }, { status: 400 }) + } + if (!isRecordLike(input.schema)) { + return Response.json( + { error: 'Invalid schema format. Schema must be a valid JSON object.' }, + { status: 400 } + ) + } + if (!hasBrowserbaseConfiguration()) { + return Response.json( + { error: 'Server configuration error: Missing required environment variables' }, + { status: 500 } + ) + } + const apiKeyError = validateProviderApiKey(input.provider, input.apiKey) + if (apiKeyError) return apiKeyError + + try { + session = await createStagehandSession({ + provider: input.provider, + apiKey: input.apiKey, + disableApi: false, + signal: context.signal, + }) + const { stagehand } = session + const page = stagehand.context.pages()[0] + await session.run(page.goto(url, { waitUntil: 'networkidle' })) + + const schemaObject = isRecordLike(input.schema.schema) ? input.schema.schema : input.schema + let zodSchema + try { + zodSchema = ensureZodObject(logger, schemaObject) + } catch (error) { + logger.error('Failed to convert JSON schema to Zod schema', { error }) + } + const data = zodSchema + ? await session.run(stagehand.extract(input.instruction, zodSchema)) + : await session.run(stagehand.extract(input.instruction)) + return Response.json({ data, schema: input.schema }) + } catch (error) { + context.signal?.throwIfAborted() + return Response.json( + { + error: getErrorMessage(error, 'Unknown error during extraction'), + details: errorDetails(error), + }, + { status: 500 } + ) + } + } catch (error) { + context.signal?.throwIfAborted() + return Response.json( + { error: 'Internal server error', details: getErrorMessage(error, 'Unknown error') }, + { status: 500 } + ) + } finally { + await session?.close() + } +} diff --git a/apps/sim/lib/internal/stagehand/schema-conversion.test.ts b/apps/sim/lib/internal/stagehand/schema-conversion.test.ts new file mode 100644 index 00000000000..4859c2561b6 --- /dev/null +++ b/apps/sim/lib/internal/stagehand/schema-conversion.test.ts @@ -0,0 +1,31 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { ensureZodObject, normalizeStagehandUrl } from '@/lib/internal/stagehand/schema-conversion' + +const logger = { + warn: () => {}, + error: () => {}, +} as Parameters[0] + +describe('Stagehand schema conversion', () => { + it('preserves required and optional JSON-schema properties', () => { + const schema = ensureZodObject(logger, { + type: 'object', + properties: { + title: { type: 'string' }, + count: { type: 'integer' }, + }, + required: ['title'], + }) + + expect(schema.parse({ title: 'Example' })).toEqual({ title: 'Example' }) + expect(schema.safeParse({ count: 1 }).success).toBe(false) + }) + + it('normalizes scheme-less URLs without changing absolute URLs', () => { + expect(normalizeStagehandUrl('example.com')).toBe('https://example.com') + expect(normalizeStagehandUrl('http://example.com')).toBe('http://example.com') + }) +}) diff --git a/apps/sim/lib/internal/stagehand/schema-conversion.ts b/apps/sim/lib/internal/stagehand/schema-conversion.ts new file mode 100644 index 00000000000..9b90cf9485d --- /dev/null +++ b/apps/sim/lib/internal/stagehand/schema-conversion.ts @@ -0,0 +1,55 @@ +import type { Logger } from '@sim/logger' +import { isRecordLike } from '@sim/utils/object' +import { z } from 'zod' + +function jsonSchemaToZod(logger: Logger, jsonSchema: Record): z.ZodType { + const type = jsonSchema.type + if (type === 'object' && isRecordLike(jsonSchema.properties)) { + const shape: Record = {} + const required = new Set( + Array.isArray(jsonSchema.required) + ? jsonSchema.required.filter((value): value is string => typeof value === 'string') + : [] + ) + for (const [key, property] of Object.entries(jsonSchema.properties)) { + const propertySchema = isRecordLike(property) ? property : {} + let fieldSchema = jsonSchemaToZod(logger, propertySchema) + if (typeof propertySchema.description === 'string') { + fieldSchema = fieldSchema.describe(propertySchema.description) + } + shape[key] = required.has(key) ? fieldSchema : fieldSchema.optional() + } + return z.object(shape) + } + if (type === 'array' && isRecordLike(jsonSchema.items)) { + const arraySchema = z.array(jsonSchemaToZod(logger, jsonSchema.items)) + return typeof jsonSchema.description === 'string' + ? arraySchema.describe(jsonSchema.description) + : arraySchema + } + if (type === 'string') return z.string() + if (type === 'number') return z.number() + if (type === 'integer') return z.number().int() + if (type === 'boolean') return z.boolean() + if (type === 'null') return z.null() + logger.warn('Unknown schema type, defaulting to any', { type }) + return z.any() +} + +export function ensureZodObject( + logger: Logger, + schema: Record +): z.ZodObject> { + const converted = jsonSchemaToZod(logger, schema) + if (schema.type !== 'object') { + logger.warn('Schema is not an object type, wrapping in an object', { type: schema.type }) + return z.object({ value: converted }) + } + if (converted instanceof z.ZodObject) return converted + return z.object({}) +} + +export function normalizeStagehandUrl(url: string): string { + if (url.startsWith('http://') || url.startsWith('https://')) return url + return `https://${url.trim()}` +} diff --git a/apps/sim/lib/internal/stagehand/schema.ts b/apps/sim/lib/internal/stagehand/schema.ts new file mode 100644 index 00000000000..f9ac03a7197 --- /dev/null +++ b/apps/sim/lib/internal/stagehand/schema.ts @@ -0,0 +1,24 @@ +import { z } from 'zod' +import { unknownRecordSchema } from '@/lib/api/contracts/primitives' + +export const stagehandAgentInputSchema = z.object({ + task: z.string().min(1), + startUrl: z.string().url(), + outputSchema: z.unknown(), + variables: z.unknown(), + provider: z.enum(['openai', 'anthropic']).optional().default('openai'), + apiKey: z.string(), + mode: z.enum(['dom', 'hybrid', 'cua']).optional().default('dom'), + maxSteps: z.number().int().min(1).max(200).optional().default(20), +}) + +export const stagehandExtractInputSchema = z.object({ + instruction: z.string(), + schema: unknownRecordSchema, + provider: z.enum(['openai', 'anthropic']).optional().default('openai'), + apiKey: z.string(), + url: z.string().url(), +}) + +export type StagehandAgentInput = z.output +export type StagehandExtractInput = z.output diff --git a/apps/sim/lib/internal/sts/client.ts b/apps/sim/lib/internal/sts/client.ts new file mode 100644 index 00000000000..d5fa70efed9 --- /dev/null +++ b/apps/sim/lib/internal/sts/client.ts @@ -0,0 +1,250 @@ +import { + AssumeRoleCommand, + AssumeRoleWithSAMLCommand, + AssumeRoleWithWebIdentityCommand, + GetAccessKeyInfoCommand, + GetCallerIdentityCommand, + GetSessionTokenCommand, + type PolicyDescriptorType, + STSClient, + type Tag, +} from '@aws-sdk/client-sts' +import type { STSConnectionConfig } from '@/tools/sts/types' + +export function createSTSClient(config: STSConnectionConfig): STSClient { + return new STSClient({ + region: config.region, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + }) +} + +/** + * Creates an STS client for AssumeRoleWithWebIdentity / AssumeRoleWithSAML, + * which authenticate the caller via the supplied token/assertion rather than + * an IAM access key — AWS does not check the request signature for these two + * operations. The SDK's signing middleware still requires a `credentials` + * value to be resolvable, though, so static placeholder credentials are + * supplied explicitly to skip the default credential provider chain (env + * vars, shared config, container/IMDS role). Without this, the client would + * throw a CredentialsProviderError before the request is even sent in + * environments with no ambient AWS identity, even though a real IAM identity + * was never required. + */ +export function createUnauthenticatedSTSClient(region: string): STSClient { + return new STSClient({ + region, + credentials: { accessKeyId: 'anonymous', secretAccessKey: 'anonymous' }, + }) +} + +function parsePolicyArns(policyArns?: string | null): PolicyDescriptorType[] | undefined { + if (!policyArns) return undefined + const arns = policyArns + .split(',') + .map((arn) => arn.trim()) + .filter((arn) => arn.length > 0) + return arns.length > 0 ? arns.map((arn) => ({ arn })) : undefined +} + +function parseTags(tags?: string | null): Tag[] | undefined { + if (!tags) return undefined + const parsed = JSON.parse(tags) as Record + const entries = Object.entries(parsed) + return entries.length > 0 + ? entries.map(([Key, Value]) => ({ Key, Value: String(Value) })) + : undefined +} + +function parseTransitiveTagKeys(transitiveTagKeys?: string | null): string[] | undefined { + if (!transitiveTagKeys) return undefined + const keys = transitiveTagKeys + .split(',') + .map((key) => key.trim()) + .filter((key) => key.length > 0) + return keys.length > 0 ? keys : undefined +} + +export async function assumeRole( + client: STSClient, + roleArn: string, + roleSessionName: string, + durationSeconds?: number | null, + policy?: string | null, + externalId?: string | null, + serialNumber?: string | null, + tokenCode?: string | null, + policyArns?: string | null, + tags?: string | null, + transitiveTagKeys?: string | null, + signal?: AbortSignal +) { + const command = new AssumeRoleCommand({ + RoleArn: roleArn, + RoleSessionName: roleSessionName, + ...(durationSeconds ? { DurationSeconds: durationSeconds } : {}), + ...(policy ? { Policy: policy } : {}), + ...(externalId ? { ExternalId: externalId } : {}), + ...(serialNumber ? { SerialNumber: serialNumber } : {}), + ...(tokenCode ? { TokenCode: tokenCode } : {}), + ...(() => { + const arns = parsePolicyArns(policyArns) + return arns ? { PolicyArns: arns } : {} + })(), + ...(() => { + const sessionTags = parseTags(tags) + return sessionTags ? { Tags: sessionTags } : {} + })(), + ...(() => { + const keys = parseTransitiveTagKeys(transitiveTagKeys) + return keys ? { TransitiveTagKeys: keys } : {} + })(), + }) + + const response = await client.send(command, { abortSignal: signal }) + + return { + accessKeyId: response.Credentials?.AccessKeyId ?? '', + secretAccessKey: response.Credentials?.SecretAccessKey ?? '', + sessionToken: response.Credentials?.SessionToken ?? '', + expiration: response.Credentials?.Expiration?.toISOString() ?? null, + assumedRoleArn: response.AssumedRoleUser?.Arn ?? '', + assumedRoleId: response.AssumedRoleUser?.AssumedRoleId ?? '', + packedPolicySize: response.PackedPolicySize ?? null, + sourceIdentity: response.SourceIdentity ?? null, + } +} + +export async function assumeRoleWithWebIdentity( + client: STSClient, + roleArn: string, + roleSessionName: string, + webIdentityToken: string, + providerId?: string | null, + policyArns?: string | null, + policy?: string | null, + durationSeconds?: number | null, + signal?: AbortSignal +) { + const command = new AssumeRoleWithWebIdentityCommand({ + RoleArn: roleArn, + RoleSessionName: roleSessionName, + WebIdentityToken: webIdentityToken, + ...(providerId ? { ProviderId: providerId } : {}), + ...(policy ? { Policy: policy } : {}), + ...(durationSeconds ? { DurationSeconds: durationSeconds } : {}), + ...(() => { + const arns = parsePolicyArns(policyArns) + return arns ? { PolicyArns: arns } : {} + })(), + }) + + const response = await client.send(command, { abortSignal: signal }) + + return { + accessKeyId: response.Credentials?.AccessKeyId ?? '', + secretAccessKey: response.Credentials?.SecretAccessKey ?? '', + sessionToken: response.Credentials?.SessionToken ?? '', + expiration: response.Credentials?.Expiration?.toISOString() ?? null, + assumedRoleArn: response.AssumedRoleUser?.Arn ?? '', + assumedRoleId: response.AssumedRoleUser?.AssumedRoleId ?? '', + subjectFromWebIdentityToken: response.SubjectFromWebIdentityToken ?? '', + audience: response.Audience ?? null, + provider: response.Provider ?? null, + packedPolicySize: response.PackedPolicySize ?? null, + sourceIdentity: response.SourceIdentity ?? null, + } +} + +export async function assumeRoleWithSAML( + client: STSClient, + roleArn: string, + principalArn: string, + samlAssertion: string, + policyArns?: string | null, + policy?: string | null, + durationSeconds?: number | null, + signal?: AbortSignal +) { + const command = new AssumeRoleWithSAMLCommand({ + RoleArn: roleArn, + PrincipalArn: principalArn, + SAMLAssertion: samlAssertion, + ...(policy ? { Policy: policy } : {}), + ...(durationSeconds ? { DurationSeconds: durationSeconds } : {}), + ...(() => { + const arns = parsePolicyArns(policyArns) + return arns ? { PolicyArns: arns } : {} + })(), + }) + + const response = await client.send(command, { abortSignal: signal }) + + return { + accessKeyId: response.Credentials?.AccessKeyId ?? '', + secretAccessKey: response.Credentials?.SecretAccessKey ?? '', + sessionToken: response.Credentials?.SessionToken ?? '', + expiration: response.Credentials?.Expiration?.toISOString() ?? null, + assumedRoleArn: response.AssumedRoleUser?.Arn ?? '', + assumedRoleId: response.AssumedRoleUser?.AssumedRoleId ?? '', + subject: response.Subject ?? null, + subjectType: response.SubjectType ?? null, + issuer: response.Issuer ?? null, + audience: response.Audience ?? null, + nameQualifier: response.NameQualifier ?? null, + packedPolicySize: response.PackedPolicySize ?? null, + sourceIdentity: response.SourceIdentity ?? null, + } +} + +export async function getCallerIdentity(client: STSClient, signal?: AbortSignal) { + const command = new GetCallerIdentityCommand({}) + const response = await client.send(command, { abortSignal: signal }) + + return { + account: response.Account ?? '', + arn: response.Arn ?? '', + userId: response.UserId ?? '', + } +} + +export async function getSessionToken( + client: STSClient, + durationSeconds?: number | null, + serialNumber?: string | null, + tokenCode?: string | null, + signal?: AbortSignal +) { + const command = new GetSessionTokenCommand({ + ...(durationSeconds ? { DurationSeconds: durationSeconds } : {}), + ...(serialNumber ? { SerialNumber: serialNumber } : {}), + ...(tokenCode ? { TokenCode: tokenCode } : {}), + }) + + const response = await client.send(command, { abortSignal: signal }) + + return { + accessKeyId: response.Credentials?.AccessKeyId ?? '', + secretAccessKey: response.Credentials?.SecretAccessKey ?? '', + sessionToken: response.Credentials?.SessionToken ?? '', + expiration: response.Credentials?.Expiration?.toISOString() ?? null, + } +} + +export async function getAccessKeyInfo( + client: STSClient, + accessKeyId: string, + signal?: AbortSignal +) { + const command = new GetAccessKeyInfoCommand({ + AccessKeyId: accessKeyId, + }) + + const response = await client.send(command, { abortSignal: signal }) + + return { + account: response.Account ?? '', + } +} diff --git a/apps/sim/lib/internal/sts/execute-tool.test.ts b/apps/sim/lib/internal/sts/execute-tool.test.ts new file mode 100644 index 00000000000..bdcf9f4538b --- /dev/null +++ b/apps/sim/lib/internal/sts/execute-tool.test.ts @@ -0,0 +1,117 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockExecuteStsAssumeRole, + mockExecuteStsAssumeRoleWithWebIdentity, + mockExecuteStsAssumeRoleWithSAML, + mockExecuteStsGetCallerIdentity, + mockExecuteStsGetSessionToken, + mockExecuteStsGetAccessKeyInfo, +} = vi.hoisted(() => ({ + mockExecuteStsAssumeRole: vi.fn(), + mockExecuteStsAssumeRoleWithWebIdentity: vi.fn(), + mockExecuteStsAssumeRoleWithSAML: vi.fn(), + mockExecuteStsGetCallerIdentity: vi.fn(), + mockExecuteStsGetSessionToken: vi.fn(), + mockExecuteStsGetAccessKeyInfo: vi.fn(), +})) + +vi.mock('@/lib/internal/sts/operations', () => ({ + executeStsAssumeRole: mockExecuteStsAssumeRole, + executeStsAssumeRoleWithWebIdentity: mockExecuteStsAssumeRoleWithWebIdentity, + executeStsAssumeRoleWithSAML: mockExecuteStsAssumeRoleWithSAML, + executeStsGetCallerIdentity: mockExecuteStsGetCallerIdentity, + executeStsGetSessionToken: mockExecuteStsGetSessionToken, + executeStsGetAccessKeyInfo: mockExecuteStsGetAccessKeyInfo, +})) + +import { executeStsTool } from '@/lib/internal/sts/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'sts_get_caller_identity', + input: { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', + }, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeStsTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('validates and executes the matching STS operation', async () => { + mockExecuteStsGetCallerIdentity.mockResolvedValue({ + account: '123456789012', + arn: 'arn:aws:iam::123456789012:user/test', + userId: 'AIDATEST', + }) + + const response = await executeStsTool(createRequest()) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + account: '123456789012', + arn: 'arn:aws:iam::123456789012:user/test', + userId: 'AIDATEST', + }) + expect(mockExecuteStsGetCallerIdentity).toHaveBeenCalledWith( + { + region: 'us-east-1', + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', + }, + undefined + ) + }) + + it('returns the route-compatible validation envelope before provider work', async () => { + const response = await executeStsTool(createRequest({ input: { region: 'invalid' } })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(mockExecuteStsGetCallerIdentity).not.toHaveBeenCalled() + }) + + it('preserves the provider error envelope', async () => { + mockExecuteStsGetCallerIdentity.mockRejectedValue(new Error('AWS rejected credentials')) + + const response = await executeStsTool(createRequest()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + error: 'Failed to get caller identity: AWS rejected credentials', + }) + }) + + it('propagates cancellation without converting it into a provider failure', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeStsTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mockExecuteStsGetCallerIdentity).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/sts/execute-tool.ts b/apps/sim/lib/internal/sts/execute-tool.ts new file mode 100644 index 00000000000..f2025444e9b --- /dev/null +++ b/apps/sim/lib/internal/sts/execute-tool.ts @@ -0,0 +1,102 @@ +import { toError } from '@sim/utils/errors' +import type { z } from 'zod' +import { + executeStsAssumeRole, + executeStsAssumeRoleWithSAML, + executeStsAssumeRoleWithWebIdentity, + executeStsGetAccessKeyInfo, + executeStsGetCallerIdentity, + executeStsGetSessionToken, +} from '@/lib/internal/sts/operations' +import { + stsAssumeRoleInputSchema, + stsAssumeRoleWithSamlInputSchema, + stsAssumeRoleWithWebIdentityInputSchema, + stsGetAccessKeyInfoInputSchema, + stsGetCallerIdentityInputSchema, + stsGetSessionTokenInputSchema, +} from '@/lib/internal/sts/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +async function executeOperation( + schema: z.ZodType, + input: unknown, + execute: (input: TInput, signal?: AbortSignal) => Promise, + errorMessage: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const parsed = schema.safeParse(input) + if (!parsed.success) { + return Response.json( + { error: 'Invalid request data', details: parsed.error.issues }, + { status: 400 } + ) + } + + try { + const result = await execute(parsed.data, signal) + signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + signal?.throwIfAborted() + return Response.json({ error: `${errorMessage}: ${toError(error).message}` }, { status: 500 }) + } +} + +export const executeStsTool: InternalToolOperationHandler = async ({ toolId, input, signal }) => { + signal?.throwIfAborted() + + switch (toolId) { + case 'sts_assume_role': + return executeOperation( + stsAssumeRoleInputSchema, + input, + executeStsAssumeRole, + 'Failed to assume role', + signal + ) + case 'sts_assume_role_with_web_identity': + return executeOperation( + stsAssumeRoleWithWebIdentityInputSchema, + input, + executeStsAssumeRoleWithWebIdentity, + 'Failed to assume role with web identity', + signal + ) + case 'sts_assume_role_with_saml': + return executeOperation( + stsAssumeRoleWithSamlInputSchema, + input, + executeStsAssumeRoleWithSAML, + 'Failed to assume role with SAML', + signal + ) + case 'sts_get_caller_identity': + return executeOperation( + stsGetCallerIdentityInputSchema, + input, + executeStsGetCallerIdentity, + 'Failed to get caller identity', + signal + ) + case 'sts_get_session_token': + return executeOperation( + stsGetSessionTokenInputSchema, + input, + executeStsGetSessionToken, + 'Failed to get session token', + signal + ) + case 'sts_get_access_key_info': + return executeOperation( + stsGetAccessKeyInfoInputSchema, + input, + executeStsGetAccessKeyInfo, + 'Failed to get access key info', + signal + ) + default: + return Response.json({ error: `Unsupported STS tool: ${toolId}` }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/sts/operations.ts b/apps/sim/lib/internal/sts/operations.ts new file mode 100644 index 00000000000..846b842b6c0 --- /dev/null +++ b/apps/sim/lib/internal/sts/operations.ts @@ -0,0 +1,125 @@ +import { + assumeRole, + assumeRoleWithSAML, + assumeRoleWithWebIdentity, + createSTSClient, + createUnauthenticatedSTSClient, + getAccessKeyInfo, + getCallerIdentity, + getSessionToken, +} from '@/lib/internal/sts/client' +import type { + StsAssumeRoleInput, + StsAssumeRoleWithSamlInput, + StsAssumeRoleWithWebIdentityInput, + StsGetAccessKeyInfoInput, + StsGetCallerIdentityInput, + StsGetSessionTokenInput, +} from '@/lib/internal/sts/schema' + +export async function executeStsAssumeRole(input: StsAssumeRoleInput, signal?: AbortSignal) { + const client = createSTSClient(input) + try { + return await assumeRole( + client, + input.roleArn, + input.roleSessionName, + input.durationSeconds, + input.policy, + input.externalId, + input.serialNumber, + input.tokenCode, + input.policyArns, + input.tags, + input.transitiveTagKeys, + signal + ) + } finally { + client.destroy() + } +} + +export async function executeStsAssumeRoleWithWebIdentity( + input: StsAssumeRoleWithWebIdentityInput, + signal?: AbortSignal +) { + const client = createUnauthenticatedSTSClient(input.region) + try { + return await assumeRoleWithWebIdentity( + client, + input.roleArn, + input.roleSessionName, + input.webIdentityToken, + input.providerId, + input.policyArns, + input.policy, + input.durationSeconds, + signal + ) + } finally { + client.destroy() + } +} + +export async function executeStsAssumeRoleWithSAML( + input: StsAssumeRoleWithSamlInput, + signal?: AbortSignal +) { + const client = createUnauthenticatedSTSClient(input.region) + try { + return await assumeRoleWithSAML( + client, + input.roleArn, + input.principalArn, + input.samlAssertion, + input.policyArns, + input.policy, + input.durationSeconds, + signal + ) + } finally { + client.destroy() + } +} + +export async function executeStsGetCallerIdentity( + input: StsGetCallerIdentityInput, + signal?: AbortSignal +) { + const client = createSTSClient(input) + try { + return await getCallerIdentity(client, signal) + } finally { + client.destroy() + } +} + +export async function executeStsGetSessionToken( + input: StsGetSessionTokenInput, + signal?: AbortSignal +) { + const client = createSTSClient(input) + try { + return await getSessionToken( + client, + input.durationSeconds, + input.serialNumber, + input.tokenCode, + signal + ) + } finally { + client.destroy() + } +} + +export async function executeStsGetAccessKeyInfo( + input: StsGetAccessKeyInfoInput, + signal?: AbortSignal +) { + const client = createSTSClient(input) + try { + return await getAccessKeyInfo(client, input.targetAccessKeyId, signal) + } finally { + client.destroy() + } +} diff --git a/apps/sim/lib/internal/sts/schema.ts b/apps/sim/lib/internal/sts/schema.ts new file mode 100644 index 00000000000..3cfa219e313 --- /dev/null +++ b/apps/sim/lib/internal/sts/schema.ts @@ -0,0 +1,104 @@ +import { isRecordLike } from '@sim/utils/object' +import { z } from 'zod' +import { validateAwsRegion } from '@/lib/core/security/input-validation' + +const regionSchema = z + .string() + .min(1, 'AWS region is required') + .refine((value) => validateAwsRegion(value).isValid, { + message: 'Invalid AWS region format (e.g., us-east-1, eu-west-2)', + }) + +const authenticatedInputSchema = z.object({ + region: regionSchema, + accessKeyId: z.string().min(1, 'AWS access key ID is required'), + secretAccessKey: z.string().min(1, 'AWS secret access key is required'), +}) + +const policyArnsSchema = z + .string() + .nullish() + .refine( + (value) => !value || value.split(',').filter((arn) => arn.trim().length > 0).length <= 10, + { message: 'A maximum of 10 policy ARNs can be provided' } + ) + +export const stsAssumeRoleInputSchema = authenticatedInputSchema.extend({ + roleArn: z.string().min(1, 'Role ARN is required'), + roleSessionName: z.string().min(1, 'Role session name is required'), + durationSeconds: z.number().int().min(900).max(43200).nullish(), + policy: z.string().max(2048).nullish(), + externalId: z.string().min(2).max(1224).nullish(), + serialNumber: z.string().nullish(), + tokenCode: z.string().nullish(), + policyArns: policyArnsSchema, + tags: z + .string() + .nullish() + .refine( + (value) => { + if (!value) return true + try { + return isRecordLike(JSON.parse(value)) + } catch { + return false + } + }, + { message: 'tags must be a valid JSON object string' } + ), + transitiveTagKeys: z + .string() + .nullish() + .refine( + (value) => !value || value.split(',').filter((key) => key.trim().length > 0).length <= 50, + { message: 'A maximum of 50 transitive tag keys can be provided' } + ), +}) + +export const stsAssumeRoleWithWebIdentityInputSchema = z.object({ + region: regionSchema, + roleArn: z.string().min(20, 'Role ARN is required').max(2048), + roleSessionName: z.string().min(2, 'Role session name is required').max(64), + webIdentityToken: z + .string() + .min(4, 'Web identity token is required') + .max(20000, 'Web identity token must not exceed 20000 characters'), + providerId: z.string().min(4).max(2048).nullish(), + policy: z.string().max(2048).nullish(), + policyArns: policyArnsSchema, + durationSeconds: z.number().int().min(900).max(43200).nullish(), +}) + +export const stsAssumeRoleWithSamlInputSchema = z.object({ + region: regionSchema, + roleArn: z.string().min(20, 'Role ARN is required').max(2048), + principalArn: z.string().min(20, 'SAML provider ARN is required').max(2048), + samlAssertion: z + .string() + .min(4, 'SAML assertion is required') + .max(100000, 'SAML assertion must not exceed 100000 characters'), + policy: z.string().max(2048).nullish(), + policyArns: policyArnsSchema, + durationSeconds: z.number().int().min(900).max(43200).nullish(), +}) + +export const stsGetCallerIdentityInputSchema = authenticatedInputSchema + +export const stsGetSessionTokenInputSchema = authenticatedInputSchema.extend({ + durationSeconds: z.number().int().min(900).max(129600).nullish(), + serialNumber: z.string().nullish(), + tokenCode: z.string().nullish(), +}) + +export const stsGetAccessKeyInfoInputSchema = authenticatedInputSchema.extend({ + targetAccessKeyId: z.string().min(1, 'Target access key ID is required'), +}) + +export type StsAssumeRoleInput = z.output +export type StsAssumeRoleWithWebIdentityInput = z.output< + typeof stsAssumeRoleWithWebIdentityInputSchema +> +export type StsAssumeRoleWithSamlInput = z.output +export type StsGetCallerIdentityInput = z.output +export type StsGetSessionTokenInput = z.output +export type StsGetAccessKeyInfoInput = z.output diff --git a/apps/sim/lib/internal/stt/execute-tool.test.ts b/apps/sim/lib/internal/stt/execute-tool.test.ts new file mode 100644 index 00000000000..79340a99fe1 --- /dev/null +++ b/apps/sim/lib/internal/stt/execute-tool.test.ts @@ -0,0 +1,499 @@ +/** + * @vitest-environment node + */ +import { inputValidationMock, inputValidationMockFns } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { PRIVATE_MODEL_INPUT_PROVENANCE_HEADER } from '@/lib/execution/model-input-provenance' +import { + RESOLVED_SECRET_PROVENANCE_FIELD, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, +} from '@/lib/execution/private-tool-metadata' + +const { + mockIsInternalFileUrl, + mockDownloadFileFromStorage, + mockIsModelSafeWorkspaceFileKey, + mockResolveInternalFileUrl, +} = vi.hoisted(() => ({ + mockIsInternalFileUrl: vi.fn(), + mockDownloadFileFromStorage: vi.fn(), + mockIsModelSafeWorkspaceFileKey: vi.fn(), + mockResolveInternalFileUrl: vi.fn(), +})) + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + extractStorageKey: vi.fn(() => 'storage-key'), + isInternalFileUrl: mockIsInternalFileUrl, + getMimeTypeFromExtension: vi.fn(() => 'application/octet-stream'), +})) +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadFileFromStorage: mockDownloadFileFromStorage, + resolveInternalFileUrl: mockResolveInternalFileUrl, +})) +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: vi.fn().mockResolvedValue(null), +})) +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({ + isModelSafeWorkspaceFileKey: mockIsModelSafeWorkspaceFileKey, + MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE: + 'File cannot be sent to a model because its secret provenance is unavailable', +})) +vi.mock('@/lib/audio/extractor', () => ({ + isVideoFile: vi.fn(() => false), + extractAudioFromVideo: vi.fn(), +})) + +import { executeSttTool } from '@/lib/internal/stt/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const PINNED_IP = '93.184.216.34' + +const baseBody = { + provider: 'whisper', + apiKey: 'test-api-key', + audioUrl: 'https://example.com/audio.mp3', +} + +function createSttRequest( + input: Record, + headers = new Headers(), + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'stt_whisper', + input, + headers, + context: { + userId: 'user-1', + workspaceId: 'workspace-1', + metadata: {}, + }, + requestId: 'request-1', + ...overrides, + } +} + +function createVerifiedSttRequest( + body: Record, + overrides: Partial = {} +) { + return createSttRequest( + { + ...body, + [RESOLVED_SECRET_PROVENANCE_FIELD]: { + version: 1, + complete: true, + entries: [], + }, + }, + new Headers({ + [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1, + }), + overrides + ) +} + +function mockSecureFetchResponse(body: { ok?: boolean; contentType?: string }) { + return { + ok: body.ok ?? true, + status: 200, + statusText: '', + headers: new Headers({ 'content-type': body.contentType ?? 'audio/mpeg' }), + body: null, + text: async () => '', + json: async () => ({}), + arrayBuffer: async () => new ArrayBuffer(8), + } +} + +describe('executeSttTool', () => { + beforeEach(() => { + vi.clearAllMocks() + inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({ + isValid: true, + resolvedIP: PINNED_IP, + originalHostname: 'example.com', + }) + mockIsInternalFileUrl.mockReturnValue(false) + mockIsModelSafeWorkspaceFileKey.mockResolvedValue(true) + mockDownloadFileFromStorage.mockResolvedValue(Buffer.from('audio')) + + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ text: 'hello world', language: 'en', duration: 1.2 }), + }) + ) + }) + + it('bounds the audioUrl download and rejects oversized responses cleanly', async () => { + inputValidationMockFns.mockSecureFetchWithPinnedIP.mockRejectedValueOnce( + new PayloadSizeLimitError({ + label: 'response body', + maxBytes: 100 * 1024 * 1024, + observedBytes: 200 * 1024 * 1024, + }) + ) + + const response = await executeSttTool(createVerifiedSttRequest(baseBody)) + + expect(response.status).toBe(413) + const data = (await response.json()) as { error: string } + expect(data.error).toMatch(/exceeds the maximum supported size/i) + + const call = inputValidationMockFns.mockSecureFetchWithPinnedIP.mock.calls[0] + expect(call[1]).toBe(PINNED_IP) + expect(call[2]).toMatchObject({ maxResponseBytes: 100 * 1024 * 1024 }) + }) + + it('transcribes a normal, well-under-cap audio download successfully', async () => { + inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( + mockSecureFetchResponse({}) + ) + + const response = await executeSttTool(createVerifiedSttRequest(baseBody)) + + expect(response.status).toBe(200) + const data = (await response.json()) as { transcript: string } + expect(data.transcript).toBe('hello world') + }) + + it('rejects an authenticated but incomplete private provenance envelope before downloading', async () => { + const response = await executeSttTool( + createSttRequest( + { + ...baseBody, + [RESOLVED_SECRET_PROVENANCE_FIELD]: { + version: 1, + complete: false, + entries: [], + }, + }, + new Headers({ + [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1, + }) + ) + ) + + expect(response.status).toBe(400) + expect(await response.json()).toEqual({ error: 'Model input provenance is unavailable' }) + expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() + expect(fetch).not.toHaveBeenCalled() + }) + + it('accepts a verified empty private provenance envelope', async () => { + inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( + mockSecureFetchResponse({}) + ) + + const response = await executeSttTool(createVerifiedSttRequest(baseBody)) + + expect(response.status).toBe(200) + expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenCalledOnce() + }) + + it('rejects a tracked unsafe workspace audio file before reading its bytes', async () => { + mockIsModelSafeWorkspaceFileKey.mockResolvedValueOnce(false) + + const response = await executeSttTool( + createVerifiedSttRequest({ + provider: 'whisper', + apiKey: 'test-api-key', + audioFile: { + id: 'file-1', + name: 'audio.mp3', + size: 5, + type: 'audio/mpeg', + key: 'workspace/workspace-1/audio.mp3', + }, + }) + ) + + expect(response.status).toBe(400) + expect(await response.json()).toEqual({ + error: 'File cannot be sent to a model because its secret provenance is unavailable', + }) + expect(mockDownloadFileFromStorage).not.toHaveBeenCalled() + expect(fetch).not.toHaveBeenCalled() + }) + + it('rejects a tracked unsafe internal audio URL after access resolution', async () => { + mockIsInternalFileUrl.mockReturnValue(true) + mockResolveInternalFileUrl.mockResolvedValueOnce({ + fileUrl: 'https://storage.example.com/signed-audio.mp3', + }) + mockIsModelSafeWorkspaceFileKey.mockResolvedValueOnce(false) + + const response = await executeSttTool( + createVerifiedSttRequest({ + ...baseBody, + audioUrl: '/api/files/serve/workspace/workspace-1/audio.mp3', + }) + ) + + expect(response.status).toBe(400) + expect(mockResolveInternalFileUrl).toHaveBeenCalledOnce() + expect(mockIsModelSafeWorkspaceFileKey).toHaveBeenCalledWith('storage-key') + expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() + expect(fetch).not.toHaveBeenCalled() + }) + + it('requires the trusted executor user before resolving audio', async () => { + const response = await executeSttTool( + createVerifiedSttRequest(baseBody, { + context: { workspaceId: 'workspace-1', metadata: {} }, + }) + ) + + expect(response.status).toBe(401) + expect(await response.json()).toEqual({ error: 'Unauthorized' }) + expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() + }) + + it('rejects unsupported STT tool IDs before doing work', async () => { + const response = await executeSttTool( + createVerifiedSttRequest(baseBody, { toolId: 'stt_unknown' }) + ) + + expect(response.status).toBe(500) + expect(await response.json()).toEqual({ + success: false, + error: 'Unsupported STT tool: stt_unknown', + }) + expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).not.toHaveBeenCalled() + }) + + it('propagates cancellation instead of converting it into a retryable tool error', async () => { + const controller = new AbortController() + controller.abort() + + await expect( + executeSttTool(createVerifiedSttRequest(baseBody, { signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) + + it('preserves the Deepgram request and result contract', async () => { + inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( + mockSecureFetchResponse({ contentType: 'audio/wav' }) + ) + vi.mocked(fetch).mockResolvedValueOnce( + new Response( + JSON.stringify({ + results: { + channels: [ + { + detected_language: 'en', + alternatives: [ + { + transcript: 'deepgram transcript', + confidence: 0.98, + words: [{ word: 'deepgram', start: 0, end: 0.5, confidence: 0.98, speaker: 1 }], + }, + ], + }, + ], + }, + metadata: { duration: 1.5 }, + }), + { status: 200 } + ) + ) + + const response = await executeSttTool( + createVerifiedSttRequest( + { + provider: 'deepgram', + apiKey: 'deepgram-key', + audioUrl: 'https://example.com/audio.wav', + language: 'auto', + timestamps: 'word', + diarization: true, + }, + { toolId: 'stt_deepgram' } + ) + ) + + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ + transcript: 'deepgram transcript', + segments: [ + { + text: 'deepgram', + start: 0, + end: 0.5, + speaker: 'Speaker 1', + confidence: 0.98, + }, + ], + language: 'en', + duration: 1.5, + confidence: 0.98, + }) + const [url, init] = vi.mocked(fetch).mock.calls[0] + expect(String(url)).toContain('detect_language=true') + expect(String(url)).toContain('diarize=true') + expect(init?.headers).toEqual({ + Authorization: 'Token deepgram-key', + 'Content-Type': 'audio/wav', + }) + }) + + it('preserves the ElevenLabs multipart contract', async () => { + inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( + mockSecureFetchResponse({}) + ) + vi.mocked(fetch).mockResolvedValueOnce( + new Response( + JSON.stringify({ + text: 'eleven transcript', + language_code: 'en', + words: [{ type: 'word', text: 'eleven', start: 0, end: 0.4, speaker_id: 'A' }], + }), + { status: 200 } + ) + ) + + const response = await executeSttTool( + createVerifiedSttRequest( + { + provider: 'elevenlabs', + apiKey: 'eleven-key', + audioUrl: 'https://example.com/audio.mp3', + language: 'en', + timestamps: 'sentence', + }, + { toolId: 'stt_elevenlabs' } + ) + ) + + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ + transcript: 'eleven transcript', + segments: [{ text: 'eleven', start: 0, end: 0.4, speaker: 'A' }], + language: 'en', + }) + const [, init] = vi.mocked(fetch).mock.calls[0] + expect(init?.headers).toEqual({ 'xi-api-key': 'eleven-key' }) + expect(init?.body).toBeInstanceOf(FormData) + const form = init?.body as FormData + expect(form.get('model_id')).toBe('scribe_v2') + expect(form.get('language_code')).toBe('en') + expect(form.get('timestamps_granularity')).toBe('word') + }) + + it('preserves AssemblyAI upload, feature flags, polling, and result fields', async () => { + inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( + mockSecureFetchResponse({}) + ) + vi.mocked(fetch) + .mockResolvedValueOnce( + new Response(JSON.stringify({ upload_url: 'https://assembly.example/uploaded' }), { + status: 200, + }) + ) + .mockResolvedValueOnce(new Response(JSON.stringify({ id: 'transcript-1' }), { status: 200 })) + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + status: 'completed', + text: 'assembly transcript', + language_code: 'en', + audio_duration: 2, + confidence: 0.91, + words: [{ text: 'assembly', start: 0, end: 500, confidence: 0.91, speaker: 'A' }], + sentiment_analysis_results: [{ sentiment: 'POSITIVE' }], + entities: [{ entity_type: 'person_name', text: 'Ada' }], + summary: 'Summary', + }), + { status: 200 } + ) + ) + + const response = await executeSttTool( + createVerifiedSttRequest( + { + provider: 'assemblyai', + apiKey: 'assembly-key', + audioUrl: 'https://example.com/audio.mp3', + model: 'universal', + language: 'auto', + timestamps: 'word', + diarization: true, + sentiment: true, + entityDetection: true, + piiRedaction: true, + summarization: true, + }, + { toolId: 'stt_assemblyai' } + ) + ) + + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ + transcript: 'assembly transcript', + segments: [ + { + text: 'assembly', + start: 0, + end: 0.5, + speaker: 'Speaker A', + confidence: 0.91, + }, + ], + language: 'en', + duration: 2, + confidence: 0.91, + sentiment: [{ sentiment: 'POSITIVE' }], + entities: [{ entity_type: 'person_name', text: 'Ada' }], + summary: 'Summary', + }) + const transcriptRequest = JSON.parse(String(vi.mocked(fetch).mock.calls[1][1]?.body)) + expect(transcriptRequest).toMatchObject({ + audio_url: 'https://assembly.example/uploaded', + speech_model: 'universal', + language_detection: true, + speaker_labels: true, + sentiment_analysis: true, + entity_detection: true, + redact_pii: true, + summarization: true, + }) + }) + + it('preserves the Gemini inline-data payload and response contract', async () => { + inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValueOnce( + mockSecureFetchResponse({ contentType: 'audio/ogg' }) + ) + vi.mocked(fetch).mockResolvedValueOnce( + new Response( + JSON.stringify({ candidates: [{ content: { parts: [{ text: 'gemini transcript' }] } }] }), + { status: 200 } + ) + ) + + const response = await executeSttTool( + createVerifiedSttRequest( + { + provider: 'gemini', + apiKey: 'gemini-key', + audioUrl: 'https://example.com/audio.ogg', + language: 'fr', + timestamps: 'sentence', + }, + { toolId: 'stt_gemini' } + ) + ) + + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ transcript: 'gemini transcript', language: 'fr' }) + const [url, init] = vi.mocked(fetch).mock.calls[0] + expect(String(url)).toContain('gemini-2.5-flash:generateContent?key=gemini-key') + const requestBody = JSON.parse(String(init?.body)) + expect(requestBody.contents[0].parts[0].inline_data.mime_type).toBe('audio/ogg') + expect(requestBody.contents[0].parts[1].text).toContain('The audio is in fr.') + expect(requestBody.contents[0].parts[1].text).toContain('Include timestamps') + }) +}) diff --git a/apps/sim/lib/internal/stt/execute-tool.ts b/apps/sim/lib/internal/stt/execute-tool.ts new file mode 100644 index 00000000000..1648acb5c18 --- /dev/null +++ b/apps/sim/lib/internal/stt/execute-tool.ts @@ -0,0 +1,76 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { executeSttOperation } from '@/lib/internal/stt/operations' +import { sttOperationInputSchema } from '@/lib/internal/stt/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +const logger = createLogger('SttToolExecution') + +const STT_TOOL_IDS = new Set([ + 'stt_assemblyai', + 'stt_assemblyai_v2', + 'stt_deepgram', + 'stt_deepgram_v2', + 'stt_elevenlabs', + 'stt_elevenlabs_v2', + 'stt_gemini', + 'stt_gemini_v2', + 'stt_whisper', + 'stt_whisper_v2', +]) + +export const executeSttTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!STT_TOOL_IDS.has(request.toolId)) { + return Response.json( + { success: false, error: `Unsupported STT tool: ${request.toolId}` }, + { status: 500 } + ) + } + + const userId = request.context.userId + if (!userId) return Response.json({ error: 'Unauthorized' }, { status: 401 }) + + let serializedInput: string + try { + serializedInput = JSON.stringify(request.input) ?? '' + } catch { + return Response.json({ error: 'Invalid request data' }, { status: 400 }) + } + if (Buffer.byteLength(serializedInput, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + + const parsed = sttOperationInputSchema.safeParse(request.input) + if (!parsed.success) { + logger.warn(`[${request.requestId}] Invalid STT request`, { issues: parsed.error.issues }) + return Response.json( + { error: 'Invalid request data', details: parsed.error.issues }, + { status: 400 } + ) + } + + try { + return await executeSttOperation(parsed.data, { + headers: request.headers, + requestId: request.requestId, + signal: request.signal, + userId, + }) + } catch (error) { + request.signal?.throwIfAborted() + const message = getErrorMessage(error, 'Unknown error') + logger.error('STT operation dispatch failed', { + error: message, + requestId: request.requestId, + toolId: request.toolId, + }) + return Response.json({ error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/stt/operations.ts b/apps/sim/lib/internal/stt/operations.ts new file mode 100644 index 00000000000..fb319967e94 --- /dev/null +++ b/apps/sim/lib/internal/stt/operations.ts @@ -0,0 +1,932 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { sleep } from '@sim/utils/helpers' +import { extractAudioFromVideo, isVideoFile } from '@/lib/audio/extractor' +import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' +import type { SttOperationInput } from '@/lib/internal/stt/schema' +import { + isModelSafeWorkspaceFileKey, + MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, +} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { + extractStorageKey, + getMimeTypeFromExtension, + isInternalFileUrl, +} from '@/lib/uploads/utils/file-utils' +import { + downloadFileFromStorage, + resolveInternalFileUrl, +} from '@/lib/uploads/utils/file-utils.server' +import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' +import { assertToolFileAccess } from '@/app/api/files/authorization' +import type { TranscriptSegment } from '@/tools/stt/types' + +const logger = createLogger('SttOperations') +const ELEVENLABS_STT_MODEL = 'scribe_v2' + +interface TimedWord { + text: string + start: number + end: number + confidence?: number + speaker?: string | number + speaker_id?: string + type?: string + word?: string + transcript?: string +} + +interface WhisperApiResponse { + text: string + segments?: TimedWord[] + words?: TimedWord[] + language?: string + duration?: number + error?: { message?: string } + message?: string +} + +interface DeepgramAlternative { + transcript: string + confidence?: number + words?: TimedWord[] +} + +interface DeepgramApiResponse { + results?: { + channels?: Array<{ + alternatives?: DeepgramAlternative[] + detected_language?: string + }> + utterances?: TimedWord[] + } + metadata?: { duration?: number } + err_msg?: string + message?: string +} + +interface ElevenLabsApiResponse { + text?: string + words?: TimedWord[] + language_code?: string + detail?: string | { message?: string } + message?: string +} + +interface AssemblyAiTranscriptRequest { + audio_url: string + speech_model?: 'best' | 'slam-1' | 'universal' + language_code?: string + language_detection?: boolean + speaker_labels?: boolean + sentiment_analysis?: boolean + entity_detection?: boolean + redact_pii?: boolean + redact_pii_policies?: string[] + summarization?: boolean + summary_model?: 'informative' + summary_type?: 'bullets' +} + +interface AssemblyAiTranscript { + status: string + error?: string + text: string + words?: TimedWord[] + language_code?: string + audio_duration?: number + confidence?: number + sentiment_analysis_results?: Record[] + entities?: Record[] + summary?: string +} + +interface AssemblyAiApiResponse extends Partial { + id?: string + upload_url?: string + error?: string +} + +interface GeminiApiResponse { + error?: { message?: string } + candidates?: Array<{ + finishReason?: string + content?: { parts?: Array<{ text?: string }> } + }> +} + +interface SttOutput { + transcript: string + segments?: TranscriptSegment[] + language?: string + duration?: number + confidence?: number + sentiment?: Record[] + entities?: Record[] + summary?: string +} + +export interface SttOperationContext { + headers: Headers + userId: string + requestId: string + signal?: AbortSignal +} + +export async function executeSttOperation( + body: SttOperationInput, + context: SttOperationContext +): Promise { + const { headers, requestId, signal, userId } = context + signal?.throwIfAborted() + logger.info(`[${requestId}] STT transcription request started`) + + try { + const modelInputProvenance = validateOpaqueModelInputProvenance({ + headers, + payload: body, + isInternalRequest: true, + }) + if (!modelInputProvenance.success) { + return Response.json( + { error: modelInputProvenance.error }, + { status: modelInputProvenance.status } + ) + } + + const { + provider, + apiKey, + model, + language, + timestamps, + diarization, + translateToEnglish, + sentiment, + entityDetection, + piiRedaction, + summarization, + } = body + + let audioBuffer: Buffer + let audioFileName: string + let audioMimeType: string + + if (body.audioFile) { + if (Array.isArray(body.audioFile) && body.audioFile.length !== 1) { + return Response.json({ error: 'audioFile must be a single file' }, { status: 400 }) + } + const file = Array.isArray(body.audioFile) ? body.audioFile[0] : body.audioFile + logger.info(`[${requestId}] Processing uploaded audio`) + + const deniedAudio = await assertToolFileAccess(file.key, userId, requestId, logger) + if (deniedAudio) return deniedAudio + if (!(await isModelSafeWorkspaceFileKey(file.key))) { + return Response.json({ error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE }, { status: 400 }) + } + audioBuffer = await downloadFileFromStorage(file, requestId, logger, { + maxBytes: MAX_FILE_SIZE, + }) + signal?.throwIfAborted() + audioFileName = file.name + const ext = file.name.split('.').pop()?.toLowerCase() || '' + audioMimeType = file.type || getMimeTypeFromExtension(ext) + } else if (body.audioFileReference) { + if (Array.isArray(body.audioFileReference) && body.audioFileReference.length !== 1) { + return Response.json({ error: 'audioFileReference must be a single file' }, { status: 400 }) + } + const file = Array.isArray(body.audioFileReference) + ? body.audioFileReference[0] + : body.audioFileReference + logger.info(`[${requestId}] Processing referenced audio`) + + const deniedRef = await assertToolFileAccess(file.key, userId, requestId, logger) + if (deniedRef) return deniedRef + if (!(await isModelSafeWorkspaceFileKey(file.key))) { + return Response.json({ error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE }, { status: 400 }) + } + audioBuffer = await downloadFileFromStorage(file, requestId, logger, { + maxBytes: MAX_FILE_SIZE, + }) + signal?.throwIfAborted() + audioFileName = file.name + + const ext = file.name.split('.').pop()?.toLowerCase() || '' + audioMimeType = file.type || getMimeTypeFromExtension(ext) + } else if (body.audioUrl) { + let audioUrl = body.audioUrl.trim() + const internalAudioUrl = isInternalFileUrl(audioUrl) + logger.info(`[${requestId}] Downloading audio source`, { internal: internalAudioUrl }) + if (audioUrl.startsWith('/') && !isInternalFileUrl(audioUrl)) { + return Response.json( + { + error: 'Invalid file path. Only uploaded files are supported for internal paths.', + }, + { status: 400 } + ) + } + + if (internalAudioUrl) { + if (!userId) { + return Response.json( + { error: 'Authentication required for internal file access' }, + { status: 401 } + ) + } + const resolution = await resolveInternalFileUrl(audioUrl, userId, requestId, logger) + if (resolution.error) { + return Response.json( + { error: resolution.error.message }, + { status: resolution.error.status } + ) + } + audioUrl = resolution.fileUrl || audioUrl + if (!(await isModelSafeWorkspaceFileKey(extractStorageKey(body.audioUrl)))) { + return Response.json( + { error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE }, + { status: 400 } + ) + } + } + + const urlValidation = await validateUrlWithDNS(audioUrl, 'audioUrl') + if (!urlValidation.isValid) { + return Response.json({ error: urlValidation.error }, { status: 400 }) + } + + const response = await secureFetchWithPinnedIP(audioUrl, urlValidation.resolvedIP!, { + method: 'GET', + maxResponseBytes: MAX_FILE_SIZE, + signal, + }) + if (!response.ok) { + await response.text().catch(() => {}) + throw new Error(`Failed to download audio from URL: ${response.statusText}`) + } + + const arrayBuffer = await response.arrayBuffer() + signal?.throwIfAborted() + audioBuffer = Buffer.from(arrayBuffer) + audioFileName = audioUrl.split('/').pop() || 'audio_file' + audioMimeType = response.headers.get('content-type') || 'audio/mpeg' + } else { + return Response.json( + { error: 'No audio source provided. Provide audioFile, audioFileReference, or audioUrl' }, + { status: 400 } + ) + } + + if (isVideoFile(audioMimeType)) { + logger.info(`[${requestId}] Extracting audio from video file`) + try { + const extracted = await extractAudioFromVideo(audioBuffer, audioMimeType, { + outputFormat: 'mp3', + sampleRate: 16000, + channels: 1, + }) + signal?.throwIfAborted() + audioBuffer = extracted.buffer + audioMimeType = 'audio/mpeg' + audioFileName = audioFileName.replace(/\.[^.]+$/, '.mp3') + } catch (error) { + logger.error(`[${requestId}] Video extraction failed:`, error) + return Response.json( + { + error: `Failed to extract audio from video: ${getErrorMessage(error, 'Unknown error')}`, + }, + { status: 500 } + ) + } + } + + logger.info(`[${requestId}] Transcribing audio`, { provider }) + + let transcript: string + let segments: TranscriptSegment[] | undefined + let detectedLanguage: string | undefined + let duration: number | undefined + let confidence: number | undefined + let sentimentResults: Record[] | undefined + let entities: Record[] | undefined + let summary: string | undefined + + try { + if (provider === 'whisper') { + const result = await transcribeWithWhisper( + audioBuffer, + apiKey, + language, + timestamps, + translateToEnglish, + model, + body.prompt, + body.temperature, + audioMimeType, + audioFileName, + signal + ) + transcript = result.transcript + segments = result.segments + detectedLanguage = result.language + duration = result.duration + } else if (provider === 'deepgram') { + const result = await transcribeWithDeepgram( + audioBuffer, + apiKey, + language, + timestamps, + diarization, + model, + audioMimeType, + signal + ) + transcript = result.transcript + segments = result.segments + detectedLanguage = result.language + duration = result.duration + confidence = result.confidence + } else if (provider === 'elevenlabs') { + const result = await transcribeWithElevenLabs( + audioBuffer, + apiKey, + language, + timestamps, + signal + ) + transcript = result.transcript + segments = result.segments + detectedLanguage = result.language + duration = result.duration + } else if (provider === 'assemblyai') { + const result = await transcribeWithAssemblyAI( + audioBuffer, + apiKey, + language, + timestamps, + diarization, + sentiment, + entityDetection, + piiRedaction, + summarization, + model, + signal + ) + transcript = result.transcript + segments = result.segments + detectedLanguage = result.language + duration = result.duration + confidence = result.confidence + sentimentResults = result.sentiment + entities = result.entities + summary = result.summary + } else if (provider === 'gemini') { + const result = await transcribeWithGemini( + audioBuffer, + apiKey, + audioMimeType, + language, + timestamps, + model, + signal + ) + transcript = result.transcript + segments = result.segments + detectedLanguage = result.language + duration = result.duration + confidence = result.confidence + } else { + return Response.json({ error: `Unknown provider: ${provider}` }, { status: 400 }) + } + } catch (error) { + signal?.throwIfAborted() + logger.error(`[${requestId}] Transcription failed:`, error) + const errorMessage = getErrorMessage(error, 'Transcription failed') + return Response.json({ error: errorMessage }, { status: 500 }) + } + + logger.info(`[${requestId}] Transcription completed successfully`) + + const response: SttOutput = { transcript } + if (segments !== undefined) response.segments = segments + if (detectedLanguage !== undefined) response.language = detectedLanguage + if (duration !== undefined) response.duration = duration + if (confidence !== undefined) response.confidence = confidence + if (sentimentResults !== undefined) response.sentiment = sentimentResults + if (entities !== undefined) response.entities = entities + if (summary !== undefined) response.summary = summary + + return Response.json(response) + } catch (error) { + signal?.throwIfAborted() + logger.error(`[${requestId}] STT proxy error:`, error) + const isSizeLimit = isPayloadSizeLimitError(error) + const errorMessage = isSizeLimit + ? 'Audio file exceeds the maximum supported size' + : getErrorMessage(error, 'Unknown error') + return Response.json({ error: errorMessage }, { status: isSizeLimit ? 413 : 500 }) + } +} + +async function transcribeWithWhisper( + audioBuffer: Buffer, + apiKey: string, + language?: string, + timestamps?: 'none' | 'sentence' | 'word', + translate?: boolean, + model?: string, + prompt?: string, + temperature?: number, + mimeType?: string, + fileName?: string, + signal?: AbortSignal +): Promise<{ + transcript: string + segments?: TranscriptSegment[] + language?: string + duration?: number +}> { + const formData = new FormData() + + const actualMimeType = mimeType || 'audio/mpeg' + const actualFileName = fileName || 'audio.mp3' + const blob = new Blob([new Uint8Array(audioBuffer)], { type: actualMimeType }) + formData.append('file', blob, actualFileName) + formData.append('model', model || 'whisper-1') + + if (language && language !== 'auto') { + formData.append('language', language) + } + + if (prompt) { + formData.append('prompt', prompt) + } + + if (temperature !== undefined) { + formData.append('temperature', temperature.toString()) + } + + formData.append('response_format', 'verbose_json') + + if (timestamps === 'word') { + formData.append('timestamp_granularities[]', 'word') + } else if (timestamps === 'sentence') { + formData.append('timestamp_granularities[]', 'segment') + } + + const endpoint = translate ? 'translations' : 'transcriptions' + const response = await fetch(`https://api.openai.com/v1/audio/${endpoint}`, { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + }, + body: formData, + signal, + }) + + if (!response.ok) { + const error = (await response.json()) as WhisperApiResponse + const errorMessage = error.error?.message || error.message || JSON.stringify(error) + throw new Error(`Whisper API error: ${errorMessage}`) + } + + const data = (await response.json()) as WhisperApiResponse + + let segments: TranscriptSegment[] | undefined + if (timestamps !== 'none') { + segments = (data.segments || data.words || []).map((seg) => ({ + text: seg.text, + start: seg.start, + end: seg.end, + })) + } + + return { + transcript: data.text, + segments, + language: data.language, + duration: data.duration, + } +} + +async function transcribeWithDeepgram( + audioBuffer: Buffer, + apiKey: string, + language?: string, + timestamps?: 'none' | 'sentence' | 'word', + diarization?: boolean, + model?: string, + mimeType?: string, + signal?: AbortSignal +): Promise<{ + transcript: string + segments?: TranscriptSegment[] + language?: string + duration?: number + confidence?: number +}> { + const params = new URLSearchParams({ + model: model || 'nova-3', + smart_format: 'true', + punctuate: 'true', + }) + + if (language && language !== 'auto') { + params.append('language', language) + } else if (language === 'auto') { + params.append('detect_language', 'true') + } + + if (timestamps === 'sentence') { + params.append('utterances', 'true') + } + + if (diarization) { + params.append('diarize', 'true') + } + + const response = await fetch(`https://api.deepgram.com/v1/listen?${params.toString()}`, { + method: 'POST', + headers: { + Authorization: `Token ${apiKey}`, + 'Content-Type': mimeType || 'audio/mpeg', + }, + body: new Uint8Array(audioBuffer), + signal, + }) + + if (!response.ok) { + const error = (await response.json()) as DeepgramApiResponse + const errorMessage = error.err_msg || error.message || JSON.stringify(error) + throw new Error(`Deepgram API error: ${errorMessage}`) + } + + const data = (await response.json()) as DeepgramApiResponse + const result = data.results?.channels?.[0]?.alternatives?.[0] + + if (!result) { + throw new Error('No transcription result from Deepgram') + } + + const transcript = result.transcript + const detectedLanguage = data.results?.channels?.[0]?.detected_language + const confidence = result.confidence + + let segments: TranscriptSegment[] | undefined + if (result.words && timestamps === 'word') { + segments = result.words.map((word) => ({ + text: word.word ?? word.text, + start: word.start, + end: word.end, + speaker: word.speaker !== undefined ? `Speaker ${word.speaker}` : undefined, + confidence: word.confidence, + })) + } else if (data.results?.utterances && timestamps === 'sentence') { + segments = data.results.utterances.map((utterance) => ({ + text: utterance.transcript ?? utterance.text, + start: utterance.start, + end: utterance.end, + speaker: utterance.speaker !== undefined ? `Speaker ${utterance.speaker}` : undefined, + confidence: utterance.confidence, + })) + } + + return { + transcript, + segments, + language: detectedLanguage, + duration: data.metadata?.duration, + confidence, + } +} + +async function transcribeWithElevenLabs( + audioBuffer: Buffer, + apiKey: string, + language?: string, + timestamps?: 'none' | 'sentence' | 'word', + signal?: AbortSignal +): Promise<{ + transcript: string + segments?: TranscriptSegment[] + language?: string + duration?: number +}> { + const formData = new FormData() + const blob = new Blob([new Uint8Array(audioBuffer)], { type: 'audio/mpeg' }) + formData.append('file', blob, 'audio.mp3') + formData.append('model_id', ELEVENLABS_STT_MODEL) + + if (language && language !== 'auto') { + formData.append('language_code', language) + } + + if (timestamps && timestamps !== 'none') { + const granularity = timestamps === 'word' ? 'word' : 'word' + formData.append('timestamps_granularity', granularity) + } else { + formData.append('timestamps_granularity', 'word') + } + + const response = await fetch('https://api.elevenlabs.io/v1/speech-to-text', { + method: 'POST', + headers: { + 'xi-api-key': apiKey, + }, + body: formData, + signal, + }) + + if (!response.ok) { + const error = (await response.json()) as ElevenLabsApiResponse + const errorMessage = + typeof error.detail === 'string' + ? error.detail + : error.detail?.message || error.message || JSON.stringify(error) + throw new Error(`ElevenLabs API error: ${errorMessage}`) + } + + const data = (await response.json()) as ElevenLabsApiResponse + + const words = data.words || [] + const segments: TranscriptSegment[] = words + .filter((word) => word.type === 'word') + .map((word) => ({ + text: word.text, + start: word.start, + end: word.end, + speaker: word.speaker_id, + })) + + return { + transcript: data.text || '', + segments: segments.length > 0 ? segments : undefined, + language: data.language_code, + duration: undefined, // ElevenLabs doesn't return duration in response + } +} + +async function transcribeWithAssemblyAI( + audioBuffer: Buffer, + apiKey: string, + language?: string, + timestamps?: 'none' | 'sentence' | 'word', + diarization?: boolean, + sentiment?: boolean, + entityDetection?: boolean, + piiRedaction?: boolean, + summarization?: boolean, + model?: string, + signal?: AbortSignal +): Promise<{ + transcript: string + segments?: TranscriptSegment[] + language?: string + duration?: number + confidence?: number + sentiment?: Record[] + entities?: Record[] + summary?: string +}> { + const uploadResponse = await fetch('https://api.assemblyai.com/v2/upload', { + method: 'POST', + headers: { + authorization: apiKey, + 'content-type': 'application/octet-stream', + }, + body: new Uint8Array(audioBuffer), + signal, + }) + + if (!uploadResponse.ok) { + const error = (await uploadResponse.json()) as AssemblyAiApiResponse + throw new Error(`AssemblyAI upload error: ${error.error || JSON.stringify(error)}`) + } + + const { upload_url } = (await uploadResponse.json()) as AssemblyAiApiResponse + if (!upload_url) throw new Error('AssemblyAI upload error: Missing upload URL') + + const transcriptRequest: AssemblyAiTranscriptRequest = { + audio_url: upload_url, + } + + if (model === 'best' || model === 'slam-1' || model === 'universal') { + transcriptRequest.speech_model = model + } + + if (language && language !== 'auto') { + transcriptRequest.language_code = language + } else if (language === 'auto') { + transcriptRequest.language_detection = true + } + + if (diarization) { + transcriptRequest.speaker_labels = true + } + + if (sentiment) { + transcriptRequest.sentiment_analysis = true + } + + if (entityDetection) { + transcriptRequest.entity_detection = true + } + + if (piiRedaction) { + transcriptRequest.redact_pii = true + transcriptRequest.redact_pii_policies = [ + 'us_social_security_number', + 'email_address', + 'phone_number', + ] + } + + if (summarization) { + transcriptRequest.summarization = true + transcriptRequest.summary_model = 'informative' + transcriptRequest.summary_type = 'bullets' + } + + const transcriptResponse = await fetch('https://api.assemblyai.com/v2/transcript', { + method: 'POST', + headers: { + authorization: apiKey, + 'content-type': 'application/json', + }, + body: JSON.stringify(transcriptRequest), + signal, + }) + + if (!transcriptResponse.ok) { + const error = (await transcriptResponse.json()) as AssemblyAiApiResponse + throw new Error(`AssemblyAI transcript error: ${error.error || JSON.stringify(error)}`) + } + + const { id } = (await transcriptResponse.json()) as AssemblyAiApiResponse + if (!id) throw new Error('AssemblyAI transcript error: Missing transcript ID') + + let transcript: AssemblyAiTranscript | undefined + let attempts = 0 + const pollIntervalMs = 5000 + const maxAttempts = Math.ceil(getMaxExecutionTimeout() / pollIntervalMs) + + while (attempts < maxAttempts) { + const statusResponse = await fetch(`https://api.assemblyai.com/v2/transcript/${id}`, { + headers: { + authorization: apiKey, + }, + signal, + }) + + if (!statusResponse.ok) { + const error = (await statusResponse.json()) as AssemblyAiApiResponse + throw new Error(`AssemblyAI status error: ${error.error || JSON.stringify(error)}`) + } + + transcript = (await statusResponse.json()) as AssemblyAiTranscript + + if (transcript.status === 'completed') { + break + } + if (transcript.status === 'error') { + throw new Error(`AssemblyAI transcription failed: ${transcript.error}`) + } + + signal?.throwIfAborted() + await sleep(pollIntervalMs) + signal?.throwIfAborted() + attempts++ + } + + if (!transcript || transcript.status !== 'completed') { + throw new Error('AssemblyAI transcription timed out') + } + + let segments: TranscriptSegment[] | undefined + if (timestamps !== 'none' && transcript.words) { + segments = transcript.words.map((word) => ({ + text: word.text, + start: word.start / 1000, + end: word.end / 1000, + speaker: word.speaker ? `Speaker ${word.speaker}` : undefined, + confidence: word.confidence, + })) + } + + const result: SttOutput = { + transcript: transcript.text, + segments, + language: transcript.language_code, + duration: transcript.audio_duration, + confidence: transcript.confidence, + } + + if (sentiment && transcript.sentiment_analysis_results) { + result.sentiment = transcript.sentiment_analysis_results + } + + if (entityDetection && transcript.entities) { + result.entities = transcript.entities + } + + if (summarization && transcript.summary) { + result.summary = transcript.summary + } + + return result +} + +async function transcribeWithGemini( + audioBuffer: Buffer, + apiKey: string, + mimeType: string, + language?: string, + timestamps?: 'none' | 'sentence' | 'word', + model?: string, + signal?: AbortSignal +): Promise<{ + transcript: string + segments?: TranscriptSegment[] + language?: string + duration?: number + confidence?: number +}> { + const modelName = model || 'gemini-2.5-flash' + + const estimatedSize = audioBuffer.length * 1.34 + if (estimatedSize > 20 * 1024 * 1024) { + throw new Error('Audio file exceeds 20MB limit for inline data') + } + + const base64Audio = audioBuffer.toString('base64') + + const languagePrompt = language && language !== 'auto' ? ` The audio is in ${language}.` : '' + + const timestampPrompt = + timestamps === 'sentence' || timestamps === 'word' + ? ' Include timestamps in MM:SS format for each sentence.' + : '' + + const requestBody = { + contents: [ + { + parts: [ + { + inline_data: { + mime_type: mimeType, + data: base64Audio, + }, + }, + { + text: `Please transcribe this audio file.${languagePrompt}${timestampPrompt} Provide the full transcript.`, + }, + ], + }, + ], + } + + const response = await fetch( + `https://generativelanguage.googleapis.com/v1beta/models/${modelName}:generateContent?key=${apiKey}`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(requestBody), + signal, + } + ) + + if (!response.ok) { + const error = (await response.json()) as GeminiApiResponse + if (response.status === 404) { + throw new Error( + `Model not found: ${modelName}. Use gemini-3.1-pro-preview, gemini-3-pro-preview, gemini-2.5-pro, gemini-2.5-flash, gemini-2.5-flash-lite, or gemini-2.0-flash-exp` + ) + } + const errorMessage = error.error?.message || JSON.stringify(error) + throw new Error(`Gemini API error: ${errorMessage}`) + } + + const data = (await response.json()) as GeminiApiResponse + + if (!data.candidates?.[0]?.content?.parts?.[0]?.text) { + const candidate = data.candidates?.[0] + if (candidate?.finishReason === 'SAFETY') { + throw new Error('Content was blocked by safety filters') + } + throw new Error('Invalid response structure from Gemini API') + } + + const transcript = data.candidates[0].content.parts[0].text + + return { + transcript, + language: language !== 'auto' ? language : undefined, + } +} diff --git a/apps/sim/lib/internal/stt/schema.ts b/apps/sim/lib/internal/stt/schema.ts new file mode 100644 index 00000000000..f6aec1fa2e7 --- /dev/null +++ b/apps/sim/lib/internal/stt/schema.ts @@ -0,0 +1,41 @@ +import { z } from 'zod' +import { resolvedSecretTraceProvenanceSchema, userFileSchema } from '@/lib/api/contracts/primitives' +import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' + +export const sttProviders = ['whisper', 'deepgram', 'elevenlabs', 'assemblyai', 'gemini'] as const +const MISSING_STT_FIELDS_ERROR = 'Missing required fields: provider and apiKey' + +export const sttUserFileSchema = userFileSchema.extend({ + type: z.string().optional().default(''), +}) + +export const sttUserFileInputSchema = z.union([sttUserFileSchema, z.array(sttUserFileSchema)]) + +export const sttOperationInputSchema = z + .object({ + provider: z + .string({ error: MISSING_STT_FIELDS_ERROR }) + .min(1, MISSING_STT_FIELDS_ERROR) + .refine((provider) => sttProviders.includes(provider as (typeof sttProviders)[number]), { + message: `Invalid provider. Must be one of: ${sttProviders.join(', ')}`, + }), + apiKey: z.string({ error: MISSING_STT_FIELDS_ERROR }).min(1, MISSING_STT_FIELDS_ERROR), + model: z.string().optional(), + audioFile: sttUserFileInputSchema.optional(), + audioFileReference: sttUserFileInputSchema.optional(), + audioUrl: z.string().optional(), + language: z.string().optional(), + timestamps: z.enum(['none', 'sentence', 'word']).optional(), + diarization: z.boolean().optional(), + translateToEnglish: z.boolean().optional(), + prompt: z.string().optional(), + temperature: z.coerce.number().optional(), + sentiment: z.boolean().optional(), + entityDetection: z.boolean().optional(), + piiRedaction: z.boolean().optional(), + summarization: z.boolean().optional(), + [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), + }) + .passthrough() + +export type SttOperationInput = z.output diff --git a/apps/sim/lib/internal/supabase/execute-tool.test.ts b/apps/sim/lib/internal/supabase/execute-tool.test.ts new file mode 100644 index 00000000000..8d7b196f4e4 --- /dev/null +++ b/apps/sim/lib/internal/supabase/execute-tool.test.ts @@ -0,0 +1,76 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockExecuteSupabaseStorageUpload } = vi.hoisted(() => ({ + mockExecuteSupabaseStorageUpload: vi.fn(), +})) + +vi.mock('@/lib/internal/supabase/operations', () => ({ + executeSupabaseStorageUpload: mockExecuteSupabaseStorageUpload, +})) + +import { executeSupabaseTool } from '@/lib/internal/supabase/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const INPUT = { + projectId: 'project1234', + apiKey: 'service-key', + bucket: 'documents', + fileName: 'hello.txt', + fileData: 'hello', +} + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'supabase_storage_upload', + input: INPUT, + headers: new Headers(), + context: { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + userId: 'user-1', + metadata: {}, + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeSupabaseTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mockExecuteSupabaseStorageUpload.mockResolvedValue(Response.json({ success: true })) + }) + + it('validates and dispatches operation input with trusted identity', async () => { + const response = await executeSupabaseTool(createRequest()) + + expect(response.status).toBe(200) + expect(mockExecuteSupabaseStorageUpload).toHaveBeenCalledWith( + { ...INPUT, path: undefined, contentType: undefined, cacheControl: undefined, upsert: false }, + { userId: 'user-1', requestId: 'request-1', signal: undefined } + ) + }) + + it('rejects malformed input before provider work', async () => { + const response = await executeSupabaseTool(createRequest({ input: { ...INPUT, bucket: '' } })) + + expect(response.status).toBe(400) + expect(mockExecuteSupabaseStorageUpload).not.toHaveBeenCalled() + }) + + it('requires trusted execution identity', async () => { + const response = await executeSupabaseTool( + createRequest({ + context: { workflowId: 'workflow-1', workspaceId: 'workspace-1', metadata: {} }, + }) + ) + + expect(response.status).toBe(401) + expect(mockExecuteSupabaseStorageUpload).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/supabase/execute-tool.ts b/apps/sim/lib/internal/supabase/execute-tool.ts new file mode 100644 index 00000000000..56a2c329abd --- /dev/null +++ b/apps/sim/lib/internal/supabase/execute-tool.ts @@ -0,0 +1,32 @@ +import { getValidationErrorMessage } from '@/lib/api/server' +import { executeSupabaseStorageUpload } from '@/lib/internal/supabase/operations' +import { supabaseStorageUploadInputSchema } from '@/lib/internal/supabase/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +export const executeSupabaseTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'supabase_storage_upload') { + return Response.json({ error: `Unsupported Supabase tool: ${request.toolId}` }, { status: 500 }) + } + if (!request.context.userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + + const parsed = supabaseStorageUploadInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { + success: false, + error: getValidationErrorMessage(parsed.error, 'Invalid request data'), + details: parsed.error.issues, + }, + { status: 400 } + ) + } + + return executeSupabaseStorageUpload(parsed.data, { + userId: request.context.userId, + requestId: request.requestId, + signal: request.signal, + }) +} diff --git a/apps/sim/lib/internal/supabase/operations.test.ts b/apps/sim/lib/internal/supabase/operations.test.ts new file mode 100644 index 00000000000..dfec9986dfa --- /dev/null +++ b/apps/sim/lib/internal/supabase/operations.test.ts @@ -0,0 +1,126 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + assertToolFileAccess: vi.fn(), + downloadServableFileFromStorage: vi.fn(), +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertToolFileAccess, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mocks.downloadServableFileFromStorage, +})) + +import { executeSupabaseStorageUpload } from '@/lib/internal/supabase/operations' + +const BASE_INPUT = { + projectId: 'project1234', + apiKey: 'service-key', + bucket: 'documents', + fileName: 'hello.txt', + path: null, + contentType: null, + cacheControl: null, + upsert: false, +} as const + +describe('executeSupabaseStorageUpload', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.assertToolFileAccess.mockResolvedValue(null) + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(Response.json({ Key: 'documents/hello.txt' }))) + }) + + it('uploads inline text with the exact provider and output paths', async () => { + const response = await executeSupabaseStorageUpload( + { ...BASE_INPUT, path: 'folder', fileData: 'hello, world' }, + { userId: 'user-1', requestId: 'request-1' } + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + success: true, + output: { + results: { + path: 'folder/hello.txt', + bucket: 'documents', + publicUrl: + 'https://project1234.supabase.co/storage/v1/object/public/documents/folder/hello.txt', + }, + }, + }) + expect(fetch).toHaveBeenCalledWith( + 'https://project1234.supabase.co/storage/v1/object/documents/folder/hello.txt', + expect.objectContaining({ method: 'POST' }) + ) + }) + + it('authorizes stored files before loading bytes', async () => { + mocks.downloadServableFileFromStorage.mockResolvedValue({ + buffer: Buffer.from('audio'), + contentType: 'text/plain', + }) + + const response = await executeSupabaseStorageUpload( + { + ...BASE_INPUT, + fileData: { + id: 'file-1', + key: 'workspace/workspace-1/hello.txt', + name: 'hello.txt', + size: 5, + type: 'text/plain', + url: '/api/files/serve?key=workspace%2Fworkspace-1%2Fhello.txt', + }, + }, + { userId: 'user-1', requestId: 'request-1' } + ) + + expect(response.status).toBe(200) + expect(mocks.assertToolFileAccess).toHaveBeenCalledWith( + 'workspace/workspace-1/hello.txt', + 'user-1', + 'request-1', + expect.anything() + ) + expect(mocks.downloadServableFileFromStorage).toHaveBeenCalledAfter(mocks.assertToolFileAccess) + }) + + it('preserves provider error details and status', async () => { + vi.mocked(fetch).mockResolvedValueOnce( + Response.json({ message: 'Bucket not found', code: '404' }, { status: 404 }) + ) + + const response = await executeSupabaseStorageUpload( + { ...BASE_INPUT, fileData: 'hello' }, + { userId: 'user-1', requestId: 'request-1' } + ) + + expect(response.status).toBe(404) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Bucket not found', + details: { message: 'Bucket not found', code: '404' }, + }) + }) + + it('forwards cancellation to the provider', async () => { + const controller = new AbortController() + vi.mocked(fetch).mockImplementationOnce(async (_url, init) => { + controller.abort(new DOMException('cancelled', 'AbortError')) + throw init?.signal?.reason + }) + + await expect( + executeSupabaseStorageUpload( + { ...BASE_INPUT, fileData: 'hello' }, + { userId: 'user-1', requestId: 'request-1', signal: controller.signal } + ) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) +}) diff --git a/apps/sim/lib/internal/supabase/operations.ts b/apps/sim/lib/internal/supabase/operations.ts new file mode 100644 index 00000000000..33856d6ea81 --- /dev/null +++ b/apps/sim/lib/internal/supabase/operations.ts @@ -0,0 +1,185 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { validateSupabaseProjectId } from '@/lib/core/security/input-validation' +import { + assertKnownSizeWithinLimit, + isPayloadSizeLimitError, + readResponseTextWithLimit, +} from '@/lib/core/utils/stream-limits' +import type { SupabaseStorageUploadInput } from '@/lib/internal/supabase/schema' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' +import { assertToolFileAccess } from '@/app/api/files/authorization' +import { encodeStoragePath, encodeStorageSegment } from '@/tools/supabase/utils' + +const logger = createLogger('SupabaseStorageUpload') +const MAX_SUPABASE_RESPONSE_BYTES = 10 * 1024 * 1024 + +export interface SupabaseOperationContext { + userId: string + requestId: string + signal?: AbortSignal +} + +function failureResponse(error: string, status: number, details?: unknown): Response { + return Response.json( + details === undefined ? { success: false, error } : { success: false, error, details }, + { status } + ) +} + +function decodeStringInput(value: string): Buffer { + let content = value + const dataUrlMatch = content.match(/^data:([^;]+);base64,(.+)$/s) + if (dataUrlMatch) content = dataUrlMatch[2] + + const cleanedContent = content.replace(/[\s\r\n]/g, '') + const isLikelyBase64 = /^[A-Za-z0-9+/]*={0,2}$/.test(cleanedContent) + if (!isLikelyBase64 || cleanedContent.length < 4) return Buffer.from(content, 'utf-8') + + try { + const decoded = Buffer.from(cleanedContent, 'base64') + const expectedMinSize = Math.floor(cleanedContent.length * 0.7) + const expectedMaxSize = Math.ceil(cleanedContent.length * 0.8) + if ( + decoded.length >= expectedMinSize && + decoded.length <= expectedMaxSize && + decoded.length > 0 + ) { + return decoded + } + return decoded.toString('base64') === cleanedContent ? decoded : Buffer.from(content, 'utf-8') + } catch { + return Buffer.from(content, 'utf-8') + } +} + +async function resolveUploadBody( + input: SupabaseStorageUploadInput, + context: SupabaseOperationContext +): Promise<{ body: Buffer; contentType: string } | Response> { + if (typeof input.fileData === 'string') { + const dataUrlType = input.fileData.match(/^data:([^;]+);base64,/s)?.[1] + const body = decodeStringInput(input.fileData) + assertKnownSizeWithinLimit(body.length, MAX_BUFFERED_TRANSFER_BYTES, 'Supabase upload file') + return { + body, + contentType: input.contentType || dataUrlType || 'application/octet-stream', + } + } + + let userFile + try { + userFile = processSingleFileToUserFile(input.fileData, context.requestId, logger) + } catch (error) { + return failureResponse(getErrorMessage(error, 'Failed to process file'), 400) + } + + const denied = await assertToolFileAccess(userFile.key, context.userId, context.requestId, logger) + context.signal?.throwIfAborted() + if (denied) return denied + + try { + const resolved = await downloadServableFileFromStorage(userFile, context.requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + signal: context.signal, + }) + return { + body: resolved.buffer, + contentType: + input.contentType || resolved.contentType || userFile.type || 'application/octet-stream', + } + } catch (error) { + context.signal?.throwIfAborted() + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + return failureResponse( + getErrorMessage(error, 'Internal server error'), + isPayloadSizeLimitError(error) ? 413 : 500 + ) + } +} + +export async function executeSupabaseStorageUpload( + input: SupabaseStorageUploadInput, + context: SupabaseOperationContext +): Promise { + try { + context.signal?.throwIfAborted() + const projectValidation = validateSupabaseProjectId(input.projectId) + if (!projectValidation.isValid) { + return failureResponse(projectValidation.error || 'Invalid Supabase project ID', 400) + } + + const upload = await resolveUploadBody(input, context) + if (upload instanceof Response) return upload + context.signal?.throwIfAborted() + + const fullPath = input.path + ? `${input.path.endsWith('/') ? input.path : `${input.path}/`}${input.fileName}` + : input.fileName + const encodedBucket = encodeStorageSegment(input.bucket) + const encodedPath = encodeStoragePath(fullPath) + const baseUrl = `https://${projectValidation.sanitized}.supabase.co/storage/v1/object` + const headers: Record = { + apikey: input.apiKey, + Authorization: `Bearer ${input.apiKey}`, + 'Content-Type': upload.contentType, + } + if (input.cacheControl) { + const cacheControl = input.cacheControl.trim() + headers['cache-control'] = /^\d+$/.test(cacheControl) + ? `max-age=${cacheControl}` + : cacheControl + } + if (input.upsert) headers['x-upsert'] = 'true' + + const response = await fetch(`${baseUrl}/${encodedBucket}/${encodedPath}`, { + method: 'POST', + headers, + body: new Uint8Array(upload.body), + signal: context.signal, + }) + const responseText = await readResponseTextWithLimit(response, { + maxBytes: MAX_SUPABASE_RESPONSE_BYTES, + label: 'Supabase upload response', + signal: context.signal, + }) + let result: Record + try { + const parsed: unknown = JSON.parse(responseText) + result = parsed && typeof parsed === 'object' ? (parsed as Record) : {} + } catch { + result = { message: responseText } + } + + if (!response.ok) { + const error = + (typeof result.message === 'string' && result.message) || + (typeof result.error === 'string' && result.error) || + `Upload failed: ${response.statusText}` + return failureResponse(error, response.status, result) + } + + return Response.json({ + success: true, + output: { + message: 'Successfully uploaded file to storage', + results: { + ...result, + path: fullPath, + bucket: input.bucket, + publicUrl: `${baseUrl}/public/${encodedBucket}/${encodedPath}`, + }, + }, + }) + } catch (error) { + context.signal?.throwIfAborted() + return failureResponse( + getErrorMessage(error, 'Internal server error'), + isPayloadSizeLimitError(error) ? 413 : 500 + ) + } +} diff --git a/apps/sim/lib/internal/supabase/schema.ts b/apps/sim/lib/internal/supabase/schema.ts new file mode 100644 index 00000000000..78ef8318c55 --- /dev/null +++ b/apps/sim/lib/internal/supabase/schema.ts @@ -0,0 +1,19 @@ +import { z } from 'zod' +import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' + +export const supabaseStorageUploadInputSchema = z.object({ + projectId: z + .string() + .min(1, 'Project ID is required') + .regex(/^[a-z0-9]+$/, 'Project ID must contain only lowercase alphanumeric characters'), + apiKey: z.string().min(1, 'API key is required'), + bucket: z.string().min(1, 'Bucket name is required'), + fileName: z.string().min(1, 'File name is required'), + path: z.string().optional().nullable(), + fileData: FileInputSchema, + contentType: z.string().optional().nullable(), + cacheControl: z.string().optional().nullable(), + upsert: z.boolean().optional().default(false), +}) + +export type SupabaseStorageUploadInput = z.output diff --git a/apps/sim/lib/internal/table/execute-tool.test.ts b/apps/sim/lib/internal/table/execute-tool.test.ts new file mode 100644 index 00000000000..db9855e0612 --- /dev/null +++ b/apps/sim/lib/internal/table/execute-tool.test.ts @@ -0,0 +1,382 @@ +/** + * @vitest-environment node + */ + +import type { WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { InvalidInternalDelegationBindingError } from '@/lib/auth/internal-delegation' +import type { ExecutionContext } from '@/executor/types' + +const mocks = vi.hoisted(() => ({ + createPrincipal: vi.fn(), + create: vi.fn(), + list: vi.fn(), + getSchema: vi.fn(), + getRow: vi.fn(), + insertRows: vi.fn(), + queryRows: vi.fn(), + queryRowsV2: vi.fn(), + updateRow: vi.fn(), + updateRowsByFilter: vi.fn(), + deleteRow: vi.fn(), + deleteRows: vi.fn(), + upsertRow: vi.fn(), +})) + +vi.mock('@/lib/internal/principals/executor', () => ({ + createExecutorPrincipalFromExecutionContext: mocks.createPrincipal, +})) + +vi.mock('@/lib/internal/table/operations', () => ({ + executeTableCreate: mocks.create, + executeTableList: mocks.list, + executeTableGetSchema: mocks.getSchema, + executeTableGetRow: mocks.getRow, + executeTableInsertRows: mocks.insertRows, + executeTableQueryRows: mocks.queryRows, + executeTableQueryRowsV2: mocks.queryRowsV2, + executeTableUpdateRow: mocks.updateRow, + executeTableUpdateRowsByFilter: mocks.updateRowsByFilter, + executeTableDeleteRow: mocks.deleteRow, + executeTableDeleteRows: mocks.deleteRows, + executeTableUpsertRow: mocks.upsertRow, +})) + +import { executeTableTool } from '@/lib/internal/table/execute-tool' +import { TableRowsValidationError, TableV2FeatureDisabledError } from '@/lib/table/application/rows' + +const PRINCIPAL: WorkflowExecutionDelegatedPrincipal = { + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'workspace-canonical', + delegationId: 'delegation-1', + audience: 'sim:tables', + issuedAt: new Date('2026-08-27T00:00:00.000Z'), + expiresAt: new Date('2026-08-27T00:05:00.000Z'), + delegationContext: { kind: 'workflow_execution', workflowId: 'workflow-1' }, +} + +const CONTEXT = { + workflowId: 'workflow-1', + userId: 'user-1', +} as ExecutionContext + +const WIRE_ROW = { + id: 'row-1', + data: { Email: 'a@example.com' }, + position: 0, + createdAt: '2026-08-27T00:00:00.000Z', + updatedAt: '2026-08-27T00:00:00.000Z', +} + +interface Case { + toolId: string + input: Record + operation: keyof typeof mocks + tableId?: string +} + +const CASES: Case[] = [ + { + toolId: 'table_create', + input: { + workspaceId: 'workspace-forged', + name: 'Contacts', + schema: { columns: [{ name: 'Email', type: 'string' }] }, + }, + operation: 'create', + }, + { + toolId: 'table_list', + input: { workspaceId: 'workspace-forged' }, + operation: 'list', + }, + { + toolId: 'table_get_schema', + input: { tableId: 'table-1', workspaceId: 'workspace-forged' }, + operation: 'getSchema', + tableId: 'table-1', + }, + { + toolId: 'table_get_row', + input: { tableId: 'table-1', rowId: 'row-1', workspaceId: 'workspace-forged' }, + operation: 'getRow', + tableId: 'table-1', + }, + { + toolId: 'table_insert_row', + input: { + tableId: 'table-1', + workspaceId: 'workspace-forged', + data: { Email: 'a@example.com' }, + }, + operation: 'insertRows', + tableId: 'table-1', + }, + { + toolId: 'table_batch_insert_rows', + input: { + tableId: 'table-1', + workspaceId: 'workspace-forged', + rows: [{ Email: 'a@example.com' }], + }, + operation: 'insertRows', + tableId: 'table-1', + }, + { + toolId: 'table_query_rows', + input: { tableId: 'table-1', workspaceId: 'workspace-forged', limit: '10' }, + operation: 'queryRows', + tableId: 'table-1', + }, + { + toolId: 'table_query_rows_v2', + input: { tableId: 'table-1', workspaceId: 'workspace-forged', limit: 10 }, + operation: 'queryRowsV2', + tableId: 'table-1', + }, + { + toolId: 'table_update_row', + input: { + tableId: 'table-1', + rowId: 'row-1', + workspaceId: 'workspace-forged', + data: { Email: 'b@example.com' }, + }, + operation: 'updateRow', + tableId: 'table-1', + }, + { + toolId: 'table_update_rows_by_filter', + input: { + tableId: 'table-1', + workspaceId: 'workspace-forged', + filter: { Email: { $eq: 'a@example.com' } }, + data: { Email: 'b@example.com' }, + }, + operation: 'updateRowsByFilter', + tableId: 'table-1', + }, + { + toolId: 'table_delete_row', + input: { tableId: 'table-1', rowId: 'row-1', workspaceId: 'workspace-forged' }, + operation: 'deleteRow', + tableId: 'table-1', + }, + { + toolId: 'table_delete_rows_by_filter', + input: { + tableId: 'table-1', + workspaceId: 'workspace-forged', + filter: { Email: { $eq: 'a@example.com' } }, + }, + operation: 'deleteRows', + tableId: 'table-1', + }, + { + toolId: 'table_upsert_row', + input: { + tableId: 'table-1', + workspaceId: 'workspace-forged', + data: { Email: 'a@example.com' }, + }, + operation: 'upsertRow', + tableId: 'table-1', + }, +] + +describe('executeTableTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.createPrincipal.mockResolvedValue(PRINCIPAL) + mocks.create.mockResolvedValue({ + body: { success: true, data: { table: {}, message: 'Table created successfully' } }, + }) + mocks.list.mockResolvedValue({ + body: { success: true, data: { tables: [], totalCount: 0 } }, + }) + mocks.getSchema.mockResolvedValue({ + body: { success: true, data: { table: {} } }, + }) + mocks.getRow.mockResolvedValue({ + body: { success: true, data: { row: WIRE_ROW } }, + }) + mocks.insertRows.mockImplementation(async (_tableId, body) => ({ + body: + 'rows' in body + ? { + success: true, + data: { rows: [WIRE_ROW], insertedCount: 1, message: 'Rows inserted successfully' }, + } + : { + success: true, + data: { row: WIRE_ROW, message: 'Row inserted successfully' }, + }, + })) + mocks.queryRows.mockResolvedValue({ + body: { + success: true, + data: { + rows: [WIRE_ROW], + rowCount: 1, + totalCount: 1, + limit: 10, + offset: 0, + nextCursor: null, + }, + }, + }) + mocks.queryRowsV2.mockResolvedValue({ + body: { + success: true, + data: { rows: [WIRE_ROW], rowCount: 1, totalCount: 1, limit: 10, nextCursor: null }, + }, + }) + mocks.updateRow.mockResolvedValue({ + body: { success: true, data: { row: WIRE_ROW, message: 'Row updated successfully' } }, + }) + mocks.updateRowsByFilter.mockResolvedValue({ + body: { + success: true, + data: { message: 'Rows updated successfully', updatedCount: 1, updatedRowIds: ['row-1'] }, + }, + }) + mocks.deleteRow.mockResolvedValue({ + body: { success: true, data: { message: 'Row deleted successfully', deletedCount: 1 } }, + }) + mocks.deleteRows.mockResolvedValue({ + body: { + success: true, + data: { message: 'Rows deleted successfully', deletedCount: 1, deletedRowIds: ['row-1'] }, + }, + }) + mocks.upsertRow.mockResolvedValue({ + body: { + success: true, + data: { + row: WIRE_ROW, + operation: 'insert', + message: 'Row inserted successfully', + }, + }, + }) + }) + + it.each(CASES)('dispatches $toolId through its canonical operation input', async (testCase) => { + const response = await executeTableTool({ + toolId: testCase.toolId, + input: testCase.input, + headers: new Headers(), + context: CONTEXT, + requestId: 'request-1', + }) + + expect(response.status).toBe(200) + expect(mocks[testCase.operation]).toHaveBeenCalledOnce() + expect(mocks.createPrincipal).toHaveBeenCalledWith({ + context: CONTEXT, + audience: 'sim:tables', + ...(testCase.tableId ? { resourceScope: { tableId: testCase.tableId } } : {}), + }) + }) + + it('authenticates before contract parsing and rejects missing trusted identity', async () => { + mocks.createPrincipal.mockRejectedValueOnce(new Error('Authentication required')) + + const response = await executeTableTool({ + toolId: 'table_create', + input: null, + headers: new Headers(), + context: CONTEXT, + requestId: 'request-1', + }) + + expect(response.status).toBe(401) + expect(await response.json()).toEqual({ error: 'Authentication required' }) + expect(mocks.create).not.toHaveBeenCalled() + }) + + it('projects a stale workflow binding as authentication failure', async () => { + mocks.createPrincipal.mockRejectedValueOnce(new InvalidInternalDelegationBindingError()) + + const response = await executeTableTool({ + toolId: 'table_list', + input: { workspaceId: 'workspace-forged' }, + headers: new Headers(), + context: CONTEXT, + requestId: 'request-1', + }) + + expect(response.status).toBe(401) + expect(await response.json()).toEqual({ error: 'Authentication required' }) + }) + + it('rejects malformed operation input', async () => { + const response = await executeTableTool({ + toolId: 'table_get_row', + input: { tableId: 'table-1', workspaceId: 'workspace-forged' }, + headers: new Headers(), + context: CONTEXT, + requestId: 'request-1', + }) + + expect(response.status).toBe(400) + expect(mocks.getRow).not.toHaveBeenCalled() + }) + + it('validates operation output against the canonical response contract', async () => { + mocks.getRow.mockResolvedValueOnce({ body: { success: true, data: { row: {} } } }) + + const response = await executeTableTool({ + toolId: 'table_get_row', + input: { tableId: 'table-1', rowId: 'row-1', workspaceId: 'workspace-forged' }, + headers: new Headers(), + context: CONTEXT, + requestId: 'request-1', + }) + + expect(response.status).toBe(500) + expect(await response.json()).toEqual({ error: 'Failed to get row' }) + }) + + it('preserves the v2 rollout-gate error contract', async () => { + mocks.queryRowsV2.mockRejectedValueOnce(new TableV2FeatureDisabledError()) + + const response = await executeTableTool({ + toolId: 'table_query_rows_v2', + input: { tableId: 'table-1', workspaceId: 'workspace-forged', limit: 10 }, + headers: new Headers(), + context: CONTEXT, + requestId: 'request-1', + }) + + expect(response.status).toBe(403) + expect(await response.json()).toEqual({ + error: 'The v2 table query API is not enabled for this workspace', + code: 'tables_v2_disabled', + }) + }) + + it('preserves v2 query validation codes', async () => { + mocks.queryRowsV2.mockRejectedValueOnce( + new TableRowsValidationError('Unknown sort column "Missing"', { + code: 'INVALID_ORDER', + }) + ) + + const response = await executeTableTool({ + toolId: 'table_query_rows_v2', + input: { tableId: 'table-1', workspaceId: 'workspace-forged', limit: 10 }, + headers: new Headers(), + context: CONTEXT, + requestId: 'request-1', + }) + + expect(response.status).toBe(400) + expect(await response.json()).toEqual({ + error: 'Unknown sort column "Missing"', + code: 'INVALID_ORDER', + }) + }) +}) diff --git a/apps/sim/lib/internal/table/execute-tool.ts b/apps/sim/lib/internal/table/execute-tool.ts new file mode 100644 index 00000000000..299d94f4a75 --- /dev/null +++ b/apps/sim/lib/internal/table/execute-tool.ts @@ -0,0 +1,308 @@ +import { createLogger } from '@sim/logger' +import type { AnyApiRouteContract } from '@/lib/api/contracts' +import { + createTableContract, + deleteTableRowContract, + deleteTableRowsContract, + getTableContract, + getTableRowContract, + insertTableRowsContract, + listTableRowsContract, + listTablesContract, + rowQueryContract, + TABLE_QUERY_MAX_BODY_BYTES, + updateTableRowContract, + updateTableRowsByFilterContract, + upsertTableRowContract, +} from '@/lib/api/contracts/tables' +import { type InternalErrorPolicy, internalOrchestrationErrorPolicy } from '@/lib/api/server/routes' +import { InvalidInternalDelegationBindingError } from '@/lib/auth/internal-delegation' +import { createExecutorPrincipalFromExecutionContext } from '@/lib/internal/principals/executor' +import { + executeTableCreate, + executeTableDeleteRow, + executeTableDeleteRows, + executeTableGetRow, + executeTableGetSchema, + executeTableInsertRows, + executeTableList, + executeTableQueryRows, + executeTableQueryRowsV2, + executeTableUpdateRow, + executeTableUpdateRowsByFilter, + executeTableUpsertRow, + type TableToolOperationContext, + type TableToolOperationResult, +} from '@/lib/internal/table/operations' +import { createTableToolResponse } from '@/lib/internal/table/provenance' +import { parseInternalContractInput } from '@/lib/internal/tool-operations/parse-contract-input' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { internalTableErrorPolicies } from '@/lib/table/api/route-policies' +import { + internalTableRowsErrorPolicy, + internalTableV2QueryErrorPolicy, +} from '@/lib/table/api/row-route-policies' +import { TABLE_DELEGATION_AUDIENCE } from '@/lib/table/application/authorization' + +const logger = createLogger('TableInternalOperation') + +const TABLE_SCOPED_TOOLS = new Set([ + 'table_get_schema', + 'table_get_row', + 'table_insert_row', + 'table_batch_insert_rows', + 'table_query_rows', + 'table_query_rows_v2', + 'table_update_row', + 'table_update_rows_by_filter', + 'table_delete_row', + 'table_delete_rows_by_filter', + 'table_upsert_row', +]) + +const ROW_TOOLS = new Set([ + 'table_get_row', + 'table_insert_row', + 'table_batch_insert_rows', + 'table_query_rows', + 'table_query_rows_v2', + 'table_update_row', + 'table_update_rows_by_filter', + 'table_delete_row', + 'table_delete_rows_by_filter', + 'table_upsert_row', +]) + +const FAILURE_MESSAGES: Record = { + table_create: 'Failed to create table', + table_list: 'Failed to list tables', + table_get_schema: 'Failed to get table', + table_get_row: 'Failed to get row', + table_insert_row: 'Failed to insert row', + table_batch_insert_rows: 'Failed to insert rows', + table_query_rows: 'Failed to query rows', + table_query_rows_v2: 'Failed to query rows', + table_update_row: 'Failed to update row', + table_update_rows_by_filter: 'Failed to update rows', + table_delete_row: 'Failed to delete row', + table_delete_rows_by_filter: 'Failed to delete rows', + table_upsert_row: 'Failed to upsert row', +} + +function errorPolicyForTool(toolId: string): InternalErrorPolicy { + if (toolId === 'table_query_rows_v2') return internalTableV2QueryErrorPolicy + if (ROW_TOOLS.has(toolId)) return internalTableRowsErrorPolicy + if (toolId === 'table_get_schema') return internalTableErrorPolicies.concealTableAuthorization + return internalOrchestrationErrorPolicy +} + +function errorResponse( + toolId: string, + error: unknown, + requestId: string, + policy: InternalErrorPolicy +): Response { + const projected = policy.project(error) + if (projected) { + return Response.json(projected.body, { + status: projected.status, + headers: projected.headers, + }) + } + logger.error(`[${requestId}] ${FAILURE_MESSAGES[toolId] ?? 'Table operation failed'}:`, error) + return Response.json( + { error: FAILURE_MESSAGES[toolId] ?? 'Table operation failed' }, + { status: 500 } + ) +} + +async function dispatchTableTool( + request: Parameters[0], + operationContext: TableToolOperationContext +): Promise<{ contract: AnyApiRouteContract; result: TableToolOperationResult } | Response> { + const dispatched = async ( + contract: AnyApiRouteContract, + result: Promise + ) => ({ contract, result: await result }) + + switch (request.toolId) { + case 'table_create': { + const parsed = parseInternalContractInput(createTableContract, request.input) + return parsed.success + ? dispatched(createTableContract, executeTableCreate(parsed.data.body, operationContext)) + : parsed.response + } + case 'table_list': { + const parsed = parseInternalContractInput(listTablesContract, request.input) + return parsed.success + ? dispatched(listTablesContract, executeTableList(parsed.data.query, operationContext)) + : parsed.response + } + case 'table_get_schema': { + const parsed = parseInternalContractInput(getTableContract, request.input) + return parsed.success + ? dispatched( + getTableContract, + executeTableGetSchema(parsed.data.params.tableId, operationContext) + ) + : parsed.response + } + case 'table_get_row': { + const parsed = parseInternalContractInput(getTableRowContract, request.input) + return parsed.success + ? dispatched( + getTableRowContract, + executeTableGetRow( + parsed.data.params.tableId, + parsed.data.params.rowId, + parsed.data.query, + operationContext + ) + ) + : parsed.response + } + case 'table_insert_row': + case 'table_batch_insert_rows': { + const parsed = parseInternalContractInput(insertTableRowsContract, request.input) + return parsed.success + ? dispatched( + insertTableRowsContract, + executeTableInsertRows(parsed.data.params.tableId, parsed.data.body, operationContext) + ) + : parsed.response + } + case 'table_query_rows': { + const parsed = parseInternalContractInput(listTableRowsContract, request.input) + return parsed.success + ? dispatched( + listTableRowsContract, + executeTableQueryRows(parsed.data.params.tableId, parsed.data.query, operationContext) + ) + : parsed.response + } + case 'table_query_rows_v2': { + const parsed = parseInternalContractInput(rowQueryContract, request.input, { + maxInputBytes: TABLE_QUERY_MAX_BODY_BYTES, + }) + return parsed.success + ? dispatched( + rowQueryContract, + executeTableQueryRowsV2(parsed.data.params.tableId, parsed.data.body, operationContext) + ) + : parsed.response + } + case 'table_update_row': { + const parsed = parseInternalContractInput(updateTableRowContract, request.input) + return parsed.success + ? dispatched( + updateTableRowContract, + executeTableUpdateRow( + parsed.data.params.tableId, + parsed.data.params.rowId, + parsed.data.body, + operationContext + ) + ) + : parsed.response + } + case 'table_update_rows_by_filter': { + const parsed = parseInternalContractInput(updateTableRowsByFilterContract, request.input) + return parsed.success + ? dispatched( + updateTableRowsByFilterContract, + executeTableUpdateRowsByFilter( + parsed.data.params.tableId, + parsed.data.body, + operationContext + ) + ) + : parsed.response + } + case 'table_delete_row': { + const parsed = parseInternalContractInput(deleteTableRowContract, request.input) + return parsed.success + ? dispatched( + deleteTableRowContract, + executeTableDeleteRow( + parsed.data.params.tableId, + parsed.data.params.rowId, + parsed.data.body, + operationContext + ) + ) + : parsed.response + } + case 'table_delete_rows_by_filter': { + const parsed = parseInternalContractInput(deleteTableRowsContract, request.input) + return parsed.success + ? dispatched( + deleteTableRowsContract, + executeTableDeleteRows(parsed.data.params.tableId, parsed.data.body, operationContext) + ) + : parsed.response + } + case 'table_upsert_row': { + const parsed = parseInternalContractInput(upsertTableRowContract, request.input) + return parsed.success + ? dispatched( + upsertTableRowContract, + executeTableUpsertRow(parsed.data.params.tableId, parsed.data.body, operationContext) + ) + : parsed.response + } + default: + return Response.json({ error: `Unsupported Table tool: ${request.toolId}` }, { status: 500 }) + } +} + +export const executeTableTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + + if (!Object.hasOwn(FAILURE_MESSAGES, request.toolId)) { + return Response.json({ error: `Unsupported Table tool: ${request.toolId}` }, { status: 500 }) + } + + const scopedInput = TABLE_SCOPED_TOOLS.has(request.toolId) + ? parseInternalContractInput(getTableContract, request.input) + : undefined + if (scopedInput && !scopedInput.success) return scopedInput.response + const tableId = scopedInput?.data.params.tableId + try { + const principal = await createExecutorPrincipalFromExecutionContext({ + context: request.context, + audience: TABLE_DELEGATION_AUDIENCE, + ...(tableId ? { resourceScope: { tableId } } : {}), + }) + request.signal?.throwIfAborted() + + const result = await dispatchTableTool(request, { + principal, + headers: request.headers, + requestId: request.requestId, + signal: request.signal, + }) + if (result instanceof Response) return result + if (result.contract.response.mode !== 'json') { + throw new Error('Table tool contract must return JSON') + } + const validatedBody = result.contract.response.schema.parse(result.result.body) as Record< + string, + unknown + > + return createTableToolResponse(validatedBody, result.result.provenance) + } catch (error) { + request.signal?.throwIfAborted() + if ( + error instanceof InvalidInternalDelegationBindingError || + (error instanceof Error && error.message === 'Authentication required') + ) { + return Response.json({ error: 'Authentication required' }, { status: 401 }) + } + return errorResponse( + request.toolId, + error, + request.requestId, + errorPolicyForTool(request.toolId) + ) + } +} diff --git a/apps/sim/lib/internal/table/operations.test.ts b/apps/sim/lib/internal/table/operations.test.ts new file mode 100644 index 00000000000..fa8ead47374 --- /dev/null +++ b/apps/sim/lib/internal/table/operations.test.ts @@ -0,0 +1,158 @@ +/** + * @vitest-environment node + */ + +import type { WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' +import { createTableDefinition } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + createRows: vi.fn(), + queryRows: vi.fn(), + updateRow: vi.fn(), +})) + +vi.mock('@/lib/table/application/rows', () => ({ + createTableRows: { execute: mocks.createRows }, + deleteTableRow: { execute: vi.fn() }, + deleteTableRows: { execute: vi.fn() }, + queryTableRows: { execute: mocks.queryRows }, + readTableRow: { execute: vi.fn() }, + updateTableRow: { execute: mocks.updateRow }, + updateTableRows: { execute: vi.fn() }, + upsertTableRow: { execute: vi.fn() }, +})) + +vi.mock('@/lib/table/application/tables', () => ({ + createTableUseCase: { execute: vi.fn() }, + listTableDefinitionsUseCase: { execute: vi.fn() }, + readTableDetailsUseCase: { execute: vi.fn() }, +})) + +import { + executeTableInsertRows, + executeTableQueryRows, + executeTableUpdateRow, + type TableToolOperationContext, +} from '@/lib/internal/table/operations' + +const PRINCIPAL: WorkflowExecutionDelegatedPrincipal = { + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'workspace-canonical', + delegationId: 'delegation-1', + audience: 'sim:tables', + issuedAt: new Date('2026-08-27T00:00:00.000Z'), + expiresAt: new Date('2026-08-27T00:05:00.000Z'), + resourceScope: { tableId: 'table-1' }, + delegationContext: { kind: 'workflow_execution', workflowId: 'workflow-1' }, +} + +const TABLE = createTableDefinition({ + id: 'table-1', + workspaceId: 'workspace-canonical', + columns: [{ id: 'column-1', name: 'Email', type: 'string' }], +}) + +const ROW = { + id: 'row-1', + data: { 'column-1': 'a@example.com' }, + position: 0, + orderKey: 'a0', + createdAt: new Date('2026-08-27T00:00:00.000Z'), + updatedAt: new Date('2026-08-27T00:00:00.000Z'), +} + +function operationContext(): TableToolOperationContext { + return { + principal: PRINCIPAL, + headers: new Headers(), + requestId: 'request-1', + } +} + +describe('Table direct operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.createRows.mockResolvedValue({ kind: 'single', table: TABLE, row: ROW }) + mocks.updateRow.mockResolvedValue({ table: TABLE, row: ROW, changed: true }) + mocks.queryRows.mockResolvedValue({ + table: TABLE, + rows: [ROW], + rowCount: 1, + totalCount: 1, + limit: 10, + offset: 0, + nextCursor: null, + }) + }) + + it('uses canonical principal workspace instead of the prepared body assertion', async () => { + await executeTableUpdateRow( + 'table-1', + 'row-1', + { workspaceId: 'workspace-forged', data: { Email: 'a@example.com' } }, + operationContext() + ) + + expect(mocks.updateRow).toHaveBeenCalledWith({ + principal: PRINCIPAL, + input: expect.objectContaining({ + tableId: 'table-1', + rowId: 'row-1', + assertedWorkspaceId: 'workspace-canonical', + dataKeying: 'names', + strictWrite: false, + secretProvenanceEnvelope: { kind: 'none' }, + }), + }) + }) + + it('hands unresolved write provenance to the authorized create use case', async () => { + await executeTableInsertRows( + 'table-1', + { workspaceId: 'workspace-forged', data: { Email: 'a@example.com' } }, + operationContext() + ) + + expect(mocks.createRows).toHaveBeenCalledWith({ + principal: PRINCIPAL, + input: expect.objectContaining({ + assertedWorkspaceId: 'workspace-canonical', + dataKeying: 'names', + secretProvenanceEnvelope: { kind: 'none' }, + }), + }) + }) + + it('preserves legacy name-keyed filter, sort, count, and offset query semantics', async () => { + await executeTableQueryRows( + 'table-1', + { + workspaceId: 'workspace-forged', + filter: { Email: { $eq: 'a@example.com' } }, + sort: { Email: 'asc' }, + limit: 10, + offset: 4, + includeTotal: true, + }, + operationContext() + ) + + expect(mocks.queryRows).toHaveBeenCalledWith({ + principal: PRINCIPAL, + input: expect.objectContaining({ + assertedWorkspaceId: 'workspace-canonical', + legacyFilter: { Email: { $eq: 'a@example.com' } }, + legacySort: { Email: 'asc' }, + legacyKeying: 'names', + limit: 10, + offset: 4, + includeTotal: true, + includeRunState: true, + allowExpandedLimit: true, + }), + }) + }) +}) diff --git a/apps/sim/lib/internal/table/operations.ts b/apps/sim/lib/internal/table/operations.ts new file mode 100644 index 00000000000..e55bab8afa5 --- /dev/null +++ b/apps/sim/lib/internal/table/operations.ts @@ -0,0 +1,477 @@ +import type { WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' +import type { ContractBody, ContractQuery } from '@/lib/api/contracts' +import type { + createTableContract, + deleteTableRowContract, + deleteTableRowsContract, + getTableRowContract, + insertTableRowsContract, + listTableRowsContract, + listTablesContract, + rowQueryContract, + updateTableRowContract, + updateTableRowsByFilterContract, + upsertTableRowContract, +} from '@/lib/api/contracts/tables' +import { + presentCreatedTable, + presentNamedTableQueryRow, + presentNamedTableRow, + presentTableDetails, + presentTableListItem, +} from '@/lib/internal/table/presentation' +import { + readTableToolProvenanceEnvelope, + tableToolRequestsProvenance, +} from '@/lib/internal/table/provenance' +import type { Filter, RowData, Sort, SortSpec, TablePredicate, TableSchema } from '@/lib/table' +import { + createTableRows, + deleteTableRow, + deleteTableRows, + queryTableRows, + readTableRow, + updateTableRow, + updateTableRows, + upsertTableRow, +} from '@/lib/table/application/rows' +import { + createTableUseCase, + listTableDefinitionsUseCase, + readTableDetailsUseCase, +} from '@/lib/table/application/tables' +import { TABLE_LIMITS } from '@/lib/table/constants' +import { isTablePredicate } from '@/lib/table/query-builder/converters' +import { normalizeColumn } from '@/lib/table/wire' + +export interface TableToolOperationContext { + principal: WorkflowExecutionDelegatedPrincipal + headers: Headers + requestId: string + signal?: AbortSignal +} + +export interface TableToolOperationResult { + body: Record + provenance?: unknown +} + +function complete(context: TableToolOperationContext, value: T): T { + context.signal?.throwIfAborted() + return value +} + +export async function executeTableCreate( + body: ContractBody, + context: TableToolOperationContext +): Promise { + const result = await createTableUseCase.execute({ + principal: context.principal, + input: { + workspaceId: context.principal.workspaceId, + name: body.name, + description: body.description, + schema: { + columns: body.schema.columns.map(normalizeColumn), + } as TableSchema, + initialRowCount: body.initialRowCount, + }, + }) + return complete(context, { + body: { + success: true, + data: { + table: presentCreatedTable(result.table), + message: 'Table created successfully', + }, + }, + }) +} + +export async function executeTableList( + _query: ContractQuery, + context: TableToolOperationContext +): Promise { + const result = await listTableDefinitionsUseCase.execute({ + principal: context.principal, + input: { + workspaceId: context.principal.workspaceId, + }, + }) + const tables = result.tables.map(presentTableListItem) + return complete(context, { + body: { success: true, data: { tables, totalCount: tables.length } }, + }) +} + +export async function executeTableGetSchema( + tableId: string, + context: TableToolOperationContext +): Promise { + const result = await readTableDetailsUseCase.execute({ + principal: context.principal, + input: { tableId, workspaceId: context.principal.workspaceId }, + }) + return complete(context, { + body: { + success: true, + data: { table: presentTableDetails(result.table, result.maxRows) }, + }, + }) +} + +export async function executeTableGetRow( + tableId: string, + rowId: string, + _query: ContractQuery, + context: TableToolOperationContext +): Promise { + const includePersistedSecretProvenance = tableToolRequestsProvenance(context.headers) + const result = await readTableRow.execute({ + principal: context.principal, + input: { + tableId, + rowId, + assertedWorkspaceId: context.principal.workspaceId, + includePersistedSecretProvenance, + requestId: context.requestId, + }, + }) + return complete(context, { + body: { + success: true, + data: { row: presentNamedTableRow(result.row, result.table) }, + }, + provenance: result.secretProvenance, + }) +} + +export async function executeTableInsertRows( + tableId: string, + body: ContractBody, + context: TableToolOperationContext +): Promise { + const includePersistedSecretProvenance = tableToolRequestsProvenance(context.headers) + const secretProvenanceEnvelope = readTableToolProvenanceEnvelope(context.headers, body) + const result = await createTableRows.execute({ + principal: context.principal, + input: + 'rows' in body + ? { + kind: 'batch', + tableId, + assertedWorkspaceId: context.principal.workspaceId, + rows: body.rows as RowData[], + orderKeys: body.orderKeys, + strictWrite: false, + dataKeying: 'names', + secretProvenanceEnvelope, + includePersistedSecretProvenance, + requestId: context.requestId, + } + : { + kind: 'single', + tableId, + assertedWorkspaceId: context.principal.workspaceId, + data: body.data as RowData, + position: body.position, + afterRowId: body.afterRowId, + beforeRowId: body.beforeRowId, + strictWrite: false, + dataKeying: 'names', + secretProvenanceEnvelope, + includePersistedSecretProvenance, + requestId: context.requestId, + }, + }) + + if (result.kind === 'single') { + return complete(context, { + body: { + success: true, + data: { + row: presentNamedTableQueryRow(result.row, result.table), + message: 'Row inserted successfully', + }, + }, + provenance: result.secretProvenance, + }) + } + + return complete(context, { + body: { + success: true, + data: { + rows: result.rows.map((row) => presentNamedTableQueryRow(row, result.table)), + insertedCount: result.rows.length, + message: `Successfully inserted ${result.rows.length} rows`, + }, + }, + provenance: result.secretProvenance, + }) +} + +export async function executeTableQueryRows( + tableId: string, + query: ContractQuery, + context: TableToolOperationContext +): Promise { + const filter = query.filter as Filter | TablePredicate | undefined + const sort = query.sort as Sort | SortSpec | undefined + const includePersistedSecretProvenance = tableToolRequestsProvenance(context.headers) + const result = await queryTableRows.execute({ + principal: context.principal, + input: { + tableId, + assertedWorkspaceId: context.principal.workspaceId, + ...(filter && isTablePredicate(filter) + ? { predicate: filter } + : { legacyFilter: filter as Filter | undefined }), + ...(Array.isArray(sort) + ? { sort: sort as SortSpec } + : { legacySort: sort as Sort | undefined }), + legacyKeying: 'names', + limit: query.limit, + offset: query.offset, + includeTotal: query.includeTotal, + includeRunState: query.limit !== undefined && query.limit <= TABLE_LIMITS.MAX_QUERY_LIMIT, + allowExpandedLimit: true, + includePersistedSecretProvenance, + requestId: context.requestId, + }, + }) + return complete(context, { + body: { + success: true, + data: { + rows: result.rows.map((row) => presentNamedTableQueryRow(row, result.table)), + rowCount: result.rowCount, + totalCount: result.totalCount, + limit: result.limit, + offset: result.offset, + nextCursor: result.nextCursor, + }, + }, + provenance: result.secretProvenance, + }) +} + +export async function executeTableQueryRowsV2( + tableId: string, + body: ContractBody, + context: TableToolOperationContext +): Promise { + const includePersistedSecretProvenance = tableToolRequestsProvenance(context.headers) + const result = await queryTableRows.execute({ + principal: context.principal, + input: { + tableId, + assertedWorkspaceId: context.principal.workspaceId, + predicate: body.predicate, + sort: body.sort, + columns: body.columns, + limit: body.limit, + cursor: body.cursor, + includeTotal: !body.cursor, + includeRunState: false, + allowExpandedLimit: true, + requireV2Feature: true, + includePersistedSecretProvenance, + requestId: context.requestId, + }, + }) + return complete(context, { + body: { + success: true, + data: { + rows: result.rows.map((row) => presentNamedTableQueryRow(row, result.table)), + rowCount: result.rowCount, + totalCount: result.totalCount, + limit: result.limit, + nextCursor: result.nextCursor, + }, + }, + provenance: result.secretProvenance, + }) +} + +export async function executeTableUpdateRow( + tableId: string, + rowId: string, + body: ContractBody, + context: TableToolOperationContext +): Promise { + const includePersistedSecretProvenance = tableToolRequestsProvenance(context.headers) + const result = await updateTableRow.execute({ + principal: context.principal, + input: { + tableId, + rowId, + assertedWorkspaceId: context.principal.workspaceId, + data: body.data as RowData, + dataKeying: 'names', + strictWrite: false, + secretProvenanceEnvelope: readTableToolProvenanceEnvelope(context.headers, body), + includePersistedSecretProvenance, + requestId: context.requestId, + }, + }) + return complete(context, { + body: { + success: true, + data: { + row: presentNamedTableRow(result.row, result.table), + message: 'Row updated successfully', + }, + }, + provenance: result.secretProvenance, + }) +} + +export async function executeTableUpdateRowsByFilter( + tableId: string, + body: ContractBody, + context: TableToolOperationContext +): Promise { + const result = await updateTableRows.execute({ + principal: context.principal, + input: { + tableId, + assertedWorkspaceId: context.principal.workspaceId, + filter: body.filter, + filterKeying: 'names', + data: body.data as RowData, + dataKeying: 'names', + strictWrite: false, + limit: body.limit, + secretProvenanceEnvelope: readTableToolProvenanceEnvelope(context.headers, body), + requestId: context.requestId, + }, + }) + const matched = result.affectedCount > 0 + return complete(context, { + body: { + success: true, + data: { + message: matched ? 'Rows updated successfully' : 'No rows matched the filter criteria', + updatedCount: result.affectedCount, + ...(matched ? { updatedRowIds: result.affectedRowIds } : {}), + }, + }, + }) +} + +export async function executeTableDeleteRow( + tableId: string, + rowId: string, + _body: ContractBody, + context: TableToolOperationContext +): Promise { + await deleteTableRow.execute({ + principal: context.principal, + input: { + tableId, + rowId, + assertedWorkspaceId: context.principal.workspaceId, + requestId: context.requestId, + }, + }) + return complete(context, { + body: { + success: true, + data: { message: 'Row deleted successfully', deletedCount: 1 }, + }, + }) +} + +export async function executeTableDeleteRows( + tableId: string, + body: ContractBody, + context: TableToolOperationContext +): Promise { + const result = await deleteTableRows.execute({ + principal: context.principal, + input: body.rowIds + ? { + kind: 'ids', + tableId, + assertedWorkspaceId: context.principal.workspaceId, + rowIds: body.rowIds, + requestId: context.requestId, + } + : { + kind: 'filter', + tableId, + assertedWorkspaceId: context.principal.workspaceId, + filter: body.filter!, + filterKeying: 'names', + limit: body.limit, + requestId: context.requestId, + }, + }) + + if (result.kind === 'ids') { + return complete(context, { + body: { + success: true, + data: { + message: + result.deletedCount === 0 + ? 'No matching rows found for the provided IDs' + : 'Rows deleted successfully', + deletedCount: result.deletedCount, + deletedRowIds: result.deletedRowIds, + requestedCount: result.requestedCount, + ...(result.missingRowIds.length > 0 ? { missingRowIds: result.missingRowIds } : {}), + }, + }, + }) + } + + return complete(context, { + body: { + success: true, + data: { + message: + result.affectedCount === 0 + ? 'No rows matched the filter criteria' + : 'Rows deleted successfully', + deletedCount: result.affectedCount, + deletedRowIds: result.affectedRowIds, + }, + }, + }) +} + +export async function executeTableUpsertRow( + tableId: string, + body: ContractBody, + context: TableToolOperationContext +): Promise { + const includePersistedSecretProvenance = tableToolRequestsProvenance(context.headers) + const result = await upsertTableRow.execute({ + principal: context.principal, + input: { + tableId, + assertedWorkspaceId: context.principal.workspaceId, + data: body.data as RowData, + dataKeying: 'names', + strictWrite: false, + conflictTarget: body.conflictTarget, + secretProvenanceEnvelope: readTableToolProvenanceEnvelope(context.headers, body), + includePersistedSecretProvenance, + requestId: context.requestId, + }, + }) + return complete(context, { + body: { + success: true, + data: { + row: presentNamedTableRow(result.row, result.table), + operation: result.operation, + message: `Row ${result.operation === 'update' ? 'updated' : 'inserted'} successfully`, + }, + }, + provenance: result.secretProvenance, + }) +} diff --git a/apps/sim/lib/internal/table/presentation.ts b/apps/sim/lib/internal/table/presentation.ts new file mode 100644 index 00000000000..0bf99aa4321 --- /dev/null +++ b/apps/sim/lib/internal/table/presentation.ts @@ -0,0 +1,71 @@ +import type { TableDefinition, TableRow, TableRowSummary } from '@/lib/table' +import { namedRowMapper } from '@/lib/table/cell-format' +import { normalizeColumn, toTableListItem, toWireTimestamp } from '@/lib/table/wire' + +export function presentTableListItem(table: TableDefinition): TableDefinition { + return toTableListItem(table) +} + +export function presentTableDetails(table: TableDefinition, maxRows = table.maxRows) { + return { + id: table.id, + name: table.name, + description: table.description, + schema: { + columns: table.schema.columns.map(normalizeColumn), + ...(table.schema.workflowGroups ? { workflowGroups: table.schema.workflowGroups } : {}), + }, + metadata: table.metadata ?? null, + rowCount: table.rowCount, + maxRows, + folderId: table.folderId ?? null, + locks: table.locks, + createdAt: toWireTimestamp(table.createdAt), + updatedAt: toWireTimestamp(table.updatedAt), + jobStatus: table.jobStatus ?? null, + jobId: table.jobId ?? null, + jobType: table.jobType ?? null, + jobError: table.jobError ?? null, + jobRowsProcessed: table.jobRowsProcessed ?? 0, + } +} + +export function presentCreatedTable(table: TableDefinition) { + return { + id: table.id, + name: table.name, + description: table.description, + schema: { columns: table.schema.columns.map(normalizeColumn) }, + rowCount: table.rowCount, + maxRows: table.maxRows, + folderId: table.folderId ?? null, + locks: table.locks, + createdAt: toWireTimestamp(table.createdAt), + updatedAt: toWireTimestamp(table.updatedAt), + } +} + +export function presentNamedTableRow( + row: Pick, + table: TableDefinition +) { + return { + id: row.id, + data: namedRowMapper(table.schema.columns)(row.data), + position: row.position, + createdAt: toWireTimestamp(row.createdAt), + updatedAt: toWireTimestamp(row.updatedAt), + } +} + +export function presentNamedTableQueryRow(row: TableRow, table: TableDefinition) { + return { + id: row.id, + data: namedRowMapper(table.schema.columns)(row.data), + executions: row.executions, + position: row.position, + orderKey: row.orderKey ?? undefined, + createdAt: toWireTimestamp(row.createdAt), + updatedAt: toWireTimestamp(row.updatedAt), + } +} diff --git a/apps/sim/lib/internal/table/provenance.ts b/apps/sim/lib/internal/table/provenance.ts new file mode 100644 index 00000000000..f0a69e56747 --- /dev/null +++ b/apps/sim/lib/internal/table/provenance.ts @@ -0,0 +1,43 @@ +import { inspectPrivateSecretProvenanceRequest } from '@/lib/execution/model-input-provenance' +import { + negotiatePrivateToolMetadataResponse, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, + serializePrivateToolMetadataResponseEnvelope, +} from '@/lib/execution/private-tool-metadata' +import { + type TableRowProvenanceEnvelope, + TableRowProvenanceError, +} from '@/lib/table/application/row-secret-provenance' + +export function readTableToolProvenanceEnvelope( + headers: Headers, + payload: unknown +): TableRowProvenanceEnvelope { + const inspection = inspectPrivateSecretProvenanceRequest(headers, payload) + if (inspection.status === 'unsupported') return { kind: 'none' } + if (inspection.status !== 'verified') throw new TableRowProvenanceError() + return { kind: 'bundle', value: inspection.value } +} + +export function tableToolRequestsProvenance(headers: Headers): boolean { + const negotiation = negotiatePrivateToolMetadataResponse( + headers, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, + true + ) + if (negotiation.status === 'rejected') throw new TableRowProvenanceError() + return negotiation.status !== 'not-requested' +} + +export function createTableToolResponse( + body: Record, + provenance?: unknown +): Response { + if (provenance === undefined) return Response.json(body) + const envelope = serializePrivateToolMetadataResponseEnvelope( + body, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, + provenance + ) + return Response.json(envelope.body, { headers: envelope.headers }) +} diff --git a/apps/sim/lib/internal/table/read-schema.test.ts b/apps/sim/lib/internal/table/read-schema.test.ts new file mode 100644 index 00000000000..ff438f37705 --- /dev/null +++ b/apps/sim/lib/internal/table/read-schema.test.ts @@ -0,0 +1,97 @@ +/** + * @vitest-environment node + */ + +import type { WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + createPrincipal: vi.fn(), + readTable: vi.fn(), +})) + +vi.mock('@/lib/internal/principals/executor', () => ({ + createExecutorPrincipalFromExecutionContext: mocks.createPrincipal, +})) + +vi.mock('@/lib/table/application/tables', () => ({ + readTableDefinitionUseCase: { execute: mocks.readTable }, +})) + +import { readTableSchemaAsExecutor } from '@/lib/internal/table/read-schema' + +const PRINCIPAL: WorkflowExecutionDelegatedPrincipal = { + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'workspace-canonical', + delegationId: 'delegation-1', + audience: 'sim:tables', + issuedAt: new Date('2026-08-27T00:00:00.000Z'), + expiresAt: new Date('2026-08-27T00:05:00.000Z'), + resourceScope: { tableId: 'table-1' }, + delegationContext: { kind: 'workflow_execution', workflowId: 'workflow-1' }, +} + +describe('readTableSchemaAsExecutor', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.createPrincipal.mockResolvedValue(PRINCIPAL) + mocks.readTable.mockResolvedValue({ + table: { + name: 'Customers', + schema: { + columns: [ + { id: 'column-email', name: 'email', type: 'string' }, + { id: 'column-score', name: 'score', type: 'number' }, + ], + }, + }, + }) + }) + + it('binds the read to the canonical delegated workspace', async () => { + const result = await readTableSchemaAsExecutor({ + tableId: 'table-1', + context: { + workflowId: 'workflow-1', + executorDelegationOrigin: { + subjectUserId: 'user-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }, + }, + }) + + expect(mocks.readTable).toHaveBeenCalledWith({ + principal: PRINCIPAL, + input: { tableId: 'table-1', workspaceId: 'workspace-canonical' }, + }) + expect(result).toEqual({ + name: 'Customers', + columns: [ + { name: 'email', type: 'string' }, + { name: 'score', type: 'number' }, + ], + }) + }) + + it('fails closed when canonical schema metadata is malformed', async () => { + mocks.readTable.mockResolvedValueOnce({ + table: { name: 'Customers', schema: { columns: [{ name: 'email', type: 'unknown' }] } }, + }) + + await expect( + readTableSchemaAsExecutor({ + tableId: 'table-1', + context: { + workflowId: 'workflow-1', + executorDelegationOrigin: { + subjectUserId: 'user-1', + workflowId: 'workflow-1', + }, + }, + }) + ).rejects.toThrow('Invalid table column 0 while enriching schema for table-1') + }) +}) diff --git a/apps/sim/lib/internal/table/read-schema.ts b/apps/sim/lib/internal/table/read-schema.ts new file mode 100644 index 00000000000..b1559501e87 --- /dev/null +++ b/apps/sim/lib/internal/table/read-schema.ts @@ -0,0 +1,39 @@ +import { createExecutorPrincipalFromExecutionContext } from '@/lib/internal/principals/executor' +import type { InternalToolOperationContext } from '@/lib/internal/tool-operations/types' +import { TABLE_DELEGATION_AUDIENCE } from '@/lib/table/application/authorization' +import { readTableDefinitionUseCase } from '@/lib/table/application/tables' +import { isColumnType } from '@/lib/table/column-types' +import type { TableSummary } from '@/lib/table/types' + +export interface ReadTableSchemaAsExecutorInput { + tableId: string + context: InternalToolOperationContext +} + +export async function readTableSchemaAsExecutor({ + tableId, + context, +}: ReadTableSchemaAsExecutorInput): Promise { + const principal = await createExecutorPrincipalFromExecutionContext({ + context, + audience: TABLE_DELEGATION_AUDIENCE, + resourceScope: { tableId }, + }) + const { table } = await readTableDefinitionUseCase.execute({ + principal, + input: { tableId, workspaceId: principal.workspaceId }, + }) + + if (!table || typeof table.name !== 'string' || !Array.isArray(table.schema?.columns)) { + throw new Error(`Invalid table metadata while enriching schema for ${tableId}`) + } + + const columns = table.schema.columns.map((column, index) => { + if (typeof column.name !== 'string' || !isColumnType(column.type)) { + throw new Error(`Invalid table column ${index} while enriching schema for ${tableId}`) + } + return { name: column.name, type: column.type } + }) + + return { name: table.name, columns } +} diff --git a/apps/sim/lib/internal/telegram/errors.ts b/apps/sim/lib/internal/telegram/errors.ts new file mode 100644 index 00000000000..dbf5d3814a8 --- /dev/null +++ b/apps/sim/lib/internal/telegram/errors.ts @@ -0,0 +1,9 @@ +export class TelegramOperationError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'TelegramOperationError' + } +} diff --git a/apps/sim/lib/internal/telegram/execute-tool.test.ts b/apps/sim/lib/internal/telegram/execute-tool.test.ts new file mode 100644 index 00000000000..8dd8f96435b --- /dev/null +++ b/apps/sim/lib/internal/telegram/execute-tool.test.ts @@ -0,0 +1,45 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ sendTelegramDocument: vi.fn() })) + +vi.mock('@/lib/internal/telegram/operations', () => ({ + sendTelegramDocument: mocks.sendTelegramDocument, +})) + +import { executeTelegramTool } from '@/lib/internal/telegram/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +describe('executeTelegramTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.sendTelegramDocument.mockResolvedValue({ success: true, output: {} }) + }) + + it('uses trusted user context for protected files', async () => { + const controller = new AbortController() + const input = { + botToken: 'token', + chatId: 'chat-1', + files: [{ key: 'workspace/file.pdf', name: 'file.pdf', size: 3 }], + } + const request: InternalToolOperationCall = { + toolId: 'telegram_send_document', + input, + headers: new Headers(), + context: { ...createExecutionContext(), userId: 'user-1' }, + requestId: 'request-1', + signal: controller.signal, + } + + expect((await executeTelegramTool(request)).status).toBe(200) + expect(mocks.sendTelegramDocument).toHaveBeenCalledWith(input, { + userId: 'user-1', + requestId: 'request-1', + signal: controller.signal, + }) + }) +}) diff --git a/apps/sim/lib/internal/telegram/execute-tool.ts b/apps/sim/lib/internal/telegram/execute-tool.ts new file mode 100644 index 00000000000..ae7f4d27200 --- /dev/null +++ b/apps/sim/lib/internal/telegram/execute-tool.ts @@ -0,0 +1,50 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { z } from 'zod' +import { TelegramOperationError } from '@/lib/internal/telegram/errors' +import { sendTelegramDocument } from '@/lib/internal/telegram/operations' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { RawFileInputArraySchema } from '@/lib/uploads/utils/file-schemas' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' + +const inputSchema = z.object({ + botToken: z.string().min(1, 'Bot token is required'), + chatId: z.string().min(1, 'Chat ID is required'), + files: RawFileInputArraySchema.optional().nullable(), + caption: z.string().optional().nullable(), +}) + +export const executeTelegramTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'telegram_send_document') { + return Response.json( + { success: false, error: `Unsupported Telegram tool: ${request.toolId}` }, + { status: 500 } + ) + } + const userId = request.context.userId + if (!userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + const parsed = inputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + try { + return Response.json( + await sendTelegramDocument(parsed.data, { + userId, + requestId: request.requestId, + signal: request.signal, + }) + ) + } catch (error) { + request.signal?.throwIfAborted() + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + const status = error instanceof TelegramOperationError ? error.status : 500 + return Response.json( + { success: false, error: getErrorMessage(error, 'Unknown error occurred') }, + { status } + ) + } +} diff --git a/apps/sim/lib/internal/telegram/operations.test.ts b/apps/sim/lib/internal/telegram/operations.test.ts new file mode 100644 index 00000000000..862acbaae0a --- /dev/null +++ b/apps/sim/lib/internal/telegram/operations.test.ts @@ -0,0 +1,76 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + assertToolFileAccess: vi.fn(), + downloadServableFileFromStorage: vi.fn(), + fetch: vi.fn(), +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertToolFileAccess, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mocks.downloadServableFileFromStorage, +})) + +import { sendTelegramDocument } from '@/lib/internal/telegram/operations' + +describe('sendTelegramDocument', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mocks.fetch) + mocks.assertToolFileAccess.mockResolvedValue(null) + mocks.downloadServableFileFromStorage.mockResolvedValue({ + buffer: Buffer.from([1, 2, 3]), + contentType: 'application/pdf', + }) + mocks.fetch.mockResolvedValue(Response.json({ ok: true, result: { message_id: 1 } })) + }) + + it('authorizes one stored file and sends one abortable provider request', async () => { + const controller = new AbortController() + const result = await sendTelegramDocument( + { + botToken: 'token', + chatId: 'chat-1', + caption: '**report**', + files: [{ key: 'workspace/file.pdf', name: 'file.pdf', size: 3 }], + }, + { userId: 'user-1', requestId: 'request-1', signal: controller.signal } + ) + + expect(mocks.assertToolFileAccess).toHaveBeenCalledWith( + 'workspace/file.pdf', + 'user-1', + 'request-1', + expect.anything() + ) + expect(mocks.fetch).toHaveBeenCalledTimes(1) + expect(mocks.fetch.mock.calls[0][1]).toEqual( + expect.objectContaining({ signal: controller.signal }) + ) + expect(result.output.files?.[0]).toEqual( + expect.objectContaining({ name: 'file.pdf', data: 'AQID', size: 3 }) + ) + }) + + it('fails closed before materialization when file access is denied', async () => { + mocks.assertToolFileAccess.mockResolvedValue(new Response(null, { status: 404 })) + + await expect( + sendTelegramDocument( + { + botToken: 'token', + chatId: 'chat-1', + files: [{ key: 'workspace/file.pdf', name: 'file.pdf', size: 3 }], + }, + { userId: 'user-1', requestId: 'request-1' } + ) + ).rejects.toMatchObject({ status: 404 }) + expect(mocks.fetch).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/telegram/operations.ts b/apps/sim/lib/internal/telegram/operations.ts new file mode 100644 index 00000000000..aa18bdaf50d --- /dev/null +++ b/apps/sim/lib/internal/telegram/operations.ts @@ -0,0 +1,136 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' +import { TelegramOperationError } from '@/lib/internal/telegram/errors' +import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' +import type { TelegramSendDocumentResponse } from '@/tools/telegram/types' +import { convertMarkdownToHTML } from '@/tools/telegram/utils' + +const logger = createLogger('TelegramSendDocumentOperation') +const MAX_TELEGRAM_DOCUMENT_BYTES = 50 * 1024 * 1024 +const MAX_TELEGRAM_RESPONSE_BYTES = 2 * 1024 * 1024 + +export interface TelegramSendDocumentInput { + botToken: string + chatId: string + files?: RawFileInput[] | null + caption?: string | null +} + +export interface TelegramOperationContext { + userId: string + requestId: string + signal?: AbortSignal +} + +interface TelegramApiResponse { + ok?: boolean + description?: string + result?: TelegramSendDocumentResponse['output']['data'] +} + +export async function sendTelegramDocument( + input: TelegramSendDocumentInput, + context: TelegramOperationContext +): Promise { + context.signal?.throwIfAborted() + if (!input.files?.length) { + throw new TelegramOperationError( + 'At least one document file is required for sendDocument operation', + 400 + ) + } + const userFiles = processFilesToUserFiles(input.files, context.requestId, logger) + if (!userFiles.length) throw new TelegramOperationError('No valid files provided for upload', 400) + const tooLarge = userFiles.filter((file) => file.size > MAX_TELEGRAM_DOCUMENT_BYTES) + if (tooLarge.length) { + const details = tooLarge + .map((file) => `${file.name} (${(file.size / (1024 * 1024)).toFixed(2)}MB)`) + .join(', ') + throw new TelegramOperationError( + `The following files exceed Telegram's 50MB limit: ${details}`, + 400 + ) + } + + const userFile = userFiles[0] + const denied = await assertToolFileAccess(userFile.key, context.userId, context.requestId, logger) + if (denied) throw new TelegramOperationError('File not found', denied.status) + + let buffer: Buffer + let contentType: string + try { + const downloaded = await downloadServableFileFromStorage(userFile, context.requestId, logger, { + maxBytes: MAX_TELEGRAM_DOCUMENT_BYTES, + }) + buffer = downloaded.buffer + contentType = downloaded.contentType + } catch (error) { + if (isPayloadSizeLimitError(error)) { + const sizeMB = ((error.observedBytes ?? userFile.size) / (1024 * 1024)).toFixed(2) + throw new TelegramOperationError( + `The following files exceed Telegram's 50MB limit: ${userFile.name} (${sizeMB}MB)`, + 400 + ) + } + throw error + } + context.signal?.throwIfAborted() + if (buffer.length > MAX_TELEGRAM_DOCUMENT_BYTES) { + const sizeMB = (buffer.length / (1024 * 1024)).toFixed(2) + throw new TelegramOperationError( + `The following files exceed Telegram's 50MB limit: ${userFile.name} (${sizeMB}MB)`, + 400 + ) + } + + const mimeType = contentType || userFile.type || 'application/octet-stream' + const form = new FormData() + form.append('chat_id', input.chatId) + form.append('document', new Blob([new Uint8Array(buffer)], { type: mimeType }), userFile.name) + if (input.caption) { + form.append('caption', convertMarkdownToHTML(input.caption)) + form.append('parse_mode', 'HTML') + } + + const response = await fetch( + `https://api.telegram.org/bot${encodeURIComponent(input.botToken)}/sendDocument`, + { method: 'POST', body: form, signal: context.signal } + ) + const data = await readResponseJsonWithLimit(response, { + maxBytes: MAX_TELEGRAM_RESPONSE_BYTES, + label: 'Telegram send document response', + signal: context.signal, + }).catch((error) => { + context.signal?.throwIfAborted() + throw new TelegramOperationError( + getErrorMessage(error, 'Failed to read Telegram response'), + response.status || 500 + ) + }) + if (!data.ok) { + throw new TelegramOperationError( + data.description || 'Failed to send document to Telegram', + response.status + ) + } + + return { + success: true, + output: { + message: 'Document sent successfully', + data: data.result, + files: [ + { + name: userFile.name, + mimeType, + data: buffer.toString('base64'), + size: buffer.length, + }, + ], + }, + } +} diff --git a/apps/sim/lib/internal/textract/document-input.test.ts b/apps/sim/lib/internal/textract/document-input.test.ts new file mode 100644 index 00000000000..27403738336 --- /dev/null +++ b/apps/sim/lib/internal/textract/document-input.test.ts @@ -0,0 +1,23 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { parseS3Uri } from '@/lib/internal/textract/document-input' +import { TextractOperationError } from '@/lib/internal/textract/errors' + +describe('parseS3Uri', () => { + it('parses a valid S3 URI', () => { + expect(parseS3Uri('s3://my-bucket/path/to/doc.pdf')).toEqual({ + bucket: 'my-bucket', + key: 'path/to/doc.pdf', + }) + }) + + it('rejects a malformed URI', () => { + expect(() => parseS3Uri('not-an-s3-uri')).toThrow(TextractOperationError) + }) + + it('rejects path traversal in the key', () => { + expect(() => parseS3Uri('s3://my-bucket/../secrets.pdf')).toThrow('path traversal') + }) +}) diff --git a/apps/sim/lib/internal/textract/document-input.ts b/apps/sim/lib/internal/textract/document-input.ts new file mode 100644 index 00000000000..ebd82384cc4 --- /dev/null +++ b/apps/sim/lib/internal/textract/document-input.ts @@ -0,0 +1,218 @@ +import type { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { NextResponse } from 'next/server' +import { validateS3BucketName } from '@/lib/core/security/input-validation' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { TextractOperationError } from '@/lib/internal/textract/errors' +import { + isModelSafeWorkspaceFileKey, + MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, +} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import type { RawFileInput } from '@/lib/uploads/utils/file-utils' +import { + extractStorageKey, + isInternalFileUrl, + processSingleFileToUserFile, +} from '@/lib/uploads/utils/file-utils' +import { + downloadServableFileFromStorage, + resolveInternalFileUrl, +} from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +type TextractLogger = ReturnType + +export interface ResolvedDocument { + bytes: Buffer + contentType: string + isPdf: boolean +} + +export type ResolveDocumentResult = + | { ok: true; document: ResolvedDocument } + | { ok: false; response: NextResponse } + +async function fetchDocumentBytes( + url: string, + signal?: AbortSignal +): Promise<{ bytes: Buffer; contentType: string }> { + signal?.throwIfAborted() + const urlValidation = await validateUrlWithDNS(url, 'Document URL') + if (!urlValidation.isValid) { + throw new TextractOperationError(urlValidation.error || 'Invalid document URL', 400) + } + + const response = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP!, { + method: 'GET', + signal, + }) + if (!response.ok) { + await response.text().catch(() => {}) + throw new TextractOperationError( + `Failed to fetch document: ${response.statusText}`, + response.status + ) + } + + const arrayBuffer = await response.arrayBuffer() + const contentType = response.headers.get('content-type') || 'application/octet-stream' + return { bytes: Buffer.from(arrayBuffer), contentType } +} + +export async function resolveDocumentInput( + input: { file?: RawFileInput; filePath?: string }, + userId: string, + requestId: string, + logger: TextractLogger, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + if (input.file) { + let userFile: ReturnType + try { + userFile = processSingleFileToUserFile(input.file, requestId, logger) + } catch (error) { + return { + ok: false, + response: NextResponse.json( + { success: false, error: getErrorMessage(error, 'Failed to process file') }, + { status: 400 } + ), + } + } + + const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) + if (denied) return { ok: false, response: denied } + if (!(await isModelSafeWorkspaceFileKey(userFile.key))) { + return { + ok: false, + response: NextResponse.json( + { success: false, error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE }, + { status: 400 } + ), + } + } + + signal?.throwIfAborted() + const { buffer, contentType } = await downloadServableFileFromStorage( + userFile, + requestId, + logger, + { maxBytes: MAX_BUFFERED_TRANSFER_BYTES } + ) + const resolvedContentType = contentType || userFile.type || 'application/octet-stream' + + return { + ok: true, + document: { + bytes: buffer, + contentType: resolvedContentType, + isPdf: + resolvedContentType.includes('pdf') || + Boolean(userFile.name?.toLowerCase().endsWith('.pdf')), + }, + } + } + + if (input.filePath) { + let fileUrl = input.filePath + const isInternalFilePath = isInternalFileUrl(fileUrl) + + if (isInternalFilePath) { + const resolution = await resolveInternalFileUrl(fileUrl, userId, requestId, logger) + if (resolution.error) { + return { + ok: false, + response: NextResponse.json( + { success: false, error: resolution.error.message }, + { status: resolution.error.status } + ), + } + } + fileUrl = resolution.fileUrl || fileUrl + if (!(await isModelSafeWorkspaceFileKey(extractStorageKey(input.filePath)))) { + return { + ok: false, + response: NextResponse.json( + { success: false, error: MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE }, + { status: 400 } + ), + } + } + } else if (fileUrl.startsWith('/')) { + logger.warn(`[${requestId}] Invalid internal path`, { + userId, + path: fileUrl.substring(0, 50), + }) + return { + ok: false, + response: NextResponse.json( + { + success: false, + error: 'Invalid file path. Only uploaded files are supported for internal paths.', + }, + { status: 400 } + ), + } + } else { + const urlValidation = await validateUrlWithDNS(fileUrl, 'Document URL') + if (!urlValidation.isValid) { + logger.warn(`[${requestId}] SSRF attempt blocked`, { + userId, + url: fileUrl.substring(0, 100), + error: urlValidation.error, + }) + return { + ok: false, + response: NextResponse.json( + { success: false, error: urlValidation.error }, + { status: 400 } + ), + } + } + } + + const fetched = await fetchDocumentBytes(fileUrl, signal) + return { + ok: true, + document: { + bytes: fetched.bytes, + contentType: fetched.contentType, + isPdf: fetched.contentType.includes('pdf') || fileUrl.toLowerCase().endsWith('.pdf'), + }, + } + } + + return { + ok: false, + response: NextResponse.json( + { success: false, error: 'Document input is required' }, + { status: 400 } + ), + } +} + +export function parseS3Uri(s3Uri: string): { bucket: string; key: string } { + const match = s3Uri.match(/^s3:\/\/([^/]+)\/(.+)$/) + if (!match) { + throw new TextractOperationError( + `Invalid S3 URI format: ${s3Uri}. Expected format: s3://bucket-name/path/to/object`, + 400 + ) + } + + const bucket = match[1] + const key = match[2] + const bucketValidation = validateS3BucketName(bucket, 'S3 bucket name') + if (!bucketValidation.isValid) { + throw new TextractOperationError(bucketValidation.error || 'Invalid S3 bucket name', 400) + } + if (key.includes('..') || key.startsWith('/')) { + throw new TextractOperationError('S3 key contains invalid path traversal sequences', 400) + } + return { bucket, key } +} diff --git a/apps/sim/lib/internal/textract/errors.test.ts b/apps/sim/lib/internal/textract/errors.test.ts new file mode 100644 index 00000000000..a05c2700d71 --- /dev/null +++ b/apps/sim/lib/internal/textract/errors.test.ts @@ -0,0 +1,59 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { mapTextractSdkError } from '@/lib/internal/textract/errors' + +describe('mapTextractSdkError', () => { + it('gives a friendly hint for unsupported PDFs in single-page mode', () => { + const mapped = mapTextractSdkError( + { name: 'UnsupportedDocumentException', message: 'Unsupported document' }, + true + ) + expect(mapped.status).toBe(400) + expect(mapped.message).toContain('Multi-Page (PDF, TIFF via S3)') + }) + + it('omits the multi-page hint for operations without an async mode', () => { + const mapped = mapTextractSdkError( + { name: 'UnsupportedDocumentException', message: 'Unsupported document' }, + true, + { hasAsyncMode: false } + ) + expect(mapped.message).not.toContain('Multi-Page') + expect(mapped.message).toContain('Only JPEG, PNG, and single-page PDF files are supported') + }) + + it('does not rewrite the message for non-PDF unsupported documents', () => { + const mapped = mapTextractSdkError( + { name: 'UnsupportedDocumentException', message: 'Unsupported document' }, + false + ) + expect(mapped.message).toBe('Unsupported document') + }) + + it('uses the SDK HTTP status', () => { + const mapped = mapTextractSdkError( + { + name: 'InvalidParameterException', + message: 'Bad param', + $metadata: { httpStatusCode: 400 }, + }, + false + ) + expect(mapped.status).toBe(400) + expect(mapped.message).toBe('Bad param') + }) + + it('passes through a 5xx SDK status for retry classification', () => { + const mapped = mapTextractSdkError( + { message: 'Internal failure', $metadata: { httpStatusCode: 500 } }, + false + ) + expect(mapped.status).toBe(500) + }) + + it('defaults to 500 without an SDK HTTP status', () => { + expect(mapTextractSdkError({ message: 'Unknown failure' }, false).status).toBe(500) + }) +}) diff --git a/apps/sim/lib/internal/textract/errors.ts b/apps/sim/lib/internal/textract/errors.ts new file mode 100644 index 00000000000..44d19f7898d --- /dev/null +++ b/apps/sim/lib/internal/textract/errors.ts @@ -0,0 +1,60 @@ +import type { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { NextResponse } from 'next/server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' + +type TextractLogger = ReturnType + +export class TextractOperationError extends Error { + status: number + + constructor(message: string, status = 500) { + super(message) + this.name = 'TextractOperationError' + this.status = status + } +} + +export function textractErrorResponse( + error: unknown, + requestId: string, + logger: TextractLogger +): NextResponse { + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + + logger.error(`[${requestId}] Error in Textract request:`, error) + const status = error instanceof TextractOperationError ? error.status : 500 + return NextResponse.json( + { success: false, error: getErrorMessage(error, 'Internal server error') }, + { status } + ) +} + +export function mapTextractSdkError( + error: unknown, + isPdf: boolean, + options?: { hasAsyncMode?: boolean } +): TextractOperationError { + const sdkError = error as { + name?: string + message?: string + $metadata?: { httpStatusCode?: number } + } + const hasAsyncMode = options?.hasAsyncMode ?? true + const isUnsupportedFormat = + sdkError.name === 'UnsupportedDocumentException' || + Boolean(sdkError.message?.toLowerCase().includes('unsupported document')) + + if (isUnsupportedFormat && isPdf) { + const hint = hasAsyncMode + ? ' If this is a multi-page PDF, please use "Multi-Page (PDF, TIFF via S3)" mode instead, which requires uploading your document to S3 first. Single Page mode only supports JPEG, PNG, and single-page PDF files.' + : ' Only JPEG, PNG, and single-page PDF files are supported.' + return new TextractOperationError(`This document format is not supported.${hint}`, 400) + } + + return new TextractOperationError( + sdkError.message || 'Textract API error', + sdkError.$metadata?.httpStatusCode ?? 500 + ) +} diff --git a/apps/sim/lib/internal/textract/execute-tool.test.ts b/apps/sim/lib/internal/textract/execute-tool.test.ts new file mode 100644 index 00000000000..5639a6e2f71 --- /dev/null +++ b/apps/sim/lib/internal/textract/execute-tool.test.ts @@ -0,0 +1,146 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mockOperations = vi.hoisted(() => ({ + executeTextractParse: vi.fn(), + executeTextractAnalyzeExpense: vi.fn(), + executeTextractAnalyzeId: vi.fn(), +})) + +vi.mock('@/lib/internal/textract/operations', () => mockOperations) + +import { executeTextractTool } from '@/lib/internal/textract/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const CONNECTION = { + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', + region: 'us-east-1', +} + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'textract_parser', + input: { + ...CONNECTION, + processingMode: 'sync', + filePath: 'https://example.com/document.png', + }, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + userId: 'user-1', + metadata: {}, + }, + requestId: 'request-1', + ...overrides, + } +} + +const TOOL_CASES = [ + { + toolId: 'textract_parser', + input: { + ...CONNECTION, + processingMode: 'sync', + filePath: 'https://example.com/document.png', + }, + operation: mockOperations.executeTextractParse, + }, + { + toolId: 'textract_parser_v2', + input: { + ...CONNECTION, + processingMode: 'sync', + filePath: 'https://example.com/document.png', + }, + operation: mockOperations.executeTextractParse, + }, + { + toolId: 'textract_analyze_expense', + input: { + ...CONNECTION, + processingMode: 'sync', + filePath: 'https://example.com/receipt.png', + }, + operation: mockOperations.executeTextractAnalyzeExpense, + }, + { + toolId: 'textract_analyze_id', + input: { ...CONNECTION, filePath: 'https://example.com/id.png' }, + operation: mockOperations.executeTextractAnalyzeId, + }, +] as const + +describe('executeTextractTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it.each(TOOL_CASES)('validates and dispatches $toolId', async ({ toolId, input, operation }) => { + const controller = new AbortController() + const headers = new Headers({ 'content-type': 'application/json' }) + operation.mockResolvedValue(Response.json({ success: true, output: { toolId } })) + + const response = await executeTextractTool( + createRequest({ + toolId, + input, + headers, + signal: controller.signal, + }) + ) + + expect(response.status).toBe(200) + expect(operation).toHaveBeenCalledWith(input, { + headers, + userId: 'user-1', + requestId: 'request-1', + signal: controller.signal, + }) + }) + + it('returns the route-compatible validation envelope before provider work', async () => { + const response = await executeTextractTool( + createRequest({ input: { ...CONNECTION, processingMode: 'sync' } }) + ) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + success: false, + details: expect.any(Array), + }) + expect(mockOperations.executeTextractParse).not.toHaveBeenCalled() + }) + + it('fails closed without a trusted execution user', async () => { + const response = await executeTextractTool( + createRequest({ + context: { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + metadata: {}, + }, + }) + ) + + expect(response.status).toBe(401) + await expect(response.json()).resolves.toEqual({ success: false, error: 'Unauthorized' }) + expect(mockOperations.executeTextractParse).not.toHaveBeenCalled() + }) + + it('propagates cancellation without starting provider work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeTextractTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mockOperations.executeTextractParse).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/textract/execute-tool.ts b/apps/sim/lib/internal/textract/execute-tool.ts new file mode 100644 index 00000000000..bea475a9cd5 --- /dev/null +++ b/apps/sim/lib/internal/textract/execute-tool.ts @@ -0,0 +1,82 @@ +import type { AnyApiRouteContract, ContractBody } from '@/lib/api/contracts' +import { + textractAnalyzeExpenseContract, + textractAnalyzeIdContract, + textractParseContract, +} from '@/lib/api/contracts/tools/media/document-parse' +import { getValidationErrorMessage } from '@/lib/api/server' +import { + executeTextractAnalyzeExpense, + executeTextractAnalyzeId, + executeTextractParse, + type TextractOperationContext, +} from '@/lib/internal/textract/operations' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +type ParseResult = + | { success: true; data: ContractBody } + | { success: false; response: Response } + +function parseTextractInput( + contract: C, + input: unknown +): ParseResult { + if (!contract.body) return { success: true, data: undefined as ContractBody } + const parsed = contract.body.safeParse(input) + if (!parsed.success) { + return { + success: false, + response: Response.json( + { + success: false, + error: getValidationErrorMessage(parsed.error, 'Invalid request data'), + details: parsed.error.issues, + }, + { status: 400 } + ), + } + } + return { success: true, data: parsed.data as ContractBody } +} + +export const executeTextractTool: InternalToolOperationHandler = async ({ + toolId, + input, + headers, + context, + requestId, + signal, +}) => { + signal?.throwIfAborted() + if (!context.userId) { + return Response.json({ success: false, error: 'Unauthorized' }, { status: 401 }) + } + + const operationContext: TextractOperationContext = { + headers, + userId: context.userId, + requestId, + signal, + } + + switch (toolId) { + case 'textract_parser': + case 'textract_parser_v2': { + const parsed = parseTextractInput(textractParseContract, input) + if (!parsed.success) return parsed.response + return executeTextractParse(parsed.data, operationContext) + } + case 'textract_analyze_expense': { + const parsed = parseTextractInput(textractAnalyzeExpenseContract, input) + if (!parsed.success) return parsed.response + return executeTextractAnalyzeExpense(parsed.data, operationContext) + } + case 'textract_analyze_id': { + const parsed = parseTextractInput(textractAnalyzeIdContract, input) + if (!parsed.success) return parsed.response + return executeTextractAnalyzeId(parsed.data, operationContext) + } + default: + return Response.json({ error: `Unsupported Textract tool: ${toolId}` }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/textract/normalizers.ts b/apps/sim/lib/internal/textract/normalizers.ts new file mode 100644 index 00000000000..91f3563494e --- /dev/null +++ b/apps/sim/lib/internal/textract/normalizers.ts @@ -0,0 +1,70 @@ +import type { ExpenseDocument, IdentityDocument } from '@aws-sdk/client-textract' + +export function normalizeExpenseField(field: { + Type?: { Text?: string; Confidence?: number } + ValueDetection?: { Text?: string; Confidence?: number } + LabelDetection?: { Text?: string; Confidence?: number } + PageNumber?: number + Currency?: { Code?: string; Confidence?: number } + GroupProperties?: { Id?: string; Types?: string[] }[] +}) { + return { + type: { text: field.Type?.Text, confidence: field.Type?.Confidence }, + valueDetection: { + text: field.ValueDetection?.Text, + confidence: field.ValueDetection?.Confidence, + }, + labelDetection: field.LabelDetection + ? { text: field.LabelDetection.Text, confidence: field.LabelDetection.Confidence } + : undefined, + pageNumber: field.PageNumber, + currency: field.Currency + ? { code: field.Currency.Code, confidence: field.Currency.Confidence } + : undefined, + groupProperties: field.GroupProperties?.map((group) => ({ + id: group.Id ?? '', + types: group.Types ?? [], + })), + } +} + +export function normalizeExpenseDocuments(documents: ExpenseDocument[]) { + return documents.map((document) => ({ + expenseIndex: document.ExpenseIndex, + summaryFields: (document.SummaryFields ?? []).map(normalizeExpenseField), + lineItemGroups: (document.LineItemGroups ?? []).map((group) => ({ + lineItemGroupIndex: group.LineItemGroupIndex, + lineItems: (group.LineItems ?? []).map((item) => ({ + lineItemExpenseFields: (item.LineItemExpenseFields ?? []).map(normalizeExpenseField), + })), + })), + })) +} + +export function normalizeIdentityDocuments(documents: IdentityDocument[]) { + return documents.map((document) => ({ + documentIndex: document.DocumentIndex, + identityDocumentFields: (document.IdentityDocumentFields ?? []).map((field) => ({ + type: { + text: field.Type?.Text, + confidence: field.Type?.Confidence, + normalizedValue: field.Type?.NormalizedValue + ? { + value: field.Type.NormalizedValue.Value, + valueType: field.Type.NormalizedValue.ValueType, + } + : undefined, + }, + valueDetection: { + text: field.ValueDetection?.Text, + confidence: field.ValueDetection?.Confidence, + normalizedValue: field.ValueDetection?.NormalizedValue + ? { + value: field.ValueDetection.NormalizedValue.Value, + valueType: field.ValueDetection.NormalizedValue.ValueType, + } + : undefined, + }, + })), + })) +} diff --git a/apps/sim/lib/internal/textract/operations.test.ts b/apps/sim/lib/internal/textract/operations.test.ts new file mode 100644 index 00000000000..12ec0c4c7b7 --- /dev/null +++ b/apps/sim/lib/internal/textract/operations.test.ts @@ -0,0 +1,111 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockSend, mockDestroy, mockResolveDocumentInput, mockDetectDocumentTextCommand } = + vi.hoisted(() => ({ + mockSend: vi.fn(), + mockDestroy: vi.fn(), + mockResolveDocumentInput: vi.fn(), + mockDetectDocumentTextCommand: vi.fn(), + })) + +vi.mock('@aws-sdk/client-textract', () => ({ + AnalyzeDocumentCommand: vi.fn(), + AnalyzeExpenseCommand: vi.fn(), + AnalyzeIDCommand: vi.fn(), + DetectDocumentTextCommand: mockDetectDocumentTextCommand, + GetDocumentAnalysisCommand: vi.fn(), + GetDocumentTextDetectionCommand: vi.fn(), + GetExpenseAnalysisCommand: vi.fn(), + StartDocumentAnalysisCommand: vi.fn(), + StartDocumentTextDetectionCommand: vi.fn(), + StartExpenseAnalysisCommand: vi.fn(), + TextractClient: class { + send = mockSend + destroy = mockDestroy + }, +})) + +vi.mock('@/lib/internal/textract/document-input', () => ({ + parseS3Uri: vi.fn(), + resolveDocumentInput: mockResolveDocumentInput, +})) + +import { PRIVATE_MODEL_INPUT_PROVENANCE_HEADER } from '@/lib/execution/model-input-provenance' +import { executeTextractParse } from '@/lib/internal/textract/operations' + +const INPUT = { + accessKeyId: 'access-key', + secretAccessKey: 'secret-key', + region: 'us-east-1', + processingMode: 'sync' as const, + filePath: 'https://example.com/document.png', +} + +function createContext(headers = new Headers(), signal?: AbortSignal) { + return { + headers, + userId: 'user-1', + requestId: 'request-1', + signal, + } +} + +describe('Textract operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mockResolveDocumentInput.mockResolvedValue({ + ok: true, + document: { + bytes: Buffer.from('document'), + contentType: 'image/png', + isPdf: false, + }, + }) + mockSend.mockResolvedValue({ + Blocks: [], + DocumentMetadata: { Pages: 1 }, + DetectDocumentTextModelVersion: '1.0', + }) + }) + + it('passes cancellation into AWS and destroys the client', async () => { + const controller = new AbortController() + + const response = await executeTextractParse( + INPUT, + createContext(new Headers(), controller.signal) + ) + + expect(response.status).toBe(200) + expect(mockSend).toHaveBeenCalledWith(expect.anything(), { + abortSignal: controller.signal, + }) + expect(mockDestroy).toHaveBeenCalledOnce() + }) + + it('destroys the client when AWS rejects the request', async () => { + mockSend.mockRejectedValue(new Error('provider failure')) + + const response = await executeTextractParse(INPUT, createContext()) + + expect(response.status).toBe(500) + expect(mockDestroy).toHaveBeenCalledOnce() + }) + + it('rejects malformed private provenance before file or AWS work', async () => { + const headers = new Headers({ [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: 'forged-protocol' }) + + const response = await executeTextractParse(INPUT, createContext(headers)) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Invalid model input provenance', + }) + expect(mockResolveDocumentInput).not.toHaveBeenCalled() + expect(mockSend).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/textract/operations.ts b/apps/sim/lib/internal/textract/operations.ts new file mode 100644 index 00000000000..7a021191c60 --- /dev/null +++ b/apps/sim/lib/internal/textract/operations.ts @@ -0,0 +1,444 @@ +import { + AnalyzeDocumentCommand, + AnalyzeExpenseCommand, + AnalyzeIDCommand, + DetectDocumentTextCommand, + type ExpenseDocument, + type FeatureType, + GetDocumentAnalysisCommand, + GetDocumentTextDetectionCommand, + GetExpenseAnalysisCommand, + StartDocumentAnalysisCommand, + StartDocumentTextDetectionCommand, + StartExpenseAnalysisCommand, + TextractClient, +} from '@aws-sdk/client-textract' +import { createLogger } from '@sim/logger' +import { NextResponse } from 'next/server' +import type { ContractBody } from '@/lib/api/contracts' +import type { + textractAnalyzeExpenseContract, + textractAnalyzeIdContract, + textractParseContract, +} from '@/lib/api/contracts/tools/media/document-parse' +import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' +import { parseS3Uri, resolveDocumentInput } from '@/lib/internal/textract/document-input' +import { mapTextractSdkError, textractErrorResponse } from '@/lib/internal/textract/errors' +import { + normalizeExpenseDocuments, + normalizeIdentityDocuments, +} from '@/lib/internal/textract/normalizers' +import { pollTextractJob } from '@/lib/internal/textract/poll-job' + +type TextractParseInput = ContractBody +type TextractAnalyzeExpenseInput = ContractBody +type TextractAnalyzeIdInput = ContractBody + +export interface TextractOperationContext { + headers: Headers + userId: string + requestId: string + signal?: AbortSignal +} + +interface TextractDocumentResult { + JobStatus?: string + StatusMessage?: string + NextToken?: string + Blocks?: unknown[] + DocumentMetadata?: { Pages?: number } + AnalyzeDocumentModelVersion?: string + DetectDocumentTextModelVersion?: string +} + +interface TextractExpenseResult { + JobStatus?: string + StatusMessage?: string + NextToken?: string + ExpenseDocuments?: ExpenseDocument[] + DocumentMetadata?: { Pages?: number } + AnalyzeExpenseModelVersion?: string +} + +const parseLogger = createLogger('TextractParseAPI') +const expenseLogger = createLogger('TextractAnalyzeExpenseAPI') +const identityLogger = createLogger('TextractAnalyzeIdAPI') + +function validateModelInputProvenance(input: unknown, headers: Headers): NextResponse | undefined { + const provenance = validateOpaqueModelInputProvenance({ + headers, + payload: input, + isInternalRequest: true, + }) + if (provenance.success) return undefined + return NextResponse.json( + { success: false, error: provenance.error }, + { status: provenance.status } + ) +} + +function createTextractClient(input: { + region: string + accessKeyId: string + secretAccessKey: string +}): TextractClient { + return new TextractClient({ + region: input.region, + credentials: { + accessKeyId: input.accessKeyId, + secretAccessKey: input.secretAccessKey, + }, + }) +} + +export async function executeTextractParse( + input: TextractParseInput, + context: TextractOperationContext +): Promise { + const { headers, userId, requestId, signal } = context + try { + const provenanceError = validateModelInputProvenance(input, headers) + if (provenanceError) return provenanceError + + const processingMode = input.processingMode || 'sync' + const featureTypes = (input.featureTypes ?? []) as FeatureType[] + const useAnalyzeDocument = featureTypes.length > 0 + const queriesConfig = + input.queries && input.queries.length > 0 && featureTypes.includes('QUERIES') + ? { + Queries: input.queries.map((query) => ({ + Text: query.Text, + Alias: query.Alias, + Pages: query.Pages, + })), + } + : undefined + + parseLogger.info(`[${requestId}] Textract parse request`, { + processingMode, + hasFile: Boolean(input.file), + hasS3Uri: Boolean(input.s3Uri), + featureTypes, + userId, + }) + + if (processingMode === 'async') { + if (!input.s3Uri) { + return NextResponse.json( + { + success: false, + error: 'S3 URI is required for multi-page processing (s3://bucket/key)', + }, + { status: 400 } + ) + } + + const { bucket, key } = parseS3Uri(input.s3Uri) + parseLogger.info(`[${requestId}] Starting async Textract job`, { + s3Bucket: bucket, + s3Key: key, + }) + const client = createTextractClient(input) + try { + const { JobId: jobId } = useAnalyzeDocument + ? await client.send( + new StartDocumentAnalysisCommand({ + DocumentLocation: { S3Object: { Bucket: bucket, Name: key } }, + FeatureTypes: featureTypes, + QueriesConfig: queriesConfig, + }), + { abortSignal: signal } + ) + : await client.send( + new StartDocumentTextDetectionCommand({ + DocumentLocation: { S3Object: { Bucket: bucket, Name: key } }, + }), + { abortSignal: signal } + ) + if (!jobId) throw new Error('Failed to start Textract job: No JobId returned') + parseLogger.info(`[${requestId}] Async job started`, { jobId }) + + const result = await pollTextractJob( + requestId, + parseLogger, + async (nextToken) => + useAnalyzeDocument + ? await client.send( + new GetDocumentAnalysisCommand({ JobId: jobId, NextToken: nextToken }), + { abortSignal: signal } + ) + : await client.send( + new GetDocumentTextDetectionCommand({ JobId: jobId, NextToken: nextToken }), + { abortSignal: signal } + ), + (accumulated, page) => ({ + ...accumulated, + ...page, + Blocks: [...(accumulated.Blocks ?? []), ...(page.Blocks ?? [])], + }), + signal + ) + + parseLogger.info(`[${requestId}] Textract async parse successful`, { + pageCount: result.DocumentMetadata?.Pages ?? 0, + blockCount: result.Blocks?.length ?? 0, + }) + return NextResponse.json({ + success: true, + output: { + blocks: result.Blocks ?? [], + documentMetadata: { pages: result.DocumentMetadata?.Pages ?? 0 }, + modelVersion: + result.AnalyzeDocumentModelVersion ?? result.DetectDocumentTextModelVersion, + }, + }) + } finally { + client.destroy() + } + } + + const resolved = await resolveDocumentInput( + { file: input.file, filePath: input.filePath }, + userId, + requestId, + parseLogger, + signal + ) + if (!resolved.ok) return resolved.response + + const client = createTextractClient(input) + try { + let result: TextractDocumentResult + try { + result = useAnalyzeDocument + ? await client.send( + new AnalyzeDocumentCommand({ + Document: { Bytes: resolved.document.bytes }, + FeatureTypes: featureTypes, + QueriesConfig: queriesConfig, + }), + { abortSignal: signal } + ) + : await client.send( + new DetectDocumentTextCommand({ Document: { Bytes: resolved.document.bytes } }), + { abortSignal: signal } + ) + } catch (error) { + signal?.throwIfAborted() + throw mapTextractSdkError(error, resolved.document.isPdf) + } + + parseLogger.info(`[${requestId}] Textract parse successful`, { + pageCount: result.DocumentMetadata?.Pages ?? 0, + blockCount: result.Blocks?.length ?? 0, + }) + return NextResponse.json({ + success: true, + output: { + blocks: result.Blocks ?? [], + documentMetadata: { pages: result.DocumentMetadata?.Pages ?? 0 }, + modelVersion: result.AnalyzeDocumentModelVersion ?? result.DetectDocumentTextModelVersion, + }, + }) + } finally { + client.destroy() + } + } catch (error) { + signal?.throwIfAborted() + return textractErrorResponse(error, requestId, parseLogger) + } +} + +export async function executeTextractAnalyzeExpense( + input: TextractAnalyzeExpenseInput, + context: TextractOperationContext +): Promise { + const { headers, userId, requestId, signal } = context + try { + const provenanceError = validateModelInputProvenance(input, headers) + if (provenanceError) return provenanceError + const processingMode = input.processingMode || 'sync' + + expenseLogger.info(`[${requestId}] Textract analyze-expense request`, { + processingMode, + hasFile: Boolean(input.file), + hasS3Uri: Boolean(input.s3Uri), + userId, + }) + + if (processingMode === 'async') { + if (!input.s3Uri) { + return NextResponse.json( + { + success: false, + error: 'S3 URI is required for multi-page processing (s3://bucket/key)', + }, + { status: 400 } + ) + } + const { bucket, key } = parseS3Uri(input.s3Uri) + expenseLogger.info(`[${requestId}] Starting async Textract expense analysis job`, { + s3Bucket: bucket, + s3Key: key, + }) + const client = createTextractClient(input) + try { + const { JobId: jobId } = await client.send( + new StartExpenseAnalysisCommand({ + DocumentLocation: { S3Object: { Bucket: bucket, Name: key } }, + }), + { abortSignal: signal } + ) + if (!jobId) { + throw new Error('Failed to start Textract expense analysis job: No JobId returned') + } + expenseLogger.info(`[${requestId}] Async expense analysis job started`, { jobId }) + + const result = await pollTextractJob( + requestId, + expenseLogger, + (nextToken) => + client.send(new GetExpenseAnalysisCommand({ JobId: jobId, NextToken: nextToken }), { + abortSignal: signal, + }), + (accumulated, page) => ({ + ...accumulated, + ...page, + ExpenseDocuments: [ + ...(accumulated.ExpenseDocuments ?? []), + ...(page.ExpenseDocuments ?? []), + ], + }), + signal + ) + + return NextResponse.json({ + success: true, + output: { + expenseDocuments: normalizeExpenseDocuments(result.ExpenseDocuments ?? []), + documentMetadata: { pages: result.DocumentMetadata?.Pages ?? 0 }, + modelVersion: result.AnalyzeExpenseModelVersion, + }, + }) + } finally { + client.destroy() + } + } + + const resolved = await resolveDocumentInput( + { file: input.file, filePath: input.filePath }, + userId, + requestId, + expenseLogger, + signal + ) + if (!resolved.ok) return resolved.response + + const client = createTextractClient(input) + try { + let result: TextractExpenseResult + try { + result = await client.send( + new AnalyzeExpenseCommand({ Document: { Bytes: resolved.document.bytes } }), + { abortSignal: signal } + ) + } catch (error) { + signal?.throwIfAborted() + throw mapTextractSdkError(error, resolved.document.isPdf) + } + + expenseLogger.info(`[${requestId}] Textract analyze-expense successful`, { + pageCount: result.DocumentMetadata?.Pages ?? 0, + expenseDocumentCount: result.ExpenseDocuments?.length ?? 0, + }) + return NextResponse.json({ + success: true, + output: { + expenseDocuments: normalizeExpenseDocuments(result.ExpenseDocuments ?? []), + documentMetadata: { pages: result.DocumentMetadata?.Pages ?? 0 }, + }, + }) + } finally { + client.destroy() + } + } catch (error) { + signal?.throwIfAborted() + return textractErrorResponse(error, requestId, expenseLogger) + } +} + +export async function executeTextractAnalyzeId( + input: TextractAnalyzeIdInput, + context: TextractOperationContext +): Promise { + const { headers, userId, requestId, signal } = context + try { + const provenanceError = validateModelInputProvenance(input, headers) + if (provenanceError) return provenanceError + + identityLogger.info(`[${requestId}] Textract analyze-id request`, { + hasFile: Boolean(input.file), + hasBackFile: Boolean(input.fileBack || input.filePathBack), + userId, + }) + + const front = await resolveDocumentInput( + { file: input.file, filePath: input.filePath }, + userId, + requestId, + identityLogger, + signal + ) + if (!front.ok) return front.response + + const documentPages = [{ Bytes: front.document.bytes }] + let isPdf = front.document.isPdf + if (input.fileBack || input.filePathBack) { + const back = await resolveDocumentInput( + { file: input.fileBack, filePath: input.filePathBack }, + userId, + requestId, + identityLogger, + signal + ) + if (!back.ok) return back.response + documentPages.push({ Bytes: back.document.bytes }) + isPdf = isPdf || back.document.isPdf + } + + const client = createTextractClient(input) + try { + let result: { + AnalyzeIDModelVersion?: string + DocumentMetadata?: { Pages?: number } + IdentityDocuments?: import('@aws-sdk/client-textract').IdentityDocument[] + } + try { + result = await client.send(new AnalyzeIDCommand({ DocumentPages: documentPages }), { + abortSignal: signal, + }) + } catch (error) { + signal?.throwIfAborted() + throw mapTextractSdkError(error, isPdf, { hasAsyncMode: false }) + } + + identityLogger.info(`[${requestId}] Textract analyze-id successful`, { + pageCount: result.DocumentMetadata?.Pages ?? 0, + documentCount: result.IdentityDocuments?.length ?? 0, + }) + return NextResponse.json({ + success: true, + output: { + identityDocuments: normalizeIdentityDocuments(result.IdentityDocuments ?? []), + documentMetadata: { pages: result.DocumentMetadata?.Pages ?? 0 }, + modelVersion: result.AnalyzeIDModelVersion, + }, + }) + } finally { + client.destroy() + } + } catch (error) { + signal?.throwIfAborted() + return textractErrorResponse(error, requestId, identityLogger) + } +} diff --git a/apps/sim/lib/internal/textract/poll-job.test.ts b/apps/sim/lib/internal/textract/poll-job.test.ts new file mode 100644 index 00000000000..9cae63b6c78 --- /dev/null +++ b/apps/sim/lib/internal/textract/poll-job.test.ts @@ -0,0 +1,105 @@ +/** + * @vitest-environment node + */ +import { createLogger } from '@sim/logger' +import { describe, expect, it } from 'vitest' +import { pollTextractJob } from '@/lib/internal/textract/poll-job' + +const logger = createLogger('TextractPollJobTest') + +describe('pollTextractJob', () => { + it('returns immediately on success without a next token', async () => { + const result = await pollTextractJob( + 'request-1', + logger, + async () => ({ JobStatus: 'SUCCEEDED', Blocks: [{ Id: '1' }] }), + (accumulated, page) => ({ + ...page, + Blocks: [...(accumulated.Blocks ?? []), ...(page.Blocks ?? [])], + }) + ) + + expect(result.JobStatus).toBe('SUCCEEDED') + expect(result.Blocks).toHaveLength(1) + }) + + it('follows pagination and merges pages', async () => { + let calls = 0 + const result = await pollTextractJob( + 'request-2', + logger, + async (nextToken) => { + calls += 1 + if (!nextToken) { + return { JobStatus: 'SUCCEEDED', Blocks: [{ Id: '1' }], NextToken: 'next' } + } + return { JobStatus: 'SUCCEEDED', Blocks: [{ Id: '2' }] } + }, + (accumulated, page) => ({ + ...accumulated, + ...page, + Blocks: [...(accumulated.Blocks ?? []), ...(page.Blocks ?? [])], + }) + ) + + expect(calls).toBe(2) + expect(result.Blocks).toHaveLength(2) + }) + + it('preserves first-page metadata omitted from later pages', async () => { + const result = await pollTextractJob<{ + JobStatus?: string + NextToken?: string + Blocks?: unknown[] + DocumentMetadata?: { Pages?: number } + }>( + 'request-3', + logger, + async (nextToken) => { + if (!nextToken) { + return { + JobStatus: 'SUCCEEDED', + Blocks: [{ Id: '1' }], + DocumentMetadata: { Pages: 3 }, + NextToken: 'next', + } + } + return { JobStatus: 'SUCCEEDED', Blocks: [{ Id: '2' }] } + }, + (accumulated, page) => ({ + ...accumulated, + ...page, + Blocks: [...(accumulated.Blocks ?? []), ...(page.Blocks ?? [])], + }) + ) + + expect(result.Blocks).toHaveLength(2) + expect(result.DocumentMetadata).toEqual({ Pages: 3 }) + }) + + it('throws when the job fails', async () => { + await expect( + pollTextractJob( + 'request-4', + logger, + async () => ({ JobStatus: 'FAILED', StatusMessage: 'boom' }), + (accumulated) => accumulated + ) + ).rejects.toThrow('Textract job failed: boom') + }) + + it('propagates cancellation before polling', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + pollTextractJob( + 'request-5', + logger, + async () => ({ JobStatus: 'SUCCEEDED' }), + (accumulated) => accumulated, + controller.signal + ) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) +}) diff --git a/apps/sim/lib/internal/textract/poll-job.ts b/apps/sim/lib/internal/textract/poll-job.ts new file mode 100644 index 00000000000..06dcc48be84 --- /dev/null +++ b/apps/sim/lib/internal/textract/poll-job.ts @@ -0,0 +1,64 @@ +import type { createLogger } from '@sim/logger' +import { sleep } from '@sim/utils/helpers' +import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' +import { TextractOperationError } from '@/lib/internal/textract/errors' + +type TextractLogger = ReturnType + +interface PollableJobResult { + JobStatus?: string + StatusMessage?: string + NextToken?: string +} + +export async function pollTextractJob( + requestId: string, + logger: TextractLogger, + getPage: (nextToken?: string) => Promise, + mergePage: (accumulated: TResult, page: TResult) => TResult, + signal?: AbortSignal +): Promise { + const pollIntervalMs = 5000 + const maxPollTimeMs = getMaxExecutionTimeout() + const maxAttempts = Math.ceil(maxPollTimeMs / pollIntervalMs) + + for (let attempt = 0; attempt < maxAttempts; attempt++) { + signal?.throwIfAborted() + const result = await getPage() + const jobStatus = result.JobStatus + + if (jobStatus === 'SUCCEEDED' || jobStatus === 'PARTIAL_SUCCESS') { + if (jobStatus === 'PARTIAL_SUCCESS') { + logger.warn(`[${requestId}] Job completed with partial success: ${result.StatusMessage}`) + } else { + logger.info(`[${requestId}] Async job completed successfully after ${attempt + 1} polls`) + } + + let merged = result + let nextToken = result.NextToken + while (nextToken) { + signal?.throwIfAborted() + const page = await getPage(nextToken) + merged = mergePage(merged, page) + nextToken = page.NextToken + } + return merged + } + + if (jobStatus === 'FAILED') { + throw new TextractOperationError( + `Textract job failed: ${result.StatusMessage || 'Unknown error'}`, + 502 + ) + } + + logger.info(`[${requestId}] Job status: ${jobStatus}, attempt ${attempt + 1}/${maxAttempts}`) + await sleep(pollIntervalMs) + signal?.throwIfAborted() + } + + throw new TextractOperationError( + `Timeout waiting for Textract job to complete (max ${maxPollTimeMs / 1000} seconds)`, + 504 + ) +} diff --git a/apps/sim/lib/internal/thinking/execute-tool.test.ts b/apps/sim/lib/internal/thinking/execute-tool.test.ts new file mode 100644 index 00000000000..276c4694d5a --- /dev/null +++ b/apps/sim/lib/internal/thinking/execute-tool.test.ts @@ -0,0 +1,41 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { executeThinkingTool } from '@/lib/internal/thinking/execute-tool' + +const context = { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + userId: 'user-1', +} + +describe('executeThinkingTool', () => { + it('returns the acknowledged thought from semantic operation input', async () => { + const response = await executeThinkingTool({ + toolId: 'thinking_tool', + input: { thought: 'Consider the edge cases' }, + headers: new Headers(), + context, + requestId: 'request-1', + }) + + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ + success: true, + output: { acknowledgedThought: 'Consider the edge cases' }, + }) + }) + + it('rejects invalid operation input', async () => { + const response = await executeThinkingTool({ + toolId: 'thinking_tool', + input: { thought: '' }, + headers: new Headers(), + context, + requestId: 'request-1', + }) + + expect(response.status).toBe(400) + }) +}) diff --git a/apps/sim/lib/internal/thinking/execute-tool.ts b/apps/sim/lib/internal/thinking/execute-tool.ts new file mode 100644 index 00000000000..2278e86b9a9 --- /dev/null +++ b/apps/sim/lib/internal/thinking/execute-tool.ts @@ -0,0 +1,31 @@ +import { z } from 'zod' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import type { ThinkingToolResponse } from '@/tools/thinking/types' + +const thinkingInputSchema = z.object({ + thought: z.string().min(1, 'The thought parameter is required and must be a string'), +}) + +export const executeThinkingTool: InternalToolOperationHandler = async ({ + toolId, + input, + signal, +}) => { + signal?.throwIfAborted() + if (toolId !== 'thinking_tool') { + return Response.json({ error: `Unsupported Thinking tool: ${toolId}` }, { status: 500 }) + } + + const parsed = thinkingInputSchema.safeParse(input) + if (!parsed.success) { + return Response.json( + { error: parsed.error.issues[0]?.message ?? 'Invalid thinking input' }, + { status: 400 } + ) + } + + return Response.json({ + success: true, + output: { acknowledgedThought: parsed.data.thought }, + } satisfies ThinkingToolResponse) +} diff --git a/apps/sim/lib/internal/tiktok/execute-tool.test.ts b/apps/sim/lib/internal/tiktok/execute-tool.test.ts new file mode 100644 index 00000000000..f374ef85d67 --- /dev/null +++ b/apps/sim/lib/internal/tiktok/execute-tool.test.ts @@ -0,0 +1,68 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockExecuteTikTokUploadVideoDraft } = vi.hoisted(() => ({ + mockExecuteTikTokUploadVideoDraft: vi.fn(), +})) + +vi.mock('@/lib/internal/tiktok/operations', () => ({ + executeTikTokUploadVideoDraft: mockExecuteTikTokUploadVideoDraft, +})) + +import { executeTikTokTool } from '@/lib/internal/tiktok/execute-tool' +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' + +const FILE = { + key: 'workspace/workspace-1/video.mp4', + name: 'video.mp4', + size: 1, + type: 'video/mp4', +} + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'tiktok_upload_video_draft', + input: { accessToken: 'access-token', file: FILE }, + headers: new Headers(), + context: { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + userId: 'user-1', + metadata: {}, + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeTikTokTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mockExecuteTikTokUploadVideoDraft.mockResolvedValue(Response.json({ success: true })) + }) + + it('dispatches typed input with trusted identity', async () => { + const response = await executeTikTokTool(createRequest()) + + expect(response.status).toBe(200) + expect(mockExecuteTikTokUploadVideoDraft).toHaveBeenCalledWith( + expect.objectContaining({ accessToken: 'access-token', file: FILE }), + { userId: 'user-1', requestId: 'request-1', signal: undefined } + ) + }) + + it('requires trusted execution identity', async () => { + const response = await executeTikTokTool( + createRequest({ + context: { workflowId: 'workflow-1', workspaceId: 'workspace-1', metadata: {} }, + }) + ) + + expect(response.status).toBe(401) + expect(mockExecuteTikTokUploadVideoDraft).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/tiktok/execute-tool.ts b/apps/sim/lib/internal/tiktok/execute-tool.ts new file mode 100644 index 00000000000..ed43ee9be4d --- /dev/null +++ b/apps/sim/lib/internal/tiktok/execute-tool.ts @@ -0,0 +1,28 @@ +import { getValidationErrorMessage } from '@/lib/api/server' +import { executeTikTokUploadVideoDraft } from '@/lib/internal/tiktok/operations' +import { tiktokUploadVideoDraftInputSchema } from '@/lib/internal/tiktok/schema' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' + +export const executeTikTokTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'tiktok_upload_video_draft') { + return Response.json({ error: `Unsupported TikTok tool: ${request.toolId}` }, { status: 500 }) + } + if (!request.context.userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + + const parsed = tiktokUploadVideoDraftInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { success: false, error: getValidationErrorMessage(parsed.error, 'Invalid request data') }, + { status: 400 } + ) + } + + return executeTikTokUploadVideoDraft(parsed.data, { + userId: request.context.userId, + requestId: request.requestId, + signal: request.signal, + }) +} diff --git a/apps/sim/lib/internal/tiktok/operations.test.ts b/apps/sim/lib/internal/tiktok/operations.test.ts new file mode 100644 index 00000000000..7d1afa786be --- /dev/null +++ b/apps/sim/lib/internal/tiktok/operations.test.ts @@ -0,0 +1,130 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' + +const mocks = vi.hoisted(() => ({ + assertToolFileAccess: vi.fn(), + computeTikTokChunkPlan: vi.fn(() => ({ chunkSize: 10_000_000, totalChunkCount: 2 })), + getStoredVideoSize: vi.fn(), + streamStoredVideoToTikTok: vi.fn(), +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertToolFileAccess, +})) + +vi.mock('@/lib/internal/tiktok/upload', () => ({ + computeTikTokChunkPlan: mocks.computeTikTokChunkPlan, + getStoredVideoSize: mocks.getStoredVideoSize, + streamStoredVideoToTikTok: mocks.streamStoredVideoToTikTok, + TIKTOK_MAX_VIDEO_BYTES: 250 * 1024 * 1024, +})) + +import { executeTikTokUploadVideoDraft } from '@/lib/internal/tiktok/operations' + +const FILE = { + key: 'workspace/workspace-1/video.mp4', + name: 'video.mp4', + size: 1, + type: 'video/mp4', +} + +describe('executeTikTokUploadVideoDraft', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.assertToolFileAccess.mockResolvedValue(null) + mocks.getStoredVideoSize.mockResolvedValue(20_000_000) + mocks.computeTikTokChunkPlan.mockReturnValue({ + chunkSize: 10_000_000, + totalChunkCount: 2, + }) + mocks.streamStoredVideoToTikTok.mockResolvedValue(undefined) + vi.stubGlobal('fetch', vi.fn()) + }) + + it('uses authoritative storage size for initialization and streaming', async () => { + const fetchMock = vi.fn().mockResolvedValue( + Response.json({ + data: { publish_id: 'publish-1', upload_url: 'https://upload.example/video' }, + error: { code: 'ok' }, + }) + ) + vi.stubGlobal('fetch', fetchMock) + const controller = new AbortController() + const response = await executeTikTokUploadVideoDraft( + { accessToken: 'access-token', file: FILE }, + { + userId: 'user-1', + requestId: 'request-1', + signal: controller.signal, + } + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: true, + output: { publishId: 'publish-1' }, + }) + expect(mocks.getStoredVideoSize).toHaveBeenCalledWith({ + key: FILE.key, + context: 'workspace', + signal: controller.signal, + }) + const init = JSON.parse(fetchMock.mock.calls[0][1]?.body as string) as { + source_info: Record + } + expect(init.source_info).toEqual({ + source: 'FILE_UPLOAD', + video_size: 20_000_000, + chunk_size: 10_000_000, + total_chunk_count: 2, + }) + expect(mocks.streamStoredVideoToTikTok).toHaveBeenCalledWith({ + key: FILE.key, + context: 'workspace', + uploadUrl: 'https://upload.example/video', + totalBytes: 20_000_000, + mimeType: 'video/mp4', + requestId: 'request-1', + signal: controller.signal, + }) + }) + + it('returns 413 when the stored object exceeds the relay limit', async () => { + mocks.getStoredVideoSize.mockRejectedValue( + new PayloadSizeLimitError({ + label: 'TikTok video upload', + maxBytes: 250 * 1024 * 1024, + observedBytes: 251 * 1024 * 1024, + }) + ) + + const response = await executeTikTokUploadVideoDraft( + { accessToken: 'access-token', file: FILE }, + { userId: 'user-1', requestId: 'request-1' } + ) + + expect(response.status).toBe(413) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Video exceeds the 250MB limit for file uploads.', + }) + }) + + it('does not start provider upload before file authorization', async () => { + mocks.assertToolFileAccess.mockResolvedValue( + Response.json({ success: false, error: 'Forbidden' }, { status: 403 }) + ) + + const response = await executeTikTokUploadVideoDraft( + { accessToken: 'access-token', file: FILE }, + { userId: 'user-1', requestId: 'request-1' } + ) + + expect(response.status).toBe(403) + expect(mocks.getStoredVideoSize).not.toHaveBeenCalled() + expect(fetch).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/tiktok/operations.ts b/apps/sim/lib/internal/tiktok/operations.ts new file mode 100644 index 00000000000..ae2c255e2bc --- /dev/null +++ b/apps/sim/lib/internal/tiktok/operations.ts @@ -0,0 +1,144 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import type { TikTokUploadVideoDraftInput } from '@/lib/internal/tiktok/schema' +import { + computeTikTokChunkPlan, + getStoredVideoSize, + streamStoredVideoToTikTok, + TIKTOK_MAX_VIDEO_BYTES, +} from '@/lib/internal/tiktok/upload' +import { + getFileExtension, + getMimeTypeFromExtension, + processSingleFileToUserFile, + resolveTrustedFileContext, +} from '@/lib/uploads/utils/file-utils' +import { assertToolFileAccess } from '@/app/api/files/authorization' +import { tiktokPublishInitApiDataSchema } from '@/tools/tiktok/api-schemas' +import { readTikTokApiResponse } from '@/tools/tiktok/utils' + +const logger = createLogger('TikTokUploadVideoDraft') +const TIKTOK_VIDEO_MIME_TYPES = new Set(['video/mp4', 'video/quicktime', 'video/webm']) + +export interface TikTokOperationContext { + userId: string + requestId: string + signal?: AbortSignal +} + +function failureResponse(error: string, status: number): Response { + return Response.json({ success: false, error }, { status }) +} + +function resolveVideoMimeType(fileName: string, fileType: string | undefined): string | null { + if (fileType && TIKTOK_VIDEO_MIME_TYPES.has(fileType)) return fileType + const fromExtension = getMimeTypeFromExtension(getFileExtension(fileName)) + return TIKTOK_VIDEO_MIME_TYPES.has(fromExtension) ? fromExtension : null +} + +export async function executeTikTokUploadVideoDraft( + input: TikTokUploadVideoDraftInput, + context: TikTokOperationContext +): Promise { + const signal = context.signal ?? new AbortController().signal + try { + signal.throwIfAborted() + let userFile + try { + userFile = processSingleFileToUserFile(input.file, context.requestId, logger) + } catch (error) { + return failureResponse(getErrorMessage(error, 'Failed to process file'), 400) + } + + const denied = await assertToolFileAccess( + userFile.key, + context.userId, + context.requestId, + logger + ) + signal.throwIfAborted() + if (denied) return denied + + const mimeType = resolveVideoMimeType(userFile.name, userFile.type) + if (!mimeType) { + return failureResponse( + 'Unsupported video type. TikTok accepts MP4, MOV/QuickTime, or WebM files.', + 400 + ) + } + + const storageContext = resolveTrustedFileContext(userFile.key, userFile.context) + const videoSize = await getStoredVideoSize({ + key: userFile.key, + context: storageContext, + signal, + }) + if (videoSize === 0) return failureResponse('The video file is empty.', 400) + + const { chunkSize, totalChunkCount } = computeTikTokChunkPlan(videoSize) + const initResponse = await fetch( + 'https://open.tiktokapis.com/v2/post/publish/inbox/video/init/', + { + method: 'POST', + headers: { + Authorization: `Bearer ${input.accessToken}`, + 'Content-Type': 'application/json; charset=UTF-8', + }, + body: JSON.stringify({ + source_info: { + source: 'FILE_UPLOAD', + video_size: videoSize, + chunk_size: chunkSize, + total_chunk_count: totalChunkCount, + }, + }), + signal, + } + ) + const { data: initData, error: initError } = await readTikTokApiResponse( + initResponse, + tiktokPublishInitApiDataSchema, + { signal } + ) + if (initError) { + return failureResponse( + initError.message || initError.code || 'Failed to initialize TikTok upload', + initResponse.status >= 400 ? initResponse.status : 502 + ) + } + + const publishId = initData?.publish_id + const uploadUrl = initData?.upload_url + if (!publishId || !uploadUrl) { + return failureResponse('TikTok did not return a publish ID and upload URL', 502) + } + + try { + await streamStoredVideoToTikTok({ + key: userFile.key, + context: storageContext, + uploadUrl, + totalBytes: videoSize, + mimeType, + requestId: context.requestId, + signal, + }) + } catch (error) { + if (signal.aborted) throw error + return failureResponse(getErrorMessage(error, 'Failed to upload video to TikTok'), 502) + } + + return Response.json({ success: true, output: { publishId } }) + } catch (error) { + if (isPayloadSizeLimitError(error)) { + const maxMb = Math.floor(TIKTOK_MAX_VIDEO_BYTES / (1024 * 1024)) + return failureResponse(`Video exceeds the ${maxMb}MB limit for file uploads.`, 413) + } + if (signal.aborted) return failureResponse('TikTok video upload was cancelled.', 499) + logger.error(`[${context.requestId}] TikTok video draft upload failed`, { + error: getErrorMessage(error), + }) + return failureResponse(getErrorMessage(error, 'Internal server error'), 500) + } +} diff --git a/apps/sim/lib/internal/tiktok/schema.ts b/apps/sim/lib/internal/tiktok/schema.ts new file mode 100644 index 00000000000..f20502210f5 --- /dev/null +++ b/apps/sim/lib/internal/tiktok/schema.ts @@ -0,0 +1,9 @@ +import { z } from 'zod' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' + +export const tiktokUploadVideoDraftInputSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + file: RawFileInputSchema, +}) + +export type TikTokUploadVideoDraftInput = z.output diff --git a/apps/sim/lib/internal/tiktok/upload.test.ts b/apps/sim/lib/internal/tiktok/upload.test.ts new file mode 100644 index 00000000000..d45050ce210 --- /dev/null +++ b/apps/sim/lib/internal/tiktok/upload.test.ts @@ -0,0 +1,195 @@ +/** + * @vitest-environment node + */ +import { Readable } from 'node:stream' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' + +const { mockBackoffWithJitter, mockDownloadFileStream, mockHeadObject, mockParseRetryAfter } = + vi.hoisted(() => ({ + mockBackoffWithJitter: vi.fn(() => 0), + mockDownloadFileStream: vi.fn(), + mockHeadObject: vi.fn(), + mockParseRetryAfter: vi.fn(() => 25), + })) + +vi.mock('@sim/utils/retry', () => ({ + backoffWithJitter: mockBackoffWithJitter, + parseRetryAfter: mockParseRetryAfter, +})) + +vi.mock('@/lib/uploads/core/storage-service', () => ({ + downloadFileStream: mockDownloadFileStream, + headObject: mockHeadObject, +})) + +import { + computeTikTokChunkPlan, + getStoredVideoSize, + streamStoredVideoToTikTok, + TIKTOK_MAX_VIDEO_BYTES, +} from '@/lib/internal/tiktok/upload' + +const baseStreamOptions = { + key: 'workspace/workspace-1/video.mp4', + context: 'workspace' as const, + uploadUrl: 'https://upload.example/video', + mimeType: 'video/mp4', + requestId: 'request-1', +} + +describe('TikTok video upload streaming', () => { + beforeEach(() => { + vi.clearAllMocks() + mockBackoffWithJitter.mockReturnValue(0) + mockParseRetryAfter.mockReturnValue(25) + }) + + it('uses provider metadata as the authoritative bounded size', async () => { + mockHeadObject.mockResolvedValue({ size: 1234, contentType: 'video/mp4' }) + + await expect( + getStoredVideoSize({ + key: baseStreamOptions.key, + context: baseStreamOptions.context, + signal: new AbortController().signal, + }) + ).resolves.toBe(1234) + expect(mockDownloadFileStream).not.toHaveBeenCalled() + }) + + it('counts a stream without accumulating it when provider metadata is unavailable', async () => { + mockHeadObject.mockResolvedValue(null) + mockDownloadFileStream.mockResolvedValue( + Readable.from([Buffer.alloc(3), Buffer.alloc(5), Buffer.alloc(7)]) + ) + + await expect( + getStoredVideoSize({ + key: baseStreamOptions.key, + context: baseStreamOptions.context, + signal: new AbortController().signal, + }) + ).resolves.toBe(15) + }) + + it('rejects an oversized provider object before opening its body', async () => { + mockHeadObject.mockResolvedValue({ size: TIKTOK_MAX_VIDEO_BYTES + 1 }) + + await expect( + getStoredVideoSize({ + key: baseStreamOptions.key, + context: baseStreamOptions.context, + signal: new AbortController().signal, + }) + ).rejects.toBeInstanceOf(PayloadSizeLimitError) + expect(mockDownloadFileStream).not.toHaveBeenCalled() + }) + + it('computes TikTok chunk counts with the final chunk absorbing the remainder', () => { + expect(computeTikTokChunkPlan(4_000_000)).toEqual({ + chunkSize: 4_000_000, + totalChunkCount: 1, + }) + expect(computeTikTokChunkPlan(20_000_001)).toEqual({ + chunkSize: 10_000_000, + totalChunkCount: 2, + }) + }) + + it('streams sequential chunks with exact 206 intermediate and 201 final ranges', async () => { + const totalBytes = 20_000_001 + mockDownloadFileStream.mockResolvedValue( + Readable.from([Buffer.alloc(7_000_000, 1), Buffer.alloc(13_000_001, 2)]) + ) + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response(null, { status: 206 })) + .mockResolvedValueOnce(new Response(null, { status: 201 })) + vi.stubGlobal('fetch', fetchMock) + const controller = new AbortController() + + await streamStoredVideoToTikTok({ + ...baseStreamOptions, + totalBytes, + signal: controller.signal, + }) + + expect(fetchMock).toHaveBeenCalledTimes(2) + expect(fetchMock.mock.calls[0][1]).toMatchObject({ + method: 'PUT', + signal: controller.signal, + headers: { + 'Content-Length': '10000000', + 'Content-Range': 'bytes 0-9999999/20000001', + 'Content-Type': 'video/mp4', + }, + }) + expect(fetchMock.mock.calls[1][1]).toMatchObject({ + method: 'PUT', + signal: controller.signal, + headers: { + 'Content-Length': '10000001', + 'Content-Range': 'bytes 10000000-20000000/20000001', + 'Content-Type': 'video/mp4', + }, + }) + }) + + it('retries only 5xx responses with the same bounded chunk', async () => { + mockDownloadFileStream.mockResolvedValue(Readable.from([Buffer.from('video')])) + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + new Response('temporary one', { status: 500, headers: { 'Retry-After': '1' } }) + ) + .mockResolvedValueOnce(new Response('temporary two', { status: 503 })) + .mockResolvedValueOnce(new Response(null, { status: 201 })) + vi.stubGlobal('fetch', fetchMock) + + await streamStoredVideoToTikTok({ + ...baseStreamOptions, + totalBytes: 5, + signal: new AbortController().signal, + }) + + expect(fetchMock).toHaveBeenCalledTimes(3) + expect(mockParseRetryAfter).toHaveBeenCalledTimes(2) + expect(mockBackoffWithJitter).toHaveBeenNthCalledWith(1, 1, 25) + expect(mockBackoffWithJitter).toHaveBeenNthCalledWith(2, 2, 25) + const uploadedBodies = fetchMock.mock.calls.map((call) => + Buffer.from(call[1]?.body as Uint8Array).toString('utf8') + ) + expect(uploadedBodies).toEqual(['video', 'video', 'video']) + }) + + it('rejects a successful but protocol-invalid final status without retrying', async () => { + mockDownloadFileStream.mockResolvedValue(Readable.from([Buffer.from('video')])) + const fetchMock = vi.fn().mockResolvedValue(new Response(null, { status: 200 })) + vi.stubGlobal('fetch', fetchMock) + + await expect( + streamStoredVideoToTikTok({ + ...baseStreamOptions, + totalBytes: 5, + signal: new AbortController().signal, + }) + ).rejects.toThrow('expected HTTP 201, received HTTP 200') + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + it('detects storage-size drift before sending the final chunk', async () => { + mockDownloadFileStream.mockResolvedValue(Readable.from([Buffer.from('video-extra')])) + const fetchMock = vi.fn() + vi.stubGlobal('fetch', fetchMock) + + await expect( + streamStoredVideoToTikTok({ + ...baseStreamOptions, + totalBytes: 5, + signal: new AbortController().signal, + }) + ).rejects.toThrow('Stored video grew after its size was resolved') + expect(fetchMock).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/api/tools/tiktok/upload-video-draft/upload.ts b/apps/sim/lib/internal/tiktok/upload.ts similarity index 100% rename from apps/sim/app/api/tools/tiktok/upload-video-draft/upload.ts rename to apps/sim/lib/internal/tiktok/upload.ts diff --git a/apps/sim/lib/internal/tool-operations/execute-json-operation.ts b/apps/sim/lib/internal/tool-operations/execute-json-operation.ts new file mode 100644 index 00000000000..10cc01c903b --- /dev/null +++ b/apps/sim/lib/internal/tool-operations/execute-json-operation.ts @@ -0,0 +1,24 @@ +import { toError } from '@sim/utils/errors' +import type { AnyApiRouteContract, ContractBody } from '@/lib/api/contracts' +import { parseInternalToolInput } from '@/lib/internal/tool-operations/parse-input' + +export async function executeInternalJsonToolOperation( + contract: C, + input: unknown, + execute: (input: ContractBody, signal?: AbortSignal) => Promise, + errorMessage: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const parsed = parseInternalToolInput(contract, input) + if (!parsed.success) return parsed.response + + try { + const result = await execute(parsed.data, signal) + signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + signal?.throwIfAborted() + return Response.json({ error: `${errorMessage}: ${toError(error).message}` }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/tool-operations/parse-contract-input.ts b/apps/sim/lib/internal/tool-operations/parse-contract-input.ts new file mode 100644 index 00000000000..b7fffc5201f --- /dev/null +++ b/apps/sim/lib/internal/tool-operations/parse-contract-input.ts @@ -0,0 +1,73 @@ +import type { z } from 'zod' +import type { + AnyApiRouteContract, + ContractBody, + ContractParams, + ContractQuery, +} from '@/lib/api/contracts' +import { serializeZodIssues } from '@/lib/api/server/validation' + +export interface ParsedInternalContractInput { + params: P + query: Q + body: B +} + +function validationError(error: z.ZodError): Response { + return Response.json( + { error: 'Validation error', details: serializeZodIssues(error) }, + { status: 400 } + ) +} + +export function parseInternalContractInput( + contract: C, + input: unknown, + options: { maxInputBytes?: number } = {} +): + | { + success: true + data: ParsedInternalContractInput, ContractQuery, ContractBody> + } + | { success: false; response: Response } { + if (options.maxInputBytes !== undefined) { + let serialized: string + try { + serialized = JSON.stringify(input) + } catch { + return { + success: false, + response: Response.json({ error: 'Operation input must be valid JSON' }, { status: 400 }), + } + } + if (Buffer.byteLength(serialized, 'utf8') > options.maxInputBytes) { + return { + success: false, + response: Response.json( + { + error: `Operation input exceeds the maximum allowed size of ${options.maxInputBytes} bytes`, + }, + { status: 413 } + ), + } + } + } + + const params = contract.params?.safeParse(input) + if (params && !params.success) return { success: false, response: validationError(params.error) } + + const query = contract.query?.safeParse(input) + if (query && !query.success) return { success: false, response: validationError(query.error) } + + const body = contract.body?.safeParse(input) + if (body && !body.success) return { success: false, response: validationError(body.error) } + + return { + success: true, + data: { + params: (params?.data ?? undefined) as ContractParams, + query: (query?.data ?? undefined) as ContractQuery, + body: (body?.data ?? undefined) as ContractBody, + }, + } +} diff --git a/apps/sim/lib/internal/tool-operations/parse-input.ts b/apps/sim/lib/internal/tool-operations/parse-input.ts new file mode 100644 index 00000000000..a7849703fe8 --- /dev/null +++ b/apps/sim/lib/internal/tool-operations/parse-input.ts @@ -0,0 +1,47 @@ +import type { AnyApiRouteContract, ContractBody } from '@/lib/api/contracts' + +interface ParseInternalToolInputOptions { + maxInputBytes?: number +} + +export type InternalToolInputParseResult = + | { success: true; data: ContractBody } + | { success: false; response: Response } + +export function parseInternalToolInput( + contract: C, + input: unknown, + options: ParseInternalToolInputOptions = {} +): InternalToolInputParseResult { + if ( + options.maxInputBytes !== undefined && + Buffer.byteLength(JSON.stringify(input) ?? '', 'utf8') > options.maxInputBytes + ) { + return { + success: false, + response: Response.json( + { + error: `Request body exceeds the maximum allowed size of ${options.maxInputBytes} bytes`, + }, + { status: 413 } + ), + } + } + + if (!contract.body) { + return { success: true, data: undefined as ContractBody } + } + + const parsed = contract.body.safeParse(input) + if (!parsed.success) { + return { + success: false, + response: Response.json( + { error: 'Invalid request data', details: parsed.error.issues }, + { status: 400 } + ), + } + } + + return { success: true, data: parsed.data as ContractBody } +} diff --git a/apps/sim/lib/internal/tool-operations/registry.server.test.ts b/apps/sim/lib/internal/tool-operations/registry.server.test.ts new file mode 100644 index 00000000000..ca91a6341c7 --- /dev/null +++ b/apps/sim/lib/internal/tool-operations/registry.server.test.ts @@ -0,0 +1,50 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it, vi } from 'vitest' +import { + getInternalToolOperationHandler, + getRegisteredInternalToolOperationIds, + isInternalToolOperationRegistered, +} from '@/lib/internal/tool-operations/registry.server' +import { tools } from '@/tools/registry' +import { getToolIds } from '@/tools/tool-ids' +import { isInternalToolConfig } from '@/tools/types' + +vi.unmock('@/tools/registry') + +describe('internal tool operation registry', () => { + it('registers only canonical internal tool definitions with loadable handlers', async () => { + const registeredIds = getRegisteredInternalToolOperationIds() + const canonicalIds = new Set(getToolIds()) + + expect(new Set(registeredIds).size).toBe(registeredIds.length) + + for (const toolId of registeredIds) { + expect(canonicalIds.has(toolId), `Missing canonical tool definition for ${toolId}`).toBe(true) + expect(await getInternalToolOperationHandler(toolId)).toBeTypeOf('function') + } + }, 30_000) + + it('registers every operation-backed tool and keeps it free of HTTP request metadata', async () => { + const operationTools = Object.entries(tools).filter(([, tool]) => isInternalToolConfig(tool)) + + expect(operationTools.length).toBeGreaterThan(0) + for (const [toolId, tool] of operationTools) { + expect(tool.request, `${toolId} must not declare an HTTP request`).toBeUndefined() + expect(tool.operation.input, `${toolId} must materialize its operation input`).toBeTypeOf( + 'function' + ) + if (toolId === 'function_execute' || toolId === 'workflow_executor') continue + expect( + isInternalToolOperationRegistered(toolId), + `${toolId} is missing its in-process operation handler` + ).toBe(true) + } + }) + + it('loads dynamic MCP operations without an HTTP route', async () => { + expect(isInternalToolOperationRegistered('mcp-server-id-tool-name')).toBe(true) + expect(await getInternalToolOperationHandler('mcp-server-id-tool-name')).toBeTypeOf('function') + }) +}) diff --git a/apps/sim/lib/internal/tool-operations/registry.server.ts b/apps/sim/lib/internal/tool-operations/registry.server.ts new file mode 100644 index 00000000000..55f825b0310 --- /dev/null +++ b/apps/sim/lib/internal/tool-operations/registry.server.ts @@ -0,0 +1,1426 @@ +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { isMcpTool } from '@/executor/constants' + +type InternalToolOperationHandlerLoader = () => Promise + +const STS_TOOL_IDS = [ + 'sts_assume_role', + 'sts_assume_role_with_web_identity', + 'sts_assume_role_with_saml', + 'sts_get_caller_identity', + 'sts_get_session_token', + 'sts_get_access_key_info', +] as const + +const APPCONFIG_TOOL_IDS = [ + 'appconfig_create_application', + 'appconfig_create_configuration_profile', + 'appconfig_create_environment', + 'appconfig_create_hosted_configuration_version', + 'appconfig_delete_application', + 'appconfig_delete_configuration_profile', + 'appconfig_delete_environment', + 'appconfig_delete_hosted_configuration_version', + 'appconfig_get_application', + 'appconfig_get_configuration', + 'appconfig_get_configuration_profile', + 'appconfig_get_deployment', + 'appconfig_get_environment', + 'appconfig_get_hosted_configuration_version', + 'appconfig_list_applications', + 'appconfig_list_configuration_profiles', + 'appconfig_list_deployment_strategies', + 'appconfig_list_deployments', + 'appconfig_list_environments', + 'appconfig_list_hosted_configuration_versions', + 'appconfig_start_deployment', + 'appconfig_stop_deployment', + 'appconfig_update_application', + 'appconfig_update_configuration_profile', + 'appconfig_update_environment', +] as const + +const IAM_TOOL_IDS = [ + 'iam_add_user_to_group', + 'iam_attach_role_policy', + 'iam_attach_user_policy', + 'iam_create_access_key', + 'iam_create_role', + 'iam_create_user', + 'iam_delete_access_key', + 'iam_delete_role', + 'iam_delete_user', + 'iam_detach_role_policy', + 'iam_detach_user_policy', + 'iam_get_role', + 'iam_get_user', + 'iam_list_attached_role_policies', + 'iam_list_attached_user_policies', + 'iam_list_groups', + 'iam_list_policies', + 'iam_list_roles', + 'iam_list_users', + 'iam_remove_user_from_group', + 'iam_simulate_principal_policy', +] as const + +const IDENTITY_CENTER_TOOL_IDS = [ + 'identity_center_list_instances', + 'identity_center_list_accounts', + 'identity_center_describe_account', + 'identity_center_list_permission_sets', + 'identity_center_get_user', + 'identity_center_get_group', + 'identity_center_list_groups', + 'identity_center_create_account_assignment', + 'identity_center_delete_account_assignment', + 'identity_center_check_assignment_status', + 'identity_center_check_assignment_deletion_status', + 'identity_center_list_account_assignments', +] as const + +const SECRETS_MANAGER_TOOL_IDS = [ + 'secrets_manager_get_secret', + 'secrets_manager_list_secrets', + 'secrets_manager_create_secret', + 'secrets_manager_update_secret', + 'secrets_manager_delete_secret', + 'secrets_manager_describe_secret', + 'secrets_manager_tag_resource', + 'secrets_manager_untag_resource', + 'secrets_manager_restore_secret', + 'secrets_manager_rotate_secret', +] as const + +const DYNAMODB_TOOL_IDS = [ + 'dynamodb_delete', + 'dynamodb_get', + 'dynamodb_introspect', + 'dynamodb_put', + 'dynamodb_query', + 'dynamodb_scan', + 'dynamodb_update', +] as const + +const SES_TOOL_IDS = [ + 'ses_create_configuration_set', + 'ses_create_email_identity', + 'ses_create_template', + 'ses_delete_email_identity', + 'ses_delete_suppressed_destination', + 'ses_delete_template', + 'ses_get_account', + 'ses_get_email_identity', + 'ses_get_suppressed_destination', + 'ses_get_template', + 'ses_list_identities', + 'ses_list_suppressed_destinations', + 'ses_list_templates', + 'ses_put_suppressed_destination', + 'ses_send_bulk_email', + 'ses_send_custom_verification_email', + 'ses_send_email', + 'ses_send_templated_email', + 'ses_update_template', +] as const + +const SQS_TOOL_IDS = ['sqs_send'] as const + +const RDS_TOOL_IDS = [ + 'rds_query', + 'rds_execute', + 'rds_insert', + 'rds_update', + 'rds_delete', + 'rds_introspect', +] as const + +const TEXTRACT_TOOL_IDS = [ + 'textract_parser', + 'textract_parser_v2', + 'textract_analyze_expense', + 'textract_analyze_id', +] as const + +const CLOUDWATCH_TOOL_IDS = [ + 'cloudwatch_describe_alarm_history', + 'cloudwatch_describe_alarms', + 'cloudwatch_describe_log_groups', + 'cloudwatch_describe_log_streams', + 'cloudwatch_filter_log_events', + 'cloudwatch_get_log_events', + 'cloudwatch_get_metric_statistics', + 'cloudwatch_list_metrics', + 'cloudwatch_mute_alarm', + 'cloudwatch_put_log_group_retention', + 'cloudwatch_put_metric_data', + 'cloudwatch_query_logs', + 'cloudwatch_unmute_alarm', +] as const + +const POSTGRESQL_TOOL_IDS = [ + 'postgresql_query', + 'postgresql_execute', + 'postgresql_insert', + 'postgresql_update', + 'postgresql_delete', + 'postgresql_introspect', +] as const + +const REDIS_TOOL_IDS = [ + 'redis_command', + 'redis_delete', + 'redis_exists', + 'redis_expire', + 'redis_get', + 'redis_hdel', + 'redis_hget', + 'redis_hgetall', + 'redis_hset', + 'redis_incr', + 'redis_incrby', + 'redis_keys', + 'redis_llen', + 'redis_lpop', + 'redis_lpush', + 'redis_lrange', + 'redis_persist', + 'redis_rpop', + 'redis_rpush', + 'redis_set', + 'redis_setnx', + 'redis_ttl', +] as const + +const CLOUDFORMATION_TOOL_IDS = [ + 'cloudformation_cancel_update_stack', + 'cloudformation_create_change_set', + 'cloudformation_create_stack', + 'cloudformation_delete_stack', + 'cloudformation_describe_change_set', + 'cloudformation_describe_stack_drift_detection_status', + 'cloudformation_describe_stack_events', + 'cloudformation_describe_stacks', + 'cloudformation_detect_stack_drift', + 'cloudformation_execute_change_set', + 'cloudformation_get_template', + 'cloudformation_get_template_summary', + 'cloudformation_list_stack_resources', + 'cloudformation_update_stack', + 'cloudformation_validate_template', +] as const + +const CODEPIPELINE_TOOL_IDS = [ + 'codepipeline_disable_stage_transition', + 'codepipeline_enable_stage_transition', + 'codepipeline_get_pipeline', + 'codepipeline_get_pipeline_execution', + 'codepipeline_get_pipeline_state', + 'codepipeline_list_action_executions', + 'codepipeline_list_pipeline_executions', + 'codepipeline_list_pipelines', + 'codepipeline_put_approval_result', + 'codepipeline_retry_stage_execution', + 'codepipeline_start_execution', + 'codepipeline_stop_execution', +] as const + +const MYSQL_TOOL_IDS = [ + 'mysql_query', + 'mysql_execute', + 'mysql_insert', + 'mysql_update', + 'mysql_delete', + 'mysql_introspect', +] as const + +const ATHENA_TOOL_IDS = [ + 'athena_batch_get_query_execution', + 'athena_create_named_query', + 'athena_delete_named_query', + 'athena_get_named_query', + 'athena_get_query_execution', + 'athena_get_query_results', + 'athena_list_databases', + 'athena_list_named_queries', + 'athena_list_query_executions', + 'athena_list_table_metadata', + 'athena_start_query', + 'athena_stop_query', +] as const + +const CLICKHOUSE_TOOL_IDS = [ + 'clickhouse_count_rows', + 'clickhouse_create_database', + 'clickhouse_create_table', + 'clickhouse_delete', + 'clickhouse_describe_table', + 'clickhouse_drop_database', + 'clickhouse_drop_partition', + 'clickhouse_drop_table', + 'clickhouse_execute', + 'clickhouse_insert_rows', + 'clickhouse_insert', + 'clickhouse_introspect', + 'clickhouse_kill_query', + 'clickhouse_list_clusters', + 'clickhouse_list_databases', + 'clickhouse_list_mutations', + 'clickhouse_list_partitions', + 'clickhouse_list_running_queries', + 'clickhouse_list_tables', + 'clickhouse_optimize_table', + 'clickhouse_query', + 'clickhouse_rename_table', + 'clickhouse_show_create_table', + 'clickhouse_table_stats', + 'clickhouse_truncate_table', + 'clickhouse_update', +] as const + +const MONGODB_TOOL_IDS = [ + 'mongodb_query', + 'mongodb_execute', + 'mongodb_insert', + 'mongodb_update', + 'mongodb_delete', + 'mongodb_introspect', +] as const + +const NEO4J_TOOL_IDS = [ + 'neo4j_query', + 'neo4j_execute', + 'neo4j_create', + 'neo4j_update', + 'neo4j_delete', + 'neo4j_merge', + 'neo4j_introspect', +] as const + +const S3_TOOL_IDS = [ + 's3_copy_object', + 's3_create_bucket', + 's3_delete_bucket', + 's3_delete_object', + 's3_delete_objects', + 's3_head_object', + 's3_list_buckets', + 's3_list_objects', + 's3_presigned_url', + 's3_put_object', +] as const + +const JUPYTER_TOOL_IDS = [ + 'jupyter_copy_content', + 'jupyter_create_file', + 'jupyter_create_session', + 'jupyter_delete_content', + 'jupyter_delete_session', + 'jupyter_get_content', + 'jupyter_interrupt_kernel', + 'jupyter_list_contents', + 'jupyter_list_kernels', + 'jupyter_list_kernelspecs', + 'jupyter_list_sessions', + 'jupyter_rename_content', + 'jupyter_restart_kernel', + 'jupyter_start_kernel', + 'jupyter_stop_kernel', + 'jupyter_upload_file', +] as const + +const MSSQL_TOOL_IDS = [ + 'mssql_query', + 'mssql_execute', + 'mssql_insert', + 'mssql_update', + 'mssql_delete', + 'mssql_introspect', +] as const + +const KNOWLEDGE_TOOL_IDS = [ + 'knowledge_create_document', + 'knowledge_delete_chunk', + 'knowledge_delete_document', + 'knowledge_get_connector', + 'knowledge_get_document', + 'knowledge_list_chunks', + 'knowledge_list_connectors', + 'knowledge_list_documents', + 'knowledge_list_tags', + 'knowledge_search', + 'knowledge_trigger_sync', + 'knowledge_update_chunk', + 'knowledge_upload_chunk', + 'knowledge_upsert_document', +] as const + +const CONFLUENCE_TOOL_IDS = [ + 'confluence_add_label', + 'confluence_create_blogpost', + 'confluence_create_comment', + 'confluence_create_page', + 'confluence_create_page_property', + 'confluence_create_space', + 'confluence_create_space_property', + 'confluence_delete_attachment', + 'confluence_delete_blogpost', + 'confluence_delete_comment', + 'confluence_delete_label', + 'confluence_delete_page', + 'confluence_delete_page_property', + 'confluence_delete_space', + 'confluence_delete_space_property', + 'confluence_get_blogpost', + 'confluence_get_page_ancestors', + 'confluence_get_page_children', + 'confluence_get_page_descendants', + 'confluence_get_page_version', + 'confluence_get_pages_by_label', + 'confluence_get_space', + 'confluence_get_task', + 'confluence_get_user', + 'confluence_list_attachments', + 'confluence_list_blogposts', + 'confluence_list_blogposts_in_space', + 'confluence_list_comments', + 'confluence_list_labels', + 'confluence_list_page_properties', + 'confluence_list_page_versions', + 'confluence_list_pages_in_space', + 'confluence_list_space_labels', + 'confluence_list_space_permissions', + 'confluence_list_space_properties', + 'confluence_list_spaces', + 'confluence_list_tasks', + 'confluence_retrieve', + 'confluence_search', + 'confluence_search_in_space', + 'confluence_update', + 'confluence_update_blogpost', + 'confluence_update_comment', + 'confluence_update_space', + 'confluence_update_task', + 'confluence_upload_attachment', +] as const + +const JSM_TOOL_IDS = [ + 'jsm_add_comment', + 'jsm_add_customer', + 'jsm_add_organization', + 'jsm_add_participants', + 'jsm_answer_approval', + 'jsm_attach_form', + 'jsm_copy_forms', + 'jsm_create_object', + 'jsm_create_organization', + 'jsm_create_request', + 'jsm_delete_form', + 'jsm_delete_object', + 'jsm_externalise_form', + 'jsm_get_approvals', + 'jsm_get_comments', + 'jsm_get_customers', + 'jsm_get_form', + 'jsm_get_form_answers', + 'jsm_get_form_structure', + 'jsm_get_form_templates', + 'jsm_get_issue_forms', + 'jsm_get_object', + 'jsm_get_object_schema', + 'jsm_get_object_type_attributes', + 'jsm_get_organizations', + 'jsm_get_participants', + 'jsm_get_queues', + 'jsm_get_request', + 'jsm_get_requests', + 'jsm_get_request_type_fields', + 'jsm_get_request_types', + 'jsm_get_service_desks', + 'jsm_get_sla', + 'jsm_get_transitions', + 'jsm_internalise_form', + 'jsm_list_object_schemas', + 'jsm_list_object_types', + 'jsm_reopen_form', + 'jsm_save_form_answers', + 'jsm_search_objects_aql', + 'jsm_submit_form', + 'jsm_transition_request', + 'jsm_update_object', +] as const + +const CROWDSTRIKE_TOOL_IDS = [ + 'crowdstrike_create_indicators', + 'crowdstrike_delete_indicators', + 'crowdstrike_delete_rtr_session', + 'crowdstrike_execute_rtr_command', + 'crowdstrike_get_alert_details', + 'crowdstrike_get_case_details', + 'crowdstrike_get_host_group_details', + 'crowdstrike_get_indicator_details', + 'crowdstrike_get_rtr_command_status', + 'crowdstrike_get_sensor_aggregates', + 'crowdstrike_get_sensor_details', + 'crowdstrike_get_vulnerability_details', + 'crowdstrike_init_rtr_session', + 'crowdstrike_perform_host_action', + 'crowdstrike_perform_host_group_action', + 'crowdstrike_query_alerts', + 'crowdstrike_query_cases', + 'crowdstrike_query_host_groups', + 'crowdstrike_query_indicators', + 'crowdstrike_query_sensors', + 'crowdstrike_query_vulnerabilities', + 'crowdstrike_update_alerts', + 'crowdstrike_update_indicators', +] as const + +const GMAIL_TOOL_IDS = [ + 'gmail_add_label', + 'gmail_add_label_v2', + 'gmail_archive', + 'gmail_archive_v2', + 'gmail_delete', + 'gmail_delete_v2', + 'gmail_draft', + 'gmail_draft_v2', + 'gmail_edit_draft_v2', + 'gmail_mark_read', + 'gmail_mark_read_v2', + 'gmail_mark_unread', + 'gmail_mark_unread_v2', + 'gmail_move', + 'gmail_move_v2', + 'gmail_remove_label', + 'gmail_remove_label_v2', + 'gmail_send', + 'gmail_send_v2', + 'gmail_unarchive', + 'gmail_unarchive_v2', +] as const + +const WINDCHILL_TOOL_IDS = [ + 'windchill_create_document', + 'windchill_create_documents', + 'windchill_update_document', + 'windchill_update_common_properties', + 'windchill_update_documents', + 'windchill_delete_document', + 'windchill_delete_documents', + 'windchill_check_out_document', + 'windchill_check_out_documents', + 'windchill_check_in_document', + 'windchill_check_in_documents', + 'windchill_undo_check_out_document', + 'windchill_undo_check_out_documents', + 'windchill_revise_document', + 'windchill_revise_documents', + 'windchill_set_lifecycle_state', + 'windchill_update_document_security_labels', + 'windchill_download_primary_content', + 'windchill_upload_primary_content', + 'windchill_download_attachment', + 'windchill_upload_attachments', +] as const + +const TABLE_TOOL_IDS = [ + 'table_create', + 'table_list', + 'table_get_schema', + 'table_get_row', + 'table_insert_row', + 'table_batch_insert_rows', + 'table_query_rows', + 'table_query_rows_v2', + 'table_update_row', + 'table_update_rows_by_filter', + 'table_delete_row', + 'table_delete_rows_by_filter', + 'table_upsert_row', +] as const + +const WORKDAY_TOOL_IDS = [ + 'workday_assign_onboarding', + 'workday_change_job', + 'workday_create_prehire', + 'workday_get_compensation', + 'workday_get_organizations', + 'workday_get_worker', + 'workday_hire_employee', + 'workday_list_workers', + 'workday_terminate_worker', + 'workday_update_worker', +] as const + +const AGILOFT_TOOL_IDS = [ + 'agiloft_async_status', + 'agiloft_attach_file', + 'agiloft_attachment_info', + 'agiloft_create_record', + 'agiloft_delete_record', + 'agiloft_get_choice_line_id', + 'agiloft_list_tables', + 'agiloft_lock_record', + 'agiloft_nlp_search', + 'agiloft_read_record', + 'agiloft_remove_attachment', + 'agiloft_retrieve_attachment', + 'agiloft_run_action_button', + 'agiloft_saved_search', + 'agiloft_search_records', + 'agiloft_select_records', + 'agiloft_update_record', + 'agiloft_upsert_record', +] as const + +const ONEPASSWORD_TOOL_IDS = [ + 'onepassword_create_item', + 'onepassword_delete_item', + 'onepassword_get_item', + 'onepassword_get_item_file', + 'onepassword_get_vault', + 'onepassword_list_items', + 'onepassword_list_vaults', + 'onepassword_replace_item', + 'onepassword_resolve_secret', + 'onepassword_update_item', +] as const + +const OUTLOOK_TOOL_IDS = [ + 'outlook_copy', + 'outlook_delete', + 'outlook_draft', + 'outlook_mark_read', + 'outlook_mark_unread', + 'outlook_move', + 'outlook_send', +] as const + +const SSH_TOOL_IDS = [ + 'ssh_check_command_exists', + 'ssh_check_file_exists', + 'ssh_create_directory', + 'ssh_delete_file', + 'ssh_download_file', + 'ssh_execute_command', + 'ssh_execute_script', + 'ssh_get_system_info', + 'ssh_list_directory', + 'ssh_move_rename', + 'ssh_read_file_content', + 'ssh_upload_file', + 'ssh_write_file_content', +] as const + +const ASANA_TOOL_IDS = [ + 'asana_add_comment', + 'asana_add_followers', + 'asana_create_project', + 'asana_create_section', + 'asana_create_subtask', + 'asana_create_task', + 'asana_delete_task', + 'asana_get_project', + 'asana_get_projects', + 'asana_get_task', + 'asana_list_sections', + 'asana_list_workspaces', + 'asana_search_tasks', + 'asana_update_task', +] as const + +const DOCUSIGN_TOOL_IDS = [ + 'docusign_create_from_template', + 'docusign_download_document', + 'docusign_get_envelope', + 'docusign_list_envelopes', + 'docusign_list_recipients', + 'docusign_list_templates', + 'docusign_send_envelope', + 'docusign_void_envelope', +] as const + +const THINKING_TOOL_IDS = ['thinking_tool'] as const + +const SLACK_TOOL_IDS = [ + 'slack_add_reaction', + 'slack_delete_message', + 'slack_download', + 'slack_ephemeral_message', + 'slack_message', + 'slack_message_reader', + 'slack_remove_reaction', + 'slack_update_message', +] as const + +const SEARCH_TOOL_IDS = ['search_tool'] as const + +const SMS_TOOL_IDS = ['sms_send'] as const + +const MICROSOFT_WORD_TOOL_IDS = [ + 'microsoft_word_append', + 'microsoft_word_create', + 'microsoft_word_create_from_template', + 'microsoft_word_export_pdf', + 'microsoft_word_read', + 'microsoft_word_replace_text', + 'microsoft_word_update', +] as const + +const TTS_TOOL_IDS = [ + 'elevenlabs_tts', + 'tts_azure', + 'tts_cartesia', + 'tts_deepgram', + 'tts_elevenlabs', + 'tts_google', + 'tts_openai', + 'tts_playht', +] as const + +const FILE_TOOL_IDS = [ + 'file_append', + 'file_write', + 'file_get', + 'file_read', + 'file_get_content', + 'file_compress', + 'file_decompress', + 'file_manage_sharing', + 'file_fetch', + 'file_parser', + 'file_parser_v2', + 'file_parser_v3', +] as const + +const UPTIMEROBOT_TOOL_IDS = ['uptimerobot_create_psp', 'uptimerobot_update_psp'] as const + +const STT_TOOL_IDS = [ + 'stt_assemblyai', + 'stt_assemblyai_v2', + 'stt_deepgram', + 'stt_deepgram_v2', + 'stt_elevenlabs', + 'stt_elevenlabs_v2', + 'stt_gemini', + 'stt_gemini_v2', + 'stt_whisper', + 'stt_whisper_v2', +] as const + +const INSTAGRAM_TOOL_IDS = [ + 'instagram_download_media', + 'instagram_publish_carousel', + 'instagram_publish_image', + 'instagram_publish_reel', + 'instagram_publish_story', + 'instagram_publish_video', +] as const + +const VIDEO_TOOL_IDS = [ + 'video_falai', + 'video_luma', + 'video_minimax', + 'video_runway', + 'video_veo', +] as const + +const A2A_TOOL_IDS = [ + 'a2a_cancel_task', + 'a2a_get_agent_card', + 'a2a_get_task', + 'a2a_send_message', +] as const + +const BUFFER_TOOL_IDS = ['buffer_create_post', 'buffer_edit_post'] as const + +const GRAFANA_TOOL_IDS = [ + 'grafana_check_data_source_health', + 'grafana_update_alert_rule', + 'grafana_update_dashboard', + 'grafana_update_folder', +] as const + +const DEPLOYMENTS_TOOL_IDS = [ + 'deployments_deploy', + 'deployments_undeploy', + 'deployments_promote', + 'deployments_list_versions', + 'deployments_get_version', +] as const + +const CURSOR_TOOL_IDS = ['cursor_download_artifact', 'cursor_download_artifact_v2'] as const + +const SFTP_TOOL_IDS = [ + 'sftp_delete', + 'sftp_download', + 'sftp_list', + 'sftp_mkdir', + 'sftp_upload', +] as const + +const ZOOM_TOOL_IDS = ['zoom_get_meeting_recordings'] as const + +const ZOHO_DESK_TOOL_IDS = ['zoho_desk_get_attachment'] as const + +const WORDPRESS_TOOL_IDS = ['wordpress_upload_media'] as const + +const ELEVENLABS_TOOL_IDS = [ + 'elevenlabs_sound_effects', + 'elevenlabs_speech_to_speech', + 'elevenlabs_audio_isolation', +] as const + +const JIRA_TOOL_IDS = ['jira_write', 'jira_update', 'jira_add_attachment'] as const + +const WHATSAPP_TOOL_IDS = [ + 'whatsapp_get_media', + 'whatsapp_send_media', + 'whatsapp_upload_media', +] as const + +const TYPEFORM_TOOL_IDS = ['typeform_files'] as const + +const DAYTONA_TOOL_IDS = ['daytona_upload_file'] as const + +const GOOGLE_SLIDES_TOOL_IDS = ['google_slides_export_presentation'] as const + +const GOOGLE_VAULT_TOOL_IDS = ['google_vault_download_export_file'] as const + +const GOOGLE_DRIVE_TOOL_IDS = [ + 'google_drive_download', + 'google_drive_export', + 'google_drive_upload', +] as const + +const STAGEHAND_TOOL_IDS = ['stagehand_agent', 'stagehand_extract'] as const + +const VISION_TOOL_IDS = ['vision_tool', 'vision_tool_v2'] as const + +const GITHUB_TOOL_IDS = ['github_latest_commit', 'github_latest_commit_v2'] as const + +const TWILIO_VOICE_TOOL_IDS = ['twilio_voice_get_recording'] as const + +const PERSONA_TOOL_IDS = ['persona_import_accounts'] as const + +const SHAREPOINT_TOOL_IDS = ['sharepoint_download_file', 'sharepoint_upload_file'] as const + +const QUIVER_TOOL_IDS = ['quiver_text_to_svg', 'quiver_image_to_svg'] as const + +const TELEGRAM_TOOL_IDS = ['telegram_send_document'] as const + +const MICROSOFT_TEAMS_TOOL_IDS = [ + 'microsoft_teams_delete_chat_message', + 'microsoft_teams_write_chat', + 'microsoft_teams_write_channel', +] as const + +const BREX_TOOL_IDS = ['brex_match_receipt', 'brex_upload_receipt'] as const + +const LATEX_TOOL_IDS = ['latex_compile'] as const + +const ONEDRIVE_TOOL_IDS = ['onedrive_download', 'onedrive_upload'] as const + +const VANTA_TOOL_IDS = [ + 'vanta_download_document_file', + 'vanta_get_control', + 'vanta_get_document', + 'vanta_get_framework', + 'vanta_get_person', + 'vanta_get_policy', + 'vanta_get_risk_scenario', + 'vanta_get_test', + 'vanta_get_vendor', + 'vanta_get_vulnerable_asset', + 'vanta_list_control_documents', + 'vanta_list_control_tests', + 'vanta_list_controls', + 'vanta_list_document_uploads', + 'vanta_list_documents', + 'vanta_list_framework_controls', + 'vanta_list_frameworks', + 'vanta_list_monitored_computers', + 'vanta_list_people', + 'vanta_list_policies', + 'vanta_list_risk_scenarios', + 'vanta_list_test_entities', + 'vanta_list_tests', + 'vanta_list_vendors', + 'vanta_list_vulnerabilities', + 'vanta_list_vulnerability_remediations', + 'vanta_list_vulnerable_assets', + 'vanta_submit_document', + 'vanta_upload_document_file', +] as const + +const SAP_S4HANA_TOOL_IDS = [ + 'sap_s4hana_create_business_partner', + 'sap_s4hana_create_purchase_order', + 'sap_s4hana_create_purchase_requisition', + 'sap_s4hana_create_sales_order', + 'sap_s4hana_delete_sales_order', + 'sap_s4hana_get_billing_document', + 'sap_s4hana_get_business_partner', + 'sap_s4hana_get_customer', + 'sap_s4hana_get_inbound_delivery', + 'sap_s4hana_get_material_document', + 'sap_s4hana_get_outbound_delivery', + 'sap_s4hana_get_product', + 'sap_s4hana_get_purchase_order', + 'sap_s4hana_get_purchase_requisition', + 'sap_s4hana_get_sales_order', + 'sap_s4hana_get_supplier', + 'sap_s4hana_get_supplier_invoice', + 'sap_s4hana_list_billing_documents', + 'sap_s4hana_list_business_partners', + 'sap_s4hana_list_customers', + 'sap_s4hana_list_inbound_deliveries', + 'sap_s4hana_list_material_documents', + 'sap_s4hana_list_material_stock', + 'sap_s4hana_list_outbound_deliveries', + 'sap_s4hana_list_products', + 'sap_s4hana_list_purchase_orders', + 'sap_s4hana_list_purchase_requisitions', + 'sap_s4hana_list_sales_orders', + 'sap_s4hana_list_supplier_invoices', + 'sap_s4hana_list_suppliers', + 'sap_s4hana_odata_query', + 'sap_s4hana_update_business_partner', + 'sap_s4hana_update_customer', + 'sap_s4hana_update_product', + 'sap_s4hana_update_purchase_order', + 'sap_s4hana_update_purchase_requisition', + 'sap_s4hana_update_sales_order', + 'sap_s4hana_update_supplier', +] as const + +const AZURE_DATA_EXPLORER_TOOL_IDS = [ + 'azure_data_explorer_create_table', + 'azure_data_explorer_drop_table', + 'azure_data_explorer_ingest_from_query', + 'azure_data_explorer_ingest_inline', + 'azure_data_explorer_list_databases', + 'azure_data_explorer_list_functions', + 'azure_data_explorer_list_tables', + 'azure_data_explorer_management', + 'azure_data_explorer_query', + 'azure_data_explorer_show_database_schema', + 'azure_data_explorer_show_ingestion_failures', + 'azure_data_explorer_show_operations', + 'azure_data_explorer_show_table_details', + 'azure_data_explorer_show_table_schema', +] as const + +const ZOOMINFO_TOOL_IDS = [ + 'zoominfo_enrich_companies', + 'zoominfo_enrich_contacts', + 'zoominfo_search_companies', + 'zoominfo_search_contacts', + 'zoominfo_search_intent', + 'zoominfo_search_news', +] as const + +const SAP_CONCUR_TOOL_IDS = [ + 'sap_concur_approve_expense_report', + 'sap_concur_associate_attendees', + 'sap_concur_create_cash_advance', + 'sap_concur_create_expected_expense', + 'sap_concur_create_expense_report', + 'sap_concur_create_list_item', + 'sap_concur_create_purchase_request', + 'sap_concur_create_quick_expense', + 'sap_concur_create_quick_expense_with_image', + 'sap_concur_create_report_comment', + 'sap_concur_create_travel_request', + 'sap_concur_create_user', + 'sap_concur_delete_expected_expense', + 'sap_concur_delete_expense', + 'sap_concur_delete_expense_report', + 'sap_concur_delete_list_item', + 'sap_concur_delete_travel_request', + 'sap_concur_delete_user', + 'sap_concur_get_allocation', + 'sap_concur_get_budget', + 'sap_concur_get_cash_advance', + 'sap_concur_get_expected_expense', + 'sap_concur_get_expense', + 'sap_concur_get_expense_report', + 'sap_concur_get_itemizations', + 'sap_concur_get_itinerary', + 'sap_concur_get_list', + 'sap_concur_get_list_item', + 'sap_concur_get_purchase_request', + 'sap_concur_get_receipt', + 'sap_concur_get_receipt_status', + 'sap_concur_get_request_cash_advance', + 'sap_concur_get_travel_profile', + 'sap_concur_get_travel_request', + 'sap_concur_get_user', + 'sap_concur_issue_cash_advance', + 'sap_concur_list_allocations', + 'sap_concur_list_attendee_associations', + 'sap_concur_list_budget_categories', + 'sap_concur_list_budgets', + 'sap_concur_list_exceptions', + 'sap_concur_list_expected_expenses', + 'sap_concur_list_expense_reports', + 'sap_concur_list_expenses', + 'sap_concur_list_itineraries', + 'sap_concur_list_list_items', + 'sap_concur_list_lists', + 'sap_concur_list_receipts', + 'sap_concur_list_report_comments', + 'sap_concur_list_reports_to_approve', + 'sap_concur_list_travel_profiles_summary', + 'sap_concur_list_travel_request_comments', + 'sap_concur_list_travel_requests', + 'sap_concur_list_users', + 'sap_concur_move_travel_request', + 'sap_concur_recall_expense_report', + 'sap_concur_remove_all_attendees', + 'sap_concur_search_locations', + 'sap_concur_search_users', + 'sap_concur_send_back_expense_report', + 'sap_concur_submit_expense_report', + 'sap_concur_update_allocation', + 'sap_concur_update_expected_expense', + 'sap_concur_update_expense', + 'sap_concur_update_expense_report', + 'sap_concur_update_list_item', + 'sap_concur_update_travel_request', + 'sap_concur_update_user', + 'sap_concur_upload_exchange_rates', + 'sap_concur_upload_receipt_image', +] as const + +const RESEND_TOOL_IDS = ['resend_send'] as const + +const SENDGRID_TOOL_IDS = ['sendgrid_send_mail'] as const + +const SMTP_TOOL_IDS = ['smtp_send_mail'] as const + +const BOX_TOOL_IDS = ['box_upload_file'] as const + +const DROPBOX_TOOL_IDS = ['dropbox_upload'] as const + +const FIREFLIES_TOOL_IDS = ['fireflies_upload_audio'] as const + +const SUPABASE_TOOL_IDS = ['supabase_storage_upload'] as const + +const SQUARE_TOOL_IDS = ['square_create_catalog_image'] as const + +const TIKTOK_TOOL_IDS = ['tiktok_upload_video_draft'] as const + +const IMAGE_TOOL_IDS = ['image_generate'] as const + +const EMBEDDINGS_TOOL_IDS = [ + 'embeddings_openai', + 'embeddings_openrouter', + 'embeddings_gemini', + 'embeddings_cohere', + 'embeddings_mistral', + 'openai_embeddings', +] as const + +const ENRICHMENT_TOOL_IDS = ['enrichment_run'] as const + +const LLM_TOOL_IDS = ['llm_chat'] as const + +const GUARDRAILS_TOOL_IDS = ['guardrails_validate'] as const + +const MISTRAL_TOOL_IDS = ['mistral_parser', 'mistral_parser_v2', 'mistral_parser_v3'] as const + +const REDUCTO_TOOL_IDS = ['reducto_parser', 'reducto_parser_v2'] as const + +const PULSE_TOOL_IDS = ['pulse_parser', 'pulse_parser_v2'] as const + +const EXTEND_TOOL_IDS = ['extend_parser', 'extend_parser_v2'] as const + +const FIRECRAWL_TOOL_IDS = ['firecrawl_parse'] as const + +const CLICKUP_TOOL_IDS = ['clickup_upload_attachment'] as const + +const DISCORD_TOOL_IDS = ['discord_send_message'] as const + +const LINQ_TOOL_IDS = ['linq_create_attachment'] as const + +const MICROSOFT_DATAVERSE_TOOL_IDS = ['microsoft_dataverse_upload_file'] as const + +const SERVICENOW_TOOL_IDS = ['servicenow_upload_attachment'] as const + +const PIPEDRIVE_TOOL_IDS = ['pipedrive_get_files'] as const + +const MEMORY_TOOL_IDS = ['memory_add', 'memory_delete', 'memory_get', 'memory_get_all'] as const + +const LOG_TOOL_IDS = [ + 'logs_get_execution', + 'logs_get', + 'logs_get_run_details', + 'logs_query', + 'logs_query_runs', +] as const + +function registerFamily( + registry: Map, + toolIds: readonly string[], + loader: InternalToolOperationHandlerLoader +): void { + for (const toolId of toolIds) { + if (registry.has(toolId)) { + throw new Error(`Duplicate internal tool execution registration: ${toolId}`) + } + registry.set(toolId, loader) + } +} + +const handlerLoaders = new Map() + +registerFamily(handlerLoaders, STS_TOOL_IDS, async () => { + return (await import('@/lib/internal/sts/execute-tool')).executeStsTool +}) +registerFamily(handlerLoaders, APPCONFIG_TOOL_IDS, async () => { + return (await import('@/lib/internal/appconfig/execute-tool')).executeAppConfigTool +}) +registerFamily(handlerLoaders, IAM_TOOL_IDS, async () => { + return (await import('@/lib/internal/iam/execute-tool')).executeIamTool +}) +registerFamily(handlerLoaders, IDENTITY_CENTER_TOOL_IDS, async () => { + return (await import('@/lib/internal/identity-center/execute-tool')).executeIdentityCenterTool +}) +registerFamily(handlerLoaders, SECRETS_MANAGER_TOOL_IDS, async () => { + return (await import('@/lib/internal/secrets-manager/execute-tool')).executeSecretsManagerTool +}) +registerFamily(handlerLoaders, DYNAMODB_TOOL_IDS, async () => { + return (await import('@/lib/internal/dynamodb/execute-tool')).executeDynamodbTool +}) +registerFamily(handlerLoaders, SES_TOOL_IDS, async () => { + return (await import('@/lib/internal/ses/execute-tool')).executeSesTool +}) +registerFamily(handlerLoaders, SQS_TOOL_IDS, async () => { + return (await import('@/lib/internal/sqs/execute-tool')).executeSqsTool +}) +registerFamily(handlerLoaders, RDS_TOOL_IDS, async () => { + return (await import('@/lib/internal/rds/execute-tool')).executeRdsTool +}) +registerFamily(handlerLoaders, TEXTRACT_TOOL_IDS, async () => { + return (await import('@/lib/internal/textract/execute-tool')).executeTextractTool +}) +registerFamily(handlerLoaders, CLOUDWATCH_TOOL_IDS, async () => { + return (await import('@/lib/internal/cloudwatch/execute-tool')).executeCloudwatchTool +}) +registerFamily(handlerLoaders, POSTGRESQL_TOOL_IDS, async () => { + return (await import('@/lib/internal/postgresql/execute-tool')).executePostgresqlTool +}) +registerFamily(handlerLoaders, REDIS_TOOL_IDS, async () => { + return (await import('@/lib/internal/redis/execute-tool')).executeRedisTool +}) +registerFamily(handlerLoaders, CLOUDFORMATION_TOOL_IDS, async () => { + return (await import('@/lib/internal/cloudformation/execute-tool')).executeCloudformationTool +}) +registerFamily(handlerLoaders, CODEPIPELINE_TOOL_IDS, async () => { + return (await import('@/lib/internal/codepipeline/execute-tool')).executeCodepipelineTool +}) +registerFamily(handlerLoaders, MYSQL_TOOL_IDS, async () => { + return (await import('@/lib/internal/mysql/execute-tool')).executeMysqlTool +}) +registerFamily(handlerLoaders, ATHENA_TOOL_IDS, async () => { + return (await import('@/lib/internal/athena/execute-tool')).executeAthenaTool +}) +registerFamily(handlerLoaders, CLICKHOUSE_TOOL_IDS, async () => { + return (await import('@/lib/internal/clickhouse/execute-tool')).executeClickHouseTool +}) +registerFamily(handlerLoaders, MONGODB_TOOL_IDS, async () => { + return (await import('@/lib/internal/mongodb/execute-tool')).executeMongodbTool +}) +registerFamily(handlerLoaders, NEO4J_TOOL_IDS, async () => { + return (await import('@/lib/internal/neo4j/execute-tool')).executeNeo4jTool +}) +registerFamily(handlerLoaders, S3_TOOL_IDS, async () => { + return (await import('@/lib/internal/s3/execute-tool')).executeS3Tool +}) +registerFamily(handlerLoaders, JUPYTER_TOOL_IDS, async () => { + return (await import('@/lib/internal/jupyter/execute-tool')).executeJupyterTool +}) +registerFamily(handlerLoaders, MSSQL_TOOL_IDS, async () => { + return (await import('@/lib/internal/mssql/execute-tool')).executeMssqlTool +}) +registerFamily(handlerLoaders, KNOWLEDGE_TOOL_IDS, async () => { + return (await import('@/lib/internal/knowledge/execute-tool')).executeKnowledgeTool +}) +registerFamily(handlerLoaders, CONFLUENCE_TOOL_IDS, async () => { + return (await import('@/lib/internal/confluence/execute-tool')).executeConfluenceTool +}) +registerFamily(handlerLoaders, JSM_TOOL_IDS, async () => { + return (await import('@/lib/internal/jsm/execute-tool')).executeJsmTool +}) +registerFamily(handlerLoaders, CROWDSTRIKE_TOOL_IDS, async () => { + return (await import('@/lib/internal/crowdstrike/execute-tool')).executeCrowdStrikeTool +}) +registerFamily(handlerLoaders, GMAIL_TOOL_IDS, async () => { + return (await import('@/lib/internal/gmail/execute-tool')).executeGmailTool +}) +registerFamily(handlerLoaders, WINDCHILL_TOOL_IDS, async () => { + return (await import('@/lib/internal/windchill/execute-tool')).executeWindchillTool +}) +registerFamily(handlerLoaders, TABLE_TOOL_IDS, async () => { + return (await import('@/lib/internal/table/execute-tool')).executeTableTool +}) +registerFamily(handlerLoaders, WORKDAY_TOOL_IDS, async () => { + return (await import('@/lib/internal/workday/execute-tool')).executeWorkdayTool +}) +registerFamily(handlerLoaders, AGILOFT_TOOL_IDS, async () => { + return (await import('@/lib/internal/agiloft/execute-tool')).executeAgiloftTool +}) +registerFamily(handlerLoaders, ONEPASSWORD_TOOL_IDS, async () => { + return (await import('@/lib/internal/onepassword/execute-tool')).executeOnePasswordTool +}) +registerFamily(handlerLoaders, OUTLOOK_TOOL_IDS, async () => { + return (await import('@/lib/internal/outlook/execute-tool')).executeOutlookTool +}) +registerFamily(handlerLoaders, SSH_TOOL_IDS, async () => { + return (await import('@/lib/internal/ssh/execute-tool')).executeSshTool +}) +registerFamily(handlerLoaders, ASANA_TOOL_IDS, async () => { + return (await import('@/lib/internal/asana/execute-tool')).executeAsanaTool +}) +registerFamily(handlerLoaders, DOCUSIGN_TOOL_IDS, async () => { + return (await import('@/lib/internal/docusign/execute-tool')).executeDocuSignTool +}) +registerFamily(handlerLoaders, THINKING_TOOL_IDS, async () => { + return (await import('@/lib/internal/thinking/execute-tool')).executeThinkingTool +}) +registerFamily(handlerLoaders, SLACK_TOOL_IDS, async () => { + return (await import('@/lib/internal/slack/execute-tool')).executeSlackTool +}) +registerFamily(handlerLoaders, SEARCH_TOOL_IDS, async () => { + return (await import('@/lib/internal/search/execute-tool')).executeSearchTool +}) +registerFamily(handlerLoaders, SMS_TOOL_IDS, async () => { + return (await import('@/lib/internal/sms/execute-tool')).executeSmsTool +}) +registerFamily(handlerLoaders, MICROSOFT_WORD_TOOL_IDS, async () => { + return (await import('@/lib/internal/microsoft-word/execute-tool')).executeMicrosoftWordTool +}) +registerFamily(handlerLoaders, TTS_TOOL_IDS, async () => { + return (await import('@/lib/internal/tts/execute-tool')).executeTtsTool +}) +registerFamily(handlerLoaders, FILE_TOOL_IDS, async () => { + return (await import('@/lib/internal/file/execute-tool')).executeFileTool +}) +registerFamily(handlerLoaders, UPTIMEROBOT_TOOL_IDS, async () => { + return (await import('@/lib/internal/uptimerobot/execute-tool')).executeUptimeRobotTool +}) +registerFamily(handlerLoaders, STT_TOOL_IDS, async () => { + return (await import('@/lib/internal/stt/execute-tool')).executeSttTool +}) +registerFamily(handlerLoaders, INSTAGRAM_TOOL_IDS, async () => { + return (await import('@/lib/internal/instagram/execute-tool')).executeInstagramTool +}) +registerFamily(handlerLoaders, VIDEO_TOOL_IDS, async () => { + return (await import('@/lib/internal/video/execute-tool')).executeVideoTool +}) +registerFamily(handlerLoaders, A2A_TOOL_IDS, async () => { + return (await import('@/lib/internal/a2a/execute-tool')).executeA2ATool +}) +registerFamily(handlerLoaders, BUFFER_TOOL_IDS, async () => { + return (await import('@/lib/internal/buffer/execute-tool')).executeBufferTool +}) +registerFamily(handlerLoaders, GRAFANA_TOOL_IDS, async () => { + return (await import('@/lib/internal/grafana/execute-tool')).executeGrafanaTool +}) +registerFamily(handlerLoaders, DEPLOYMENTS_TOOL_IDS, async () => { + return (await import('@/lib/internal/deployments/execute-tool')).executeDeploymentsTool +}) +registerFamily(handlerLoaders, CURSOR_TOOL_IDS, async () => { + return (await import('@/lib/internal/cursor/execute-tool')).executeCursorTool +}) +registerFamily(handlerLoaders, SFTP_TOOL_IDS, async () => { + return (await import('@/lib/internal/sftp/execute-tool')).executeSftpTool +}) +registerFamily(handlerLoaders, ZOOM_TOOL_IDS, async () => { + return (await import('@/lib/internal/zoom/execute-tool')).executeZoomTool +}) +registerFamily(handlerLoaders, ZOHO_DESK_TOOL_IDS, async () => { + return (await import('@/lib/internal/zoho-desk/execute-tool')).executeZohoDeskTool +}) +registerFamily(handlerLoaders, WORDPRESS_TOOL_IDS, async () => { + return (await import('@/lib/internal/wordpress/execute-tool')).executeWordPressTool +}) +registerFamily(handlerLoaders, ELEVENLABS_TOOL_IDS, async () => { + return (await import('@/lib/internal/elevenlabs/execute-tool')).executeElevenLabsTool +}) +registerFamily(handlerLoaders, JIRA_TOOL_IDS, async () => { + return (await import('@/lib/internal/jira/execute-tool')).executeJiraTool +}) +registerFamily(handlerLoaders, WHATSAPP_TOOL_IDS, async () => { + return (await import('@/lib/internal/whatsapp/execute-tool')).executeWhatsAppTool +}) +registerFamily(handlerLoaders, TYPEFORM_TOOL_IDS, async () => { + return (await import('@/lib/internal/typeform/execute-tool')).executeTypeformTool +}) +registerFamily(handlerLoaders, DAYTONA_TOOL_IDS, async () => { + return (await import('@/lib/internal/daytona/execute-tool')).executeDaytonaTool +}) +registerFamily(handlerLoaders, GOOGLE_SLIDES_TOOL_IDS, async () => { + return (await import('@/lib/internal/google-slides/execute-tool')).executeGoogleSlidesTool +}) +registerFamily(handlerLoaders, GOOGLE_VAULT_TOOL_IDS, async () => { + return (await import('@/lib/internal/google-vault/execute-tool')).executeGoogleVaultTool +}) +registerFamily(handlerLoaders, GOOGLE_DRIVE_TOOL_IDS, async () => { + return (await import('@/lib/internal/google-drive/execute-tool')).executeGoogleDriveTool +}) +registerFamily(handlerLoaders, STAGEHAND_TOOL_IDS, async () => { + return (await import('@/lib/internal/stagehand/execute-tool')).executeStagehandTool +}) +registerFamily(handlerLoaders, VISION_TOOL_IDS, async () => { + return (await import('@/lib/internal/vision/execute-tool')).executeVisionTool +}) +registerFamily(handlerLoaders, GITHUB_TOOL_IDS, async () => { + return (await import('@/lib/internal/github/execute-tool')).executeGitHubTool +}) +registerFamily(handlerLoaders, TWILIO_VOICE_TOOL_IDS, async () => { + return (await import('@/lib/internal/twilio-voice/execute-tool')).executeTwilioVoiceTool +}) +registerFamily(handlerLoaders, PERSONA_TOOL_IDS, async () => { + return (await import('@/lib/internal/persona/execute-tool')).executePersonaTool +}) +registerFamily(handlerLoaders, SHAREPOINT_TOOL_IDS, async () => { + return (await import('@/lib/internal/sharepoint/execute-tool')).executeSharePointTool +}) +registerFamily(handlerLoaders, QUIVER_TOOL_IDS, async () => { + return (await import('@/lib/internal/quiver/execute-tool')).executeQuiverTool +}) +registerFamily(handlerLoaders, TELEGRAM_TOOL_IDS, async () => { + return (await import('@/lib/internal/telegram/execute-tool')).executeTelegramTool +}) +registerFamily(handlerLoaders, MICROSOFT_TEAMS_TOOL_IDS, async () => { + return (await import('@/lib/internal/microsoft-teams/execute-tool')).executeMicrosoftTeamsTool +}) +registerFamily(handlerLoaders, BREX_TOOL_IDS, async () => { + return (await import('@/lib/internal/brex/execute-tool')).executeBrexTool +}) +registerFamily(handlerLoaders, LATEX_TOOL_IDS, async () => { + return (await import('@/lib/internal/latex/execute-tool')).executeLatexTool +}) +registerFamily(handlerLoaders, ONEDRIVE_TOOL_IDS, async () => { + return (await import('@/lib/internal/onedrive/execute-tool')).executeOneDriveTool +}) +registerFamily(handlerLoaders, VANTA_TOOL_IDS, async () => { + return (await import('@/lib/internal/vanta/execute-tool')).executeVantaTool +}) +registerFamily(handlerLoaders, SAP_S4HANA_TOOL_IDS, async () => { + return (await import('@/lib/internal/sap-s4hana/execute-tool')).executeSapS4HanaTool +}) +registerFamily(handlerLoaders, AZURE_DATA_EXPLORER_TOOL_IDS, async () => { + return (await import('@/lib/internal/azure-data-explorer/execute-tool')) + .executeAzureDataExplorerTool +}) +registerFamily(handlerLoaders, ZOOMINFO_TOOL_IDS, async () => { + return (await import('@/lib/internal/zoominfo/execute-tool')).executeZoomInfoTool +}) +registerFamily(handlerLoaders, SAP_CONCUR_TOOL_IDS, async () => { + return (await import('@/lib/internal/sap-concur/execute-tool')).executeSapConcurTool +}) +registerFamily(handlerLoaders, RESEND_TOOL_IDS, async () => { + return (await import('@/lib/internal/resend/execute-tool')).executeResendTool +}) +registerFamily(handlerLoaders, SENDGRID_TOOL_IDS, async () => { + return (await import('@/lib/internal/sendgrid/execute-tool')).executeSendGridTool +}) +registerFamily(handlerLoaders, SMTP_TOOL_IDS, async () => { + return (await import('@/lib/internal/smtp/execute-tool')).executeSmtpTool +}) +registerFamily(handlerLoaders, BOX_TOOL_IDS, async () => { + return (await import('@/lib/internal/box/execute-tool')).executeBoxTool +}) +registerFamily(handlerLoaders, DROPBOX_TOOL_IDS, async () => { + return (await import('@/lib/internal/dropbox/execute-tool')).executeDropboxTool +}) +registerFamily(handlerLoaders, FIREFLIES_TOOL_IDS, async () => { + return (await import('@/lib/internal/fireflies/execute-tool')).executeFirefliesTool +}) +registerFamily(handlerLoaders, SUPABASE_TOOL_IDS, async () => { + return (await import('@/lib/internal/supabase/execute-tool')).executeSupabaseTool +}) +registerFamily(handlerLoaders, SQUARE_TOOL_IDS, async () => { + return (await import('@/lib/internal/square/execute-tool')).executeSquareTool +}) +registerFamily(handlerLoaders, TIKTOK_TOOL_IDS, async () => { + return (await import('@/lib/internal/tiktok/execute-tool')).executeTikTokTool +}) +registerFamily(handlerLoaders, IMAGE_TOOL_IDS, async () => { + return (await import('@/lib/internal/image/execute-tool')).executeImageTool +}) +registerFamily(handlerLoaders, EMBEDDINGS_TOOL_IDS, async () => { + return (await import('@/lib/internal/embeddings/execute-tool')).executeEmbeddingsTool +}) +registerFamily(handlerLoaders, ENRICHMENT_TOOL_IDS, async () => { + return (await import('@/lib/internal/enrichment/execute-tool')).executeEnrichmentTool +}) +registerFamily(handlerLoaders, LLM_TOOL_IDS, async () => { + return (await import('@/lib/internal/llm/execute-tool')).executeLlmTool +}) +registerFamily(handlerLoaders, GUARDRAILS_TOOL_IDS, async () => { + return (await import('@/lib/internal/guardrails/execute-tool')).executeGuardrailsTool +}) +registerFamily(handlerLoaders, MISTRAL_TOOL_IDS, async () => { + return (await import('@/lib/internal/mistral/execute-tool')).executeMistralTool +}) +registerFamily(handlerLoaders, REDUCTO_TOOL_IDS, async () => { + return (await import('@/lib/internal/reducto/execute-tool')).executeReductoTool +}) +registerFamily(handlerLoaders, PULSE_TOOL_IDS, async () => { + return (await import('@/lib/internal/pulse/execute-tool')).executePulseTool +}) +registerFamily(handlerLoaders, EXTEND_TOOL_IDS, async () => { + return (await import('@/lib/internal/extend/execute-tool')).executeExtendTool +}) +registerFamily(handlerLoaders, FIRECRAWL_TOOL_IDS, async () => { + return (await import('@/lib/internal/firecrawl/execute-tool')).executeFirecrawlTool +}) +registerFamily(handlerLoaders, CLICKUP_TOOL_IDS, async () => { + return (await import('@/lib/internal/clickup/execute-tool')).executeClickUpTool +}) +registerFamily(handlerLoaders, DISCORD_TOOL_IDS, async () => { + return (await import('@/lib/internal/discord/execute-tool')).executeDiscordTool +}) +registerFamily(handlerLoaders, LINQ_TOOL_IDS, async () => { + return (await import('@/lib/internal/linq/execute-tool')).executeLinqTool +}) +registerFamily(handlerLoaders, MICROSOFT_DATAVERSE_TOOL_IDS, async () => { + return (await import('@/lib/internal/microsoft-dataverse/execute-tool')) + .executeMicrosoftDataverseTool +}) +registerFamily(handlerLoaders, SERVICENOW_TOOL_IDS, async () => { + return (await import('@/lib/internal/servicenow/execute-tool')).executeServiceNowTool +}) +registerFamily(handlerLoaders, PIPEDRIVE_TOOL_IDS, async () => { + return (await import('@/lib/internal/pipedrive/execute-tool')).executePipedriveTool +}) +registerFamily(handlerLoaders, MEMORY_TOOL_IDS, async () => { + return (await import('@/lib/internal/memory/execute-tool')).executeMemoryTool +}) +registerFamily(handlerLoaders, LOG_TOOL_IDS, async () => { + return (await import('@/lib/internal/logs/execute-tool')).executeLogsTool +}) + +export function isInternalToolOperationRegistered(toolId: string): boolean { + return handlerLoaders.has(toolId) || isMcpTool(toolId) +} + +export function getRegisteredInternalToolOperationIds(): string[] { + return [...handlerLoaders.keys()] +} + +export async function getInternalToolOperationHandler( + toolId: string +): Promise { + const loader = handlerLoaders.get(toolId) + if (loader) return loader() + if (isMcpTool(toolId)) { + return (await import('@/lib/internal/mcp/execute-tool')).executeMcpTool + } + return null +} diff --git a/apps/sim/lib/internal/tool-operations/types.ts b/apps/sim/lib/internal/tool-operations/types.ts new file mode 100644 index 00000000000..635172c66b5 --- /dev/null +++ b/apps/sim/lib/internal/tool-operations/types.ts @@ -0,0 +1,34 @@ +import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' +import type { ExecutorDelegationOrigin } from '@/executor/types' +import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' + +/** Trusted runtime scope shared by every in-process tool operation. */ +export interface InternalToolOperationContext { + workflowId: string + workspaceId?: string + executionId?: string + userId?: string + executorDelegationOrigin?: ExecutorDelegationOrigin + copilotToolExecution?: boolean + copilotInteractionMode?: 'interactive' | 'headless' + chatId?: string + toolCallId?: string + billingAttribution?: BillingAttributionSnapshot + callChain?: string[] + resolvedSecretTraceRegistry?: ResolvedSecretTraceRegistry + largeValueExecutionIds?: string[] + largeValueKeys?: string[] + fileKeys?: string[] + allowLargeValueWorkflowScope?: boolean +} + +export interface InternalToolOperationCall { + toolId: string + input?: unknown + headers: Headers + context: InternalToolOperationContext + requestId: string + signal?: AbortSignal +} + +export type InternalToolOperationHandler = (request: InternalToolOperationCall) => Promise diff --git a/apps/sim/lib/internal/tts/client.test.ts b/apps/sim/lib/internal/tts/client.test.ts new file mode 100644 index 00000000000..d85af2ef383 --- /dev/null +++ b/apps/sim/lib/internal/tts/client.test.ts @@ -0,0 +1,132 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + MAX_TTS_AUDIO_BYTES, + MAX_TTS_TEXT_BYTES, + synthesizeDeepgram, + synthesizeGoogle, + synthesizeLegacyElevenLabs, + synthesizeOpenAi, +} from '@/lib/internal/tts/client' +import { TtsOperationError } from '@/lib/internal/tts/errors' + +const fetchMock = vi.fn() + +describe('TTS provider client', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', fetchMock) + }) + + afterEach(() => vi.unstubAllGlobals()) + + it('forwards cancellation and clamps OpenAI speech speed', async () => { + const controller = new AbortController() + fetchMock.mockResolvedValue(new Response(Buffer.from('audio'))) + + const result = await synthesizeOpenAi( + { + text: 'Hello', + apiKey: 'openai-key', + model: 'tts-1-hd', + voice: 'coral', + responseFormat: 'wav', + speed: 9, + }, + controller.signal + ) + + expect(result).toMatchObject({ format: 'wav', mimeType: 'audio/wav' }) + expect(fetchMock).toHaveBeenCalledWith( + 'https://api.openai.com/v1/audio/speech', + expect.objectContaining({ signal: controller.signal }) + ) + const body = JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body)) + expect(body).toEqual({ + model: 'tts-1-hd', + voice: 'coral', + input: 'Hello', + response_format: 'wav', + speed: 4, + }) + }) + + it('uses the configured Deepgram model and container query', async () => { + fetchMock.mockResolvedValue(new Response(Buffer.from('audio'))) + + const result = await synthesizeDeepgram({ + text: 'Hello', + apiKey: 'deepgram-key', + model: 'aura-2-luna-en', + encoding: 'linear16', + sampleRate: 24000, + container: 'wav', + }) + + expect(result.format).toBe('wav') + expect(fetchMock.mock.calls[0]?.[0]).toBe( + 'https://api.deepgram.com/v1/speak?model=aura-2-luna-en&encoding=linear16&sample_rate=24000&container=wav' + ) + }) + + it('bounds provider audio before materializing it', async () => { + fetchMock.mockResolvedValue( + new Response(Buffer.from('audio'), { + headers: { 'Content-Length': String(MAX_TTS_AUDIO_BYTES + 1) }, + }) + ) + + await expect(synthesizeOpenAi({ text: 'Hello', apiKey: 'key' })).rejects.toThrow( + /exceeds maximum size/i + ) + }) + + it('bounds and decodes Google base64 JSON responses', async () => { + fetchMock.mockResolvedValue( + Response.json({ audioContent: Buffer.from('google-audio').toString('base64') }) + ) + + const result = await synthesizeGoogle({ + text: 'Hello', + apiKey: 'google-key', + languageCode: 'en-US', + audioEncoding: 'OGG_OPUS', + }) + + expect(result.audioBuffer).toEqual(Buffer.from('google-audio')) + expect(result).toMatchObject({ format: 'oggopus', mimeType: 'audio/mpeg' }) + }) + + it('preserves legacy ElevenLabs provider status errors', async () => { + fetchMock.mockResolvedValue( + new Response(null, { status: 429, statusText: 'Too Many Requests' }) + ) + + await expect( + synthesizeLegacyElevenLabs({ + text: 'Hello', + apiKey: 'elevenlabs-key', + voiceId: 'voice_1', + }) + ).rejects.toEqual(new TtsOperationError('Failed to generate TTS: 429 Too Many Requests', 429)) + }) + + it('does not start a provider request when already cancelled', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + synthesizeOpenAi({ text: 'Hello', apiKey: 'key' }, controller.signal) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('rejects oversized text before serializing or contacting a provider', async () => { + await expect( + synthesizeOpenAi({ text: 'x'.repeat(MAX_TTS_TEXT_BYTES + 1), apiKey: 'key' }) + ).rejects.toThrow(/TTS text exceeds maximum size/i) + expect(fetchMock).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/tts/client.ts b/apps/sim/lib/internal/tts/client.ts new file mode 100644 index 00000000000..c88a9b4b23c --- /dev/null +++ b/apps/sim/lib/internal/tts/client.ts @@ -0,0 +1,468 @@ +import { createLogger } from '@sim/logger' +import { isRecordLike } from '@sim/utils/object' +import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' +import { validateAlphanumericId } from '@/lib/core/security/input-validation' +import { + assertKnownSizeWithinLimit, + readResponseJsonWithLimit, + readResponseTextWithLimit, + readResponseToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' +import { TtsOperationError } from '@/lib/internal/tts/errors' +import { getTtsMimeType } from '@/lib/internal/tts/formats' +import type { ElevenLabsTtsParams } from '@/tools/elevenlabs/types' +import type { + AzureTtsParams, + CartesiaTtsParams, + DeepgramTtsParams, + ElevenLabsTtsUnifiedParams, + GoogleTtsParams, + OpenAiTtsParams, + PlayHtTtsParams, +} from '@/tools/tts/types' + +const logger = createLogger('TtsClient') +export const MAX_TTS_AUDIO_BYTES = 25 * 1024 * 1024 +export const MAX_TTS_TEXT_BYTES = 10 * 1024 * 1024 +const MAX_TTS_ERROR_BYTES = 64 * 1024 +const MAX_TTS_JSON_BYTES = Math.ceil((MAX_TTS_AUDIO_BYTES * 4) / 3) + 256 * 1024 + +export interface TtsAudioResult { + audioBuffer: Buffer + format: string + mimeType: string + duration?: number +} + +export type OpenAiTtsOperationInput = Omit & { voice?: string } +export type CartesiaTtsOperationInput = Omit & { + outputFormat?: Record | string | null +} +export type AzureTtsOperationInput = Omit & { + outputFormat?: string + pitch?: number | string + style?: number | string +} + +async function providerFetch( + input: string, + init: RequestInit, + signal?: AbortSignal, + timeoutMs?: number +): Promise { + signal?.throwIfAborted() + if (!timeoutMs) return fetch(input, { ...init, signal }) + + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(new Error('TTS request timed out')), timeoutMs) + const abort = () => controller.abort(signal?.reason ?? new Error('Request aborted')) + signal?.addEventListener('abort', abort, { once: true }) + try { + return await fetch(input, { ...init, signal: controller.signal }) + } finally { + clearTimeout(timeout) + signal?.removeEventListener('abort', abort) + } +} + +async function readTtsErrorJson( + response: Response, + label: string, + signal?: AbortSignal +): Promise> { + return readResponseJsonWithLimit>(response, { + maxBytes: MAX_TTS_ERROR_BYTES, + label, + signal, + }).catch(() => ({})) +} + +function getTtsErrorMessage(error: Record, fallback: string): string { + const nested = error.error + if (isRecordLike(nested) && typeof nested.message === 'string') return nested.message + for (const key of ['message', 'err_msg', 'error_message', 'error', 'detail']) { + const value = error[key] + if (typeof value === 'string') return value + if (isRecordLike(value) && typeof value.message === 'string') return value.message + } + return fallback +} + +async function readAudio(response: Response, label: string, signal?: AbortSignal): Promise { + return readResponseToBufferWithLimit(response, { + maxBytes: MAX_TTS_AUDIO_BYTES, + label, + signal, + }) +} + +function assertTextWithinLimit(text: string): void { + assertKnownSizeWithinLimit(Buffer.byteLength(text), MAX_TTS_TEXT_BYTES, 'TTS text') +} + +export async function synthesizeOpenAi( + input: OpenAiTtsOperationInput, + signal?: AbortSignal +): Promise { + assertTextWithinLimit(input.text) + const model = input.model || 'tts-1' + const voice = input.voice || 'alloy' + const format = input.responseFormat || 'mp3' + const response = await providerFetch( + 'https://api.openai.com/v1/audio/speech', + { + method: 'POST', + headers: { + Authorization: `Bearer ${input.apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model, + voice, + input: input.text, + response_format: format, + speed: Math.max(0.25, Math.min(4, input.speed ?? 1)), + }), + }, + signal + ) + if (!response.ok) { + const error = await readTtsErrorJson(response, 'OpenAI TTS error response', signal) + throw new Error(`OpenAI TTS API error: ${getTtsErrorMessage(error, response.statusText)}`) + } + return { + audioBuffer: await readAudio(response, 'OpenAI TTS audio response', signal), + format, + mimeType: getTtsMimeType(format), + } +} + +export async function synthesizeDeepgram( + input: DeepgramTtsParams, + signal?: AbortSignal +): Promise { + assertTextWithinLimit(input.text) + const model = input.model || input.voice || 'aura-asteria-en' + const encoding = input.encoding || 'mp3' + const query = new URLSearchParams({ model, encoding }) + if (input.sampleRate && encoding === 'linear16') { + query.append('sample_rate', input.sampleRate.toString()) + } + if (input.bitRate) query.append('bit_rate', input.bitRate.toString()) + if (input.container && input.container !== 'none') query.append('container', input.container) + const response = await providerFetch( + `https://api.deepgram.com/v1/speak?${query}`, + { + method: 'POST', + headers: { + Authorization: `Token ${input.apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ text: input.text }), + }, + signal + ) + if (!response.ok) { + const error = await readTtsErrorJson(response, 'Deepgram TTS error response', signal) + throw new Error(`Deepgram TTS API error: ${getTtsErrorMessage(error, response.statusText)}`) + } + const format = input.container === 'wav' || input.container === 'ogg' ? input.container : encoding + return { + audioBuffer: await readAudio(response, 'Deepgram TTS audio response', signal), + format, + mimeType: getTtsMimeType(format), + } +} + +export async function synthesizeElevenLabs( + input: ElevenLabsTtsUnifiedParams, + signal?: AbortSignal +): Promise { + assertTextWithinLimit(input.text) + const voiceIdValidation = validateAlphanumericId(input.voiceId, 'voiceId') + if (!voiceIdValidation.isValid) { + throw new TtsOperationError(voiceIdValidation.error || 'Invalid voiceId', 400) + } + const voiceSettings: Record = { + stability: Math.max(0, Math.min(1, input.stability ?? 0.5)), + similarity_boost: Math.max(0, Math.min(1, input.similarityBoost ?? 0.8)), + use_speaker_boost: input.useSpeakerBoost ?? true, + } + if (input.style !== undefined) { + voiceSettings.style = Math.max(0, Math.min(1, input.style)) + } + const response = await providerFetch( + `https://api.elevenlabs.io/v1/text-to-speech/${input.voiceId}`, + { + method: 'POST', + headers: { + Accept: 'audio/mpeg', + 'Content-Type': 'application/json', + 'xi-api-key': input.apiKey, + }, + body: JSON.stringify({ + text: input.text, + model_id: input.modelId || 'eleven_turbo_v2_5', + voice_settings: voiceSettings, + }), + }, + signal + ) + if (!response.ok) { + const error = await readTtsErrorJson(response, 'ElevenLabs TTS error response', signal) + throw new Error(`ElevenLabs TTS API error: ${getTtsErrorMessage(error, response.statusText)}`) + } + return { + audioBuffer: await readAudio(response, 'ElevenLabs TTS audio response', signal), + format: 'mp3', + mimeType: 'audio/mpeg', + } +} + +export async function synthesizeLegacyElevenLabs( + input: ElevenLabsTtsParams, + signal?: AbortSignal +): Promise { + assertTextWithinLimit(input.text) + const voiceIdValidation = validateAlphanumericId(input.voiceId, 'voiceId', 255) + if (!voiceIdValidation.isValid) { + throw new TtsOperationError(voiceIdValidation.error || 'Invalid voiceId', 400) + } + const hasVoiceSetting = input.stability !== undefined || input.similarityBoost !== undefined + const voiceSettings = hasVoiceSetting + ? { + stability: input.stability ?? 0.5, + similarity_boost: input.similarityBoost ?? 0.75, + } + : undefined + const response = await providerFetch( + `https://api.elevenlabs.io/v1/text-to-speech/${input.voiceId}`, + { + method: 'POST', + headers: { + Accept: 'audio/mpeg', + 'Content-Type': 'application/json', + 'xi-api-key': input.apiKey, + }, + body: JSON.stringify({ + text: input.text, + model_id: input.modelId || 'eleven_monolingual_v1', + ...(voiceSettings ? { voice_settings: voiceSettings } : {}), + }), + }, + signal, + DEFAULT_EXECUTION_TIMEOUT_MS + ) + if (!response.ok) { + await response.body?.cancel().catch(() => {}) + throw new TtsOperationError( + `Failed to generate TTS: ${response.status} ${response.statusText}`, + response.status + ) + } + const audioBuffer = await readAudio(response, 'TTS audio response', signal) + if (audioBuffer.length === 0) throw new TtsOperationError('Empty audio received', 422) + return { audioBuffer, format: 'mp3', mimeType: 'audio/mpeg' } +} + +export async function synthesizeCartesia( + input: CartesiaTtsOperationInput, + signal?: AbortSignal +): Promise { + assertTextWithinLimit(input.text) + const requestBody: Record = { + model_id: input.modelId || 'sonic-3', + transcript: input.text, + language: input.language || 'en', + } + if (input.voice) requestBody.voice = { mode: 'id', id: input.voice } + const generationConfig: Record = {} + if (input.speed !== undefined) generationConfig.speed = input.speed + if (input.emotion !== undefined) generationConfig.emotion = input.emotion + if (Object.keys(generationConfig).length > 0) requestBody.generation_config = generationConfig + const outputFormat = isRecordLike(input.outputFormat) ? input.outputFormat : undefined + requestBody.output_format = outputFormat || { + container: 'wav', + encoding: 'pcm_s16le', + sample_rate: 24000, + } + logger.info('Sending Cartesia TTS request', { + modelId: requestBody.model_id, + hasVoice: Boolean(requestBody.voice), + language: requestBody.language, + hasGenerationConfig: Boolean(requestBody.generation_config), + }) + const response = await providerFetch( + 'https://api.cartesia.ai/tts/bytes', + { + method: 'POST', + headers: { + Authorization: `Bearer ${input.apiKey}`, + 'Content-Type': 'application/json', + 'Cartesia-Version': '2025-04-16', + }, + body: JSON.stringify(requestBody), + }, + signal + ) + if (!response.ok) { + const error = await readTtsErrorJson(response, 'Cartesia TTS error response', signal) + const message = getTtsErrorMessage(error, response.statusText) + const detail = typeof error.detail === 'string' ? error.detail : '' + logger.error('Cartesia TTS request failed', { status: response.status, error: message, detail }) + throw new Error(`Cartesia TTS API error: ${message}${detail ? ` - ${detail}` : ''}`) + } + const format = typeof outputFormat?.container === 'string' ? outputFormat.container : 'mp3' + return { + audioBuffer: await readAudio(response, 'Cartesia TTS audio response', signal), + format, + mimeType: getTtsMimeType(format), + } +} + +export async function synthesizeGoogle( + input: GoogleTtsParams, + signal?: AbortSignal +): Promise { + assertTextWithinLimit(input.text) + if (!input.languageCode) { + throw new Error('text, apiKey, and languageCode are required for Google Cloud TTS') + } + const audioEncoding = input.audioEncoding || 'MP3' + const audioConfig: Record = { + audioEncoding, + speakingRate: Math.max(0.25, Math.min(2, input.speakingRate ?? 1)), + pitch: input.pitch ?? 0, + } + if (input.volumeGainDb !== undefined) audioConfig.volumeGainDb = input.volumeGainDb + if (input.sampleRateHertz) audioConfig.sampleRateHertz = input.sampleRateHertz + if (input.effectsProfileId?.length) audioConfig.effectsProfileId = input.effectsProfileId + const voice: Record = { languageCode: input.languageCode } + if (input.voiceId) voice.name = input.voiceId + if (input.gender) voice.ssmlGender = input.gender + if (!input.voiceId && !input.gender) voice.name = 'en-US-Neural2-C' + const response = await providerFetch( + `https://texttospeech.googleapis.com/v1/text:synthesize?key=${input.apiKey}`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ input: { text: input.text }, voice, audioConfig }), + }, + signal + ) + if (!response.ok) { + const error = await readTtsErrorJson(response, 'Google TTS error response', signal) + throw new Error(`Google Cloud TTS API error: ${getTtsErrorMessage(error, response.statusText)}`) + } + const data = await readResponseJsonWithLimit<{ audioContent?: string }>(response, { + maxBytes: MAX_TTS_JSON_BYTES, + label: 'Google TTS JSON response', + signal, + }) + if (!data.audioContent) throw new Error('No audio content returned from Google Cloud TTS') + const audioBuffer = Buffer.from(data.audioContent, 'base64') + assertKnownSizeWithinLimit(audioBuffer.length, MAX_TTS_AUDIO_BYTES, 'Google TTS audio response') + const format = audioEncoding.toLowerCase().replace('_', '') + return { audioBuffer, format, mimeType: getTtsMimeType(format) } +} + +export async function synthesizeAzure( + input: AzureTtsOperationInput, + signal?: AbortSignal +): Promise { + assertTextWithinLimit(input.text) + const voiceId = input.voiceId || 'en-US-JennyNeural' + const region = input.region || 'eastus' + const outputFormat = input.outputFormat || 'audio-24khz-96kbitrate-mono-mp3' + const regionPattern = /^[a-z][a-z0-9-]{1,30}[a-z0-9]$/ + if (!regionPattern.test(region)) { + throw new Error( + 'Invalid Azure region: must match /^[a-z][a-z0-9-]{1,30}[a-z0-9]$/ (e.g. eastus, westeurope)' + ) + } + let ssml = `` + if (input.style) { + ssml += ` { + assertTextWithinLimit(input.text) + const format = input.outputFormat || 'mp3' + const requestBody: Record = { + text: input.text, + quality: input.quality || 'standard', + output_format: format, + speed: input.speed ?? 1, + } + if (input.voice) requestBody.voice = input.voice + if (input.temperature !== undefined) requestBody.temperature = input.temperature + if (input.voiceGuidance !== undefined) requestBody.voice_guidance = input.voiceGuidance + if (input.textGuidance !== undefined) requestBody.text_guidance = input.textGuidance + if (input.sampleRate) requestBody.sample_rate = input.sampleRate + const response = await providerFetch( + 'https://api.play.ht/api/v2/tts/stream', + { + method: 'POST', + headers: { + AUTHORIZATION: input.apiKey, + 'X-USER-ID': input.userId, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(requestBody), + }, + signal + ) + if (!response.ok) { + const error = await readTtsErrorJson(response, 'PlayHT TTS error response', signal) + throw new Error(`PlayHT TTS API error: ${getTtsErrorMessage(error, response.statusText)}`) + } + return { + audioBuffer: await readAudio(response, 'PlayHT TTS audio response', signal), + format, + mimeType: getTtsMimeType(format), + } +} diff --git a/apps/sim/lib/internal/tts/errors.ts b/apps/sim/lib/internal/tts/errors.ts new file mode 100644 index 00000000000..b0fe921a5b2 --- /dev/null +++ b/apps/sim/lib/internal/tts/errors.ts @@ -0,0 +1,10 @@ +export class TtsOperationError extends Error { + constructor( + message: string, + readonly status: number, + readonly body: Record = { error: message } + ) { + super(message) + this.name = 'TtsOperationError' + } +} diff --git a/apps/sim/lib/internal/tts/execute-tool.test.ts b/apps/sim/lib/internal/tts/execute-tool.test.ts new file mode 100644 index 00000000000..253e9572f8a --- /dev/null +++ b/apps/sim/lib/internal/tts/execute-tool.test.ts @@ -0,0 +1,157 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operations = vi.hoisted(() => ({ + executeAzureTts: vi.fn(), + executeCartesiaTts: vi.fn(), + executeDeepgramTts: vi.fn(), + executeElevenLabsTts: vi.fn(), + executeGoogleTts: vi.fn(), + executeLegacyElevenLabsTts: vi.fn(), + executeOpenAiTts: vi.fn(), + executePlayHtTts: vi.fn(), +})) + +vi.mock('@/lib/internal/tts/operations', () => operations) + +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' +import { TtsOperationError } from '@/lib/internal/tts/errors' +import { executeTtsTool } from '@/lib/internal/tts/execute-tool' + +function request(overrides: Partial = {}): InternalToolOperationCall { + return { + toolId: 'tts_openai', + input: { text: 'Hello', apiKey: 'key' }, + headers: new Headers(), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + executionId: 'execution-1', + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } as InternalToolOperationCall +} + +const CASES = [ + [ + 'elevenlabs_tts', + { text: 'Hello', apiKey: 'key', voiceId: 'voice_1' }, + operations.executeLegacyElevenLabsTts, + ], + ['tts_openai', { text: 'Hello', apiKey: 'key' }, operations.executeOpenAiTts], + ['tts_deepgram', { text: 'Hello', apiKey: 'key' }, operations.executeDeepgramTts], + [ + 'tts_elevenlabs', + { text: 'Hello', apiKey: 'key', voiceId: 'voice_1' }, + operations.executeElevenLabsTts, + ], + ['tts_cartesia', { text: 'Hello', apiKey: 'key' }, operations.executeCartesiaTts], + ['tts_google', { text: 'Hello', apiKey: 'key' }, operations.executeGoogleTts], + ['tts_azure', { text: 'Hello', apiKey: 'key' }, operations.executeAzureTts], + [ + 'tts_playht', + { text: 'Hello', apiKey: 'key', userId: 'playht-user' }, + operations.executePlayHtTts, + ], +] as const + +describe('executeTtsTool', () => { + beforeEach(() => vi.clearAllMocks()) + + it.each(CASES)( + 'dispatches %s with trusted scope and cancellation', + async (toolId, input, operation) => { + const controller = new AbortController() + operation.mockResolvedValue({ audioUrl: 'https://audio.example/file.mp3' }) + + const response = await executeTtsTool(request({ toolId, input, signal: controller.signal })) + + expect(response.status).toBe(200) + expect(operation).toHaveBeenCalledWith(input, { + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }) + } + ) + + it('authenticates before validating provider input', async () => { + const response = await executeTtsTool( + request({ + input: null, + context: createExecutionContext({ workflowId: 'workflow-1' }), + }) + ) + + expect(response.status).toBe(401) + await expect(response.json()).resolves.toEqual({ error: 'Unauthorized' }) + expect(operations.executeOpenAiTts).not.toHaveBeenCalled() + }) + + it('preserves unified provider-specific required errors', async () => { + const elevenLabs = await executeTtsTool( + request({ toolId: 'tts_elevenlabs', input: { text: 'Hello', apiKey: 'key' } }) + ) + expect(elevenLabs.status).toBe(400) + await expect(elevenLabs.json()).resolves.toEqual({ + error: 'voiceId is required for ElevenLabs provider', + }) + + const playHt = await executeTtsTool( + request({ toolId: 'tts_playht', input: { text: 'Hello', apiKey: 'key' } }) + ) + expect(playHt.status).toBe(400) + await expect(playHt.json()).resolves.toEqual({ + error: 'userId is required for PlayHT provider', + }) + }) + + it('preserves unified contract validation details for common required fields', async () => { + const response = await executeTtsTool(request({ input: {} })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Missing required fields: provider, text, and apiKey', + details: expect.any(Array), + }) + }) + + it('preserves legacy validation and provider status envelopes', async () => { + const invalid = await executeTtsTool( + request({ toolId: 'elevenlabs_tts', input: { text: 'Hello', apiKey: 'key' } }) + ) + expect(invalid.status).toBe(400) + await expect(invalid.json()).resolves.toEqual({ error: 'Missing required parameters' }) + + operations.executeLegacyElevenLabsTts.mockRejectedValue( + new TtsOperationError('Failed to generate TTS: 429 Too Many Requests', 429) + ) + const limited = await executeTtsTool( + request({ + toolId: 'elevenlabs_tts', + input: { text: 'Hello', apiKey: 'key', voiceId: 'voice_1' }, + }) + ) + expect(limited.status).toBe(429) + await expect(limited.json()).resolves.toEqual({ + error: 'Failed to generate TTS: 429 Too Many Requests', + }) + }) + + it('stops before provider work after cancellation', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + await expect(executeTtsTool(request({ signal: controller.signal }))).rejects.toMatchObject({ + name: 'AbortError', + }) + expect(operations.executeOpenAiTts).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/tts/execute-tool.ts b/apps/sim/lib/internal/tts/execute-tool.ts new file mode 100644 index 00000000000..d7c3f8d40d9 --- /dev/null +++ b/apps/sim/lib/internal/tts/execute-tool.ts @@ -0,0 +1,212 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { z } from 'zod' +import { getValidationErrorMessage } from '@/lib/api/server' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import type { + InternalToolOperationCall, + InternalToolOperationHandler, +} from '@/lib/internal/tool-operations/types' +import { TtsOperationError } from '@/lib/internal/tts/errors' +import { + executeAzureTts, + executeCartesiaTts, + executeDeepgramTts, + executeElevenLabsTts, + executeGoogleTts, + executeLegacyElevenLabsTts, + executeOpenAiTts, + executePlayHtTts, + type TtsOperationContext, +} from '@/lib/internal/tts/operations' + +const auth = { + text: z + .string({ error: 'Missing required fields: provider, text, and apiKey' }) + .min(1, 'Missing required fields: provider, text, and apiKey'), + apiKey: z + .string({ error: 'Missing required fields: provider, text, and apiKey' }) + .min(1, 'Missing required fields: provider, text, and apiKey'), +} + +const schemas = { + elevenlabs_tts: z.object({ + ...auth, + voiceId: z.string().min(1), + modelId: z.string().optional(), + stability: z.coerce.number().min(0).max(1).optional(), + similarityBoost: z.coerce.number().min(0).max(1).optional(), + }), + tts_openai: z.object({ + ...auth, + model: z.enum(['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts']).optional(), + voice: z.string().optional(), + responseFormat: z.enum(['mp3', 'opus', 'aac', 'flac', 'wav', 'pcm']).optional(), + speed: z.coerce.number().optional(), + }), + tts_deepgram: z.object({ + ...auth, + model: z.string().optional(), + voice: z.string().optional(), + encoding: z.enum(['linear16', 'mp3', 'opus', 'aac', 'flac', 'mulaw', 'alaw']).optional(), + sampleRate: z.coerce.number().optional(), + bitRate: z.coerce.number().optional(), + container: z.enum(['none', 'wav', 'ogg']).optional(), + }), + tts_elevenlabs: z.object({ + ...auth, + voiceId: z.string().min(1), + modelId: z.string().optional(), + stability: z.coerce.number().optional(), + similarityBoost: z.coerce.number().optional(), + style: z.coerce.number().optional(), + useSpeakerBoost: z.boolean().optional(), + }), + tts_cartesia: z.object({ + ...auth, + modelId: z.string().optional(), + voice: z.string().optional(), + language: z.string().optional(), + outputFormat: z + .union([z.record(z.string(), z.unknown()), z.string()]) + .optional() + .nullable(), + speed: z.coerce.number().optional(), + emotion: z.array(z.string()).optional(), + }), + tts_google: z.object({ + ...auth, + voiceId: z.string().optional(), + languageCode: z.string().optional(), + gender: z.enum(['MALE', 'FEMALE', 'NEUTRAL']).optional(), + audioEncoding: z.enum(['LINEAR16', 'MP3', 'OGG_OPUS', 'MULAW', 'ALAW']).optional(), + speakingRate: z.coerce.number().optional(), + pitch: z.coerce.number().optional(), + volumeGainDb: z.coerce.number().optional(), + sampleRateHertz: z.coerce.number().optional(), + effectsProfileId: z.array(z.string()).optional(), + }), + tts_azure: z.object({ + ...auth, + voiceId: z.string().optional(), + region: z + .string() + .regex( + /^[a-z][a-z0-9-]{1,30}[a-z0-9]$/, + 'region must be a valid Azure region identifier (e.g. eastus, westeurope)' + ) + .optional(), + outputFormat: z.string().optional(), + rate: z.string().optional(), + pitch: z.union([z.number(), z.string()]).optional(), + style: z.union([z.number(), z.string()]).optional(), + styleDegree: z.coerce.number().optional(), + role: z.string().optional(), + }), + tts_playht: z.object({ + ...auth, + userId: z.string().min(1), + voice: z.string().optional(), + quality: z.enum(['draft', 'standard', 'premium']).optional(), + outputFormat: z.enum(['mp3', 'wav', 'ogg', 'flac', 'mulaw']).optional(), + speed: z.coerce.number().optional(), + temperature: z.coerce.number().optional(), + voiceGuidance: z.coerce.number().optional(), + textGuidance: z.coerce.number().optional(), + sampleRate: z.coerce.number().optional(), + }), +} as const + +type TtsToolId = keyof typeof schemas + +function isTtsToolId(toolId: string): toolId is TtsToolId { + return Object.hasOwn(schemas, toolId) +} + +function requiredInputError(toolId: TtsToolId, input: unknown): string | undefined { + if (!input || typeof input !== 'object' || Array.isArray(input)) return undefined + const value = input as Record + if (toolId === 'elevenlabs_tts') { + if (!value.text || !value.voiceId || !value.apiKey) return 'Missing required parameters' + return undefined + } + if (toolId === 'tts_elevenlabs' && !value.voiceId) { + return 'voiceId is required for ElevenLabs provider' + } + if (toolId === 'tts_playht' && !value.userId) { + return 'userId is required for PlayHT provider' + } + return undefined +} + +async function executeOperation( + schema: z.ZodType, + request: InternalToolOperationCall, + execute: (input: I, context: TtsOperationContext) => Promise, + legacy = false +): Promise { + request.signal?.throwIfAborted() + const requiredError = requiredInputError(request.toolId as TtsToolId, request.input) + if (requiredError) return Response.json({ error: requiredError }, { status: 400 }) + const parsed = schema.safeParse(request.input) + if (!parsed.success) { + const error = getValidationErrorMessage( + parsed.error, + legacy ? 'Missing required parameters' : 'Invalid request data' + ) + return Response.json(legacy ? { error } : { error, details: parsed.error.issues }, { + status: 400, + }) + } + const userId = request.context.userId + if (!userId) return Response.json({ error: 'Unauthorized' }, { status: 401 }) + try { + const result = await execute(parsed.data, { + requestId: request.requestId, + signal: request.signal, + userId, + workspaceId: request.context.workspaceId, + workflowId: request.context.workflowId, + executionId: request.context.executionId, + }) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof TtsOperationError) { + return Response.json(error.body, { status: error.status }) + } + const message = getErrorMessage(error, legacy ? 'Unknown error' : 'TTS synthesis failed') + return Response.json( + { error: legacy ? `Internal Server Error: ${message}` : message }, + { status: isPayloadSizeLimitError(error) ? 413 : 500 } + ) + } +} + +export const executeTtsTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!request.context.userId) { + return Response.json({ error: 'Unauthorized' }, { status: 401 }) + } + if (!isTtsToolId(request.toolId)) { + return Response.json({ error: `Unsupported TTS tool: ${request.toolId}` }, { status: 500 }) + } + switch (request.toolId) { + case 'elevenlabs_tts': + return executeOperation(schemas.elevenlabs_tts, request, executeLegacyElevenLabsTts, true) + case 'tts_openai': + return executeOperation(schemas.tts_openai, request, executeOpenAiTts) + case 'tts_deepgram': + return executeOperation(schemas.tts_deepgram, request, executeDeepgramTts) + case 'tts_elevenlabs': + return executeOperation(schemas.tts_elevenlabs, request, executeElevenLabsTts) + case 'tts_cartesia': + return executeOperation(schemas.tts_cartesia, request, executeCartesiaTts) + case 'tts_google': + return executeOperation(schemas.tts_google, request, executeGoogleTts) + case 'tts_azure': + return executeOperation(schemas.tts_azure, request, executeAzureTts) + case 'tts_playht': + return executeOperation(schemas.tts_playht, request, executePlayHtTts) + } +} diff --git a/apps/sim/lib/internal/tts/formats.ts b/apps/sim/lib/internal/tts/formats.ts new file mode 100644 index 00000000000..e7c41b149d3 --- /dev/null +++ b/apps/sim/lib/internal/tts/formats.ts @@ -0,0 +1,33 @@ +const AUDIO_MIME_TYPES: Readonly> = { + mp3: 'audio/mpeg', + opus: 'audio/opus', + aac: 'audio/aac', + flac: 'audio/flac', + wav: 'audio/wav', + pcm: 'audio/pcm', + linear16: 'audio/pcm', + mulaw: 'audio/basic', + alaw: 'audio/basic', + ogg: 'audio/ogg', +} + +const AUDIO_FILE_EXTENSIONS: Readonly> = { + mp3: 'mp3', + opus: 'opus', + aac: 'aac', + flac: 'flac', + wav: 'wav', + pcm: 'pcm', + linear16: 'wav', + mulaw: 'wav', + alaw: 'wav', + ogg: 'ogg', +} + +export function getTtsFileExtension(format: string): string { + return AUDIO_FILE_EXTENSIONS[format] || 'mp3' +} + +export function getTtsMimeType(format: string): string { + return AUDIO_MIME_TYPES[format] || 'audio/mpeg' +} diff --git a/apps/sim/lib/internal/tts/operations.test.ts b/apps/sim/lib/internal/tts/operations.test.ts new file mode 100644 index 00000000000..2da3b55eccd --- /dev/null +++ b/apps/sim/lib/internal/tts/operations.test.ts @@ -0,0 +1,185 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + synthesizeAzure: vi.fn(), + synthesizeCartesia: vi.fn(), + synthesizeDeepgram: vi.fn(), + synthesizeElevenLabs: vi.fn(), + synthesizeGoogle: vi.fn(), + synthesizeLegacyElevenLabs: vi.fn(), + synthesizeOpenAi: vi.fn(), + synthesizePlayHt: vi.fn(), + uploadExecutionFile: vi.fn(), + uploadFile: vi.fn(), +})) + +vi.mock('@/lib/internal/tts/client', () => ({ + synthesizeAzure: mocks.synthesizeAzure, + synthesizeCartesia: mocks.synthesizeCartesia, + synthesizeDeepgram: mocks.synthesizeDeepgram, + synthesizeElevenLabs: mocks.synthesizeElevenLabs, + synthesizeGoogle: mocks.synthesizeGoogle, + synthesizeLegacyElevenLabs: mocks.synthesizeLegacyElevenLabs, + synthesizeOpenAi: mocks.synthesizeOpenAi, + synthesizePlayHt: mocks.synthesizePlayHt, +})) +vi.mock('@/lib/uploads/contexts/execution', () => ({ + uploadExecutionFile: mocks.uploadExecutionFile, +})) +vi.mock('@/lib/uploads', () => ({ + StorageService: { uploadFile: mocks.uploadFile }, +})) +vi.mock('@/lib/core/utils/urls', () => ({ getBaseUrl: () => 'https://sim.example' })) + +import { + executeAzureTts, + executeCartesiaTts, + executeDeepgramTts, + executeElevenLabsTts, + executeGoogleTts, + executeLegacyElevenLabsTts, + executeOpenAiTts, + executePlayHtTts, +} from '@/lib/internal/tts/operations' + +const CONTEXT = { + requestId: 'request-1', + userId: 'user-1', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', +} + +const AUDIO = { + audioBuffer: Buffer.from('audio'), + format: 'mp3', + mimeType: 'audio/mpeg', +} + +describe('TTS operations', () => { + beforeEach(() => { + vi.clearAllMocks() + for (const synthesize of [ + mocks.synthesizeAzure, + mocks.synthesizeCartesia, + mocks.synthesizeDeepgram, + mocks.synthesizeElevenLabs, + mocks.synthesizeGoogle, + mocks.synthesizeLegacyElevenLabs, + mocks.synthesizeOpenAi, + mocks.synthesizePlayHt, + ]) { + synthesize.mockResolvedValue(AUDIO) + } + mocks.uploadExecutionFile.mockResolvedValue({ + id: 'file-1', + name: 'speech.mp3', + url: 'https://execution.example/speech.mp3', + size: AUDIO.audioBuffer.length, + type: AUDIO.mimeType, + key: 'execution/speech.mp3', + }) + mocks.uploadFile.mockResolvedValue({ + key: 'copilot/speech.mp3', + path: '/api/files/serve/copilot/speech.mp3', + size: AUDIO.audioBuffer.length, + }) + }) + + it.each([ + [executeOpenAiTts, mocks.synthesizeOpenAi, { text: 'Hello', apiKey: 'key' }, 'openai'], + [executeDeepgramTts, mocks.synthesizeDeepgram, { text: 'Hello', apiKey: 'key' }, 'deepgram'], + [ + executeElevenLabsTts, + mocks.synthesizeElevenLabs, + { text: 'Hello', apiKey: 'key', voiceId: 'voice_1' }, + 'elevenlabs', + ], + [executeCartesiaTts, mocks.synthesizeCartesia, { text: 'Hello', apiKey: 'key' }, 'cartesia'], + [executeGoogleTts, mocks.synthesizeGoogle, { text: 'Hello', apiKey: 'key' }, 'google'], + [executeAzureTts, mocks.synthesizeAzure, { text: 'Hello', apiKey: 'key' }, 'azure'], + [ + executePlayHtTts, + mocks.synthesizePlayHt, + { text: 'Hello', apiKey: 'key', userId: 'playht-user' }, + 'playht', + ], + ] as const)( + 'selects the provider client and stores %s output in trusted execution scope', + async (execute, synthesize, input, provider) => { + const controller = new AbortController() + const result = await execute(input, { ...CONTEXT, signal: controller.signal }) + + expect(synthesize).toHaveBeenCalledWith(input, controller.signal) + expect(mocks.uploadExecutionFile).toHaveBeenCalledWith( + { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }, + AUDIO.audioBuffer, + expect.stringMatching(new RegExp(`^tts-${provider}-\\d+\\.mp3$`)), + AUDIO.mimeType, + 'user-1' + ) + expect(result).toMatchObject({ + audioUrl: 'https://execution.example/speech.mp3', + audioFile: { id: 'file-1' }, + characterCount: 5, + format: 'mp3', + provider, + }) + } + ) + + it('uses copilot storage when no complete execution scope exists', async () => { + const result = await executeOpenAiTts( + { text: 'Hello', apiKey: 'key' }, + { requestId: 'request-1', userId: 'user-1' } + ) + + expect(mocks.uploadExecutionFile).not.toHaveBeenCalled() + expect(mocks.uploadFile).toHaveBeenCalledWith( + expect.objectContaining({ context: 'copilot', file: AUDIO.audioBuffer }) + ) + expect(result).toEqual({ + audioUrl: 'https://sim.example/api/files/serve/copilot/speech.mp3', + characterCount: 5, + format: 'mp3', + provider: 'openai', + }) + }) + + it('preserves the legacy ElevenLabs output contract', async () => { + const result = await executeLegacyElevenLabsTts( + { text: 'Hello', apiKey: 'key', voiceId: 'voice_1' }, + CONTEXT + ) + + expect(mocks.synthesizeLegacyElevenLabs).toHaveBeenCalledWith( + { text: 'Hello', apiKey: 'key', voiceId: 'voice_1' }, + undefined + ) + expect(result).toEqual({ + audioFile: expect.objectContaining({ id: 'file-1' }), + audioUrl: 'https://execution.example/speech.mp3', + }) + }) + + it('does not start storage after synthesis cancellation', async () => { + const controller = new AbortController() + mocks.synthesizeOpenAi.mockImplementation(async () => { + controller.abort(new DOMException('cancelled', 'AbortError')) + return AUDIO + }) + + await expect( + executeOpenAiTts({ text: 'Hello', apiKey: 'key' }, { ...CONTEXT, signal: controller.signal }) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mocks.uploadExecutionFile).not.toHaveBeenCalled() + expect(mocks.uploadFile).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/tts/operations.ts b/apps/sim/lib/internal/tts/operations.ts new file mode 100644 index 00000000000..d38db9670d4 --- /dev/null +++ b/apps/sim/lib/internal/tts/operations.ts @@ -0,0 +1,224 @@ +import { createLogger } from '@sim/logger' +import { getBaseUrl } from '@/lib/core/utils/urls' +import { + type AzureTtsOperationInput, + type CartesiaTtsOperationInput, + type OpenAiTtsOperationInput, + synthesizeAzure, + synthesizeCartesia, + synthesizeDeepgram, + synthesizeElevenLabs, + synthesizeGoogle, + synthesizeLegacyElevenLabs, + synthesizeOpenAi, + synthesizePlayHt, + type TtsAudioResult, +} from '@/lib/internal/tts/client' +import { getTtsFileExtension } from '@/lib/internal/tts/formats' +import { StorageService } from '@/lib/uploads' +import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' +import type { ElevenLabsTtsParams } from '@/tools/elevenlabs/types' +import type { + DeepgramTtsParams, + ElevenLabsTtsUnifiedParams, + GoogleTtsParams, + PlayHtTtsParams, + TtsProvider, + TtsResponse, +} from '@/tools/tts/types' + +const logger = createLogger('TtsOperations') + +export interface TtsOperationContext { + requestId: string + signal?: AbortSignal + userId: string + workspaceId?: string + workflowId?: string + executionId?: string +} + +async function storeUnifiedAudio( + provider: TtsProvider, + text: string, + audio: TtsAudioResult, + context: TtsOperationContext +): Promise { + context.signal?.throwIfAborted() + const fileName = `tts-${provider}-${Date.now()}.${getTtsFileExtension(audio.format)}` + if (context.workspaceId && context.workflowId && context.executionId) { + const audioFile = await uploadExecutionFile( + { + workspaceId: context.workspaceId, + workflowId: context.workflowId, + executionId: context.executionId, + }, + audio.audioBuffer, + fileName, + audio.mimeType, + context.userId + ) + context.signal?.throwIfAborted() + logger.info('Stored TTS audio in execution context', { + requestId: context.requestId, + provider, + executionId: context.executionId, + fileName, + size: audioFile.size, + }) + return { + audioUrl: audioFile.url, + audioFile, + characterCount: text.length, + format: audio.format, + provider, + ...(audio.duration ? { duration: audio.duration } : {}), + } + } + + const file = await StorageService.uploadFile({ + file: audio.audioBuffer, + fileName, + contentType: audio.mimeType, + context: 'copilot', + }) + context.signal?.throwIfAborted() + logger.info('Stored TTS audio in copilot context', { + requestId: context.requestId, + provider, + fileName, + size: file.size, + }) + return { + audioUrl: `${getBaseUrl()}${file.path}`, + characterCount: text.length, + format: audio.format, + provider, + ...(audio.duration ? { duration: audio.duration } : {}), + } +} + +async function storeLegacyElevenLabsAudio( + audio: TtsAudioResult, + context: TtsOperationContext +): Promise> { + context.signal?.throwIfAborted() + const fileName = `tts-${Date.now()}.mp3` + if (context.workspaceId && context.workflowId && context.executionId) { + const audioFile = await uploadExecutionFile( + { + workspaceId: context.workspaceId, + workflowId: context.workflowId, + executionId: context.executionId, + }, + audio.audioBuffer, + fileName, + audio.mimeType, + context.userId + ) + context.signal?.throwIfAborted() + return { audioFile, audioUrl: audioFile.url } + } + const file = await StorageService.uploadFile({ + file: audio.audioBuffer, + fileName, + contentType: audio.mimeType, + context: 'copilot', + }) + context.signal?.throwIfAborted() + return { audioUrl: `${getBaseUrl()}${file.path}`, size: file.size } +} + +export async function executeOpenAiTts( + input: OpenAiTtsOperationInput, + context: TtsOperationContext +): Promise { + return storeUnifiedAudio( + 'openai', + input.text, + await synthesizeOpenAi(input, context.signal), + context + ) +} + +export async function executeDeepgramTts( + input: DeepgramTtsParams, + context: TtsOperationContext +): Promise { + return storeUnifiedAudio( + 'deepgram', + input.text, + await synthesizeDeepgram(input, context.signal), + context + ) +} + +export async function executeElevenLabsTts( + input: ElevenLabsTtsUnifiedParams, + context: TtsOperationContext +): Promise { + return storeUnifiedAudio( + 'elevenlabs', + input.text, + await synthesizeElevenLabs(input, context.signal), + context + ) +} + +export async function executeLegacyElevenLabsTts( + input: ElevenLabsTtsParams, + context: TtsOperationContext +): Promise> { + return storeLegacyElevenLabsAudio( + await synthesizeLegacyElevenLabs(input, context.signal), + context + ) +} + +export async function executeCartesiaTts( + input: CartesiaTtsOperationInput, + context: TtsOperationContext +): Promise { + return storeUnifiedAudio( + 'cartesia', + input.text, + await synthesizeCartesia(input, context.signal), + context + ) +} + +export async function executeGoogleTts( + input: GoogleTtsParams, + context: TtsOperationContext +): Promise { + return storeUnifiedAudio( + 'google', + input.text, + await synthesizeGoogle(input, context.signal), + context + ) +} + +export async function executeAzureTts( + input: AzureTtsOperationInput, + context: TtsOperationContext +): Promise { + return storeUnifiedAudio( + 'azure', + input.text, + await synthesizeAzure(input, context.signal), + context + ) +} + +export async function executePlayHtTts( + input: PlayHtTtsParams, + context: TtsOperationContext +): Promise { + return storeUnifiedAudio( + 'playht', + input.text, + await synthesizePlayHt(input, context.signal), + context + ) +} diff --git a/apps/sim/lib/internal/twilio-voice/errors.ts b/apps/sim/lib/internal/twilio-voice/errors.ts new file mode 100644 index 00000000000..fd3412dc9d1 --- /dev/null +++ b/apps/sim/lib/internal/twilio-voice/errors.ts @@ -0,0 +1,9 @@ +export class TwilioVoiceOperationError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'TwilioVoiceOperationError' + } +} diff --git a/apps/sim/lib/internal/twilio-voice/execute-tool.test.ts b/apps/sim/lib/internal/twilio-voice/execute-tool.test.ts new file mode 100644 index 00000000000..0270e2af21f --- /dev/null +++ b/apps/sim/lib/internal/twilio-voice/execute-tool.test.ts @@ -0,0 +1,40 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ getTwilioRecording: vi.fn() })) + +vi.mock('@/lib/internal/twilio-voice/operations', () => ({ + getTwilioRecording: mocks.getTwilioRecording, +})) + +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' +import { executeTwilioVoiceTool } from '@/lib/internal/twilio-voice/execute-tool' + +describe('executeTwilioVoiceTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.getTwilioRecording.mockResolvedValue({ success: true, output: { success: true } }) + }) + + it('dispatches typed input with cancellation', async () => { + const controller = new AbortController() + const input = { accountSid: 'AC123', authToken: 'secret', recordingSid: 'RE123' } + const request: InternalToolOperationCall = { + toolId: 'twilio_voice_get_recording', + input, + headers: new Headers(), + context: createExecutionContext(), + requestId: 'request-1', + signal: controller.signal, + } + + expect((await executeTwilioVoiceTool(request)).status).toBe(200) + expect(mocks.getTwilioRecording).toHaveBeenCalledWith(input, { + requestId: 'request-1', + signal: controller.signal, + }) + }) +}) diff --git a/apps/sim/lib/internal/twilio-voice/execute-tool.ts b/apps/sim/lib/internal/twilio-voice/execute-tool.ts new file mode 100644 index 00000000000..8b9a555b9f4 --- /dev/null +++ b/apps/sim/lib/internal/twilio-voice/execute-tool.ts @@ -0,0 +1,45 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { z } from 'zod' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { TwilioVoiceOperationError } from '@/lib/internal/twilio-voice/errors' +import { getTwilioRecording } from '@/lib/internal/twilio-voice/operations' + +const inputSchema = z.object({ + accountSid: z.string().min(1, 'Account SID is required'), + authToken: z.string().min(1, 'Auth token is required'), + recordingSid: z.string().min(1, 'Recording SID is required'), +}) + +export const executeTwilioVoiceTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'twilio_voice_get_recording') { + return Response.json( + { success: false, error: `Unsupported Twilio Voice tool: ${request.toolId}` }, + { status: 500 } + ) + } + const parsed = inputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + try { + return Response.json( + await getTwilioRecording(parsed.data, { + requestId: request.requestId, + signal: request.signal, + }) + ) + } catch (error) { + request.signal?.throwIfAborted() + const status = isPayloadSizeLimitError(error) + ? 413 + : error instanceof TwilioVoiceOperationError + ? error.status + : 500 + return Response.json( + { success: false, error: getErrorMessage(error, 'Unknown error occurred') }, + { status } + ) + } +} diff --git a/apps/sim/lib/internal/twilio-voice/operations.test.ts b/apps/sim/lib/internal/twilio-voice/operations.test.ts new file mode 100644 index 00000000000..dd952e46dbb --- /dev/null +++ b/apps/sim/lib/internal/twilio-voice/operations.test.ts @@ -0,0 +1,67 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + secureFetchWithPinnedIP: vi.fn(), + validateUrlWithDNS: vi.fn(), +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + secureFetchWithPinnedIP: mocks.secureFetchWithPinnedIP, + validateUrlWithDNS: mocks.validateUrlWithDNS, +})) + +import { getTwilioRecording } from '@/lib/internal/twilio-voice/operations' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' + +describe('getTwilioRecording', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.validateUrlWithDNS.mockResolvedValue({ isValid: true, resolvedIP: '203.0.113.1' }) + mocks.secureFetchWithPinnedIP + .mockResolvedValueOnce( + Response.json({ + sid: 'RE123', + call_sid: 'CA123', + duration: '42', + status: 'completed', + channels: 1, + source: 'RecordVerb', + uri: '/2010-04-01/Accounts/AC123/Recordings/RE123.json', + }) + ) + .mockResolvedValueOnce( + Response.json({ + transcriptions: [{ transcription_text: 'hello', status: 'completed' }], + }) + ) + .mockResolvedValueOnce( + new Response(new Uint8Array([1, 2, 3]), { headers: { 'content-type': 'audio/mpeg' } }) + ) + }) + + it('pins all provider requests and bounds the recording media', async () => { + const controller = new AbortController() + const result = await getTwilioRecording( + { accountSid: 'AC123', authToken: 'secret', recordingSid: 'RE123' }, + { requestId: 'request-1', signal: controller.signal } + ) + + expect(mocks.secureFetchWithPinnedIP).toHaveBeenCalledTimes(3) + expect(mocks.secureFetchWithPinnedIP.mock.calls[2][2]).toEqual( + expect.objectContaining({ + maxResponseBytes: MAX_BUFFERED_TRANSFER_BYTES, + signal: controller.signal, + }) + ) + expect(result.output).toEqual( + expect.objectContaining({ + duration: 42, + transcriptionText: 'hello', + file: expect.objectContaining({ name: 'RE123.mp3', data: 'AQID', size: 3 }), + }) + ) + }) +}) diff --git a/apps/sim/lib/internal/twilio-voice/operations.ts b/apps/sim/lib/internal/twilio-voice/operations.ts new file mode 100644 index 00000000000..a94793bcf1c --- /dev/null +++ b/apps/sim/lib/internal/twilio-voice/operations.ts @@ -0,0 +1,187 @@ +import { createLogger } from '@sim/logger' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + readResponseJsonWithLimit, + readResponseToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' +import { TwilioVoiceOperationError } from '@/lib/internal/twilio-voice/errors' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { getExtensionFromMimeType } from '@/lib/uploads/utils/file-utils' +import type { TwilioGetRecordingOutput, TwilioGetRecordingParams } from '@/tools/twilio_voice/types' + +const logger = createLogger('TwilioGetRecordingOperation') +const MAX_TWILIO_JSON_BYTES = 2 * 1024 * 1024 + +interface TwilioRecordingResponse { + sid?: string + call_sid?: string + duration?: string + status?: string + channels?: number + source?: string + price?: string + price_unit?: string + uri?: string + error_code?: number + message?: string + error_message?: string +} + +interface TwilioTranscription { + transcription_text?: string + status?: string + price?: string + price_unit?: string +} + +export interface TwilioVoiceOperationContext { + requestId: string + signal?: AbortSignal +} + +async function fetchPinned( + url: string, + label: string, + authHeader: string, + context: TwilioVoiceOperationContext, + maxResponseBytes: number +) { + const validation = await validateUrlWithDNS(url, label) + context.signal?.throwIfAborted() + if (!validation.isValid || !validation.resolvedIP) { + throw new TwilioVoiceOperationError(validation.error || `Invalid ${label}`, 400) + } + return secureFetchWithPinnedIP(url, validation.resolvedIP, { + method: 'GET', + headers: { Authorization: authHeader }, + maxResponseBytes, + signal: context.signal, + }) +} + +export async function getTwilioRecording( + input: TwilioGetRecordingParams, + context: TwilioVoiceOperationContext +): Promise { + context.signal?.throwIfAborted() + if (!input.accountSid.startsWith('AC')) { + throw new TwilioVoiceOperationError( + `Invalid Account SID format. Account SID must start with "AC" (you provided: ${input.accountSid.substring(0, 2)}...)`, + 400 + ) + } + const authHeader = `Basic ${Buffer.from(`${input.accountSid}:${input.authToken}`).toString('base64')}` + const infoUrl = `https://api.twilio.com/2010-04-01/Accounts/${encodeURIComponent(input.accountSid)}/Recordings/${encodeURIComponent(input.recordingSid)}.json` + const infoResponse = await fetchPinned( + infoUrl, + 'infoUrl', + authHeader, + context, + MAX_TWILIO_JSON_BYTES + ) + if (!infoResponse.ok) { + const error = await readResponseJsonWithLimit<{ message?: string }>(infoResponse, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Twilio recording error response', + signal: context.signal, + }).catch(() => ({ message: undefined })) + throw new TwilioVoiceOperationError( + error.message || `Twilio API error: ${infoResponse.status}`, + 400 + ) + } + const data = await readResponseJsonWithLimit(infoResponse, { + maxBytes: MAX_TWILIO_JSON_BYTES, + label: 'Twilio recording response', + signal: context.signal, + }) + if (data.error_code) { + const error = data.message || data.error_message || 'Failed to retrieve recording' + return { success: false, output: { success: false, error } } + } + + const mediaUrl = data.uri ? `https://api.twilio.com${data.uri.replace('.json', '')}` : undefined + let transcription: TwilioTranscription | undefined + try { + const url = `https://api.twilio.com/2010-04-01/Accounts/${encodeURIComponent(input.accountSid)}/Transcriptions.json?RecordingSid=${encodeURIComponent(data.sid || input.recordingSid)}` + const response = await fetchPinned( + url, + 'transcriptionUrl', + authHeader, + context, + MAX_TWILIO_JSON_BYTES + ) + if (response.ok) { + const payload = await readResponseJsonWithLimit<{ + transcriptions?: TwilioTranscription[] + }>(response, { + maxBytes: MAX_TWILIO_JSON_BYTES, + label: 'Twilio transcription response', + signal: context.signal, + }) + transcription = payload.transcriptions?.[0] + } + } catch (error) { + context.signal?.throwIfAborted() + logger.warn('Failed to fetch Twilio transcription', { requestId: context.requestId, error }) + } + + let file: TwilioGetRecordingOutput['output']['file'] + if (mediaUrl) { + try { + const response = await fetchPinned( + mediaUrl, + 'mediaUrl', + authHeader, + context, + MAX_BUFFERED_TRANSFER_BYTES + ) + if (response.ok) { + const mimeType = response.headers.get('content-type') || 'application/octet-stream' + const buffer = await readResponseToBufferWithLimit(response, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + label: 'Twilio recording media', + signal: context.signal, + }) + file = { + name: `${data.sid || input.recordingSid}.${getExtensionFromMimeType(mimeType) || 'dat'}`, + mimeType, + data: buffer.toString('base64'), + size: buffer.length, + } + } + } catch (error) { + context.signal?.throwIfAborted() + logger.warn('Failed to download Twilio recording media', { + requestId: context.requestId, + error, + }) + } + } + + return { + success: true, + output: { + success: true, + recordingSid: data.sid, + callSid: data.call_sid, + duration: data.duration ? Number.parseInt(data.duration, 10) : undefined, + status: data.status, + channels: data.channels, + source: data.source, + mediaUrl, + file, + price: data.price, + priceUnit: data.price_unit, + uri: data.uri, + transcriptionText: transcription?.transcription_text, + transcriptionStatus: transcription?.status, + transcriptionPrice: transcription?.price, + transcriptionPriceUnit: transcription?.price_unit, + }, + } +} diff --git a/apps/sim/lib/internal/typeform/errors.ts b/apps/sim/lib/internal/typeform/errors.ts new file mode 100644 index 00000000000..1097e9eaea5 --- /dev/null +++ b/apps/sim/lib/internal/typeform/errors.ts @@ -0,0 +1,9 @@ +export class TypeformOperationError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'TypeformOperationError' + } +} diff --git a/apps/sim/lib/internal/typeform/execute-tool.ts b/apps/sim/lib/internal/typeform/execute-tool.ts new file mode 100644 index 00000000000..e811e06ea1c --- /dev/null +++ b/apps/sim/lib/internal/typeform/execute-tool.ts @@ -0,0 +1,55 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { z } from 'zod' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { TypeformOperationError } from '@/lib/internal/typeform/errors' +import { downloadTypeformFile } from '@/lib/internal/typeform/operations' + +const inputSchema = z.object({ + formId: z.string().min(1, 'Form ID is required'), + responseId: z.string().min(1, 'Response ID is required'), + fieldId: z.string().min(1, 'Field ID is required'), + filename: z.string().min(1, 'Filename is required'), + inline: z.boolean().optional(), + apiKey: z.string().min(1, 'API key is required'), +}) + +export const executeTypeformTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'typeform_files') { + return Response.json( + { success: false, error: `Unsupported Typeform tool: ${request.toolId}` }, + { status: 500 } + ) + } + const userId = request.context.userId + if (!userId) { + return Response.json({ success: false, error: 'Unauthorized' }, { status: 401 }) + } + const parsed = inputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + try { + return Response.json( + await downloadTypeformFile(parsed.data, { + userId, + workspaceId: request.context.workspaceId, + workflowId: request.context.workflowId, + executionId: request.context.executionId, + signal: request.signal, + }) + ) + } catch (error) { + request.signal?.throwIfAborted() + const status = isPayloadSizeLimitError(error) + ? 413 + : error instanceof TypeformOperationError + ? error.status + : 500 + return Response.json( + { success: false, error: getErrorMessage(error, 'Failed to download Typeform file') }, + { status } + ) + } +} diff --git a/apps/sim/lib/internal/typeform/operations.ts b/apps/sim/lib/internal/typeform/operations.ts new file mode 100644 index 00000000000..32ade526aeb --- /dev/null +++ b/apps/sim/lib/internal/typeform/operations.ts @@ -0,0 +1,112 @@ +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + readResponseTextWithLimit, + readResponseToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' +import { TypeformOperationError } from '@/lib/internal/typeform/errors' +import { uploadCopilotFile } from '@/lib/uploads/contexts/copilot' +import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' +import type { TypeformFilesParams, TypeformFilesResponse } from '@/tools/typeform/types' + +const MAX_TYPEFORM_FILE_BYTES = 10 * 1024 * 1024 + +export interface TypeformOperationContext { + userId: string + workspaceId?: string + workflowId?: string + executionId?: string + signal?: AbortSignal +} + +function buildTypeformFileUrl(input: TypeformFilesParams): string { + const url = new URL( + `https://api.typeform.com/forms/${encodeURIComponent(input.formId)}/responses/${encodeURIComponent(input.responseId)}/fields/${encodeURIComponent(input.fieldId)}/files/${encodeURIComponent(input.filename)}` + ) + if (input.inline !== undefined) url.searchParams.set('inline', String(input.inline)) + return url.toString() +} + +function getFilename( + response: { headers: { get(name: string): string | null } }, + fallback: string +): string { + const disposition = response.headers.get('content-disposition') || '' + return disposition.match(/filename="(.+?)"/)?.[1] || fallback || 'typeform-file' +} + +export async function downloadTypeformFile( + input: TypeformFilesParams, + context: TypeformOperationContext +): Promise { + context.signal?.throwIfAborted() + const fileUrl = buildTypeformFileUrl(input) + const validation = await validateUrlWithDNS(fileUrl, 'typeformFileUrl') + context.signal?.throwIfAborted() + if (!validation.isValid || !validation.resolvedIP) { + throw new TypeformOperationError(validation.error || 'Invalid Typeform file URL', 400) + } + const response = await secureFetchWithPinnedIP(fileUrl, validation.resolvedIP, { + headers: { Authorization: `Bearer ${input.apiKey}` }, + maxResponseBytes: MAX_TYPEFORM_FILE_BYTES, + signal: context.signal, + }) + if (!response.ok) { + const errorText = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Typeform file error response', + signal: context.signal, + }).catch(() => '') + throw new TypeformOperationError( + `Failed to download Typeform file: ${response.status} ${errorText}`, + response.status + ) + } + const buffer = await readResponseToBufferWithLimit(response, { + maxBytes: MAX_TYPEFORM_FILE_BYTES, + label: 'Typeform file download', + signal: context.signal, + }) + const contentType = response.headers.get('content-type') || 'application/octet-stream' + const filename = getFilename(response, input.filename) + context.signal?.throwIfAborted() + + if (context.workspaceId && context.workflowId && context.executionId) { + const file = await uploadExecutionFile( + { + workspaceId: context.workspaceId, + workflowId: context.workflowId, + executionId: context.executionId, + }, + buffer, + filename, + contentType, + context.userId + ) + context.signal?.throwIfAborted() + return { + success: true, + output: { + fileUrl: file.url, + file: { ...file, mimeType: contentType }, + contentType, + filename, + }, + } + } + + const file = await uploadCopilotFile({ + buffer, + fileName: filename, + contentType, + userId: context.userId, + }) + context.signal?.throwIfAborted() + return { + success: true, + output: { fileUrl: file.url || fileUrl, file, contentType, filename }, + } +} diff --git a/apps/sim/lib/internal/uptimerobot/client.ts b/apps/sim/lib/internal/uptimerobot/client.ts new file mode 100644 index 00000000000..cfa44d61505 --- /dev/null +++ b/apps/sim/lib/internal/uptimerobot/client.ts @@ -0,0 +1,69 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' +import { UptimeRobotOperationError } from '@/lib/internal/uptimerobot/errors' +import { mapPsp, UPTIMEROBOT_API_BASE, type UptimeRobotPsp } from '@/tools/uptimerobot/types' + +const logger = createLogger('UptimeRobotClient') + +function providerMessage(text: string, status: number): string { + try { + const parsed = JSON.parse(text) as { message?: unknown } + if (typeof parsed.message === 'string' && parsed.message) return parsed.message + } catch {} + return `UptimeRobot API error (HTTP ${status})` +} + +export async function requestUptimeRobotPsp(args: { + apiKey: string + method: 'POST' | 'PATCH' + path: string + form: FormData + signal?: AbortSignal +}): Promise { + const { apiKey, method, path, form, signal } = args + signal?.throwIfAborted() + const response = await fetch(`${UPTIMEROBOT_API_BASE}${path}`, { + method, + headers: { Authorization: `Bearer ${apiKey}`, Accept: 'application/json' }, + body: form, + signal, + }) + const text = await readResponseTextWithLimit(response, { + maxBytes: MAX_JSON_API_RESPONSE_BYTES, + label: 'UptimeRobot response', + signal, + }) + signal?.throwIfAborted() + + if (!response.ok) { + const message = providerMessage(text, response.status) + logger.error('UptimeRobot PSP request failed', { status: response.status, message }) + throw new UptimeRobotOperationError(message, response.status) + } + if (!text) { + logger.error('UptimeRobot returned an empty PSP response') + throw new UptimeRobotOperationError('UptimeRobot returned an unexpected response', 502) + } + + let data: Record + try { + const parsed = JSON.parse(text) + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error('Expected a PSP object response') + } + data = parsed as Record + } catch (error) { + logger.error('UptimeRobot returned an unexpected PSP response', { + error: getErrorMessage(error), + }) + throw new UptimeRobotOperationError('UptimeRobot returned an unexpected response', 502) + } + + if (typeof data.id !== 'number' || data.id < 1 || !data.friendlyName) { + logger.error('UptimeRobot returned a PSP response without core fields') + throw new UptimeRobotOperationError('UptimeRobot returned an unexpected response', 502) + } + return mapPsp(data) +} diff --git a/apps/sim/lib/internal/uptimerobot/errors.ts b/apps/sim/lib/internal/uptimerobot/errors.ts new file mode 100644 index 00000000000..0caa7f97a73 --- /dev/null +++ b/apps/sim/lib/internal/uptimerobot/errors.ts @@ -0,0 +1,9 @@ +export class UptimeRobotOperationError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'UptimeRobotOperationError' + } +} diff --git a/apps/sim/lib/internal/uptimerobot/execute-tool.test.ts b/apps/sim/lib/internal/uptimerobot/execute-tool.test.ts new file mode 100644 index 00000000000..e02bdf709bb --- /dev/null +++ b/apps/sim/lib/internal/uptimerobot/execute-tool.test.ts @@ -0,0 +1,107 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operations = vi.hoisted(() => ({ + createUptimeRobotPsp: vi.fn(), + updateUptimeRobotPsp: vi.fn(), +})) + +vi.mock('@/lib/internal/uptimerobot/operations', () => operations) + +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' +import { UptimeRobotOperationError } from '@/lib/internal/uptimerobot/errors' +import { executeUptimeRobotTool } from '@/lib/internal/uptimerobot/execute-tool' + +function request(overrides: Partial = {}): InternalToolOperationCall { + return { + toolId: 'uptimerobot_create_psp', + input: { apiKey: 'key', friendlyName: 'Status' }, + headers: new Headers(), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeUptimeRobotTool', () => { + beforeEach(() => { + vi.clearAllMocks() + operations.createUptimeRobotPsp.mockResolvedValue({ + success: true, + output: { psp: { id: 1, friendlyName: 'Status' } }, + }) + operations.updateUptimeRobotPsp.mockResolvedValue({ + success: true, + output: { psp: { id: 1, friendlyName: 'Updated' } }, + }) + }) + + it('dispatches create with trusted identity and cancellation', async () => { + const controller = new AbortController() + const input = { apiKey: 'key', friendlyName: 'Status' } + + const response = await executeUptimeRobotTool(request({ input, signal: controller.signal })) + + expect(response.status).toBe(200) + expect(operations.createUptimeRobotPsp).toHaveBeenCalledWith(input, { + userId: 'user-1', + requestId: 'request-1', + signal: controller.signal, + }) + }) + + it('dispatches update without HTTP-shaped request metadata', async () => { + const input = { apiKey: 'key', pspId: 1, friendlyName: 'Updated' } + + const response = await executeUptimeRobotTool( + request({ toolId: 'uptimerobot_update_psp', input }) + ) + + expect(response.status).toBe(200) + expect(operations.updateUptimeRobotPsp).toHaveBeenCalledWith(input, { + userId: 'user-1', + requestId: 'request-1', + signal: undefined, + }) + }) + + it('authenticates before validating input', async () => { + const response = await executeUptimeRobotTool( + request({ + input: null, + context: createExecutionContext({ workflowId: 'workflow-1' }), + }) + ) + + expect(response.status).toBe(401) + expect(operations.createUptimeRobotPsp).not.toHaveBeenCalled() + }) + + it('preserves operation error status and message', async () => { + operations.createUptimeRobotPsp.mockRejectedValue( + new UptimeRobotOperationError('provider limited', 429) + ) + + const response = await executeUptimeRobotTool(request()) + + expect(response.status).toBe(429) + await expect(response.json()).resolves.toEqual({ success: false, error: 'provider limited' }) + }) + + it('propagates cancellation before provider work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeUptimeRobotTool(request({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(operations.createUptimeRobotPsp).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/uptimerobot/execute-tool.ts b/apps/sim/lib/internal/uptimerobot/execute-tool.ts new file mode 100644 index 00000000000..aea8545a946 --- /dev/null +++ b/apps/sim/lib/internal/uptimerobot/execute-tool.ts @@ -0,0 +1,76 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { z } from 'zod' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { UptimeRobotOperationError } from '@/lib/internal/uptimerobot/errors' +import { createUptimeRobotPsp, updateUptimeRobotPsp } from '@/lib/internal/uptimerobot/operations' +import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' +import type { UptimeRobotPspResponse } from '@/tools/uptimerobot/types' + +const sharedFields = { + apiKey: z.string().min(1), + monitorIds: z.string().optional(), + status: z.enum(['ENABLED', 'PAUSED']).optional(), + password: z.string().max(255).optional(), + customDomain: z.string().max(255).optional(), + hideUrlLinks: z.boolean().optional(), + noIndex: z.boolean().optional(), + logo: FileInputSchema.optional(), + icon: FileInputSchema.optional(), +} + +const createPspSchema = z.object({ + ...sharedFields, + friendlyName: z.string().min(1).max(255), +}) + +const updatePspSchema = z.object({ + ...sharedFields, + pspId: z.number().int().min(1), + friendlyName: z.string().max(255).optional(), +}) + +export const executeUptimeRobotTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!request.context.userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + if (request.toolId !== 'uptimerobot_create_psp' && request.toolId !== 'uptimerobot_update_psp') { + return Response.json( + { success: false, error: `Unsupported UptimeRobot tool: ${request.toolId}` }, + { status: 500 } + ) + } + + try { + const context = { + userId: request.context.userId, + requestId: request.requestId, + signal: request.signal, + } + let result: UptimeRobotPspResponse + if (request.toolId === 'uptimerobot_create_psp') { + const parsed = createPspSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + result = await createUptimeRobotPsp(parsed.data, context) + } else { + const parsed = updatePspSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + result = await updateUptimeRobotPsp(parsed.data, context) + } + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof UptimeRobotOperationError) { + return Response.json({ success: false, error: error.message }, { status: error.status }) + } + return Response.json( + { success: false, error: getErrorMessage(error, 'Unknown error') }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/uptimerobot/file-input.ts b/apps/sim/lib/internal/uptimerobot/file-input.ts new file mode 100644 index 00000000000..6076593cc5d --- /dev/null +++ b/apps/sim/lib/internal/uptimerobot/file-input.ts @@ -0,0 +1,55 @@ +import type { Logger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { UptimeRobotOperationError } from '@/lib/internal/uptimerobot/errors' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +export async function appendUptimeRobotPspImage(args: { + form: FormData + field: 'logo' | 'icon' + file: unknown + userId: string + requestId: string + logger: Logger + signal?: AbortSignal +}): Promise { + const { form, field, file, userId, requestId, logger, signal } = args + signal?.throwIfAborted() + const userFiles = processFilesToUserFiles([file as RawFileInput], requestId, logger) + if (userFiles.length === 0) { + throw new UptimeRobotOperationError( + `Invalid ${field} file: expected an uploaded file reference`, + 400 + ) + } + + const userFile = userFiles[0] + const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) + signal?.throwIfAborted() + if (denied) { + let message = 'File not found' + try { + const body = (await denied.json()) as { error?: unknown } + if (typeof body.error === 'string') message = body.error + } catch (error) { + logger.warn('Failed to read denied file-access response', { + error: getErrorMessage(error), + field, + requestId, + }) + } + throw new UptimeRobotOperationError(message, denied.status) + } + + const { buffer, contentType } = await downloadServableFileFromStorage( + userFile, + requestId, + logger, + { maxBytes: MAX_BUFFERED_TRANSFER_BYTES, signal } + ) + signal?.throwIfAborted() + const mimeType = contentType || userFile.type || 'application/octet-stream' + form.append(field, new Blob([new Uint8Array(buffer)], { type: mimeType }), userFile.name) +} diff --git a/apps/sim/lib/internal/uptimerobot/operations.test.ts b/apps/sim/lib/internal/uptimerobot/operations.test.ts new file mode 100644 index 00000000000..cdf0a44763b --- /dev/null +++ b/apps/sim/lib/internal/uptimerobot/operations.test.ts @@ -0,0 +1,110 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + appendUptimeRobotPspImage: vi.fn(), + requestUptimeRobotPsp: vi.fn(), +})) + +vi.mock('@/lib/internal/uptimerobot/client', () => ({ + requestUptimeRobotPsp: mocks.requestUptimeRobotPsp, +})) + +vi.mock('@/lib/internal/uptimerobot/file-input', () => ({ + appendUptimeRobotPspImage: mocks.appendUptimeRobotPspImage, +})) + +import { createUptimeRobotPsp, updateUptimeRobotPsp } from '@/lib/internal/uptimerobot/operations' + +const PSP = { + id: 1, + friendlyName: 'Status', + customDomain: null, + isPasswordSet: null, + monitorIds: [], + tagIds: [], + monitorsCount: null, + status: null, + urlKey: null, + homepageLink: null, + gaCode: null, + icon: null, + logo: null, + noIndex: null, + hideUrlLinks: null, + subscription: null, +} + +describe('UptimeRobot PSP operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.requestUptimeRobotPsp.mockResolvedValue(PSP) + }) + + it('builds create multipart input and authorizes each provided image', async () => { + const controller = new AbortController() + const logo = { key: 'logo-key' } + const icon = { key: 'icon-key' } + + const result = await createUptimeRobotPsp( + { + apiKey: 'key', + friendlyName: 'Status', + monitorIds: '1, 2, ,3', + hideUrlLinks: false, + noIndex: true, + logo, + icon, + }, + { + userId: 'user-1', + requestId: 'request-1', + signal: controller.signal, + } + ) + + expect(result).toEqual({ success: true, output: { psp: PSP } }) + expect(mocks.appendUptimeRobotPspImage).toHaveBeenCalledTimes(2) + const providerCall = mocks.requestUptimeRobotPsp.mock.calls[0][0] + expect(providerCall).toMatchObject({ + apiKey: 'key', + method: 'POST', + path: '/psps', + signal: controller.signal, + }) + expect(providerCall.form.getAll('monitorIds')).toEqual(['1', '2', '3']) + expect(providerCall.form.get('hideUrlLinks')).toBe('false') + expect(providerCall.form.get('noIndex')).toBe('true') + }) + + it('uses the canonical update path and omits absent fields', async () => { + await updateUptimeRobotPsp( + { apiKey: 'key', pspId: 42 }, + { userId: 'user-1', requestId: 'request-1' } + ) + + const providerCall = mocks.requestUptimeRobotPsp.mock.calls[0][0] + expect(providerCall).toMatchObject({ apiKey: 'key', method: 'PATCH', path: '/psps/42' }) + expect([...providerCall.form.entries()]).toEqual([]) + }) + + it('stops before file or provider work after cancellation', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + createUptimeRobotPsp( + { apiKey: 'key', friendlyName: 'Status' }, + { + userId: 'user-1', + requestId: 'request-1', + signal: controller.signal, + } + ) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mocks.appendUptimeRobotPspImage).not.toHaveBeenCalled() + expect(mocks.requestUptimeRobotPsp).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/uptimerobot/operations.ts b/apps/sim/lib/internal/uptimerobot/operations.ts new file mode 100644 index 00000000000..61ef5ec9f88 --- /dev/null +++ b/apps/sim/lib/internal/uptimerobot/operations.ts @@ -0,0 +1,97 @@ +import { createLogger } from '@sim/logger' +import { requestUptimeRobotPsp } from '@/lib/internal/uptimerobot/client' +import { appendUptimeRobotPspImage } from '@/lib/internal/uptimerobot/file-input' +import type { + UptimeRobotCreatePspParams, + UptimeRobotPspResponse, + UptimeRobotUpdatePspParams, +} from '@/tools/uptimerobot/types' + +const logger = createLogger('UptimeRobotOperations') + +export interface UptimeRobotOperationContext { + userId: string + requestId: string + signal?: AbortSignal +} + +function appendTextFields( + form: FormData, + input: UptimeRobotCreatePspParams | UptimeRobotUpdatePspParams +): void { + if (input.friendlyName) form.append('friendlyName', input.friendlyName) + if (input.status) form.append('status', input.status) + if (input.password) form.append('password', input.password) + if (input.customDomain) form.append('customDomain', input.customDomain) + if (typeof input.hideUrlLinks === 'boolean') { + form.append('hideUrlLinks', String(input.hideUrlLinks)) + } + if (typeof input.noIndex === 'boolean') form.append('noIndex', String(input.noIndex)) + if (input.monitorIds) { + for (const id of input.monitorIds.split(',')) { + const trimmed = id.trim() + if (trimmed) form.append('monitorIds', trimmed) + } + } +} + +async function executePspOperation(args: { + input: UptimeRobotCreatePspParams | UptimeRobotUpdatePspParams + method: 'POST' | 'PATCH' + path: string + context: UptimeRobotOperationContext +}): Promise { + const { input, method, path, context } = args + context.signal?.throwIfAborted() + const form = new FormData() + appendTextFields(form, input) + if (input.logo) { + await appendUptimeRobotPspImage({ + form, + field: 'logo', + file: input.logo, + userId: context.userId, + requestId: context.requestId, + logger, + signal: context.signal, + }) + } + if (input.icon) { + await appendUptimeRobotPspImage({ + form, + field: 'icon', + file: input.icon, + userId: context.userId, + requestId: context.requestId, + logger, + signal: context.signal, + }) + } + const psp = await requestUptimeRobotPsp({ + apiKey: input.apiKey, + method, + path, + form, + signal: context.signal, + }) + return { success: true, output: { psp } } +} + +export function createUptimeRobotPsp( + input: UptimeRobotCreatePspParams, + context: UptimeRobotOperationContext +): Promise { + return executePspOperation({ input, method: 'POST', path: '/psps', context }) +} + +export function updateUptimeRobotPsp( + input: UptimeRobotUpdatePspParams, + context: UptimeRobotOperationContext +): Promise { + return executePspOperation({ + input, + method: 'PATCH', + path: `/psps/${input.pspId}`, + context, + }) +} diff --git a/apps/sim/lib/internal/vanta/client.test.ts b/apps/sim/lib/internal/vanta/client.test.ts new file mode 100644 index 00000000000..e66394e5c88 --- /dev/null +++ b/apps/sim/lib/internal/vanta/client.test.ts @@ -0,0 +1,111 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { fetchVantaWithAuth } from '@/lib/internal/vanta/client' + +describe('Vanta provider client', () => { + const fetchMock = vi.fn() + + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', fetchMock) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('passes caller cancellation through token exchange and provider work', async () => { + fetchMock.mockResolvedValue( + Response.json({ access_token: 'token', expires_in: 0 }, { status: 200 }) + ) + const controller = new AbortController() + const provider = vi.fn().mockResolvedValue(new Response(null, { status: 200 })) + + await fetchVantaWithAuth( + { + clientId: 'client-cancellation', + clientSecret: 'secret-cancellation', + scope: 'scope', + }, + provider, + { signal: controller.signal } + ) + + expect(fetchMock).toHaveBeenCalledWith( + 'https://api.vanta.com/oauth/token', + expect.objectContaining({ signal: expect.any(AbortSignal) }) + ) + expect(provider).toHaveBeenCalledWith('token') + }) + + it('aborts the shared token request when its last waiter cancels', async () => { + let tokenSignal: AbortSignal | undefined + fetchMock.mockImplementation( + (_input, init) => + new Promise((_resolve, reject) => { + tokenSignal = init?.signal ?? undefined + tokenSignal?.addEventListener('abort', () => reject(tokenSignal?.reason), { once: true }) + }) + ) + const controller = new AbortController() + const provider = vi.fn() + const pending = fetchVantaWithAuth( + { + clientId: 'client-abort', + clientSecret: 'secret-abort', + scope: 'scope', + }, + provider, + { signal: controller.signal } + ) + await vi.waitFor(() => expect(tokenSignal).toBeDefined()) + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect(pending).rejects.toMatchObject({ name: 'AbortError' }) + expect(tokenSignal?.aborted).toBe(true) + expect(provider).not.toHaveBeenCalled() + }) + + it('consumes a 401 response and retries once with a fresh token', async () => { + fetchMock + .mockResolvedValueOnce( + Response.json({ access_token: 'token-1', expires_in: 0 }, { status: 200 }) + ) + .mockResolvedValueOnce( + Response.json({ access_token: 'token-2', expires_in: 0 }, { status: 200 }) + ) + let cancelled = false + const unauthorized = new Response( + new ReadableStream({ + start: (controller) => { + controller.enqueue(new TextEncoder().encode('unauthorized')) + controller.close() + }, + cancel: () => { + cancelled = true + }, + }), + { status: 401 } + ) + const provider = vi + .fn() + .mockResolvedValueOnce(unauthorized) + .mockResolvedValueOnce(new Response(null, { status: 200 })) + + const response = await fetchVantaWithAuth( + { + clientId: 'client-retry', + clientSecret: 'secret-retry', + scope: 'scope', + }, + provider + ) + + expect(response.status).toBe(200) + expect(provider.mock.calls).toEqual([['token-1'], ['token-2']]) + expect(cancelled).toBe(false) + expect(unauthorized.bodyUsed).toBe(true) + }) +}) diff --git a/apps/sim/lib/internal/vanta/client.ts b/apps/sim/lib/internal/vanta/client.ts new file mode 100644 index 00000000000..7cc9f479d7d --- /dev/null +++ b/apps/sim/lib/internal/vanta/client.ts @@ -0,0 +1,205 @@ +import { LRUCache } from 'lru-cache' +import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { consumeOrCancelBody, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' +import { extractVantaError } from '@/lib/internal/vanta/normalizers' +import type { VantaRegion } from '@/tools/vanta/types' + +export const VANTA_API_BASE_URLS: Record = { + us: 'https://api.vanta.com', + gov: 'https://api.vanta-gov.com', +} + +export const VANTA_READ_SCOPE = 'vanta-api.all:read' +export const VANTA_WRITE_SCOPE = 'vanta-api.all:read vanta-api.all:write' +export const VANTA_DOCUMENT_UPLOAD_SCOPE = + 'vanta-api.all:read vanta-api.all:write vanta-api.documents:upload' + +const VANTA_TOKEN_EXPIRY_BUFFER_MS = 10 * 60 * 1000 +const VANTA_TOKEN_EXCHANGE_TIMEOUT_MS = 15_000 +const VANTA_TOKEN_CACHE_MAX_ENTRIES = 128 +const VANTA_TOKEN_EXCHANGE_MAX_ENTRIES = 128 + +export interface VantaTokenParams { + clientId: string + clientSecret: string + region?: VantaRegion + scope: string +} + +interface VantaCachedToken { + token: string + expiresAt: number +} + +interface VantaTokenExchange { + controller: AbortController + promise: Promise + settled: boolean + waiters: number +} + +const vantaTokenCache = new LRUCache({ + max: VANTA_TOKEN_CACHE_MAX_ENTRIES, +}) +const vantaTokenExchanges = new Map() + +export function getVantaBaseUrl(region: VantaRegion | undefined): string { + return VANTA_API_BASE_URLS[region ?? 'us'] +} + +async function vantaTokenCacheKey(params: VantaTokenParams): Promise { + const digest = await crypto.subtle.digest( + 'SHA-256', + new TextEncoder().encode(`${params.clientId}:${params.clientSecret}`) + ) + const secretHash = Array.from(new Uint8Array(digest)) + .map((byte) => byte.toString(16).padStart(2, '0')) + .join('') + return [params.region ?? 'us', params.scope, params.clientId, secretHash].join('|') +} + +async function exchangeVantaToken( + params: VantaTokenParams, + cacheKey: string, + signal: AbortSignal +): Promise { + const requestSignal = AbortSignal.any([ + signal, + AbortSignal.timeout(VANTA_TOKEN_EXCHANGE_TIMEOUT_MS), + ]) + const response = await fetch(`${getVantaBaseUrl(params.region)}/oauth/token`, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + client_id: params.clientId, + client_secret: params.clientSecret, + scope: params.scope, + grant_type: 'client_credentials', + }), + cache: 'no-store', + signal: requestSignal, + }) + + const data = await readResponseJsonWithLimit(response, { + maxBytes: MAX_JSON_API_RESPONSE_BYTES, + label: 'Vanta authentication response', + signal: requestSignal, + }).catch(() => null) + if (!response.ok) { + throw new Error(extractVantaError(data, 'Failed to authenticate with Vanta')) + } + if (!data || typeof data !== 'object' || Array.isArray(data)) { + throw new Error('Vanta authentication did not return an access token') + } + const accessToken = Reflect.get(data, 'access_token') + if (typeof accessToken !== 'string') { + throw new Error('Vanta authentication did not return an access token') + } + + const expiresIn = Reflect.get(data, 'expires_in') + const expiresInMs = + (typeof expiresIn === 'number' && Number.isFinite(expiresIn) ? expiresIn : 0) * 1000 + if (expiresInMs > VANTA_TOKEN_EXPIRY_BUFFER_MS) { + vantaTokenCache.set(cacheKey, { + token: accessToken, + expiresAt: Date.now() + expiresInMs - VANTA_TOKEN_EXPIRY_BUFFER_MS, + }) + } + + return accessToken +} + +function waitForExchange(exchange: VantaTokenExchange, signal?: AbortSignal): Promise { + if (!signal) return exchange.promise + signal.throwIfAborted() + return new Promise((resolve, reject) => { + const cleanup = () => signal.removeEventListener('abort', onAbort) + const onAbort = () => { + cleanup() + reject(signal.reason) + } + signal.addEventListener('abort', onAbort, { once: true }) + exchange.promise.then( + (token) => { + cleanup() + resolve(token) + }, + (error: unknown) => { + cleanup() + reject(error) + } + ) + }) +} + +function createVantaTokenExchange(params: VantaTokenParams, cacheKey: string): VantaTokenExchange { + if (vantaTokenExchanges.size >= VANTA_TOKEN_EXCHANGE_MAX_ENTRIES) { + throw new Error('Too many concurrent Vanta authentication requests') + } + const controller = new AbortController() + const exchange: VantaTokenExchange = { + controller, + promise: Promise.resolve(''), + settled: false, + waiters: 0, + } + exchange.promise = exchangeVantaToken(params, cacheKey, controller.signal).finally(() => { + exchange.settled = true + if (vantaTokenExchanges.get(cacheKey) === exchange) { + vantaTokenExchanges.delete(cacheKey) + } + }) + vantaTokenExchanges.set(cacheKey, exchange) + return exchange +} + +export async function getVantaAccessToken( + params: VantaTokenParams, + options: { forceRefresh?: boolean; signal?: AbortSignal } = {} +): Promise { + options.signal?.throwIfAborted() + const cacheKey = await vantaTokenCacheKey(params) + options.signal?.throwIfAborted() + if (!options.forceRefresh) { + const cached = vantaTokenCache.get(cacheKey) + if (cached && cached.expiresAt > Date.now()) return cached.token + } + + vantaTokenCache.delete(cacheKey) + const exchange = vantaTokenExchanges.get(cacheKey) ?? createVantaTokenExchange(params, cacheKey) + exchange.waiters += 1 + try { + return await waitForExchange(exchange, options.signal) + } finally { + exchange.waiters -= 1 + if (exchange.waiters === 0 && !exchange.settled) { + if (vantaTokenExchanges.get(cacheKey) === exchange) vantaTokenExchanges.delete(cacheKey) + exchange.controller.abort(options.signal?.reason) + } + } +} + +export async function fetchVantaWithAuth( + tokenParams: VantaTokenParams, + doFetch: (accessToken: string) => Promise, + options: { signal?: AbortSignal } = {} +): Promise { + options.signal?.throwIfAborted() + const accessToken = await getVantaAccessToken(tokenParams, { signal: options.signal }) + options.signal?.throwIfAborted() + const response = await doFetch(accessToken) + options.signal?.throwIfAborted() + if (response.status !== 401) return response + await consumeOrCancelBody(response) + options.signal?.throwIfAborted() + + const freshToken = await getVantaAccessToken(tokenParams, { + forceRefresh: true, + signal: options.signal, + }) + options.signal?.throwIfAborted() + return doFetch(freshToken) +} diff --git a/apps/sim/lib/internal/vanta/errors.ts b/apps/sim/lib/internal/vanta/errors.ts new file mode 100644 index 00000000000..78f7338157f --- /dev/null +++ b/apps/sim/lib/internal/vanta/errors.ts @@ -0,0 +1,9 @@ +export class VantaOperationError extends Error { + constructor( + readonly status: number, + readonly body: unknown + ) { + super('Vanta operation failed') + this.name = 'VantaOperationError' + } +} diff --git a/apps/sim/lib/internal/vanta/execute-tool.test.ts b/apps/sim/lib/internal/vanta/execute-tool.test.ts new file mode 100644 index 00000000000..56c2d4a6be0 --- /dev/null +++ b/apps/sim/lib/internal/vanta/execute-tool.test.ts @@ -0,0 +1,155 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + download: vi.fn(), + query: vi.fn(), + upload: vi.fn(), +})) + +vi.mock('@/lib/internal/vanta/operations', () => ({ + executeVantaDownloadDocumentFile: mocks.download, + executeVantaQuery: mocks.query, + executeVantaUploadDocumentFile: mocks.upload, +})) + +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' +import { VantaOperationError } from '@/lib/internal/vanta/errors' +import { executeVantaTool } from '@/lib/internal/vanta/execute-tool' + +const INPUTS = { + vanta_download_document_file: { + clientId: 'client', + clientSecret: 'secret', + documentId: 'document-1', + uploadedFileId: 'upload-1', + }, + vanta_upload_document_file: { + clientId: 'client', + clientSecret: 'secret', + documentId: 'document-1', + fileContent: Buffer.from('hello').toString('base64'), + }, + vanta_list_frameworks: { + operation: 'vanta_list_frameworks', + clientId: 'client', + clientSecret: 'secret', + pageSize: 25, + }, +} as const + +function request( + toolId: keyof typeof INPUTS, + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId, + input: INPUTS[toolId], + headers: new Headers(), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + executionId: 'execution-1', + userId: 'user-1', + workspaceId: 'workspace-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeVantaTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.download.mockResolvedValue({ success: true, output: { file: {} } }) + mocks.query.mockResolvedValue({ success: true, output: { frameworks: [] } }) + mocks.upload.mockResolvedValue({ success: true, output: { upload: {} } }) + }) + + it.each(Object.keys(INPUTS) as Array)( + 'validates and dispatches %s with trusted context', + async (toolId) => { + const controller = new AbortController() + const response = await executeVantaTool(request(toolId, { signal: controller.signal })) + + expect(response.status).toBe(200) + const operation = + toolId === 'vanta_upload_document_file' + ? mocks.upload + : toolId === 'vanta_download_document_file' + ? mocks.download + : mocks.query + if (toolId === 'vanta_list_frameworks') { + expect(operation).toHaveBeenCalledWith( + expect.objectContaining(INPUTS[toolId]), + controller.signal + ) + } else { + expect(operation).toHaveBeenCalledWith(expect.objectContaining(INPUTS[toolId]), { + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + }) + } + } + ) + + it('preserves authentication, validation, and provider errors', async () => { + const unauthorized = await executeVantaTool( + request('vanta_upload_document_file', { + context: createExecutionContext({ workflowId: 'workflow-1' }), + }) + ) + expect(unauthorized.status).toBe(401) + await expect(unauthorized.json()).resolves.toEqual({ success: false, error: 'Unauthorized' }) + + const invalid = await executeVantaTool( + request('vanta_download_document_file', { input: { clientId: 'client' } }) + ) + expect(invalid.status).toBe(400) + await expect(invalid.json()).resolves.toMatchObject({ error: 'Validation error' }) + + mocks.download.mockRejectedValueOnce( + new VantaOperationError(404, { success: false, error: 'Not found' }) + ) + const provider = await executeVantaTool(request('vanta_download_document_file')) + expect(provider.status).toBe(404) + await expect(provider.json()).resolves.toEqual({ success: false, error: 'Not found' }) + }) + + it('rejects a query operation that does not match the registered tool ID', async () => { + const response = await executeVantaTool( + request('vanta_list_frameworks', { + input: { + operation: 'vanta_get_framework', + clientId: 'client', + clientSecret: 'secret', + frameworkId: 'framework-1', + }, + }) + ) + + expect(response.status).toBe(400) + expect(mocks.query).not.toHaveBeenCalled() + }) + + it('propagates cancellation before and after operation work', async () => { + const before = new AbortController() + before.abort(new DOMException('cancelled', 'AbortError')) + await expect( + executeVantaTool(request('vanta_download_document_file', { signal: before.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mocks.download).not.toHaveBeenCalled() + + const after = new AbortController() + mocks.download.mockImplementationOnce(async () => { + after.abort(new DOMException('cancelled', 'AbortError')) + return { success: true } + }) + await expect( + executeVantaTool(request('vanta_download_document_file', { signal: after.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) +}) diff --git a/apps/sim/lib/internal/vanta/execute-tool.ts b/apps/sim/lib/internal/vanta/execute-tool.ts new file mode 100644 index 00000000000..b510d7453ae --- /dev/null +++ b/apps/sim/lib/internal/vanta/execute-tool.ts @@ -0,0 +1,78 @@ +import { getErrorMessage } from '@sim/utils/errors' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { VantaOperationError } from '@/lib/internal/vanta/errors' +import { + vantaDownloadDocumentFileInputSchema, + vantaUploadDocumentFileInputSchema, +} from '@/lib/internal/vanta/input' +import { + executeVantaDownloadDocumentFile, + executeVantaQuery, + executeVantaUploadDocumentFile, +} from '@/lib/internal/vanta/operations' +import { vantaQueryBodySchema } from '@/lib/internal/vanta/schema' + +/** Executes the Vanta tool family without a same-origin HTTP hop. */ +export const executeVantaTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + const schema = + request.toolId === 'vanta_upload_document_file' + ? vantaUploadDocumentFileInputSchema + : request.toolId === 'vanta_download_document_file' + ? vantaDownloadDocumentFileInputSchema + : vantaQueryBodySchema + const parsed = schema.safeParse(request.input) + if (!parsed.success) { + return Response.json({ success: false, error: 'Validation error' }, { status: 400 }) + } + + try { + if ( + request.toolId === 'vanta_upload_document_file' || + request.toolId === 'vanta_download_document_file' + ) { + const userId = request.context.userId + if (!userId) { + return Response.json({ success: false, error: 'Unauthorized' }, { status: 401 }) + } + const context = { requestId: request.requestId, signal: request.signal, userId } + const result = + request.toolId === 'vanta_upload_document_file' + ? await executeVantaUploadDocumentFile( + vantaUploadDocumentFileInputSchema.parse(parsed.data), + context + ) + : await executeVantaDownloadDocumentFile( + vantaDownloadDocumentFileInputSchema.parse(parsed.data), + context + ) + request.signal?.throwIfAborted() + return Response.json(result) + } + + const query = vantaQueryBodySchema.parse(parsed.data) + if (query.operation !== request.toolId) { + return Response.json( + { + success: false, + error: `Vanta operation ${query.operation} does not match ${request.toolId}`, + }, + { status: 400 } + ) + } + const result = await executeVantaQuery(query, request.signal) + request.signal?.throwIfAborted() + return result.success + ? Response.json({ success: true, output: result.output }) + : Response.json({ success: false, error: result.error }, { status: result.status }) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof VantaOperationError) { + return Response.json(error.body, { status: error.status }) + } + return Response.json( + { success: false, error: getErrorMessage(error, 'Vanta request failed') }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/vanta/file-input.test.ts b/apps/sim/lib/internal/vanta/file-input.test.ts new file mode 100644 index 00000000000..fdf16e3d579 --- /dev/null +++ b/apps/sim/lib/internal/vanta/file-input.test.ts @@ -0,0 +1,117 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + assertAccess: vi.fn(), + download: vi.fn(), + process: vi.fn(), +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertAccess, +})) + +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + processFilesToUserFiles: mocks.process, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mocks.download, +})) + +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { resolveVantaUploadFile } from '@/lib/internal/vanta/file-input' +import { VANTA_MAX_TRANSFER_BYTES } from '@/lib/internal/vanta/input' + +const baseInput = { + clientId: 'client', + clientSecret: 'secret', + documentId: 'document-1', +} +const file = { key: 'workspace/file.txt', name: 'file.txt', size: 4 } + +describe('resolveVantaUploadFile', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.process.mockReturnValue([{ ...file, type: 'text/plain' }]) + mocks.assertAccess.mockResolvedValue(null) + mocks.download.mockResolvedValue({ buffer: Buffer.from('test'), contentType: 'text/plain' }) + }) + + it('fails closed on denied stored-file access', async () => { + mocks.assertAccess.mockResolvedValue( + Response.json({ success: false, error: 'File not found' }, { status: 404 }) + ) + + await expect( + resolveVantaUploadFile({ ...baseInput, file }, { requestId: 'request-1', userId: 'user-1' }) + ).rejects.toMatchObject({ + status: 404, + body: { success: false, error: 'File not found' }, + }) + expect(mocks.download).not.toHaveBeenCalled() + }) + + it('passes the upload cap and cancellation to storage', async () => { + const controller = new AbortController() + await resolveVantaUploadFile( + { ...baseInput, file, fileName: 'evidence.txt' }, + { requestId: 'request-1', signal: controller.signal, userId: 'user-1' } + ) + + expect(mocks.assertAccess).toHaveBeenCalledWith( + file.key, + 'user-1', + 'request-1', + expect.anything() + ) + expect(mocks.download).toHaveBeenCalledWith( + expect.objectContaining({ key: file.key }), + 'request-1', + expect.anything(), + { maxBytes: VANTA_MAX_TRANSFER_BYTES, signal: controller.signal } + ) + }) + + it('preserves exact size errors for declared and streamed oversized files', async () => { + mocks.process.mockReturnValueOnce([ + { ...file, size: VANTA_MAX_TRANSFER_BYTES + 1, type: 'text/plain' }, + ]) + await expect( + resolveVantaUploadFile({ ...baseInput, file }, { requestId: 'request-1', userId: 'user-1' }) + ).rejects.toMatchObject({ + status: 400, + body: { success: false, error: 'File size (100.00MB) exceeds upload limit of 100MB' }, + }) + + mocks.download.mockRejectedValueOnce( + new PayloadSizeLimitError({ + label: 'file', + maxBytes: VANTA_MAX_TRANSFER_BYTES, + observedBytes: VANTA_MAX_TRANSFER_BYTES + 1024 * 1024, + }) + ) + await expect( + resolveVantaUploadFile({ ...baseInput, file }, { requestId: 'request-1', userId: 'user-1' }) + ).rejects.toMatchObject({ + status: 400, + body: { success: false, error: 'File size (101.00MB) exceeds upload limit of 100MB' }, + }) + }) + + it('supports bounded base64 content without protected-file lookup', async () => { + await expect( + resolveVantaUploadFile( + { ...baseInput, fileContent: Buffer.from('hello').toString('base64') }, + { requestId: 'request-1', userId: 'user-1' } + ) + ).resolves.toMatchObject({ + buffer: Buffer.from('hello'), + fileName: 'file', + mimeType: 'application/octet-stream', + }) + expect(mocks.assertAccess).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/vanta/file-input.ts b/apps/sim/lib/internal/vanta/file-input.ts new file mode 100644 index 00000000000..1665358fb84 --- /dev/null +++ b/apps/sim/lib/internal/vanta/file-input.ts @@ -0,0 +1,112 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { VantaOperationError } from '@/lib/internal/vanta/errors' +import { + VANTA_MAX_TRANSFER_BYTES, + type VantaUploadDocumentFileInput, +} from '@/lib/internal/vanta/input' +import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +const logger = createLogger('VantaFileInput') + +export interface VantaResolvedUploadFile { + buffer: Buffer + fileName: string + mimeType: string +} + +function uploadSizeError(bytes: number): VantaOperationError { + const sizeMB = (bytes / (1024 * 1024)).toFixed(2) + return new VantaOperationError(400, { + success: false, + error: `File size (${sizeMB}MB) exceeds upload limit of 100MB`, + }) +} + +async function responseBody(response: Response): Promise { + try { + return await response.json() + } catch { + return { success: false, error: response.statusText || 'File operation failed' } + } +} + +export async function resolveVantaUploadFile( + input: VantaUploadDocumentFileInput, + context: { requestId: string; signal?: AbortSignal; userId: string } +): Promise { + context.signal?.throwIfAborted() + if (input.file) { + if (typeof input.file === 'string') { + throw new VantaOperationError(400, { success: false, error: 'Invalid file input' }) + } + const userFiles = processFilesToUserFiles( + [input.file as RawFileInput], + context.requestId, + logger + ) + if (userFiles.length === 0) { + throw new VantaOperationError(400, { success: false, error: 'Invalid file input' }) + } + const userFile = userFiles[0] + const denied = await assertToolFileAccess( + userFile.key, + context.userId, + context.requestId, + logger + ) + context.signal?.throwIfAborted() + if (denied) throw new VantaOperationError(denied.status, await responseBody(denied)) + if (userFile.size > VANTA_MAX_TRANSFER_BYTES) throw uploadSizeError(userFile.size) + + try { + const resolved = await downloadServableFileFromStorage(userFile, context.requestId, logger, { + maxBytes: VANTA_MAX_TRANSFER_BYTES, + signal: context.signal, + }) + context.signal?.throwIfAborted() + /** + * Every return path of `downloadServableFileFromStorage` yields a non-empty content + * type, so `resolved.contentType` always wins. The remaining operands are defensive + * fallbacks kept in place in case that guarantee is ever relaxed. + */ + return { + buffer: resolved.buffer, + fileName: input.fileName || userFile.name, + mimeType: + resolved.contentType || userFile.type || input.mimeType || 'application/octet-stream', + } + } catch (error) { + context.signal?.throwIfAborted() + const notReady = docNotReadyResponse(error) + if (notReady) throw new VantaOperationError(notReady.status, await responseBody(notReady)) + if (isPayloadSizeLimitError(error)) { + throw uploadSizeError(error.observedBytes ?? userFile.size) + } + logger.error('Failed to download Vanta upload file', { + error: getErrorMessage(error), + requestId: context.requestId, + }) + throw new VantaOperationError(500, { + success: false, + error: getErrorMessage(error, 'Failed to download file'), + }) + } + } + + if (!input.fileContent) { + throw new VantaOperationError(400, { success: false, error: 'File is required' }) + } + const buffer = Buffer.from(input.fileContent, 'base64') + if (buffer.length > VANTA_MAX_TRANSFER_BYTES) throw uploadSizeError(buffer.length) + return { + buffer, + fileName: input.fileName || 'file', + mimeType: input.mimeType || 'application/octet-stream', + } +} diff --git a/apps/sim/lib/internal/vanta/input-size.test.ts b/apps/sim/lib/internal/vanta/input-size.test.ts new file mode 100644 index 00000000000..776495d9800 --- /dev/null +++ b/apps/sim/lib/internal/vanta/input-size.test.ts @@ -0,0 +1,25 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { isJsonInputWithinLimit } from '@/lib/internal/vanta/input-size' + +describe('Vanta operation input sizing', () => { + it.each([ + null, + { value: 'plain text' }, + { value: 'quotes " and slashes \\' }, + { value: 'emoji 🚀 and control\n' }, + { nested: [{ enabled: true }, undefined, 42] }, + ])('matches JSON byte boundaries without materializing the entire input', (input) => { + const bytes = Buffer.byteLength(JSON.stringify(input) ?? '', 'utf8') + expect(isJsonInputWithinLimit(input, bytes)).toBe(true) + if (bytes > 0) expect(isJsonInputWithinLimit(input, bytes - 1)).toBe(false) + }) + + it('rejects cyclic inputs as invalid JSON', () => { + const cyclic: { self?: unknown } = {} + cyclic.self = cyclic + expect(() => isJsonInputWithinLimit(cyclic, 1024)).toThrow(/circular/i) + }) +}) diff --git a/apps/sim/lib/internal/vanta/input-size.ts b/apps/sim/lib/internal/vanta/input-size.ts new file mode 100644 index 00000000000..4e58ccd25e8 --- /dev/null +++ b/apps/sim/lib/internal/vanta/input-size.ts @@ -0,0 +1,104 @@ +function jsonStringBytes(value: string): number { + let bytes = 2 + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index) + if ( + code === 0x22 || + code === 0x5c || + code === 0x08 || + code === 0x09 || + code === 0x0a || + code === 0x0c || + code === 0x0d + ) { + bytes += 2 + } else if (code <= 0x1f) { + bytes += 6 + } else if (code <= 0x7f) { + bytes += 1 + } else if (code <= 0x7ff) { + bytes += 2 + } else if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(index + 1) + if (next >= 0xdc00 && next <= 0xdfff) { + bytes += 4 + index += 1 + } else { + bytes += 6 + } + } else if (code >= 0xdc00 && code <= 0xdfff) { + bytes += 6 + } else { + bytes += 3 + } + } + return bytes +} + +function primitiveJsonBytes(value: unknown): number | null { + if (value === null) return 4 + switch (typeof value) { + case 'string': + return jsonStringBytes(value) + case 'boolean': + return value ? 4 : 5 + case 'number': + return Number.isFinite(value) ? String(value).length : 4 + case 'bigint': + throw new TypeError('Do not know how to serialize a BigInt') + case 'undefined': + case 'function': + case 'symbol': + return null + default: + return null + } +} + +function addJsonBytes( + value: unknown, + limit: number, + seen: Set, + arrayEntry = false +): number { + const primitiveBytes = primitiveJsonBytes(value) + if (primitiveBytes !== null) return primitiveBytes + if (value === undefined || typeof value === 'function' || typeof value === 'symbol') { + return arrayEntry ? 4 : 0 + } + if (value instanceof Date) return jsonStringBytes(value.toJSON()) + if (!value || typeof value !== 'object') return 0 + if (seen.has(value)) throw new TypeError('Converting circular structure to JSON') + seen.add(value) + + let bytes = 2 + let emitted = false + if (Array.isArray(value)) { + for (const entry of value) { + if (emitted) bytes += 1 + bytes += addJsonBytes(entry, limit - bytes, seen, true) + emitted = true + if (bytes > limit) break + } + } else { + for (const [key, entry] of Object.entries(value)) { + const entryBytes = addJsonBytes(entry, limit - bytes, seen) + if ( + entryBytes === 0 && + (entry === undefined || typeof entry === 'function' || typeof entry === 'symbol') + ) { + continue + } + if (emitted) bytes += 1 + bytes += jsonStringBytes(key) + 1 + entryBytes + emitted = true + if (bytes > limit) break + } + } + seen.delete(value) + return bytes +} + +export function isJsonInputWithinLimit(input: unknown, limit: number): boolean { + return addJsonBytes(input, limit, new Set()) <= limit +} diff --git a/apps/sim/lib/internal/vanta/input.ts b/apps/sim/lib/internal/vanta/input.ts new file mode 100644 index 00000000000..53fcc6c7fe8 --- /dev/null +++ b/apps/sim/lib/internal/vanta/input.ts @@ -0,0 +1,34 @@ +import { z } from 'zod' +import { FileInputSchema } from '@/lib/uploads/utils/file-schemas' + +export const VANTA_MAX_TRANSFER_BYTES = 100 * 1024 * 1024 +export const VANTA_MAX_UPLOAD_BASE64_LENGTH = Math.ceil(VANTA_MAX_TRANSFER_BYTES / 3) * 4 + +const vantaCredentialsSchema = z.object({ + clientId: z.string().min(1, 'Client ID is required'), + clientSecret: z.string().min(1, 'Client secret is required'), + region: z.enum(['us', 'gov']).optional(), +}) + +const requiredId = (label: string) => z.string().trim().min(1, `${label} is required`) + +export const vantaUploadDocumentFileInputSchema = vantaCredentialsSchema.extend({ + documentId: requiredId('Document ID'), + file: FileInputSchema.optional().nullable(), + fileContent: z + .string() + .max(VANTA_MAX_UPLOAD_BASE64_LENGTH, 'fileContent exceeds the 100MB upload limit') + .nullish(), + fileName: z.string().nullish(), + mimeType: z.string().nullish(), + description: z.string().nullish(), + effectiveAtDate: z.string().nullish(), +}) + +export const vantaDownloadDocumentFileInputSchema = vantaCredentialsSchema.extend({ + documentId: requiredId('Document ID'), + uploadedFileId: requiredId('Uploaded file ID'), +}) + +export type VantaUploadDocumentFileInput = z.output +export type VantaDownloadDocumentFileInput = z.output diff --git a/apps/sim/lib/internal/vanta/normalizers.ts b/apps/sim/lib/internal/vanta/normalizers.ts new file mode 100644 index 00000000000..2976835a35c --- /dev/null +++ b/apps/sim/lib/internal/vanta/normalizers.ts @@ -0,0 +1,618 @@ +import { isRecordLike } from '@sim/utils/object' +import type { + VantaControl, + VantaControlDetail, + VantaCustomField, + VantaDocument, + VantaDocumentDetail, + VantaFramework, + VantaFrameworkDetail, + VantaFrameworkRequirement, + VantaFrameworkRequirementCategory, + VantaFrameworkRequirementControl, + VantaMonitoredComputer, + VantaOwner, + VantaPageInfo, + VantaPerson, + VantaPolicy, + VantaPolicyDocument, + VantaRiskScenario, + VantaTest, + VantaTestEntity, + VantaUploadedFile, + VantaVendor, + VantaVulnerability, + VantaVulnerabilityRemediation, + VantaVulnerableAsset, + VantaVulnerableAssetScanner, +} from '@/tools/vanta/types' + +type JsonRecord = Record + +/** + * Coerces an unknown single-resource response body to a record so the + * normalizers can run on it; non-object bodies normalize to all-null fields. + */ +export function asVantaRecord(value: unknown): JsonRecord { + return isRecordLike(value) ? value : {} +} + +function getString(value: unknown): string | null { + return typeof value === 'string' ? value : null +} + +function getNumber(value: unknown): number | null { + return typeof value === 'number' && Number.isFinite(value) ? value : null +} + +function getBoolean(value: unknown): boolean | null { + return typeof value === 'boolean' ? value : null +} + +function getStringArray(value: unknown): string[] { + if (!Array.isArray(value)) return [] + return value.filter((entry): entry is string => typeof entry === 'string') +} + +function getRecordArray(value: unknown): JsonRecord[] { + if (!Array.isArray(value)) return [] + return value.filter(isRecordLike) +} + +/** + * Extracts a human-readable error message from a Vanta API error body. + */ +export function extractVantaError(data: unknown, fallback: string): string { + if (!isRecordLike(data)) return fallback + + if (isRecordLike(data.error)) { + const nested = getString(data.error.message) ?? getString(data.error.code) + if (nested) return nested + } + + return ( + getString(data.message) ?? + getString(data.error_description) ?? + getString(data.error) ?? + fallback + ) +} + +/** + * Builds a Vanta v1 API URL, appending only query parameters that have a + * value. Array values are appended as repeated parameters. + */ +export function buildVantaUrl( + baseUrl: string, + path: string, + query?: Record +): string { + const url = new URL(`${baseUrl}/v1${path}`) + if (query) { + for (const [key, value] of Object.entries(query)) { + if (value == null || value === '') continue + if (Array.isArray(value)) { + for (const entry of value) { + url.searchParams.append(key, entry) + } + } else { + url.searchParams.set(key, String(value)) + } + } + } + return url.toString() +} + +/** + * Splits a comma-separated filter value into trimmed entries, returning + * undefined when no usable entries remain. + */ +export function splitVantaCommaList(value: string | null | undefined): string[] | undefined { + if (!value) return undefined + const entries = value + .split(',') + .map((entry) => entry.trim()) + .filter(Boolean) + return entries.length > 0 ? entries : undefined +} + +/** + * Unwraps the `{ results: { data, pageInfo } }` envelope that every Vanta + * list endpoint returns. + */ +export function getVantaListResults(data: unknown): { + data: JsonRecord[] + pageInfo: VantaPageInfo | null +} { + if (!isRecordLike(data) || !isRecordLike(data.results)) { + return { data: [], pageInfo: null } + } + return { + data: getRecordArray(data.results.data), + pageInfo: normalizeVantaPageInfo(data.results.pageInfo), + } +} + +export function normalizeVantaPageInfo(value: unknown): VantaPageInfo | null { + if (!isRecordLike(value)) return null + return { + startCursor: getString(value.startCursor), + endCursor: getString(value.endCursor), + hasNextPage: getBoolean(value.hasNextPage) ?? false, + hasPreviousPage: getBoolean(value.hasPreviousPage) ?? false, + } +} + +function normalizeVantaOwner(value: unknown): VantaOwner | null { + if (!isRecordLike(value)) return null + return { + id: getString(value.id), + displayName: getString(value.displayName), + emailAddress: getString(value.emailAddress), + } +} + +function normalizeVantaCustomFields(value: unknown): VantaCustomField[] { + return getRecordArray(value).map((field) => ({ + label: getString(field.label), + value: Array.isArray(field.value) ? getStringArray(field.value) : getString(field.value), + })) +} + +export function normalizeVantaFramework(resource: JsonRecord): VantaFramework { + return { + id: getString(resource.id), + displayName: getString(resource.displayName), + shorthandName: getString(resource.shorthandName), + description: getString(resource.description), + numControlsCompleted: getNumber(resource.numControlsCompleted), + numControlsTotal: getNumber(resource.numControlsTotal), + numDocumentsPassing: getNumber(resource.numDocumentsPassing), + numDocumentsTotal: getNumber(resource.numDocumentsTotal), + numTestsPassing: getNumber(resource.numTestsPassing), + numTestsTotal: getNumber(resource.numTestsTotal), + } +} + +function normalizeVantaFrameworkRequirementControl( + resource: JsonRecord +): VantaFrameworkRequirementControl { + return { + id: getString(resource.id), + externalId: getString(resource.externalId), + name: getString(resource.name), + description: getString(resource.description), + } +} + +function normalizeVantaFrameworkRequirement(resource: JsonRecord): VantaFrameworkRequirement { + return { + id: getString(resource.id), + name: getString(resource.name), + shorthand: getString(resource.shorthand), + description: getString(resource.description), + controls: getRecordArray(resource.controls).map(normalizeVantaFrameworkRequirementControl), + } +} + +function normalizeVantaFrameworkRequirementCategory( + resource: JsonRecord +): VantaFrameworkRequirementCategory { + return { + id: getString(resource.id), + name: getString(resource.name), + shorthand: getString(resource.shorthand), + requirements: getRecordArray(resource.requirements).map(normalizeVantaFrameworkRequirement), + } +} + +export function normalizeVantaFrameworkDetail(resource: JsonRecord): VantaFrameworkDetail { + return { + ...normalizeVantaFramework(resource), + requirementCategories: getRecordArray(resource.requirementCategories).map( + normalizeVantaFrameworkRequirementCategory + ), + } +} + +export function normalizeVantaControl(resource: JsonRecord): VantaControl { + return { + id: getString(resource.id), + externalId: getString(resource.externalId), + name: getString(resource.name), + description: getString(resource.description), + source: getString(resource.source), + domains: getStringArray(resource.domains), + owner: normalizeVantaOwner(resource.owner), + role: getString(resource.role), + customFields: normalizeVantaCustomFields(resource.customFields), + creationDate: getString(resource.creationDate), + modificationDate: getString(resource.modificationDate), + } +} + +export function normalizeVantaControlDetail(resource: JsonRecord): VantaControlDetail { + return { + ...normalizeVantaControl(resource), + note: getString(resource.note), + status: getString(resource.status), + numDocumentsPassing: getNumber(resource.numDocumentsPassing), + numDocumentsTotal: getNumber(resource.numDocumentsTotal), + numTestsPassing: getNumber(resource.numTestsPassing), + numTestsTotal: getNumber(resource.numTestsTotal), + } +} + +export function normalizeVantaTest(resource: JsonRecord): VantaTest { + const version = isRecordLike(resource.version) + ? { major: getNumber(resource.version.major), minor: getNumber(resource.version.minor) } + : null + const deactivatedStatusInfo = isRecordLike(resource.deactivatedStatusInfo) + ? { + isDeactivated: getBoolean(resource.deactivatedStatusInfo.isDeactivated), + deactivatedReason: getString(resource.deactivatedStatusInfo.deactivatedReason), + lastUpdatedDate: getString(resource.deactivatedStatusInfo.lastUpdatedDate), + } + : null + const remediationStatusInfo = isRecordLike(resource.remediationStatusInfo) + ? { + status: getString(resource.remediationStatusInfo.status), + soonestRemediateByDate: getString(resource.remediationStatusInfo.soonestRemediateByDate), + itemCount: getNumber(resource.remediationStatusInfo.itemCount), + } + : null + + return { + id: getString(resource.id), + name: getString(resource.name), + description: getString(resource.description), + failureDescription: getString(resource.failureDescription), + remediationDescription: getString(resource.remediationDescription), + category: getString(resource.category), + status: getString(resource.status), + integrations: getStringArray(resource.integrations), + lastTestRunDate: getString(resource.lastTestRunDate), + latestFlipDate: getString(resource.latestFlipDate), + version, + deactivatedStatusInfo, + remediationStatusInfo, + owner: normalizeVantaOwner(resource.owner), + } +} + +export function normalizeVantaTestEntity(resource: JsonRecord): VantaTestEntity { + return { + id: getString(resource.id), + entityStatus: getString(resource.entityStatus), + displayName: getString(resource.displayName), + responseType: getString(resource.responseType), + deactivatedReason: getString(resource.deactivatedReason), + createdDate: getString(resource.createdDate), + lastUpdatedDate: getString(resource.lastUpdatedDate), + } +} + +export function normalizeVantaDocument(resource: JsonRecord): VantaDocument { + return { + id: getString(resource.id), + title: getString(resource.title), + description: getString(resource.description), + category: getString(resource.category), + ownerId: getString(resource.ownerId), + isSensitive: getBoolean(resource.isSensitive), + uploadStatus: getString(resource.uploadStatus), + uploadStatusDate: getString(resource.uploadStatusDate), + url: getString(resource.url), + } +} + +export function normalizeVantaDocumentDetail(resource: JsonRecord): VantaDocumentDetail { + const deactivatedStatus = isRecordLike(resource.deactivatedStatus) + ? { + isDeactivated: getBoolean(resource.deactivatedStatus.isDeactivated), + reason: getString(resource.deactivatedStatus.reason), + creationDate: getString(resource.deactivatedStatus.creationDate), + expiration: getString(resource.deactivatedStatus.expiration), + } + : null + + return { + ...normalizeVantaDocument(resource), + note: getString(resource.note), + nextRenewalDate: getString(resource.nextRenewalDate), + renewalCadence: getString(resource.renewalCadence), + reminderWindow: getString(resource.reminderWindow), + subscribers: getStringArray(resource.subscribers), + deactivatedStatus, + } +} + +export function normalizeVantaUploadedFile(resource: JsonRecord): VantaUploadedFile { + const uploadedBy = isRecordLike(resource.uploadedBy) + ? { id: getString(resource.uploadedBy.id), type: getString(resource.uploadedBy.type) } + : null + + return { + id: getString(resource.id), + fileName: getString(resource.fileName), + title: getString(resource.title), + description: getString(resource.description), + mimeType: getString(resource.mimeType), + uploadedBy, + creationDate: getString(resource.creationDate), + updatedDate: getString(resource.updatedDate), + deletionDate: getString(resource.deletionDate), + effectiveDate: getString(resource.effectiveDate), + url: getString(resource.url), + } +} + +export function normalizeVantaPerson(resource: JsonRecord): VantaPerson { + const name = isRecordLike(resource.name) + ? { + first: getString(resource.name.first), + last: getString(resource.name.last), + display: getString(resource.name.display), + } + : null + const employment = isRecordLike(resource.employment) + ? { + status: getString(resource.employment.status), + startDate: getString(resource.employment.startDate), + endDate: getString(resource.employment.endDate), + jobTitle: getString(resource.employment.jobTitle), + } + : null + const leaveInfo = isRecordLike(resource.leaveInfo) + ? { + status: getString(resource.leaveInfo.status), + startDate: getString(resource.leaveInfo.startDate), + endDate: getString(resource.leaveInfo.endDate), + } + : null + const tasksSummary = isRecordLike(resource.tasksSummary) + ? { + status: getString(resource.tasksSummary.status), + dueDate: getString(resource.tasksSummary.dueDate), + completionDate: getString(resource.tasksSummary.completionDate), + } + : null + + return { + id: getString(resource.id), + userId: getString(resource.userId), + emailAddress: getString(resource.emailAddress), + name, + employment, + leaveInfo, + groupIds: getStringArray(resource.groupIds), + tasksSummary, + } +} + +function normalizeVantaPolicyDocument(resource: JsonRecord): VantaPolicyDocument { + return { + language: getString(resource.language), + slugId: getString(resource.slugId), + url: getString(resource.url), + } +} + +export function normalizeVantaPolicy(resource: JsonRecord): VantaPolicy { + const latestApprovedVersion = isRecordLike(resource.latestApprovedVersion) + ? { + versionId: getString(resource.latestApprovedVersion.versionId), + documents: getRecordArray(resource.latestApprovedVersion.documents).map( + normalizeVantaPolicyDocument + ), + } + : null + + return { + id: getString(resource.id), + name: getString(resource.name), + description: getString(resource.description), + status: getString(resource.status), + approvedAtDate: getString(resource.approvedAtDate), + latestVersionStatus: isRecordLike(resource.latestVersion) + ? getString(resource.latestVersion.status) + : null, + latestApprovedVersion, + } +} + +export function normalizeVantaVendor(resource: JsonRecord): VantaVendor { + const authDetails = isRecordLike(resource.authDetails) + ? { + method: getString(resource.authDetails.method), + passwordMFA: getBoolean(resource.authDetails.passwordMFA), + passwordMinimumLength: getNumber(resource.authDetails.passwordMinimumLength), + passwordRequiresNumber: getBoolean(resource.authDetails.passwordRequiresNumber), + passwordRequiresSymbol: getBoolean(resource.authDetails.passwordRequiresSymbol), + } + : null + const contractAmount = isRecordLike(resource.contractAmount) + ? { + amount: getNumber(resource.contractAmount.amount), + currency: getString(resource.contractAmount.currency), + } + : null + const latestDecision = isRecordLike(resource.latestDecision) + ? { + status: getString(resource.latestDecision.status), + lastUpdatedAt: getString(resource.latestDecision.lastUpdatedAt), + } + : null + const procurementRequest = isRecordLike(resource.linkedTaskTrackerTaskProcurementRequest) + ? { + url: getString(resource.linkedTaskTrackerTaskProcurementRequest.url), + service: getString(resource.linkedTaskTrackerTaskProcurementRequest.service), + } + : null + + return { + id: getString(resource.id), + name: getString(resource.name), + status: getString(resource.status), + websiteUrl: getString(resource.websiteUrl), + category: isRecordLike(resource.category) ? getString(resource.category.displayName) : null, + servicesProvided: getString(resource.servicesProvided), + additionalNotes: getString(resource.additionalNotes), + accountManagerName: getString(resource.accountManagerName), + accountManagerEmail: getString(resource.accountManagerEmail), + securityOwnerUserId: getString(resource.securityOwnerUserId), + businessOwnerUserId: getString(resource.businessOwnerUserId), + inherentRiskLevel: getString(resource.inherentRiskLevel), + residualRiskLevel: getString(resource.residualRiskLevel), + isRiskAutoScored: getBoolean(resource.isRiskAutoScored), + isVisibleToAuditors: getBoolean(resource.isVisibleToAuditors), + riskAttributeIds: getStringArray(resource.riskAttributeIds), + vendorHeadquarters: getString(resource.vendorHeadquarters), + contractStartDate: getString(resource.contractStartDate), + contractRenewalDate: getString(resource.contractRenewalDate), + contractTerminationDate: getString(resource.contractTerminationDate), + contractAmount, + nextSecurityReviewDueDate: getString(resource.nextSecurityReviewDueDate), + lastSecurityReviewCompletionDate: getString(resource.lastSecurityReviewCompletionDate), + authDetails, + customFields: normalizeVantaCustomFields(resource.customFields), + latestDecision, + linkedTaskTrackerTaskProcurementRequest: procurementRequest, + } +} + +function getComputerStatusOutcome(value: unknown): string | null { + return isRecordLike(value) ? getString(value.outcome) : null +} + +export function normalizeVantaMonitoredComputer(resource: JsonRecord): VantaMonitoredComputer { + const operatingSystem = isRecordLike(resource.operatingSystem) + ? { + type: getString(resource.operatingSystem.type), + version: getString(resource.operatingSystem.version), + } + : null + + return { + id: getString(resource.id), + integrationId: getString(resource.integrationId), + lastCheckDate: getString(resource.lastCheckDate), + screenlock: getComputerStatusOutcome(resource.screenlock), + diskEncryption: getComputerStatusOutcome(resource.diskEncryption), + passwordManager: getComputerStatusOutcome(resource.passwordManager), + antivirusInstallation: getComputerStatusOutcome(resource.antivirusInstallation), + operatingSystem, + owner: normalizeVantaOwner(resource.owner), + serialNumber: getString(resource.serialNumber), + udid: getString(resource.udid), + } +} + +export function normalizeVantaRiskScenario(resource: JsonRecord): VantaRiskScenario { + return { + riskId: getString(resource.riskId), + description: getString(resource.description), + likelihood: getNumber(resource.likelihood), + impact: getNumber(resource.impact), + residualLikelihood: getNumber(resource.residualLikelihood), + residualImpact: getNumber(resource.residualImpact), + categories: getStringArray(resource.categories), + ciaCategories: getStringArray(resource.ciaCategories), + treatment: getString(resource.treatment), + owner: getString(resource.owner), + note: getString(resource.note), + riskRegister: getString(resource.riskRegister), + customFields: normalizeVantaCustomFields(resource.customFields), + isArchived: getBoolean(resource.isArchived), + reviewStatus: getString(resource.reviewStatus), + requiredApprovers: getStringArray(resource.requiredApprovers), + type: getString(resource.type), + identificationDate: getString(resource.identificationDate), + } +} + +export function normalizeVantaVulnerabilityRemediation( + resource: JsonRecord +): VantaVulnerabilityRemediation { + return { + id: getString(resource.id), + vulnerabilityId: getString(resource.vulnerabilityId), + vulnerableAssetId: getString(resource.vulnerableAssetId), + severity: getString(resource.severity), + detectedDate: getString(resource.detectedDate), + slaDeadlineDate: getString(resource.slaDeadlineDate), + remediationDate: getString(resource.remediationDate), + } +} + +function normalizeVantaVulnerableAssetScanner(resource: JsonRecord): VantaVulnerableAssetScanner { + return { + resourceId: getString(resource.resourceId), + integrationId: getString(resource.integrationId), + targetId: getString(resource.targetId), + imageDigest: getString(resource.imageDigest), + imagePushedAtDate: getString(resource.imagePushedAtDate), + imageTags: getStringArray(resource.imageTags), + assetTags: getRecordArray(resource.assetTags).map((tag) => ({ + key: getString(tag.key), + value: getString(tag.value), + })), + parentAccountOrOrganization: getString(resource.parentAccountOrOrganization), + biosUuid: getString(resource.biosUuid), + ipv4s: getStringArray(resource.ipv4s), + ipv6s: getStringArray(resource.ipv6s), + macAddresses: getStringArray(resource.macAddresses), + hostnames: getStringArray(resource.hostnames), + fqdns: getStringArray(resource.fqdns), + operatingSystems: getStringArray(resource.operatingSystems), + } +} + +export function normalizeVantaVulnerableAsset(resource: JsonRecord): VantaVulnerableAsset { + return { + id: getString(resource.id), + name: getString(resource.name), + assetType: getString(resource.assetType), + hasBeenScanned: getBoolean(resource.hasBeenScanned), + imageScanTag: getString(resource.imageScanTag), + scanners: getRecordArray(resource.scanners).map(normalizeVantaVulnerableAssetScanner), + } +} + +export function normalizeVantaVulnerability(resource: JsonRecord): VantaVulnerability { + const deactivateMetadata = isRecordLike(resource.deactivateMetadata) + ? { + isVulnDeactivatedIndefinitely: getBoolean( + resource.deactivateMetadata.isVulnDeactivatedIndefinitely + ), + deactivatedUntilDate: getString(resource.deactivateMetadata.deactivatedUntilDate), + deactivationReason: getString(resource.deactivateMetadata.deactivationReason), + deactivatedOnDate: getString(resource.deactivateMetadata.deactivatedOnDate), + deactivatedBy: getString(resource.deactivateMetadata.deactivatedBy), + } + : null + + return { + id: getString(resource.id), + name: getString(resource.name), + description: getString(resource.description), + severity: getString(resource.severity), + vulnerabilityType: getString(resource.vulnerabilityType), + integrationId: getString(resource.integrationId), + targetId: getString(resource.targetId), + packageIdentifier: getString(resource.packageIdentifier), + cvssSeverityScore: getNumber(resource.cvssSeverityScore), + scannerScore: getNumber(resource.scannerScore), + isFixable: getBoolean(resource.isFixable), + fixedVersion: getString(resource.fixedVersion), + remediateByDate: getString(resource.remediateByDate), + firstDetectedDate: getString(resource.firstDetectedDate), + sourceDetectedDate: getString(resource.sourceDetectedDate), + lastDetectedDate: getString(resource.lastDetectedDate), + scanSource: getString(resource.scanSource), + externalURL: getString(resource.externalURL), + relatedVulns: getStringArray(resource.relatedVulns), + relatedUrls: getStringArray(resource.relatedUrls), + deactivateMetadata, + } +} diff --git a/apps/sim/lib/internal/vanta/operations.test.ts b/apps/sim/lib/internal/vanta/operations.test.ts new file mode 100644 index 00000000000..732f6561972 --- /dev/null +++ b/apps/sim/lib/internal/vanta/operations.test.ts @@ -0,0 +1,232 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + fetchAuth: vi.fn(), + resolveFile: vi.fn(), +})) + +vi.mock('@/lib/internal/vanta/client', () => ({ + fetchVantaWithAuth: mocks.fetchAuth, + getVantaBaseUrl: (region?: string) => + region === 'gov' ? 'https://api.vanta-gov.com' : 'https://api.vanta.com', + VANTA_DOCUMENT_UPLOAD_SCOPE: 'vanta-api.all:read vanta-api.all:write vanta-api.documents:upload', + VANTA_READ_SCOPE: 'vanta-api.all:read', +})) + +vi.mock('@/lib/internal/vanta/file-input', () => ({ + resolveVantaUploadFile: mocks.resolveFile, +})) + +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { VANTA_MAX_TRANSFER_BYTES } from '@/lib/internal/vanta/input' +import { + executeVantaDownloadDocumentFile, + executeVantaQuery, + executeVantaUploadDocumentFile, +} from '@/lib/internal/vanta/operations' + +const context = { + requestId: 'request-1', + signal: new AbortController().signal, + userId: 'user-1', +} + +describe('Vanta operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.resolveFile.mockResolvedValue({ + buffer: Buffer.from('file'), + fileName: 'evidence.txt', + mimeType: 'text/plain', + }) + }) + + it('uploads authorized files with provider cancellation and exact output', async () => { + mocks.fetchAuth.mockImplementation( + async (_params: unknown, perform: (token: string) => Promise) => perform('token') + ) + const fetchMock = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValue( + Response.json({ id: 'upload-1', fileName: 'evidence.txt', mimeType: 'text/plain' }) + ) + + const result = await executeVantaUploadDocumentFile( + { + clientId: 'client', + clientSecret: 'secret', + documentId: 'document-1', + file: { key: 'workspace/file.txt', name: 'file.txt', size: 4 }, + description: 'Evidence', + }, + context + ) + + expect(mocks.resolveFile).toHaveBeenCalledWith(expect.anything(), context) + expect(fetchMock).toHaveBeenCalledWith( + 'https://api.vanta.com/v1/documents/document-1/uploads', + expect.objectContaining({ method: 'POST', signal: context.signal }) + ) + expect(result).toMatchObject({ + success: true, + output: { upload: { id: 'upload-1', fileName: 'evidence.txt' } }, + }) + fetchMock.mockRestore() + }) + + it('downloads files with exact binary projection and filename parsing', async () => { + mocks.fetchAuth.mockResolvedValue( + new Response('hello', { + headers: { + 'Content-Disposition': "attachment; filename*=UTF-8''report%20final.pdf", + 'Content-Type': 'application/pdf', + }, + }) + ) + + const result = await executeVantaDownloadDocumentFile( + { + clientId: 'client', + clientSecret: 'secret', + documentId: 'document-1', + uploadedFileId: 'upload-1', + }, + context + ) + + expect(mocks.fetchAuth.mock.calls[0]?.[2]).toEqual({ signal: context.signal }) + expect(result).toEqual({ + success: true, + output: { + file: { + name: 'report final.pdf', + mimeType: 'application/pdf', + data: Buffer.from('hello').toString('base64'), + size: 5, + }, + name: 'report final.pdf', + mimeType: 'application/pdf', + size: 5, + }, + }) + }) + + it('preserves provider and download-size error envelopes', async () => { + mocks.fetchAuth.mockResolvedValueOnce( + Response.json({ error: { message: 'Not found' } }, { status: 404 }) + ) + await expect( + executeVantaDownloadDocumentFile( + { + clientId: 'client', + clientSecret: 'secret', + documentId: 'document-1', + uploadedFileId: 'missing', + }, + context + ) + ).rejects.toMatchObject({ + status: 404, + body: { success: false, error: 'Not found' }, + }) + + let cancelled = false + mocks.fetchAuth.mockResolvedValueOnce( + new Response( + new ReadableStream({ + cancel: () => { + cancelled = true + }, + }), + { headers: { 'Content-Length': String(VANTA_MAX_TRANSFER_BYTES + 1) } } + ) + ) + await expect( + executeVantaDownloadDocumentFile( + { + clientId: 'client', + clientSecret: 'secret', + documentId: 'document-1', + uploadedFileId: 'large', + }, + context + ) + ).rejects.toMatchObject({ + status: 400, + body: { success: false, error: 'File size (100.00MB) exceeds download limit of 100MB' }, + }) + expect(cancelled).toBe(true) + }) + + it('maps unknown-length streamed overflow to the legacy error', async () => { + const streamError = new PayloadSizeLimitError({ + label: 'Vanta document file', + maxBytes: VANTA_MAX_TRANSFER_BYTES, + observedBytes: VANTA_MAX_TRANSFER_BYTES + 1, + }) + const body = new ReadableStream({ + pull: (controller) => controller.error(streamError), + }) + mocks.fetchAuth.mockResolvedValue(new Response(body)) + + await expect( + executeVantaDownloadDocumentFile( + { + clientId: 'client', + clientSecret: 'secret', + documentId: 'document-1', + uploadedFileId: 'large', + }, + context + ) + ).rejects.toMatchObject({ + status: 400, + body: { success: false, error: 'File exceeds download limit of 100MB' }, + }) + }) + + it('executes query operations directly with exact provider URL and normalized output', async () => { + mocks.fetchAuth.mockImplementation( + async (_params: unknown, perform: (token: string) => Promise) => perform('token') + ) + const fetchMock = vi.spyOn(globalThis, 'fetch').mockResolvedValue( + Response.json({ + results: { + data: [{ id: 'framework-1', displayName: 'SOC 2' }], + pageInfo: { endCursor: 'cursor-1', hasNextPage: true }, + }, + }) + ) + + const result = await executeVantaQuery( + { + operation: 'vanta_list_frameworks', + clientId: 'client', + clientSecret: 'secret', + pageSize: 25, + pageCursor: 'cursor-0', + }, + context.signal + ) + + expect(fetchMock).toHaveBeenCalledWith( + 'https://api.vanta.com/v1/frameworks?pageSize=25&pageCursor=cursor-0', + expect.objectContaining({ + method: 'GET', + headers: expect.objectContaining({ Authorization: 'Bearer token' }), + signal: context.signal, + }) + ) + expect(result).toMatchObject({ + success: true, + output: { + frameworks: [{ id: 'framework-1', displayName: 'SOC 2' }], + pageInfo: { endCursor: 'cursor-1', hasNextPage: true }, + }, + }) + fetchMock.mockRestore() + }) +}) diff --git a/apps/sim/lib/internal/vanta/operations.ts b/apps/sim/lib/internal/vanta/operations.ts new file mode 100644 index 00000000000..ced9bece5cd --- /dev/null +++ b/apps/sim/lib/internal/vanta/operations.ts @@ -0,0 +1,577 @@ +import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { + isPayloadSizeLimitError, + readResponseJsonWithLimit, + readResponseToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' +import { + fetchVantaWithAuth, + getVantaBaseUrl, + VANTA_DOCUMENT_UPLOAD_SCOPE, + VANTA_READ_SCOPE, + VANTA_WRITE_SCOPE, +} from '@/lib/internal/vanta/client' +import { VantaOperationError } from '@/lib/internal/vanta/errors' +import { resolveVantaUploadFile } from '@/lib/internal/vanta/file-input' +import { + VANTA_MAX_TRANSFER_BYTES, + type VantaDownloadDocumentFileInput, + type VantaUploadDocumentFileInput, +} from '@/lib/internal/vanta/input' +import { + asVantaRecord, + buildVantaUrl, + extractVantaError, + getVantaListResults, + normalizeVantaControl, + normalizeVantaControlDetail, + normalizeVantaDocument, + normalizeVantaDocumentDetail, + normalizeVantaFramework, + normalizeVantaFrameworkDetail, + normalizeVantaMonitoredComputer, + normalizeVantaPerson, + normalizeVantaPolicy, + normalizeVantaRiskScenario, + normalizeVantaTest, + normalizeVantaTestEntity, + normalizeVantaUploadedFile, + normalizeVantaVendor, + normalizeVantaVulnerability, + normalizeVantaVulnerabilityRemediation, + normalizeVantaVulnerableAsset, + splitVantaCommaList, +} from '@/lib/internal/vanta/normalizers' +import type { VantaQueryBody } from '@/lib/internal/vanta/schema' + +interface VantaFileOperationContext { + requestId: string + signal?: AbortSignal + userId: string +} + +function downloadSizeError(bytes?: number): VantaOperationError { + return new VantaOperationError(400, { + success: false, + error: + bytes === undefined + ? 'File exceeds download limit of 100MB' + : `File size (${(bytes / (1024 * 1024)).toFixed(2)}MB) exceeds download limit of 100MB`, + }) +} + +function fileNameFromContentDisposition(header: string | null): string | null { + if (!header) return null + const utf8Match = header.match(/filename\*=UTF-8''([^;]+)/i) + if (utf8Match) { + try { + return decodeURIComponent(utf8Match[1]) + } catch { + return null + } + } + return header.match(/filename="?([^";]+)"?/i)?.[1] ?? null +} + +async function readVantaJson(response: Response, signal?: AbortSignal): Promise { + return readResponseJsonWithLimit(response, { + maxBytes: MAX_JSON_API_RESPONSE_BYTES, + label: 'Vanta API response', + signal, + }).catch(() => null) +} + +interface VantaApiRequest { + method: 'GET' | 'POST' + url: string +} + +/** + * Maps a validated query operation to the Vanta API request it performs. + */ +function buildVantaApiRequest(baseUrl: string, params: VantaQueryBody): VantaApiRequest { + const id = encodeURIComponent + + switch (params.operation) { + case 'vanta_list_frameworks': + return { + method: 'GET', + url: buildVantaUrl(baseUrl, '/frameworks', { + pageSize: params.pageSize, + pageCursor: params.pageCursor, + }), + } + case 'vanta_get_framework': + return { method: 'GET', url: buildVantaUrl(baseUrl, `/frameworks/${id(params.frameworkId)}`) } + case 'vanta_list_framework_controls': + return { + method: 'GET', + url: buildVantaUrl(baseUrl, `/frameworks/${id(params.frameworkId)}/controls`, { + pageSize: params.pageSize, + pageCursor: params.pageCursor, + }), + } + case 'vanta_list_controls': + return { + method: 'GET', + url: buildVantaUrl(baseUrl, '/controls', { + frameworkMatchesAny: splitVantaCommaList(params.frameworkMatchesAny), + pageSize: params.pageSize, + pageCursor: params.pageCursor, + }), + } + case 'vanta_get_control': + return { method: 'GET', url: buildVantaUrl(baseUrl, `/controls/${id(params.controlId)}`) } + case 'vanta_list_control_tests': + return { + method: 'GET', + url: buildVantaUrl(baseUrl, `/controls/${id(params.controlId)}/tests`, { + pageSize: params.pageSize, + pageCursor: params.pageCursor, + }), + } + case 'vanta_list_control_documents': + return { + method: 'GET', + url: buildVantaUrl(baseUrl, `/controls/${id(params.controlId)}/documents`, { + pageSize: params.pageSize, + pageCursor: params.pageCursor, + }), + } + case 'vanta_list_tests': + return { + method: 'GET', + url: buildVantaUrl(baseUrl, '/tests', { + statusFilter: params.statusFilter, + frameworkFilter: params.frameworkFilter, + integrationFilter: params.integrationFilter, + controlFilter: params.controlFilter, + ownerFilter: params.ownerFilter, + categoryFilter: params.categoryFilter, + isInRollout: params.isInRollout, + pageSize: params.pageSize, + pageCursor: params.pageCursor, + }), + } + case 'vanta_get_test': + return { method: 'GET', url: buildVantaUrl(baseUrl, `/tests/${id(params.testId)}`) } + case 'vanta_list_test_entities': + return { + method: 'GET', + url: buildVantaUrl(baseUrl, `/tests/${id(params.testId)}/entities`, { + entityStatus: params.entityStatus, + pageSize: params.pageSize, + pageCursor: params.pageCursor, + }), + } + case 'vanta_list_documents': + return { + method: 'GET', + url: buildVantaUrl(baseUrl, '/documents', { + frameworkMatchesAny: splitVantaCommaList(params.frameworkMatchesAny), + statusMatchesAny: splitVantaCommaList(params.statusMatchesAny), + pageSize: params.pageSize, + pageCursor: params.pageCursor, + }), + } + case 'vanta_get_document': + return { method: 'GET', url: buildVantaUrl(baseUrl, `/documents/${id(params.documentId)}`) } + case 'vanta_list_document_uploads': + return { + method: 'GET', + url: buildVantaUrl(baseUrl, `/documents/${id(params.documentId)}/uploads`, { + pageSize: params.pageSize, + pageCursor: params.pageCursor, + }), + } + case 'vanta_submit_document': + return { + method: 'POST', + url: buildVantaUrl(baseUrl, `/documents/${id(params.documentId)}/submit`), + } + case 'vanta_list_people': + return { + method: 'GET', + url: buildVantaUrl(baseUrl, '/people', { + emailAndNameFilter: params.emailAndNameFilter, + employmentStatus: params.employmentStatus, + groupIdsMatchesAny: splitVantaCommaList(params.groupIdsMatchesAny), + tasksSummaryStatusMatchesAny: splitVantaCommaList(params.tasksSummaryStatusMatchesAny), + taskTypeMatchesAny: splitVantaCommaList(params.taskTypeMatchesAny), + taskStatusMatchesAny: splitVantaCommaList(params.taskStatusMatchesAny), + pageSize: params.pageSize, + pageCursor: params.pageCursor, + }), + } + case 'vanta_get_person': + return { method: 'GET', url: buildVantaUrl(baseUrl, `/people/${id(params.personId)}`) } + case 'vanta_list_policies': + return { + method: 'GET', + url: buildVantaUrl(baseUrl, '/policies', { + pageSize: params.pageSize, + pageCursor: params.pageCursor, + }), + } + case 'vanta_get_policy': + return { method: 'GET', url: buildVantaUrl(baseUrl, `/policies/${id(params.policyId)}`) } + case 'vanta_list_vendors': + return { + method: 'GET', + url: buildVantaUrl(baseUrl, '/vendors', { + name: params.name, + statusMatchesAny: splitVantaCommaList(params.statusMatchesAny), + pageSize: params.pageSize, + pageCursor: params.pageCursor, + }), + } + case 'vanta_get_vendor': + return { method: 'GET', url: buildVantaUrl(baseUrl, `/vendors/${id(params.vendorId)}`) } + case 'vanta_list_monitored_computers': + return { + method: 'GET', + url: buildVantaUrl(baseUrl, '/monitored-computers', { + complianceStatusFilterMatchesAny: splitVantaCommaList( + params.complianceStatusFilterMatchesAny + ), + pageSize: params.pageSize, + pageCursor: params.pageCursor, + }), + } + case 'vanta_list_vulnerabilities': + return { + method: 'GET', + url: buildVantaUrl(baseUrl, '/vulnerabilities', { + q: params.q, + severity: params.severity, + isFixAvailable: params.isFixAvailable, + isDeactivated: params.isDeactivated, + includeVulnerabilitiesWithoutSlas: params.includeVulnerabilitiesWithoutSlas, + packageIdentifier: params.packageIdentifier, + externalVulnerabilityId: params.externalVulnerabilityId, + integrationId: params.integrationId, + vulnerableAssetId: params.vulnerableAssetId, + slaDeadlineAfterDate: params.slaDeadlineAfterDate, + slaDeadlineBeforeDate: params.slaDeadlineBeforeDate, + pageSize: params.pageSize, + pageCursor: params.pageCursor, + }), + } + case 'vanta_list_vulnerability_remediations': + return { + method: 'GET', + url: buildVantaUrl(baseUrl, '/vulnerability-remediations', { + integrationId: params.integrationId, + severity: params.severity, + isRemediatedOnTime: params.isRemediatedOnTime, + remediatedAfterDate: params.remediatedAfterDate, + remediatedBeforeDate: params.remediatedBeforeDate, + pageSize: params.pageSize, + pageCursor: params.pageCursor, + }), + } + case 'vanta_list_vulnerable_assets': + return { + method: 'GET', + url: buildVantaUrl(baseUrl, '/vulnerable-assets', { + q: params.q, + integrationId: params.integrationId, + assetType: params.assetType, + assetExternalAccountId: params.assetExternalAccountId, + pageSize: params.pageSize, + pageCursor: params.pageCursor, + }), + } + case 'vanta_get_vulnerable_asset': + return { + method: 'GET', + url: buildVantaUrl(baseUrl, `/vulnerable-assets/${id(params.vulnerableAssetId)}`), + } + case 'vanta_list_risk_scenarios': + return { + method: 'GET', + url: buildVantaUrl(baseUrl, '/risk-scenarios', { + searchString: params.searchString, + includeIgnored: params.includeIgnored, + type: params.type, + ownerMatchesAny: splitVantaCommaList(params.ownerMatchesAny), + categoryMatchesAny: splitVantaCommaList(params.categoryMatchesAny), + ciaCategoryMatchesAny: splitVantaCommaList(params.ciaCategoryMatchesAny), + treatmentTypeMatchesAny: splitVantaCommaList(params.treatmentTypeMatchesAny), + inherentScoreGroupMatchesAny: splitVantaCommaList(params.inherentScoreGroupMatchesAny), + residualScoreGroupMatchesAny: splitVantaCommaList(params.residualScoreGroupMatchesAny), + reviewStatusMatchesAny: splitVantaCommaList(params.reviewStatusMatchesAny), + orderBy: params.orderBy, + pageSize: params.pageSize, + pageCursor: params.pageCursor, + }), + } + case 'vanta_get_risk_scenario': + return { + method: 'GET', + url: buildVantaUrl(baseUrl, `/risk-scenarios/${id(params.riskScenarioId)}`), + } + } +} + +/** + * Normalizes a successful Vanta API response body into the operation's + * documented output shape. + */ +function buildVantaOutput(params: VantaQueryBody, data: unknown): Record { + switch (params.operation) { + case 'vanta_list_frameworks': { + const { data: items, pageInfo } = getVantaListResults(data) + return { frameworks: items.map(normalizeVantaFramework), pageInfo } + } + case 'vanta_get_framework': + return { framework: normalizeVantaFrameworkDetail(asVantaRecord(data)) } + case 'vanta_list_framework_controls': + case 'vanta_list_controls': { + const { data: items, pageInfo } = getVantaListResults(data) + return { controls: items.map(normalizeVantaControl), pageInfo } + } + case 'vanta_get_control': + return { control: normalizeVantaControlDetail(asVantaRecord(data)) } + case 'vanta_list_control_tests': + case 'vanta_list_tests': { + const { data: items, pageInfo } = getVantaListResults(data) + return { tests: items.map(normalizeVantaTest), pageInfo } + } + case 'vanta_get_test': + return { test: normalizeVantaTest(asVantaRecord(data)) } + case 'vanta_list_test_entities': { + const { data: items, pageInfo } = getVantaListResults(data) + return { entities: items.map(normalizeVantaTestEntity), pageInfo } + } + case 'vanta_list_control_documents': + case 'vanta_list_documents': { + const { data: items, pageInfo } = getVantaListResults(data) + return { documents: items.map(normalizeVantaDocument), pageInfo } + } + case 'vanta_get_document': + return { document: normalizeVantaDocumentDetail(asVantaRecord(data)) } + case 'vanta_list_document_uploads': { + const { data: items, pageInfo } = getVantaListResults(data) + return { uploads: items.map(normalizeVantaUploadedFile), pageInfo } + } + case 'vanta_submit_document': + return { documentId: params.documentId, submitted: true } + case 'vanta_list_people': { + const { data: items, pageInfo } = getVantaListResults(data) + return { people: items.map(normalizeVantaPerson), pageInfo } + } + case 'vanta_get_person': + return { person: normalizeVantaPerson(asVantaRecord(data)) } + case 'vanta_list_policies': { + const { data: items, pageInfo } = getVantaListResults(data) + return { policies: items.map(normalizeVantaPolicy), pageInfo } + } + case 'vanta_get_policy': + return { policy: normalizeVantaPolicy(asVantaRecord(data)) } + case 'vanta_list_vendors': { + const { data: items, pageInfo } = getVantaListResults(data) + return { vendors: items.map(normalizeVantaVendor), pageInfo } + } + case 'vanta_get_vendor': + return { vendor: normalizeVantaVendor(asVantaRecord(data)) } + case 'vanta_list_monitored_computers': { + const { data: items, pageInfo } = getVantaListResults(data) + return { computers: items.map(normalizeVantaMonitoredComputer), pageInfo } + } + case 'vanta_list_vulnerabilities': { + const { data: items, pageInfo } = getVantaListResults(data) + return { vulnerabilities: items.map(normalizeVantaVulnerability), pageInfo } + } + case 'vanta_list_vulnerability_remediations': { + const { data: items, pageInfo } = getVantaListResults(data) + return { remediations: items.map(normalizeVantaVulnerabilityRemediation), pageInfo } + } + case 'vanta_list_vulnerable_assets': { + const { data: items, pageInfo } = getVantaListResults(data) + return { assets: items.map(normalizeVantaVulnerableAsset), pageInfo } + } + case 'vanta_get_vulnerable_asset': + return { asset: normalizeVantaVulnerableAsset(asVantaRecord(data)) } + case 'vanta_list_risk_scenarios': { + const { data: items, pageInfo } = getVantaListResults(data) + return { riskScenarios: items.map(normalizeVantaRiskScenario), pageInfo } + } + case 'vanta_get_risk_scenario': + return { riskScenario: normalizeVantaRiskScenario(asVantaRecord(data)) } + } +} + +export type VantaQueryResult = + | { success: true; output: Record } + | { success: false; error: string; status: number } + +/** Executes one canonical Vanta query directly against the provider. */ +export async function executeVantaQuery( + params: VantaQueryBody, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const baseUrl = getVantaBaseUrl(params.region) + const scope = params.operation === 'vanta_submit_document' ? VANTA_WRITE_SCOPE : VANTA_READ_SCOPE + const apiRequest = buildVantaApiRequest(baseUrl, params) + const response = await fetchVantaWithAuth( + { + clientId: params.clientId, + clientSecret: params.clientSecret, + region: params.region, + scope, + }, + (accessToken) => + fetch(apiRequest.url, { + method: apiRequest.method, + headers: { + Accept: 'application/json', + Authorization: `Bearer ${accessToken}`, + }, + cache: 'no-store', + signal, + }), + { signal } + ) + signal?.throwIfAborted() + + const data = + response.status === 204 + ? null + : await readResponseJsonWithLimit(response, { + maxBytes: MAX_JSON_API_RESPONSE_BYTES, + label: 'Vanta API response', + signal, + }).catch(() => null) + signal?.throwIfAborted() + + if (!response.ok) { + return { + success: false, + error: extractVantaError(data, 'Vanta request failed'), + status: response.status, + } + } + return { success: true, output: buildVantaOutput(params, data) } +} + +export async function executeVantaUploadDocumentFile( + input: VantaUploadDocumentFileInput, + context: VantaFileOperationContext +): Promise> { + context.signal?.throwIfAborted() + const file = await resolveVantaUploadFile(input, context) + context.signal?.throwIfAborted() + const uploadUrl = buildVantaUrl( + getVantaBaseUrl(input.region), + `/documents/${encodeURIComponent(input.documentId)}/uploads` + ) + const response = await fetchVantaWithAuth( + { + clientId: input.clientId, + clientSecret: input.clientSecret, + region: input.region, + scope: VANTA_DOCUMENT_UPLOAD_SCOPE, + }, + (accessToken) => { + const formData = new FormData() + formData.append( + 'file', + new Blob([new Uint8Array(file.buffer)], { type: file.mimeType }), + file.fileName + ) + if (input.description) formData.append('description', input.description) + if (input.effectiveAtDate) formData.append('effectiveAtDate', input.effectiveAtDate) + return fetch(uploadUrl, { + method: 'POST', + headers: { Authorization: `Bearer ${accessToken}` }, + body: formData, + cache: 'no-store', + signal: context.signal, + }) + }, + { signal: context.signal } + ) + context.signal?.throwIfAborted() + const data = await readVantaJson(response, context.signal) + context.signal?.throwIfAborted() + if (!response.ok) { + throw new VantaOperationError(response.status, { + success: false, + error: extractVantaError(data, 'Failed to upload file to Vanta document'), + }) + } + return { + success: true, + output: { upload: normalizeVantaUploadedFile(asVantaRecord(data)) }, + } +} + +export async function executeVantaDownloadDocumentFile( + input: VantaDownloadDocumentFileInput, + context: VantaFileOperationContext +): Promise> { + context.signal?.throwIfAborted() + const mediaUrl = buildVantaUrl( + getVantaBaseUrl(input.region), + `/documents/${encodeURIComponent(input.documentId)}/uploads/${encodeURIComponent(input.uploadedFileId)}/media` + ) + const response = await fetchVantaWithAuth( + { + clientId: input.clientId, + clientSecret: input.clientSecret, + region: input.region, + scope: VANTA_READ_SCOPE, + }, + (accessToken) => + fetch(mediaUrl, { + method: 'GET', + headers: { Authorization: `Bearer ${accessToken}` }, + cache: 'no-store', + signal: context.signal, + }), + { signal: context.signal } + ) + context.signal?.throwIfAborted() + if (!response.ok) { + const errorData = await readVantaJson(response, context.signal) + throw new VantaOperationError(response.status, { + success: false, + error: extractVantaError(errorData, 'Failed to download Vanta document file'), + }) + } + + const contentLengthHeader = response.headers.get('content-length') + const contentLength = contentLengthHeader === null ? undefined : Number(contentLengthHeader) + const knownContentLength = + contentLength !== undefined && Number.isFinite(contentLength) ? contentLength : undefined + let buffer: Buffer + try { + buffer = await readResponseToBufferWithLimit(response, { + maxBytes: VANTA_MAX_TRANSFER_BYTES, + label: 'Vanta document file', + signal: context.signal, + }) + } catch (error) { + context.signal?.throwIfAborted() + if (isPayloadSizeLimitError(error)) { + throw downloadSizeError(knownContentLength) + } + throw error + } + context.signal?.throwIfAborted() + const mimeType = response.headers.get('content-type') || 'application/octet-stream' + const name = + fileNameFromContentDisposition(response.headers.get('content-disposition')) || + `vanta-document-file-${input.uploadedFileId}` + return { + success: true, + output: { + file: { name, mimeType, data: buffer.toString('base64'), size: buffer.length }, + name, + mimeType, + size: buffer.length, + }, + } +} diff --git a/apps/sim/lib/internal/vanta/schema.ts b/apps/sim/lib/internal/vanta/schema.ts new file mode 100644 index 00000000000..ad28745ae37 --- /dev/null +++ b/apps/sim/lib/internal/vanta/schema.ts @@ -0,0 +1,664 @@ +import { z } from 'zod' + +const nullableString = z.string().nullable() +const nullableNumber = z.number().nullable() +const nullableBoolean = z.boolean().nullable() + +const vantaPageInfoSchema = z + .object({ + startCursor: nullableString, + endCursor: nullableString, + hasNextPage: z.boolean(), + hasPreviousPage: z.boolean(), + }) + .nullable() + +const vantaOwnerSchema = z + .object({ + id: nullableString, + displayName: nullableString, + emailAddress: nullableString, + }) + .nullable() + +const vantaCustomFieldsSchema = z.array( + z.object({ + label: nullableString, + value: z.union([z.string(), z.array(z.string())]).nullable(), + }) +) + +const vantaFrameworkSchema = z.object({ + id: nullableString, + displayName: nullableString, + shorthandName: nullableString, + description: nullableString, + numControlsCompleted: nullableNumber, + numControlsTotal: nullableNumber, + numDocumentsPassing: nullableNumber, + numDocumentsTotal: nullableNumber, + numTestsPassing: nullableNumber, + numTestsTotal: nullableNumber, +}) + +const vantaFrameworkDetailSchema = vantaFrameworkSchema.extend({ + requirementCategories: z.array( + z.object({ + id: nullableString, + name: nullableString, + shorthand: nullableString, + requirements: z.array( + z.object({ + id: nullableString, + name: nullableString, + shorthand: nullableString, + description: nullableString, + controls: z.array( + z.object({ + id: nullableString, + externalId: nullableString, + name: nullableString, + description: nullableString, + }) + ), + }) + ), + }) + ), +}) + +const vantaControlSchema = z.object({ + id: nullableString, + externalId: nullableString, + name: nullableString, + description: nullableString, + source: nullableString, + domains: z.array(z.string()), + owner: vantaOwnerSchema, + role: nullableString, + customFields: vantaCustomFieldsSchema, + creationDate: nullableString, + modificationDate: nullableString, +}) + +const vantaControlDetailSchema = vantaControlSchema.extend({ + note: nullableString, + status: nullableString, + numDocumentsPassing: nullableNumber, + numDocumentsTotal: nullableNumber, + numTestsPassing: nullableNumber, + numTestsTotal: nullableNumber, +}) + +const vantaTestSchema = z.object({ + id: nullableString, + name: nullableString, + description: nullableString, + failureDescription: nullableString, + remediationDescription: nullableString, + category: nullableString, + status: nullableString, + integrations: z.array(z.string()), + lastTestRunDate: nullableString, + latestFlipDate: nullableString, + version: z.object({ major: nullableNumber, minor: nullableNumber }).nullable(), + deactivatedStatusInfo: z + .object({ + isDeactivated: nullableBoolean, + deactivatedReason: nullableString, + lastUpdatedDate: nullableString, + }) + .nullable(), + remediationStatusInfo: z + .object({ + status: nullableString, + soonestRemediateByDate: nullableString, + itemCount: nullableNumber, + }) + .nullable(), + owner: vantaOwnerSchema, +}) + +const vantaTestEntitySchema = z.object({ + id: nullableString, + entityStatus: nullableString, + displayName: nullableString, + responseType: nullableString, + deactivatedReason: nullableString, + createdDate: nullableString, + lastUpdatedDate: nullableString, +}) + +const vantaDocumentSchema = z.object({ + id: nullableString, + title: nullableString, + description: nullableString, + category: nullableString, + ownerId: nullableString, + isSensitive: nullableBoolean, + uploadStatus: nullableString, + uploadStatusDate: nullableString, + url: nullableString, +}) + +const vantaDocumentDetailSchema = vantaDocumentSchema.extend({ + note: nullableString, + nextRenewalDate: nullableString, + renewalCadence: nullableString, + reminderWindow: nullableString, + subscribers: z.array(z.string()), + deactivatedStatus: z + .object({ + isDeactivated: nullableBoolean, + reason: nullableString, + creationDate: nullableString, + expiration: nullableString, + }) + .nullable(), +}) + +const vantaUploadedFileSchema = z.object({ + id: nullableString, + fileName: nullableString, + title: nullableString, + description: nullableString, + mimeType: nullableString, + uploadedBy: z.object({ id: nullableString, type: nullableString }).nullable(), + creationDate: nullableString, + updatedDate: nullableString, + deletionDate: nullableString, + effectiveDate: nullableString, + url: nullableString, +}) + +const vantaPersonSchema = z.object({ + id: nullableString, + userId: nullableString, + emailAddress: nullableString, + name: z + .object({ first: nullableString, last: nullableString, display: nullableString }) + .nullable(), + employment: z + .object({ + status: nullableString, + startDate: nullableString, + endDate: nullableString, + jobTitle: nullableString, + }) + .nullable(), + leaveInfo: z + .object({ status: nullableString, startDate: nullableString, endDate: nullableString }) + .nullable(), + groupIds: z.array(z.string()), + tasksSummary: z + .object({ + status: nullableString, + dueDate: nullableString, + completionDate: nullableString, + }) + .nullable(), +}) + +const vantaPolicySchema = z.object({ + id: nullableString, + name: nullableString, + description: nullableString, + status: nullableString, + approvedAtDate: nullableString, + latestVersionStatus: nullableString, + latestApprovedVersion: z + .object({ + versionId: nullableString, + documents: z.array( + z.object({ language: nullableString, slugId: nullableString, url: nullableString }) + ), + }) + .nullable(), +}) + +const vantaVendorSchema = z.object({ + id: nullableString, + name: nullableString, + status: nullableString, + websiteUrl: nullableString, + category: nullableString, + servicesProvided: nullableString, + additionalNotes: nullableString, + accountManagerName: nullableString, + accountManagerEmail: nullableString, + securityOwnerUserId: nullableString, + businessOwnerUserId: nullableString, + inherentRiskLevel: nullableString, + residualRiskLevel: nullableString, + isRiskAutoScored: nullableBoolean, + isVisibleToAuditors: nullableBoolean, + riskAttributeIds: z.array(z.string()), + vendorHeadquarters: nullableString, + contractStartDate: nullableString, + contractRenewalDate: nullableString, + contractTerminationDate: nullableString, + contractAmount: z.object({ amount: nullableNumber, currency: nullableString }).nullable(), + nextSecurityReviewDueDate: nullableString, + lastSecurityReviewCompletionDate: nullableString, + authDetails: z + .object({ + method: nullableString, + passwordMFA: nullableBoolean, + passwordMinimumLength: nullableNumber, + passwordRequiresNumber: nullableBoolean, + passwordRequiresSymbol: nullableBoolean, + }) + .nullable(), + customFields: vantaCustomFieldsSchema, + latestDecision: z.object({ status: nullableString, lastUpdatedAt: nullableString }).nullable(), + linkedTaskTrackerTaskProcurementRequest: z + .object({ url: nullableString, service: nullableString }) + .nullable(), +}) + +const vantaMonitoredComputerSchema = z.object({ + id: nullableString, + integrationId: nullableString, + lastCheckDate: nullableString, + screenlock: nullableString, + diskEncryption: nullableString, + passwordManager: nullableString, + antivirusInstallation: nullableString, + operatingSystem: z.object({ type: nullableString, version: nullableString }).nullable(), + owner: vantaOwnerSchema, + serialNumber: nullableString, + udid: nullableString, +}) + +const vantaVulnerabilitySchema = z.object({ + id: nullableString, + name: nullableString, + description: nullableString, + severity: nullableString, + vulnerabilityType: nullableString, + integrationId: nullableString, + targetId: nullableString, + packageIdentifier: nullableString, + cvssSeverityScore: nullableNumber, + scannerScore: nullableNumber, + isFixable: nullableBoolean, + fixedVersion: nullableString, + remediateByDate: nullableString, + firstDetectedDate: nullableString, + sourceDetectedDate: nullableString, + lastDetectedDate: nullableString, + scanSource: nullableString, + externalURL: nullableString, + relatedVulns: z.array(z.string()), + relatedUrls: z.array(z.string()), + deactivateMetadata: z + .object({ + isVulnDeactivatedIndefinitely: nullableBoolean, + deactivatedUntilDate: nullableString, + deactivationReason: nullableString, + deactivatedOnDate: nullableString, + deactivatedBy: nullableString, + }) + .nullable(), +}) + +const vantaVulnerabilityRemediationSchema = z.object({ + id: nullableString, + vulnerabilityId: nullableString, + vulnerableAssetId: nullableString, + severity: nullableString, + detectedDate: nullableString, + slaDeadlineDate: nullableString, + remediationDate: nullableString, +}) + +const vantaVulnerableAssetSchema = z.object({ + id: nullableString, + name: nullableString, + assetType: nullableString, + hasBeenScanned: nullableBoolean, + imageScanTag: nullableString, + scanners: z.array( + z.object({ + resourceId: nullableString, + integrationId: nullableString, + targetId: nullableString, + imageDigest: nullableString, + imagePushedAtDate: nullableString, + imageTags: z.array(z.string()), + assetTags: z.array(z.object({ key: nullableString, value: nullableString })), + parentAccountOrOrganization: nullableString, + biosUuid: nullableString, + ipv4s: z.array(z.string()), + ipv6s: z.array(z.string()), + macAddresses: z.array(z.string()), + hostnames: z.array(z.string()), + fqdns: z.array(z.string()), + operatingSystems: z.array(z.string()), + }) + ), +}) + +const vantaRiskScenarioSchema = z.object({ + riskId: nullableString, + description: nullableString, + likelihood: nullableNumber, + impact: nullableNumber, + residualLikelihood: nullableNumber, + residualImpact: nullableNumber, + categories: z.array(z.string()), + ciaCategories: z.array(z.string()), + treatment: nullableString, + owner: nullableString, + note: nullableString, + riskRegister: nullableString, + customFields: vantaCustomFieldsSchema, + isArchived: nullableBoolean, + reviewStatus: nullableString, + requiredApprovers: z.array(z.string()), + type: nullableString, + identificationDate: nullableString, +}) + +const VANTA_REGIONS = ['us', 'gov'] as const + +const vantaBaseBodySchema = z.object({ + clientId: z.string().min(1, 'Client ID is required'), + clientSecret: z.string().min(1, 'Client secret is required'), + region: z.enum(VANTA_REGIONS).optional(), +}) + +const vantaPaginationBodySchema = z.object({ + pageSize: z + .number() + .int() + .min(1, 'pageSize must be at least 1') + .max(100, 'pageSize must be at most 100') + .optional(), + pageCursor: z.string().min(1, 'pageCursor cannot be empty').optional(), +}) + +const vantaListBaseBodySchema = vantaBaseBodySchema.extend(vantaPaginationBodySchema.shape) + +const requiredId = (label: string) => z.string().trim().min(1, `${label} is required`) + +const listFrameworksSchema = vantaListBaseBodySchema.extend({ + operation: z.literal('vanta_list_frameworks'), +}) + +const getFrameworkSchema = vantaBaseBodySchema.extend({ + operation: z.literal('vanta_get_framework'), + frameworkId: requiredId('Framework ID'), +}) + +const listFrameworkControlsSchema = vantaListBaseBodySchema.extend({ + operation: z.literal('vanta_list_framework_controls'), + frameworkId: requiredId('Framework ID'), +}) + +const listControlsSchema = vantaListBaseBodySchema.extend({ + operation: z.literal('vanta_list_controls'), + frameworkMatchesAny: z.string().optional(), +}) + +const getControlSchema = vantaBaseBodySchema.extend({ + operation: z.literal('vanta_get_control'), + controlId: requiredId('Control ID'), +}) + +const listControlTestsSchema = vantaListBaseBodySchema.extend({ + operation: z.literal('vanta_list_control_tests'), + controlId: requiredId('Control ID'), +}) + +const listControlDocumentsSchema = vantaListBaseBodySchema.extend({ + operation: z.literal('vanta_list_control_documents'), + controlId: requiredId('Control ID'), +}) + +const listTestsSchema = vantaListBaseBodySchema.extend({ + operation: z.literal('vanta_list_tests'), + statusFilter: z + .enum(['OK', 'DEACTIVATED', 'NEEDS_ATTENTION', 'IN_PROGRESS', 'INVALID', 'NOT_APPLICABLE']) + .optional(), + frameworkFilter: z.string().optional(), + integrationFilter: z.string().optional(), + controlFilter: z.string().optional(), + ownerFilter: z.string().optional(), + categoryFilter: z + .enum([ + 'ACCOUNTS_ACCESS', + 'ACCOUNT_SECURITY', + 'ACCOUNT_SETUP', + 'COMPUTERS', + 'CUSTOM', + 'DATA_STORAGE', + 'EMPLOYEES', + 'INFRASTRUCTURE', + 'IT', + 'LOGGING', + 'MONITORING_ALERTS', + 'PEOPLE', + 'POLICIES', + 'RISK_ANALYSIS', + 'SECURITY_ALERT_MANAGEMENT', + 'SOFTWARE_DEVELOPMENT', + 'VENDORS', + 'VULNERABILITY_MANAGEMENT', + ]) + .optional(), + isInRollout: z.boolean().optional(), +}) + +const getTestSchema = vantaBaseBodySchema.extend({ + operation: z.literal('vanta_get_test'), + testId: requiredId('Test ID'), +}) + +const listTestEntitiesSchema = vantaListBaseBodySchema.extend({ + operation: z.literal('vanta_list_test_entities'), + testId: requiredId('Test ID'), + entityStatus: z.enum(['FAILING', 'DEACTIVATED']).optional(), +}) + +const listDocumentsSchema = vantaListBaseBodySchema.extend({ + operation: z.literal('vanta_list_documents'), + frameworkMatchesAny: z.string().optional(), + statusMatchesAny: z.string().optional(), +}) + +const getDocumentSchema = vantaBaseBodySchema.extend({ + operation: z.literal('vanta_get_document'), + documentId: requiredId('Document ID'), +}) + +const listDocumentUploadsSchema = vantaListBaseBodySchema.extend({ + operation: z.literal('vanta_list_document_uploads'), + documentId: requiredId('Document ID'), +}) + +const submitDocumentSchema = vantaBaseBodySchema.extend({ + operation: z.literal('vanta_submit_document'), + documentId: requiredId('Document ID'), +}) + +const listPeopleSchema = vantaListBaseBodySchema.extend({ + operation: z.literal('vanta_list_people'), + emailAndNameFilter: z.string().optional(), + employmentStatus: z.enum(['UPCOMING', 'CURRENT', 'ON_LEAVE', 'INACTIVE', 'FORMER']).optional(), + groupIdsMatchesAny: z.string().optional(), + tasksSummaryStatusMatchesAny: z.string().optional(), + taskTypeMatchesAny: z.string().optional(), + taskStatusMatchesAny: z.string().optional(), +}) + +const getPersonSchema = vantaBaseBodySchema.extend({ + operation: z.literal('vanta_get_person'), + personId: requiredId('Person ID'), +}) + +const listPoliciesSchema = vantaListBaseBodySchema.extend({ + operation: z.literal('vanta_list_policies'), +}) + +const getPolicySchema = vantaBaseBodySchema.extend({ + operation: z.literal('vanta_get_policy'), + policyId: requiredId('Policy ID'), +}) + +const listVendorsSchema = vantaListBaseBodySchema.extend({ + operation: z.literal('vanta_list_vendors'), + name: z.string().optional(), + statusMatchesAny: z.string().optional(), +}) + +const getVendorSchema = vantaBaseBodySchema.extend({ + operation: z.literal('vanta_get_vendor'), + vendorId: requiredId('Vendor ID'), +}) + +const listMonitoredComputersSchema = vantaListBaseBodySchema.extend({ + operation: z.literal('vanta_list_monitored_computers'), + complianceStatusFilterMatchesAny: z.string().optional(), +}) + +const listVulnerabilitiesSchema = vantaListBaseBodySchema.extend({ + operation: z.literal('vanta_list_vulnerabilities'), + q: z.string().optional(), + severity: z.enum(['LOW', 'MEDIUM', 'HIGH', 'CRITICAL']).optional(), + isFixAvailable: z.boolean().optional(), + isDeactivated: z.boolean().optional(), + includeVulnerabilitiesWithoutSlas: z.boolean().optional(), + packageIdentifier: z.string().optional(), + externalVulnerabilityId: z.string().optional(), + integrationId: z.string().optional(), + vulnerableAssetId: z.string().optional(), + slaDeadlineAfterDate: z.string().optional(), + slaDeadlineBeforeDate: z.string().optional(), +}) + +const listVulnerabilityRemediationsSchema = vantaListBaseBodySchema.extend({ + operation: z.literal('vanta_list_vulnerability_remediations'), + integrationId: z.string().optional(), + severity: z.enum(['LOW', 'MEDIUM', 'HIGH', 'CRITICAL']).optional(), + isRemediatedOnTime: z.boolean().optional(), + remediatedAfterDate: z.string().optional(), + remediatedBeforeDate: z.string().optional(), +}) + +const listVulnerableAssetsSchema = vantaListBaseBodySchema.extend({ + operation: z.literal('vanta_list_vulnerable_assets'), + q: z.string().optional(), + integrationId: z.string().optional(), + assetType: z + .enum([ + 'SERVER', + 'SERVERLESS_FUNCTION', + 'CONTAINER', + 'CONTAINER_REPOSITORY', + 'CONTAINER_REPOSITORY_IMAGE', + 'CODE_REPOSITORY', + 'MANIFEST_FILE', + 'WORKSTATION', + 'OTHER', + ]) + .optional(), + assetExternalAccountId: z.string().optional(), +}) + +const getVulnerableAssetSchema = vantaBaseBodySchema.extend({ + operation: z.literal('vanta_get_vulnerable_asset'), + vulnerableAssetId: requiredId('Vulnerable asset ID'), +}) + +const listRiskScenariosSchema = vantaListBaseBodySchema.extend({ + operation: z.literal('vanta_list_risk_scenarios'), + searchString: z.string().optional(), + includeIgnored: z.boolean().optional(), + type: z.enum(['Risk Scenario', 'Enterprise Risk']).optional(), + ownerMatchesAny: z.string().optional(), + categoryMatchesAny: z.string().optional(), + ciaCategoryMatchesAny: z.string().optional(), + treatmentTypeMatchesAny: z.string().optional(), + inherentScoreGroupMatchesAny: z.string().optional(), + residualScoreGroupMatchesAny: z.string().optional(), + reviewStatusMatchesAny: z.string().optional(), + orderBy: z.enum(['description', 'createdAt']).optional(), +}) + +const getRiskScenarioSchema = vantaBaseBodySchema.extend({ + operation: z.literal('vanta_get_risk_scenario'), + riskScenarioId: requiredId('Risk scenario ID'), +}) + +export const vantaQueryBodySchema = z.discriminatedUnion('operation', [ + listFrameworksSchema, + getFrameworkSchema, + listFrameworkControlsSchema, + listControlsSchema, + getControlSchema, + listControlTestsSchema, + listControlDocumentsSchema, + listTestsSchema, + getTestSchema, + listTestEntitiesSchema, + listDocumentsSchema, + getDocumentSchema, + listDocumentUploadsSchema, + submitDocumentSchema, + listPeopleSchema, + getPersonSchema, + listPoliciesSchema, + getPolicySchema, + listVendorsSchema, + getVendorSchema, + listMonitoredComputersSchema, + listVulnerabilitiesSchema, + listVulnerabilityRemediationsSchema, + listVulnerableAssetsSchema, + getVulnerableAssetSchema, + listRiskScenariosSchema, + getRiskScenarioSchema, +]) + +export const vantaQueryOutputSchema = z.union([ + z.object({ frameworks: z.array(vantaFrameworkSchema), pageInfo: vantaPageInfoSchema }), + z.object({ framework: vantaFrameworkDetailSchema }), + z.object({ controls: z.array(vantaControlSchema), pageInfo: vantaPageInfoSchema }), + z.object({ control: vantaControlDetailSchema }), + z.object({ tests: z.array(vantaTestSchema), pageInfo: vantaPageInfoSchema }), + z.object({ test: vantaTestSchema }), + z.object({ entities: z.array(vantaTestEntitySchema), pageInfo: vantaPageInfoSchema }), + z.object({ documents: z.array(vantaDocumentSchema), pageInfo: vantaPageInfoSchema }), + z.object({ document: vantaDocumentDetailSchema }), + z.object({ uploads: z.array(vantaUploadedFileSchema), pageInfo: vantaPageInfoSchema }), + z.object({ documentId: z.string(), submitted: z.boolean() }), + z.object({ people: z.array(vantaPersonSchema), pageInfo: vantaPageInfoSchema }), + z.object({ person: vantaPersonSchema }), + z.object({ policies: z.array(vantaPolicySchema), pageInfo: vantaPageInfoSchema }), + z.object({ policy: vantaPolicySchema }), + z.object({ vendors: z.array(vantaVendorSchema), pageInfo: vantaPageInfoSchema }), + z.object({ vendor: vantaVendorSchema }), + z.object({ computers: z.array(vantaMonitoredComputerSchema), pageInfo: vantaPageInfoSchema }), + z.object({ + vulnerabilities: z.array(vantaVulnerabilitySchema), + pageInfo: vantaPageInfoSchema, + }), + z.object({ + remediations: z.array(vantaVulnerabilityRemediationSchema), + pageInfo: vantaPageInfoSchema, + }), + z.object({ assets: z.array(vantaVulnerableAssetSchema), pageInfo: vantaPageInfoSchema }), + z.object({ asset: vantaVulnerableAssetSchema }), + z.object({ riskScenarios: z.array(vantaRiskScenarioSchema), pageInfo: vantaPageInfoSchema }), + z.object({ riskScenario: vantaRiskScenarioSchema }), +]) + +export const vantaQueryResponseSchema = z.object({ + success: z.literal(true), + output: vantaQueryOutputSchema, +}) + +export type VantaQueryBody = z.output +export type VantaQueryBodyInput = z.input +export type VantaQueryResponse = z.output diff --git a/apps/sim/lib/internal/video/client.test.ts b/apps/sim/lib/internal/video/client.test.ts new file mode 100644 index 00000000000..07698228b44 --- /dev/null +++ b/apps/sim/lib/internal/video/client.test.ts @@ -0,0 +1,241 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/execution-limits', () => ({ getMaxExecutionTimeout: () => 5000 })) + +import { generateVideo } from '@/lib/internal/video/client' + +function jsonResponse(value: unknown, status = 200): Response { + return new Response(JSON.stringify(value), { + status, + headers: { 'Content-Type': 'application/json' }, + }) +} + +describe('Video provider client', () => { + beforeEach(() => vi.useFakeTimers()) + + afterEach(() => { + vi.useRealTimers() + vi.unstubAllGlobals() + }) + + it('submits a Runway job once and only polls the returned task', async () => { + const mockFetch = vi + .fn() + .mockResolvedValueOnce(jsonResponse({ id: 'task-1' })) + .mockResolvedValueOnce( + jsonResponse({ status: 'SUCCEEDED', output: ['https://cdn.example/video.mp4'] }) + ) + .mockResolvedValueOnce(new Response(Buffer.from('video'))) + vi.stubGlobal('fetch', mockFetch) + + const resultPromise = generateVideo( + { + provider: 'runway', + apiKey: 'key', + prompt: 'A cinematic sunrise', + duration: 5, + aspectRatio: '16:9', + resolution: '720p', + }, + { requestId: 'request-1' } + ) + await vi.advanceTimersByTimeAsync(5000) + const result = await resultPromise + + expect(result).toMatchObject({ + buffer: Buffer.from('video'), + width: 1280, + height: 720, + jobId: 'task-1', + duration: 5, + }) + const requests = mockFetch.mock.calls.map(([url, init]) => ({ + method: (init as RequestInit | undefined)?.method, + url: String(url), + })) + expect(requests).toEqual([ + { + method: 'POST', + url: 'https://api.dev.runwayml.com/v1/image_to_video', + }, + { + method: undefined, + url: 'https://api.dev.runwayml.com/v1/tasks/task-1', + }, + { method: undefined, url: 'https://cdn.example/video.mp4' }, + ]) + expect(requests.filter((request) => request.method === 'POST')).toHaveLength(1) + }) + + it.each([ + { + provider: 'veo' as const, + responses: [ + jsonResponse({ name: 'operations/veo-1' }), + jsonResponse({ + done: true, + response: { + generateVideoResponse: { + generatedSamples: [{ video: { uri: 'https://cdn.example/veo.mp4' } }], + }, + }, + }), + new Response(Buffer.from('video')), + ], + submitUrl: + 'https://generativelanguage.googleapis.com/v1beta/models/veo-3.0-generate-001:predictLongRunning', + }, + { + provider: 'luma' as const, + responses: [ + jsonResponse({ id: 'luma-1' }), + jsonResponse({ state: 'completed', assets: { video: 'https://cdn.example/luma.mp4' } }), + new Response(Buffer.from('video')), + ], + submitUrl: 'https://api.lumalabs.ai/dream-machine/v1/generations', + }, + { + provider: 'minimax' as const, + responses: [ + jsonResponse({ base_resp: { status_code: 0 }, task_id: 'minimax-1' }), + jsonResponse({ base_resp: { status_code: 0 }, status: 'Success', file_id: 'file-1' }), + jsonResponse({ file: { download_url: 'https://cdn.example/minimax.mp4' } }), + new Response(Buffer.from('video')), + ], + submitUrl: 'https://api.minimax.io/v1/video_generation', + }, + { + provider: 'falai' as const, + responses: [ + jsonResponse({ + request_id: 'fal-1', + status_url: 'https://queue.fal.run/status/fal-1', + response_url: 'https://queue.fal.run/response/fal-1', + }), + jsonResponse({ status: 'COMPLETED' }), + jsonResponse({ + video: { url: 'https://cdn.example/fal.mp4', width: 1920, height: 1080, duration: 8 }, + }), + new Response(Buffer.from('video')), + ], + submitUrl: 'https://queue.fal.run/fal-ai/veo3.1', + }, + ])('submits $provider once and polls only its returned provider job', async (testCase) => { + const mockFetch = vi.fn() + for (const response of testCase.responses) mockFetch.mockResolvedValueOnce(response) + vi.stubGlobal('fetch', mockFetch) + + const resultPromise = generateVideo( + { + provider: testCase.provider, + apiKey: 'key', + model: testCase.provider === 'falai' ? 'veo-3.1' : undefined, + prompt: 'A cinematic sunrise', + }, + { requestId: 'request-1' } + ) + await vi.advanceTimersByTimeAsync(5000) + await resultPromise + + expect(String(mockFetch.mock.calls[0]?.[0])).toBe(testCase.submitUrl) + expect((mockFetch.mock.calls[0]?.[1] as RequestInit).method).toBe('POST') + expect( + mockFetch.mock.calls.filter( + ([, init]) => (init as RequestInit | undefined)?.method === 'POST' + ) + ).toHaveLength(1) + }) + + it('cancels during the provider wait without polling or resubmitting', async () => { + const controller = new AbortController() + const mockFetch = vi.fn().mockResolvedValueOnce(jsonResponse({ id: 'task-1' })) + vi.stubGlobal('fetch', mockFetch) + + const resultPromise = generateVideo( + { + provider: 'runway', + apiKey: 'key', + prompt: 'A cinematic sunrise', + }, + { requestId: 'request-1', signal: controller.signal } + ) + await vi.advanceTimersByTimeAsync(0) + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect(resultPromise).rejects.toMatchObject({ name: 'AbortError' }) + expect(mockFetch).toHaveBeenCalledTimes(1) + }) + + it('forwards cancellation to submission, polling, and download requests', async () => { + const controller = new AbortController() + const mockFetch = vi + .fn() + .mockResolvedValueOnce(jsonResponse({ id: 'task-1' })) + .mockResolvedValueOnce( + jsonResponse({ status: 'SUCCEEDED', output: ['https://cdn.example/video.mp4'] }) + ) + .mockResolvedValueOnce(new Response(Buffer.from('video'))) + vi.stubGlobal('fetch', mockFetch) + + const resultPromise = generateVideo( + { + provider: 'runway', + apiKey: 'key', + prompt: 'A cinematic sunrise', + }, + { requestId: 'request-1', signal: controller.signal } + ) + await vi.advanceTimersByTimeAsync(5000) + await resultPromise + + for (const [, init] of mockFetch.mock.calls) { + expect((init as RequestInit | undefined)?.signal).toBe(controller.signal) + } + }) + + it('rejects a generated video whose declared size exceeds the 250 MiB cap', async () => { + const mockFetch = vi + .fn() + .mockResolvedValueOnce(jsonResponse({ id: 'task-1' })) + .mockResolvedValueOnce( + jsonResponse({ status: 'SUCCEEDED', output: ['https://cdn.example/video.mp4'] }) + ) + .mockResolvedValueOnce( + new Response(Buffer.from('video'), { + headers: { 'Content-Length': String(250 * 1024 * 1024 + 1) }, + }) + ) + vi.stubGlobal('fetch', mockFetch) + + const resultPromise = generateVideo( + { provider: 'runway', apiKey: 'key', prompt: 'A cinematic sunrise' }, + { requestId: 'request-1' } + ) + const rejection = expect(resultPromise).rejects.toMatchObject({ + name: 'PayloadSizeLimitError', + }) + await vi.advanceTimersByTimeAsync(5000) + await rejection + }) + + it('times out after the execution deadline without resubmitting', async () => { + const mockFetch = vi + .fn() + .mockResolvedValueOnce(jsonResponse({ id: 'task-1' })) + .mockResolvedValueOnce(jsonResponse({ status: 'RUNNING' })) + vi.stubGlobal('fetch', mockFetch) + + const resultPromise = generateVideo( + { provider: 'runway', apiKey: 'key', prompt: 'A cinematic sunrise' }, + { requestId: 'request-1' } + ) + const rejection = expect(resultPromise).rejects.toThrow('Runway generation timed out') + await vi.advanceTimersByTimeAsync(5000) + await rejection + expect(mockFetch).toHaveBeenCalledTimes(2) + }) +}) diff --git a/apps/sim/lib/internal/video/client.ts b/apps/sim/lib/internal/video/client.ts new file mode 100644 index 00000000000..50a204bf37e --- /dev/null +++ b/apps/sim/lib/internal/video/client.ts @@ -0,0 +1,1220 @@ +import { createLogger, type Logger } from '@sim/logger' +import { interruptibleSleep } from '@sim/utils/helpers' +import { isRecordLike } from '@sim/utils/object' +import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' +import { + assertKnownSizeWithinLimit, + DEFAULT_MAX_ERROR_BODY_BYTES, + PayloadSizeLimitError, + readResponseJsonWithLimit, + readResponseTextWithLimit, + readResponseToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' +import { type FalAICostMetadata, getFalAICostMetadata } from '@/lib/tools/falai-pricing' +import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import type { UserFile } from '@/executor/types' + +const logger = createLogger('VideoProviderClient') +const MAX_VIDEO_OUTPUT_BYTES = 250 * 1024 * 1024 +const MAX_VIDEO_REFERENCE_IMAGE_BYTES = 25 * 1024 * 1024 +const MAX_VIDEO_JSON_BYTES = 2 * 1024 * 1024 +const POLL_INTERVAL_MS = 5000 + +export type VideoProvider = 'runway' | 'veo' | 'luma' | 'minimax' | 'falai' + +export interface VideoGenerationInput { + provider: VideoProvider + apiKey: string + model?: string + prompt: string + duration?: number + aspectRatio?: string + resolution?: string + visualReference?: UserFile + cameraControl?: unknown + endpoint?: string + promptOptimizer?: boolean + generateAudio?: boolean + useHostedCostTracking?: boolean +} + +export interface VideoGenerationResult { + buffer: Buffer + width?: number + height?: number + jobId?: string + duration?: number + falaiCost?: FalAICostMetadata +} + +export interface VideoGenerationContext { + requestId: string + signal?: AbortSignal +} + +async function readVideoResponseBuffer(response: Response, label: string): Promise { + return readResponseToBufferWithLimit(response, { + maxBytes: MAX_VIDEO_OUTPUT_BYTES, + label, + }) +} + +async function readVideoJson>( + response: Response, + label: string +): Promise { + return readResponseJsonWithLimit(response, { + maxBytes: MAX_VIDEO_JSON_BYTES, + label, + }) +} + +async function readVideoErrorText(response: Response, label: string): Promise { + return readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label, + }).catch(() => '') +} + +async function waitForProvider(context: VideoGenerationContext): Promise { + await interruptibleSleep(POLL_INTERVAL_MS, context.signal) + context.signal?.throwIfAborted() +} + +export function getVideoInputValidationError(input: VideoGenerationInput): string | undefined { + if (input.prompt.length < 3 || input.prompt.length > 2000) { + return 'Prompt must be between 3 and 2000 characters' + } + if ( + input.provider === 'veo' && + input.duration !== undefined && + ![4, 6, 8].includes(input.duration) + ) { + return 'Duration must be 4, 6, or 8 seconds for Veo' + } + if ( + input.provider === 'minimax' && + input.duration !== undefined && + ![6, 10].includes(input.duration) + ) { + return 'Duration must be 6 or 10 seconds for MiniMax' + } + if ( + input.provider !== 'falai' && + input.provider !== 'veo' && + input.provider !== 'minimax' && + input.duration !== undefined && + (input.duration < 5 || input.duration > 10) + ) { + return 'Duration must be between 5 and 10 seconds' + } + if (input.provider !== 'falai' && input.aspectRatio) { + const valid = input.provider === 'veo' ? ['16:9', '9:16'] : ['16:9', '9:16', '1:1'] + if (!valid.includes(input.aspectRatio)) return `Aspect ratio must be ${valid.join(', ')}` + } + if (input.provider === 'falai') { + if (!input.model) return 'Model is required for Fal.ai provider' + return getFalAIValidationError(input.model, input.duration, input.aspectRatio, input.resolution) + } + return undefined +} + +export async function generateVideo( + input: VideoGenerationInput, + context: VideoGenerationContext +): Promise { + context.signal?.throwIfAborted() + logger.info(`[${context.requestId}] Generating video with ${input.provider}`, { + model: input.model || 'default', + }) + switch (input.provider) { + case 'runway': + return generateWithRunway( + input.apiKey, + input.prompt, + input.duration || 5, + input.aspectRatio || '16:9', + input.resolution || '1080p', + input.visualReference, + context, + logger + ) + case 'veo': + return generateWithVeo( + input.apiKey, + input.model || 'veo-3', + input.prompt, + input.duration || 8, + input.aspectRatio || '16:9', + input.resolution || '1080p', + context, + logger + ) + case 'luma': + return generateWithLuma( + input.apiKey, + input.model || 'ray-2', + input.prompt, + input.duration || 5, + input.aspectRatio || '16:9', + input.resolution || '1080p', + input.cameraControl, + context, + logger + ) + case 'minimax': + return generateWithMiniMax( + input.apiKey, + input.model || 'hailuo-2.3', + input.prompt, + input.duration || 6, + input.promptOptimizer !== false, + input.endpoint, + context, + logger + ) + case 'falai': + return generateWithFalAI( + input.apiKey, + input.model as string, + input.prompt, + input.duration, + input.aspectRatio, + input.resolution, + input.promptOptimizer, + input.generateAudio, + input.useHostedCostTracking === true, + context, + logger + ) + } +} + +async function generateWithRunway( + apiKey: string, + prompt: string, + duration: number, + aspectRatio: string, + resolution: string, + visualReference: UserFile | undefined, + context: VideoGenerationContext, + logger: Logger +): Promise<{ buffer: Buffer; width: number; height: number; jobId: string; duration: number }> { + const { requestId, signal } = context + signal?.throwIfAborted() + logger.info(`[${requestId}] Starting Runway Gen-4 generation`) + + const dimensions = getVideoDimensions(aspectRatio, resolution) + + const ratioMap: Record = { + '16:9': '1280:720', + '9:16': '720:1280', + '1:1': '960:960', + } + const runwayRatio = ratioMap[aspectRatio] || '1280:720' + + const createPayload: Record = { + promptText: prompt, + duration, + ratio: runwayRatio, + model: 'gen4_turbo', + } + + if (visualReference) { + if (visualReference.size > MAX_VIDEO_REFERENCE_IMAGE_BYTES) { + throw new PayloadSizeLimitError({ + label: 'video visual reference', + maxBytes: MAX_VIDEO_REFERENCE_IMAGE_BYTES, + observedBytes: visualReference.size, + }) + } + const refBuffer = await downloadFileFromStorage(visualReference, requestId, logger, { + maxBytes: MAX_VIDEO_REFERENCE_IMAGE_BYTES, + }) + assertKnownSizeWithinLimit( + refBuffer.length, + MAX_VIDEO_REFERENCE_IMAGE_BYTES, + 'video visual reference' + ) + const refBase64 = refBuffer.toString('base64') + createPayload.promptImage = `data:${visualReference.type};base64,${refBase64}` + } + + const createResponse = await fetch('https://api.dev.runwayml.com/v1/image_to_video', { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + 'X-Runway-Version': '2024-11-06', + }, + body: JSON.stringify(createPayload), + signal, + }) + + if (!createResponse.ok) { + const error = await readVideoErrorText(createResponse, 'Runway create error response') + throw new Error(`Runway API error: ${createResponse.status} - ${error}`) + } + + const createData = await readVideoJson<{ id: string }>(createResponse, 'Runway create response') + const taskId = createData.id + + logger.info(`[${requestId}] Runway task created: ${taskId}`) + + const maxAttempts = Math.ceil(getMaxExecutionTimeout() / POLL_INTERVAL_MS) + let attempts = 0 + + while (attempts < maxAttempts) { + await waitForProvider(context) + + const statusResponse = await fetch(`https://api.dev.runwayml.com/v1/tasks/${taskId}`, { + headers: { + Authorization: `Bearer ${apiKey}`, + 'X-Runway-Version': '2024-11-06', + }, + signal, + }) + + if (!statusResponse.ok) { + await readVideoErrorText(statusResponse, 'Runway status error response') + throw new Error(`Runway status check failed: ${statusResponse.status}`) + } + + const statusData = await readVideoJson<{ + status?: string + output?: string[] + failure?: string + }>(statusResponse, 'Runway status response') + + if (statusData.status === 'SUCCEEDED') { + logger.info(`[${requestId}] Runway generation completed after ${attempts * 5}s`) + + const videoUrl = statusData.output?.[0] + if (!videoUrl) { + throw new Error('No video URL in response') + } + + const videoResponse = await fetch(videoUrl, { signal }) + if (!videoResponse.ok) { + await readVideoErrorText(videoResponse, 'Runway video error response') + throw new Error(`Failed to download video: ${videoResponse.status}`) + } + + return { + buffer: await readVideoResponseBuffer(videoResponse, 'Runway video response'), + width: dimensions.width, + height: dimensions.height, + jobId: taskId, + duration, + } + } + + if (statusData.status === 'FAILED') { + throw new Error(`Runway generation failed: ${statusData.failure || 'Unknown error'}`) + } + + attempts++ + } + + throw new Error('Runway generation timed out') +} + +async function generateWithVeo( + apiKey: string, + model: string, + prompt: string, + duration: number, + aspectRatio: string, + resolution: string, + context: VideoGenerationContext, + logger: Logger +): Promise<{ buffer: Buffer; width: number; height: number; jobId: string; duration: number }> { + const { requestId, signal } = context + signal?.throwIfAborted() + logger.info(`[${requestId}] Starting Google Veo generation`) + + const dimensions = getVideoDimensions(aspectRatio, resolution) + + const modelNameMap: Record = { + 'veo-3': 'veo-3.0-generate-001', + 'veo-3-fast': 'veo-3.0-fast-generate-001', + 'veo-3.1': 'veo-3.1-generate-preview', + } + const modelName = modelNameMap[model] || 'veo-3.1-generate-preview' + + const createPayload = { + instances: [ + { + prompt, + }, + ], + parameters: { + aspectRatio, + resolution: resolution, + durationSeconds: duration, + }, + } + + const createResponse = await fetch( + `https://generativelanguage.googleapis.com/v1beta/models/${modelName}:predictLongRunning`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'x-goog-api-key': apiKey, + }, + body: JSON.stringify(createPayload), + signal, + } + ) + + if (!createResponse.ok) { + const error = await readVideoErrorText(createResponse, 'Veo create error response') + throw new Error(`Veo API error: ${createResponse.status} - ${error}`) + } + + const createData = await readVideoJson<{ name: string }>(createResponse, 'Veo create response') + const operationName = createData.name + + logger.info(`[${requestId}] Veo operation created: ${operationName}`) + + const maxAttempts = Math.ceil(getMaxExecutionTimeout() / POLL_INTERVAL_MS) + let attempts = 0 + + while (attempts < maxAttempts) { + await waitForProvider(context) + + const statusResponse = await fetch( + `https://generativelanguage.googleapis.com/v1beta/${operationName}`, + { + headers: { + 'x-goog-api-key': apiKey, + }, + signal, + } + ) + + if (!statusResponse.ok) { + await readVideoErrorText(statusResponse, 'Veo status error response') + throw new Error(`Veo status check failed: ${statusResponse.status}`) + } + + const statusData = await readVideoJson<{ + done?: boolean + error?: { message?: string } + response?: { + generateVideoResponse?: { generatedSamples?: Array<{ video?: { uri?: string } }> } + } + }>(statusResponse, 'Veo status response') + + if (statusData.done) { + if (statusData.error) { + throw new Error(`Veo generation failed: ${statusData.error.message}`) + } + + logger.info(`[${requestId}] Veo generation completed after ${attempts * 5}s`) + + const videoUri = statusData.response?.generateVideoResponse?.generatedSamples?.[0]?.video?.uri + if (!videoUri) { + throw new Error('No video URI in response') + } + + const videoResponse = await fetch(videoUri, { + headers: { + 'x-goog-api-key': apiKey, + }, + signal, + }) + + if (!videoResponse.ok) { + await readVideoErrorText(videoResponse, 'Veo video error response') + throw new Error(`Failed to download video: ${videoResponse.status}`) + } + + return { + buffer: await readVideoResponseBuffer(videoResponse, 'Veo video response'), + width: dimensions.width, + height: dimensions.height, + jobId: operationName, + duration, + } + } + + attempts++ + } + + throw new Error('Veo generation timed out') +} + +async function generateWithLuma( + apiKey: string, + model: string, + prompt: string, + duration: number, + aspectRatio: string, + resolution: string, + cameraControl: unknown, + context: VideoGenerationContext, + logger: Logger +): Promise<{ buffer: Buffer; width: number; height: number; jobId: string; duration: number }> { + const { requestId, signal } = context + signal?.throwIfAborted() + logger.info(`[${requestId}] Starting Luma Dream Machine generation`) + + const dimensions = getVideoDimensions(aspectRatio, resolution) + + const createPayload: Record = { + prompt, + model: model || 'ray-2', + aspect_ratio: aspectRatio, + loop: false, + } + + if (duration) { + createPayload.duration = `${duration}s` + } + + if (resolution) { + createPayload.resolution = resolution + } + + if (cameraControl) { + createPayload.concepts = Array.isArray(cameraControl) ? cameraControl : [{ key: cameraControl }] + } + + const createResponse = await fetch('https://api.lumalabs.ai/dream-machine/v1/generations', { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(createPayload), + signal, + }) + + if (!createResponse.ok) { + const error = await readVideoErrorText(createResponse, 'Luma create error response') + throw new Error(`Luma API error: ${createResponse.status} - ${error}`) + } + + const createData = await readVideoJson<{ id: string }>(createResponse, 'Luma create response') + const generationId = createData.id + + logger.info(`[${requestId}] Luma generation created: ${generationId}`) + + const maxAttempts = Math.ceil(getMaxExecutionTimeout() / POLL_INTERVAL_MS) + let attempts = 0 + + while (attempts < maxAttempts) { + await waitForProvider(context) + + const statusResponse = await fetch( + `https://api.lumalabs.ai/dream-machine/v1/generations/${generationId}`, + { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + signal, + } + ) + + if (!statusResponse.ok) { + await readVideoErrorText(statusResponse, 'Luma status error response') + throw new Error(`Luma status check failed: ${statusResponse.status}`) + } + + const statusData = await readVideoJson<{ + state?: string + failure_reason?: string + assets?: { video?: string } + }>(statusResponse, 'Luma status response') + + if (statusData.state === 'completed') { + logger.info(`[${requestId}] Luma generation completed after ${attempts * 5}s`) + + const videoUrl = statusData.assets?.video + if (!videoUrl) { + throw new Error('No video URL in response') + } + + const videoResponse = await fetch(videoUrl, { signal }) + if (!videoResponse.ok) { + await readVideoErrorText(videoResponse, 'Luma video error response') + throw new Error(`Failed to download video: ${videoResponse.status}`) + } + + return { + buffer: await readVideoResponseBuffer(videoResponse, 'Luma video response'), + width: dimensions.width, + height: dimensions.height, + jobId: generationId, + duration, + } + } + + if (statusData.state === 'failed') { + throw new Error(`Luma generation failed: ${statusData.failure_reason || 'Unknown error'}`) + } + + attempts++ + } + + throw new Error('Luma generation timed out') +} + +async function generateWithMiniMax( + apiKey: string, + model: string, + prompt: string, + duration: number, + promptOptimizer: boolean, + endpoint: string | undefined, + context: VideoGenerationContext, + logger: Logger +): Promise<{ buffer: Buffer; width: number; height: number; jobId: string; duration: number }> { + const { requestId, signal } = context + signal?.throwIfAborted() + logger.info(`[${requestId}] Starting MiniMax Hailuo generation via MiniMax Platform API`) + logger.info( + `[${requestId}] Request params - model: ${model}, duration: ${duration}, endpoint: ${endpoint || 'standard'}, promptOptimizer: ${promptOptimizer}` + ) + + const useProResolution = endpoint === 'pro' && duration === 6 + const resolution = useProResolution ? '1080P' : '768P' + const dimensions = useProResolution ? { width: 1920, height: 1080 } : { width: 1360, height: 768 } + + logger.info( + `[${requestId}] Using resolution: ${resolution}, dimensions: ${dimensions.width}x${dimensions.height}` + ) + + const minimaxModel = model === 'hailuo-02' ? 'MiniMax-Hailuo-02' : 'MiniMax-Hailuo-2.3' + + const createResponse = await fetch('https://api.minimax.io/v1/video_generation', { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: minimaxModel, + prompt: prompt, + duration: duration, + resolution: resolution, + prompt_optimizer: promptOptimizer, + }), + signal, + }) + + if (!createResponse.ok) { + const errorText = await readVideoErrorText(createResponse, 'MiniMax create error response') + if (createResponse.status === 401 || createResponse.status === 1004) { + throw new Error( + `MiniMax API authentication failed (${createResponse.status}). Please ensure you're using a valid MiniMax API key from platform.minimax.io. Error: ${errorText}` + ) + } + throw new Error(`MiniMax API error: ${createResponse.status} - ${errorText}`) + } + + const createData = await readVideoJson<{ + base_resp?: { status_code?: number; status_msg?: string } + task_id?: string + }>(createResponse, 'MiniMax create response') + + if (createData.base_resp?.status_code !== 0) { + throw new Error(`MiniMax API error: ${createData.base_resp?.status_msg || 'Unknown error'}`) + } + + const taskId = createData.task_id + if (!taskId) { + throw new Error('MiniMax response missing task_id') + } + + logger.info(`[${requestId}] MiniMax task created: ${taskId}`) + + const maxAttempts = Math.ceil(getMaxExecutionTimeout() / POLL_INTERVAL_MS) + let attempts = 0 + + while (attempts < maxAttempts) { + await waitForProvider(context) + + const statusResponse = await fetch( + `https://api.minimax.io/v1/query/video_generation?task_id=${taskId}`, + { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + signal, + } + ) + + if (!statusResponse.ok) { + await readVideoErrorText(statusResponse, 'MiniMax status error response') + throw new Error(`MiniMax status check failed: ${statusResponse.status}`) + } + + const statusData = await readVideoJson<{ + base_resp?: { status_code?: number; status_msg?: string } + status?: string + file_id?: string + error?: string + }>(statusResponse, 'MiniMax status response') + + if ( + statusData.base_resp?.status_code !== 0 && + statusData.base_resp?.status_code !== undefined + ) { + throw new Error( + `MiniMax status query error: ${statusData.base_resp?.status_msg || 'Unknown error'}` + ) + } + + if (statusData.status === 'Success' || statusData.status === 'success') { + logger.info(`[${requestId}] MiniMax generation completed after ${attempts * 5}s`) + + const fileId = statusData.file_id + if (!fileId) { + throw new Error('No file_id in response') + } + + const fileResponse = await fetch( + `https://api.minimax.io/v1/files/retrieve?file_id=${fileId}`, + { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + signal, + } + ) + + if (!fileResponse.ok) { + await readVideoErrorText(fileResponse, 'MiniMax file error response') + throw new Error(`Failed to download video: ${fileResponse.status}`) + } + + const fileData = await readVideoJson<{ file?: { download_url?: string } }>( + fileResponse, + 'MiniMax file response' + ) + const videoUrl = fileData.file?.download_url + + if (!videoUrl) { + throw new Error('No download URL in file response') + } + + const videoResponse = await fetch(videoUrl, { signal }) + if (!videoResponse.ok) { + await readVideoErrorText(videoResponse, 'MiniMax video error response') + throw new Error(`Failed to download video from URL: ${videoResponse.status}`) + } + + return { + buffer: await readVideoResponseBuffer(videoResponse, 'MiniMax video response'), + width: dimensions.width, + height: dimensions.height, + jobId: taskId, + duration, + } + } + + if (statusData.status === 'Failed' || statusData.status === 'failed') { + throw new Error(`MiniMax generation failed: ${statusData.error || 'Unknown error'}`) + } + + attempts++ + } + + throw new Error('MiniMax generation timed out') +} + +type FalAIDurationFormat = 'number' | 'seconds' | 'string' + +interface FalAIModelConfig { + endpoint: string + durationFormat?: FalAIDurationFormat + durationOptions?: readonly number[] + supportsAspectRatio?: boolean + aspectRatioOptions?: readonly string[] + supportsResolution?: boolean + resolutionOptions?: readonly string[] + supportsPromptOptimizer?: boolean + supportsGenerateAudio?: boolean +} + +interface FalAIRequestBody { + prompt: string + duration?: number | string + aspect_ratio?: string + resolution?: string + prompt_optimizer?: boolean + generate_audio?: boolean +} + +const FALAI_MODEL_CONFIGS: Record = { + 'veo-3.1': { + endpoint: 'fal-ai/veo3.1', + durationFormat: 'seconds', + durationOptions: [4, 6, 8], + supportsAspectRatio: true, + aspectRatioOptions: ['16:9', '9:16'], + supportsResolution: true, + resolutionOptions: ['720p', '1080p', '4k'], + supportsGenerateAudio: true, + }, + 'veo-3.1-fast': { + endpoint: 'fal-ai/veo3.1/fast', + durationFormat: 'seconds', + durationOptions: [4, 6, 8], + supportsAspectRatio: true, + aspectRatioOptions: ['16:9', '9:16'], + supportsResolution: true, + resolutionOptions: ['720p', '1080p', '4k'], + supportsGenerateAudio: true, + }, + 'sora-2': { + endpoint: 'fal-ai/sora-2/text-to-video', + durationFormat: 'number', + durationOptions: [4, 8, 12, 16, 20], + supportsAspectRatio: true, + aspectRatioOptions: ['16:9', '9:16'], + supportsResolution: true, + resolutionOptions: ['720p'], + }, + 'sora-2-pro': { + endpoint: 'fal-ai/sora-2/text-to-video/pro', + durationFormat: 'number', + durationOptions: [4, 8, 12, 16, 20], + supportsAspectRatio: true, + aspectRatioOptions: ['16:9', '9:16'], + supportsResolution: true, + resolutionOptions: ['720p', '1080p', 'true_1080p'], + }, + 'seedance-2.0': { + endpoint: 'bytedance/seedance-2.0/text-to-video', + durationFormat: 'string', + durationOptions: [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + supportsAspectRatio: true, + aspectRatioOptions: ['auto', '21:9', '16:9', '4:3', '1:1', '3:4', '9:16'], + supportsResolution: true, + resolutionOptions: ['480p', '720p', '1080p'], + supportsGenerateAudio: true, + }, + 'seedance-2.0-fast': { + endpoint: 'bytedance/seedance-2.0/fast/text-to-video', + durationFormat: 'string', + durationOptions: [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + supportsAspectRatio: true, + aspectRatioOptions: ['auto', '21:9', '16:9', '4:3', '1:1', '3:4', '9:16'], + supportsResolution: true, + resolutionOptions: ['480p', '720p'], + supportsGenerateAudio: true, + }, + 'kling-v3-pro': { + endpoint: 'fal-ai/kling-video/v3/pro/text-to-video', + durationFormat: 'string', + durationOptions: [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + supportsAspectRatio: true, + aspectRatioOptions: ['16:9', '9:16', '1:1'], + supportsGenerateAudio: true, + }, + 'kling-v3-4k': { + endpoint: 'fal-ai/kling-video/v3/4k/text-to-video', + durationFormat: 'string', + durationOptions: [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + supportsAspectRatio: true, + aspectRatioOptions: ['16:9', '9:16', '1:1'], + supportsGenerateAudio: true, + }, + 'kling-o3-pro': { + endpoint: 'fal-ai/kling-video/o3/pro/text-to-video', + durationFormat: 'string', + durationOptions: [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + supportsAspectRatio: true, + aspectRatioOptions: ['16:9', '9:16', '1:1'], + supportsGenerateAudio: true, + }, + 'kling-o3-4k': { + endpoint: 'fal-ai/kling-video/o3/4k/text-to-video', + durationFormat: 'string', + durationOptions: [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + supportsAspectRatio: true, + aspectRatioOptions: ['16:9', '9:16', '1:1'], + supportsGenerateAudio: true, + }, + 'kling-2.5-turbo-pro': { + endpoint: 'fal-ai/kling-video/v2.5-turbo/pro/text-to-video', + durationFormat: 'string', + supportsAspectRatio: true, + supportsResolution: true, + }, + 'kling-2.1-pro': { + endpoint: 'fal-ai/kling-video/v2.1/master/text-to-video', + durationFormat: 'string', + supportsAspectRatio: true, + supportsResolution: true, + }, + 'minimax-hailuo-2.3-pro': { + endpoint: 'fal-ai/minimax/hailuo-2.3/pro/text-to-video', + supportsPromptOptimizer: true, + }, + 'minimax-hailuo-2.3-standard': { + endpoint: 'fal-ai/minimax/hailuo-2.3/standard/text-to-video', + durationFormat: 'string', + durationOptions: [6, 10], + supportsPromptOptimizer: true, + }, + 'minimax-hailuo-02-pro': { + endpoint: 'fal-ai/minimax/hailuo-02/pro/text-to-video', + durationFormat: 'string', + supportsAspectRatio: true, + supportsResolution: true, + supportsPromptOptimizer: true, + }, + 'minimax-hailuo-02-standard': { + endpoint: 'fal-ai/minimax/hailuo-02/standard/text-to-video', + durationFormat: 'string', + supportsAspectRatio: true, + supportsResolution: true, + supportsPromptOptimizer: true, + }, + 'wan-2.2-a14b-turbo': { + endpoint: 'fal-ai/wan/v2.2-a14b/text-to-video/turbo', + supportsAspectRatio: true, + aspectRatioOptions: ['16:9', '9:16', '1:1'], + supportsResolution: true, + resolutionOptions: ['480p', '580p', '720p'], + }, + 'wan-2.1': { + endpoint: 'fal-ai/wan-t2v', + }, + 'ltx-2.3': { + endpoint: 'fal-ai/ltx-2.3/text-to-video', + durationFormat: 'number', + durationOptions: [6, 8, 10], + supportsAspectRatio: true, + aspectRatioOptions: ['16:9', '9:16'], + supportsResolution: true, + resolutionOptions: ['1080p', '1440p', '2160p'], + supportsGenerateAudio: true, + }, + 'ltx-2.3-fast': { + endpoint: 'fal-ai/ltx-2.3/text-to-video/fast', + durationFormat: 'number', + durationOptions: [6, 8, 10, 12, 14, 16, 18, 20], + supportsAspectRatio: true, + aspectRatioOptions: ['16:9', '9:16'], + supportsResolution: true, + resolutionOptions: ['1080p', '1440p', '2160p'], + supportsGenerateAudio: true, + }, + 'ltxv-0.9.8': { + endpoint: 'fal-ai/ltxv-13b-098-distilled', + }, +} + +function formatFalAIDuration( + format: FalAIDurationFormat | undefined, + duration: number | undefined +): string | number | undefined { + if (!format || duration === undefined) return undefined + + if (format === 'number') return duration + if (format === 'seconds') return `${duration}s` + return String(duration) +} + +function getStringProperty( + record: Record | undefined, + key: string +): string | undefined { + const value = record?.[key] + return typeof value === 'string' ? value : undefined +} + +function getNumberProperty( + record: Record | undefined, + key: string +): number | undefined { + const value = record?.[key] + return typeof value === 'number' ? value : undefined +} + +function formatAllowedValues(allowed: readonly (number | string)[]): string { + return allowed.map(String).join(', ') +} + +function getFalAIValidationError( + model: string, + duration: number | undefined, + aspectRatio: string | undefined, + resolution: string | undefined +): string | undefined { + const modelConfig = FALAI_MODEL_CONFIGS[model] + if (!modelConfig) { + return `Unknown Fal.ai model: ${model}` + } + + if ( + duration !== undefined && + modelConfig.durationOptions && + !modelConfig.durationOptions.includes(duration) + ) { + return `Invalid duration for Fal.ai model ${model}. Supported durations: ${formatAllowedValues(modelConfig.durationOptions)}` + } + + if (aspectRatio) { + if (!modelConfig.supportsAspectRatio) { + return `Fal.ai model ${model} does not support aspect ratio` + } + + if (modelConfig.aspectRatioOptions && !modelConfig.aspectRatioOptions.includes(aspectRatio)) { + return `Invalid aspect ratio for Fal.ai model ${model}. Supported aspect ratios: ${formatAllowedValues(modelConfig.aspectRatioOptions)}` + } + } + + if (resolution) { + if (!modelConfig.supportsResolution) { + return `Fal.ai model ${model} does not support resolution` + } + + if (modelConfig.resolutionOptions && !modelConfig.resolutionOptions.includes(resolution)) { + return `Invalid resolution for Fal.ai model ${model}. Supported resolutions: ${formatAllowedValues(modelConfig.resolutionOptions)}` + } + } + + if ( + model === 'ltx-2.3-fast' && + duration !== undefined && + duration > 10 && + resolution && + resolution !== '1080p' + ) { + return 'Fal.ai model ltx-2.3-fast only supports durations over 10 seconds with 1080p resolution' + } + + return undefined +} + +function getFalAIErrorMessage(error: unknown): string { + if (typeof error === 'string') return error + if (isRecordLike(error)) return getStringProperty(error, 'message') || JSON.stringify(error) + return 'Unknown error' +} + +function buildFalAIQueueUrl( + endpoint: string, + requestId: string, + path: 'response' | 'status' +): string { + return `https://queue.fal.run/${endpoint}/requests/${requestId}/${path}` +} + +async function generateWithFalAI( + apiKey: string, + model: string, + prompt: string, + duration: number | undefined, + aspectRatio: string | undefined, + resolution: string | undefined, + promptOptimizer: boolean | undefined, + generateAudio: boolean | undefined, + useHostedCostTracking: boolean, + context: VideoGenerationContext, + logger: Logger +): Promise<{ + buffer: Buffer + width: number + height: number + jobId: string + duration: number + falaiCost?: FalAICostMetadata +}> { + const { requestId, signal } = context + signal?.throwIfAborted() + logger.info(`[${requestId}] Starting Fal.ai generation with model: ${model}`) + + const modelConfig = FALAI_MODEL_CONFIGS[model] + if (!modelConfig) { + throw new Error(`Unknown Fal.ai model: ${model}`) + } + + const requestBody: FalAIRequestBody = { prompt } + const formattedDuration = formatFalAIDuration(modelConfig.durationFormat, duration) + + if (formattedDuration !== undefined) { + requestBody.duration = formattedDuration + } + + if (modelConfig.supportsAspectRatio && aspectRatio) { + requestBody.aspect_ratio = aspectRatio + } + + if (modelConfig.supportsResolution && resolution) { + requestBody.resolution = resolution + } + + if (modelConfig.supportsPromptOptimizer && promptOptimizer !== undefined) { + requestBody.prompt_optimizer = promptOptimizer + } + + if (modelConfig.supportsGenerateAudio && generateAudio !== undefined) { + requestBody.generate_audio = generateAudio + } + + const createResponse = await fetch(`https://queue.fal.run/${modelConfig.endpoint}`, { + method: 'POST', + headers: { + Authorization: `Key ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(requestBody), + signal, + }) + + if (!createResponse.ok) { + const error = await readVideoErrorText(createResponse, 'Fal.ai create error response') + throw new Error(`Fal.ai API error: ${createResponse.status} - ${error}`) + } + + const createData = await readVideoJson(createResponse, 'Fal.ai queue response') + if (!isRecordLike(createData)) { + throw new Error('Invalid Fal.ai queue response') + } + + const requestIdFal = getStringProperty(createData, 'request_id') + if (!requestIdFal) { + throw new Error('Fal.ai queue response missing request_id') + } + + const statusUrl = + getStringProperty(createData, 'status_url') || + buildFalAIQueueUrl(modelConfig.endpoint, requestIdFal, 'status') + const responseUrl = + getStringProperty(createData, 'response_url') || + buildFalAIQueueUrl(modelConfig.endpoint, requestIdFal, 'response') + + logger.info(`[${requestId}] Fal.ai request created: ${requestIdFal}`) + + const maxAttempts = Math.ceil(getMaxExecutionTimeout() / POLL_INTERVAL_MS) + let attempts = 0 + + while (attempts < maxAttempts) { + await waitForProvider(context) + + const statusResponse = await fetch(statusUrl, { + headers: { + Authorization: `Key ${apiKey}`, + }, + signal, + }) + + if (!statusResponse.ok) { + await readVideoErrorText(statusResponse, 'Fal.ai status error response') + throw new Error(`Fal.ai status check failed: ${statusResponse.status}`) + } + + const statusData = await readVideoJson(statusResponse, 'Fal.ai status response') + if (!isRecordLike(statusData)) { + throw new Error('Invalid Fal.ai status response') + } + + if (getStringProperty(statusData, 'status') === 'COMPLETED') { + const statusError = statusData.error + if (statusError) { + throw new Error(`Fal.ai generation failed: ${getFalAIErrorMessage(statusError)}`) + } + + logger.info(`[${requestId}] Fal.ai generation completed after ${attempts * 5}s`) + + const resultResponse = await fetch( + getStringProperty(statusData, 'response_url') || responseUrl, + { + headers: { + Authorization: `Key ${apiKey}`, + }, + signal, + } + ) + + if (!resultResponse.ok) { + await readVideoErrorText(resultResponse, 'Fal.ai result error response') + throw new Error(`Failed to fetch result: ${resultResponse.status}`) + } + + const resultData = await readVideoJson(resultResponse, 'Fal.ai result response') + if (!isRecordLike(resultData)) { + throw new Error('Invalid Fal.ai result response') + } + + const videoOutput = isRecordLike(resultData.video) ? resultData.video : undefined + const fallbackOutput = isRecordLike(resultData.output) ? resultData.output : undefined + const videoUrl = + getStringProperty(videoOutput, 'url') || getStringProperty(fallbackOutput, 'url') + if (!videoUrl) { + throw new Error('No video URL in response') + } + + const videoResponse = await fetch(videoUrl, { signal }) + if (!videoResponse.ok) { + await readVideoErrorText(videoResponse, 'Fal.ai video error response') + throw new Error(`Failed to download video: ${videoResponse.status}`) + } + + let width = getNumberProperty(videoOutput, 'width') || 1920 + let height = getNumberProperty(videoOutput, 'height') || 1080 + + if (!getNumberProperty(videoOutput, 'width') && aspectRatio?.includes(':')) { + const dims = getVideoDimensions(aspectRatio, resolution || '1080p') + width = dims.width + height = dims.height + } + + return { + buffer: await readVideoResponseBuffer(videoResponse, 'Fal.ai video response'), + width, + height, + jobId: requestIdFal, + duration: getNumberProperty(videoOutput, 'duration') || duration || 5, + falaiCost: useHostedCostTracking + ? await getFalAICostMetadata({ + apiKey, + endpointId: modelConfig.endpoint, + requestId: requestIdFal, + signal, + }) + : undefined, + } + } + + if (['ERROR', 'FAILED', 'CANCELLED'].includes(getStringProperty(statusData, 'status') || '')) { + throw new Error(`Fal.ai generation failed: ${getFalAIErrorMessage(statusData.error)}`) + } + + attempts++ + } + + throw new Error('Fal.ai generation timed out') +} + +function getVideoDimensions( + aspectRatio: string, + resolution: string +): { width: number; height: number } { + let height: number + if (resolution === '4k' || resolution === '2160p') { + height = 2160 + } else if (resolution === 'true_1080p') { + height = 1080 + } else { + const parsedHeight = Number.parseInt(resolution.replace('p', '')) + height = Number.isFinite(parsedHeight) ? parsedHeight : 1080 + } + + const [ratioW, ratioH] = aspectRatio.split(':').map(Number) + if (!Number.isFinite(ratioW) || !Number.isFinite(ratioH) || ratioH === 0) { + return { width: Math.round((height * 16) / 9), height } + } + + const width = Math.round((height * ratioW) / ratioH) + + return { width, height } +} diff --git a/apps/sim/lib/internal/video/errors.ts b/apps/sim/lib/internal/video/errors.ts new file mode 100644 index 00000000000..bbe3886a8a4 --- /dev/null +++ b/apps/sim/lib/internal/video/errors.ts @@ -0,0 +1,10 @@ +export class VideoOperationError extends Error { + constructor( + message: string, + readonly status: number, + readonly body: Record = { error: message } + ) { + super(message) + this.name = 'VideoOperationError' + } +} diff --git a/apps/sim/lib/internal/video/execute-tool.test.ts b/apps/sim/lib/internal/video/execute-tool.test.ts new file mode 100644 index 00000000000..782b86321c2 --- /dev/null +++ b/apps/sim/lib/internal/video/execute-tool.test.ts @@ -0,0 +1,110 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operations = vi.hoisted(() => ({ executeVideoOperation: vi.fn() })) + +vi.mock('@/lib/internal/video/operations', () => operations) + +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' +import { VideoOperationError } from '@/lib/internal/video/errors' +import { executeVideoTool } from '@/lib/internal/video/execute-tool' + +function request(overrides: Partial = {}): InternalToolOperationCall { + return { + toolId: 'video_luma', + input: { provider: 'luma', apiKey: 'key', prompt: 'A cinematic sunrise' }, + headers: new Headers(), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + executionId: 'execution-1', + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } as InternalToolOperationCall +} + +const CASES = [ + ['video_falai', 'falai'], + ['video_luma', 'luma'], + ['video_minimax', 'minimax'], + ['video_runway', 'runway'], + ['video_veo', 'veo'], +] as const + +describe('executeVideoTool', () => { + beforeEach(() => { + vi.clearAllMocks() + operations.executeVideoOperation.mockResolvedValue({ + videoUrl: 'https://files.example/video.mp4', + provider: 'luma', + model: 'ray-2', + }) + }) + + it.each(CASES)('dispatches %s through the typed %s operation', async (toolId, provider) => { + const controller = new AbortController() + const input = { provider, apiKey: 'key', prompt: 'A cinematic sunrise' } + + const response = await executeVideoTool(request({ toolId, input, signal: controller.signal })) + + expect(response.status).toBe(200) + expect(operations.executeVideoOperation).toHaveBeenCalledWith(input, { + headers: expect.any(Headers), + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }) + }) + + it('authenticates before parsing operation input', async () => { + const response = await executeVideoTool( + request({ input: null, context: createExecutionContext({ workflowId: 'workflow-1' }) }) + ) + + expect(response.status).toBe(401) + await expect(response.json()).resolves.toEqual({ error: 'Unauthorized' }) + expect(operations.executeVideoOperation).not.toHaveBeenCalled() + }) + + it('preserves contract validation messages and details', async () => { + const response = await executeVideoTool(request({ input: {} })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Missing required fields: provider, apiKey, and prompt', + details: expect.any(Array), + }) + }) + + it('preserves exact operation error bodies and statuses', async () => { + operations.executeVideoOperation.mockRejectedValue( + new VideoOperationError('File not found', 404, { + success: false, + error: 'File not found', + }) + ) + + const response = await executeVideoTool(request()) + + expect(response.status).toBe(404) + await expect(response.json()).resolves.toEqual({ success: false, error: 'File not found' }) + }) + + it('does no provider work after cancellation', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect(executeVideoTool(request({ signal: controller.signal }))).rejects.toMatchObject({ + name: 'AbortError', + }) + expect(operations.executeVideoOperation).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/video/execute-tool.ts b/apps/sim/lib/internal/video/execute-tool.ts new file mode 100644 index 00000000000..e55522e0349 --- /dev/null +++ b/apps/sim/lib/internal/video/execute-tool.ts @@ -0,0 +1,85 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { videoToolBodySchema } from '@/lib/api/contracts/tools/media/video' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import type { VideoProvider } from '@/lib/internal/video/client' +import { VideoOperationError } from '@/lib/internal/video/errors' +import { executeVideoOperation } from '@/lib/internal/video/operations' + +const VIDEO_TOOL_IDS = new Set([ + 'video_falai', + 'video_luma', + 'video_minimax', + 'video_runway', + 'video_veo', +]) + +function parseInput(input: unknown): Response | ReturnType { + let serialized: string + try { + serialized = JSON.stringify(input) ?? '' + } catch { + return Response.json({ error: 'Invalid request data' }, { status: 400 }) + } + if (Buffer.byteLength(serialized, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + const parsed = videoToolBodySchema.safeParse(input) + if (!parsed.success) { + return Response.json( + { + error: getValidationErrorMessage(parsed.error, 'Invalid request data'), + details: parsed.error.issues, + }, + { status: 400 } + ) + } + return parsed.data +} + +export const executeVideoTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!VIDEO_TOOL_IDS.has(request.toolId)) { + return Response.json( + { success: false, error: `Unsupported Video tool: ${request.toolId}` }, + { status: 500 } + ) + } + const userId = request.context.userId + if (!userId) return Response.json({ error: 'Unauthorized' }, { status: 401 }) + const input = parseInput(request.input) + if (input instanceof Response) return input + + try { + const result = await executeVideoOperation( + { ...input, provider: input.provider as VideoProvider }, + { + headers: request.headers, + requestId: request.requestId, + signal: request.signal, + userId, + workspaceId: request.context.workspaceId, + workflowId: request.context.workflowId, + executionId: request.context.executionId, + } + ) + request.signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof VideoOperationError) { + return Response.json(error.body, { status: error.status }) + } + return Response.json( + { error: getErrorMessage(error, 'Video generation failed') }, + { status: isPayloadSizeLimitError(error) ? 413 : 500 } + ) + } +} diff --git a/apps/sim/lib/internal/video/operations.test.ts b/apps/sim/lib/internal/video/operations.test.ts new file mode 100644 index 00000000000..da1e2902a5f --- /dev/null +++ b/apps/sim/lib/internal/video/operations.test.ts @@ -0,0 +1,191 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + assertToolFileAccess: vi.fn(), + generateVideo: vi.fn(), + isModelSafeWorkspaceFileKey: vi.fn(), + uploadExecutionFile: vi.fn(), + uploadFile: vi.fn(), + validateOpaqueModelInputProvenance: vi.fn(), +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertToolFileAccess, +})) +vi.mock('@/lib/execution/model-input-provenance', () => ({ + validateOpaqueModelInputProvenance: mocks.validateOpaqueModelInputProvenance, +})) +vi.mock('@/lib/internal/video/client', () => ({ + generateVideo: mocks.generateVideo, + getVideoInputValidationError: vi.fn().mockReturnValue(undefined), +})) +vi.mock('@/lib/uploads', () => ({ + StorageService: { uploadFile: mocks.uploadFile }, +})) +vi.mock('@/lib/uploads/contexts/execution', () => ({ + uploadExecutionFile: mocks.uploadExecutionFile, +})) +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({ + isModelSafeWorkspaceFileKey: mocks.isModelSafeWorkspaceFileKey, + MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE: + 'File cannot be sent to a model because its secret provenance is unavailable', +})) + +import { executeVideoOperation } from '@/lib/internal/video/operations' + +const file = { + id: 'file-1', + name: 'reference.png', + size: 5, + type: 'image/png', + key: 'workspace/workspace-1/reference.png', +} + +describe('executeVideoOperation', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.assertToolFileAccess.mockResolvedValue(null) + mocks.isModelSafeWorkspaceFileKey.mockResolvedValue(true) + mocks.validateOpaqueModelInputProvenance.mockReturnValue({ success: true }) + mocks.generateVideo.mockResolvedValue({ + buffer: Buffer.from('video'), + width: 1280, + height: 720, + duration: 5, + jobId: 'job-1', + }) + mocks.uploadExecutionFile.mockResolvedValue({ + ...file, + name: 'video.mp4', + type: 'video/mp4', + url: '/api/files/serve/video.mp4', + }) + }) + + it('uses trusted execution scope and returns the legacy output contract', async () => { + const controller = new AbortController() + const result = await executeVideoOperation( + { + provider: 'runway', + apiKey: 'key', + model: 'gen-4-turbo', + prompt: 'A cinematic sunrise', + visualReference: file, + }, + { + headers: new Headers(), + requestId: 'request-1', + signal: controller.signal, + userId: 'user-1', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + } + ) + + expect(mocks.validateOpaqueModelInputProvenance).toHaveBeenCalled() + expect(mocks.assertToolFileAccess).toHaveBeenCalledWith( + file.key, + 'user-1', + 'request-1', + expect.anything() + ) + expect(mocks.generateVideo).toHaveBeenCalledWith( + expect.objectContaining({ provider: 'runway', visualReference: file }), + { requestId: 'request-1', signal: controller.signal } + ) + expect(mocks.uploadExecutionFile).toHaveBeenCalledWith( + { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }, + Buffer.from('video'), + expect.stringMatching(/^video-runway-/), + 'video/mp4', + 'user-1' + ) + expect(result).toMatchObject({ + videoUrl: '/api/files/serve/video.mp4', + videoFile: expect.objectContaining({ type: 'video/mp4' }), + duration: 5, + width: 1280, + height: 720, + provider: 'runway', + model: 'gen-4-turbo', + jobId: 'job-1', + }) + }) + + it('fails opaque provenance before inspecting or downloading a Runway file', async () => { + mocks.validateOpaqueModelInputProvenance.mockReturnValue({ + success: false, + error: 'Model input provenance is unavailable', + status: 400, + }) + + await expect( + executeVideoOperation( + { + provider: 'runway', + apiKey: 'key', + prompt: 'A cinematic sunrise', + visualReference: file, + }, + { headers: new Headers(), requestId: 'request-1', userId: 'user-1' } + ) + ).rejects.toMatchObject({ status: 400, message: 'Model input provenance is unavailable' }) + expect(mocks.assertToolFileAccess).not.toHaveBeenCalled() + expect(mocks.generateVideo).not.toHaveBeenCalled() + }) + + it('fails closed for model-unsafe workspace files', async () => { + mocks.isModelSafeWorkspaceFileKey.mockResolvedValue(false) + + await expect( + executeVideoOperation( + { + provider: 'runway', + apiKey: 'key', + prompt: 'A cinematic sunrise', + visualReference: file, + }, + { headers: new Headers(), requestId: 'request-1', userId: 'user-1' } + ) + ).rejects.toMatchObject({ + status: 400, + message: 'File cannot be sent to a model because its secret provenance is unavailable', + }) + expect(mocks.generateVideo).not.toHaveBeenCalled() + }) + + it('preserves Fal.ai hosted cost metadata', async () => { + mocks.generateVideo.mockResolvedValue({ + buffer: Buffer.from('video'), + falaiCost: { + endpointId: 'fal-ai/veo3.1', + requestId: 'fal-request-1', + costDollars: 0.4, + source: 'billing_events', + }, + }) + mocks.uploadFile.mockResolvedValue({ path: '/api/files/video.mp4', size: 5 }) + + const result = await executeVideoOperation( + { + provider: 'falai', + apiKey: 'key', + model: 'veo-3.1', + prompt: 'A cinematic sunrise', + useHostedCostTracking: true, + }, + { headers: new Headers(), requestId: 'request-1', userId: 'user-1' } + ) + + expect(result.__falaiCostDollars).toBe(0.4) + expect(result.__falaiBilling).toMatchObject({ source: 'billing_events' }) + }) +}) diff --git a/apps/sim/lib/internal/video/operations.ts b/apps/sim/lib/internal/video/operations.ts new file mode 100644 index 00000000000..0da2c9e2d1e --- /dev/null +++ b/apps/sim/lib/internal/video/operations.ts @@ -0,0 +1,163 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { getBaseUrl } from '@/lib/core/utils/urls' +import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' +import { + generateVideo, + getVideoInputValidationError, + type VideoGenerationInput, +} from '@/lib/internal/video/client' +import { VideoOperationError } from '@/lib/internal/video/errors' +import { StorageService } from '@/lib/uploads' +import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' +import { + isModelSafeWorkspaceFileKey, + MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, +} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { assertToolFileAccess } from '@/app/api/files/authorization' +import type { UserFile } from '@/executor/types' + +const logger = createLogger('VideoOperations') + +export interface VideoOperationInput extends VideoGenerationInput { + visualReference?: UserFile +} + +export interface VideoOperationContext { + headers: Headers + requestId: string + signal?: AbortSignal + userId: string + workspaceId?: string + workflowId?: string + executionId?: string +} + +export interface VideoOperationResult { + videoUrl: string + videoFile?: UserFile + duration?: number + width?: number + height?: number + provider: VideoOperationInput['provider'] + model: string + jobId?: string + __falaiCostDollars?: number + __falaiBilling?: unknown +} + +async function authorizeVisualReference( + input: VideoOperationInput, + context: VideoOperationContext +): Promise { + const file = input.visualReference + if (!file) return + context.signal?.throwIfAborted() + const denied = await assertToolFileAccess(file.key, context.userId, context.requestId, logger) + context.signal?.throwIfAborted() + if (denied) { + const body = (await denied.json()) as Record + throw new VideoOperationError('File not found', denied.status, body) + } + if (!(await isModelSafeWorkspaceFileKey(file.key))) { + throw new VideoOperationError(MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, 400) + } + context.signal?.throwIfAborted() +} + +function validateModelInputProvenance( + input: VideoOperationInput, + context: VideoOperationContext +): void { + if (input.provider !== 'runway') return + const provenance = validateOpaqueModelInputProvenance({ + headers: context.headers, + payload: input, + isInternalRequest: true, + }) + if (!provenance.success) { + throw new VideoOperationError(provenance.error, provenance.status) + } +} + +async function storeVideo( + input: VideoOperationInput, + buffer: Buffer, + context: VideoOperationContext +): Promise<{ videoUrl: string; videoFile?: UserFile }> { + context.signal?.throwIfAborted() + const fileName = `video-${input.provider}-${Date.now()}.mp4` + try { + if (context.workspaceId && context.workflowId && context.executionId) { + const videoFile = await uploadExecutionFile( + { + workspaceId: context.workspaceId, + workflowId: context.workflowId, + executionId: context.executionId, + }, + buffer, + fileName, + 'video/mp4', + context.userId + ) + context.signal?.throwIfAborted() + logger.info('Stored generated video in execution context', { + requestId: context.requestId, + provider: input.provider, + executionId: context.executionId, + size: videoFile.size, + }) + return { videoUrl: videoFile.url, videoFile } + } + + const file = await StorageService.uploadFile({ + file: buffer, + fileName, + contentType: 'video/mp4', + context: 'copilot', + }) + context.signal?.throwIfAborted() + logger.info('Stored generated video in copilot context', { + requestId: context.requestId, + provider: input.provider, + size: file.size, + }) + return { videoUrl: `${getBaseUrl()}${file.path}` } + } catch (error) { + context.signal?.throwIfAborted() + throw new VideoOperationError( + `Failed to store video: ${getErrorMessage(error, 'Unknown error')}`, + 500 + ) + } +} + +export async function executeVideoOperation( + input: VideoOperationInput, + context: VideoOperationContext +): Promise { + context.signal?.throwIfAborted() + validateModelInputProvenance(input, context) + const validationError = getVideoInputValidationError(input) + if (validationError) throw new VideoOperationError(validationError, 400) + await authorizeVisualReference(input, context) + + const generated = await generateVideo(input, { + requestId: context.requestId, + signal: context.signal, + }) + context.signal?.throwIfAborted() + const stored = await storeVideo(input, generated.buffer, context) + + return { + ...stored, + duration: generated.duration || input.duration, + width: generated.width, + height: generated.height, + provider: input.provider, + model: input.model || 'default', + jobId: generated.jobId, + __falaiCostDollars: generated.falaiCost?.costDollars, + __falaiBilling: generated.falaiCost, + } +} diff --git a/apps/sim/lib/internal/vision/client.test.ts b/apps/sim/lib/internal/vision/client.test.ts new file mode 100644 index 00000000000..78d52407f0f --- /dev/null +++ b/apps/sim/lib/internal/vision/client.test.ts @@ -0,0 +1,221 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' + +const mocks = vi.hoisted(() => ({ + generateContent: vi.fn(), + secureFetchWithPinnedIP: vi.fn(), +})) + +vi.mock('@google/genai', () => ({ + GoogleGenAI: class { + models = { generateContent: mocks.generateContent } + }, +})) +vi.mock('@/lib/core/security/input-validation.server', () => ({ + MAX_JSON_API_RESPONSE_BYTES: 10 * 1024 * 1024, + secureFetchWithPinnedIP: mocks.secureFetchWithPinnedIP, +})) + +import { analyzeVision } from '@/lib/internal/vision/client' + +describe('Vision client', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.restoreAllMocks() + }) + + it('preserves the OpenAI request and usage projection with cancellation', async () => { + const controller = new AbortController() + const fetchMock = vi.spyOn(globalThis, 'fetch').mockResolvedValue( + Response.json({ + model: 'gpt-5.2', + choices: [{ message: { content: 'A lighthouse' } }], + usage: { input_tokens: 10, output_tokens: 4, total_tokens: 14 }, + }) + ) + + await expect( + analyzeVision( + { + apiKey: 'secret', + imageSource: 'https://images.example.com/a.png', + model: 'gpt-5.2', + prompt: 'Describe it', + }, + controller.signal + ) + ).resolves.toEqual({ + content: 'A lighthouse', + model: 'gpt-5.2', + tokens: 14, + usage: { input_tokens: 10, output_tokens: 4, total_tokens: 14 }, + }) + + expect(fetchMock).toHaveBeenCalledWith( + 'https://api.openai.com/v1/chat/completions', + expect.objectContaining({ + method: 'POST', + signal: controller.signal, + headers: { + Authorization: 'Bearer secret', + 'Content-Type': 'application/json', + }, + }) + ) + const body = JSON.parse(fetchMock.mock.calls[0][1]?.body as string) + expect(body).toEqual({ + model: 'gpt-5.2', + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'Describe it' }, + { + type: 'image_url', + image_url: { url: 'https://images.example.com/a.png' }, + }, + ], + }, + ], + max_completion_tokens: 1000, + }) + }) + + it('preserves Anthropic base64 payloads and token totals', async () => { + const fetchMock = vi.spyOn(globalThis, 'fetch').mockResolvedValue( + Response.json({ + model: 'claude-3-opus-20240229', + content: [{ text: 'A lighthouse' }], + usage: { input_tokens: 8, output_tokens: 3 }, + }) + ) + + await expect( + analyzeVision({ + apiKey: 'secret', + imageSource: 'data:image/png;base64,YQ==', + model: 'claude-3-opus-20240229', + prompt: 'Describe it', + }) + ).resolves.toEqual({ + content: 'A lighthouse', + model: 'claude-3-opus-20240229', + tokens: 11, + usage: { input_tokens: 8, output_tokens: 3, total_tokens: 11 }, + }) + + expect(fetchMock.mock.calls[0][0]).toBe('https://api.anthropic.com/v1/messages') + expect(fetchMock.mock.calls[0][1]?.headers).toEqual({ + 'Content-Type': 'application/json', + 'anthropic-version': '2023-06-01', + 'x-api-key': 'secret', + }) + expect(JSON.parse(fetchMock.mock.calls[0][1]?.body as string)).toMatchObject({ + max_tokens: 1024, + messages: [ + { + content: [ + { type: 'text', text: 'Describe it' }, + { + type: 'image', + source: { type: 'base64', media_type: 'image/png', data: 'YQ==' }, + }, + ], + }, + ], + }) + }) + + it('pins and bounds Gemini remote image downloads and forwards cancellation', async () => { + const controller = new AbortController() + mocks.secureFetchWithPinnedIP.mockResolvedValue( + new Response(new Uint8Array([1, 2, 3]), { + status: 200, + headers: { 'content-type': 'image/png' }, + }) + ) + mocks.generateContent.mockResolvedValue({ + candidates: [{ content: { parts: [{ text: 'A lighthouse' }] } }], + usageMetadata: { promptTokenCount: 7, candidatesTokenCount: 2, totalTokenCount: 9 }, + }) + + await expect( + analyzeVision( + { + apiKey: 'secret', + imageSource: 'https://images.example.com/a.png', + model: 'gemini-2.5-pro', + prompt: 'Describe it', + remoteImageResolvedIP: '203.0.113.10', + }, + controller.signal + ) + ).resolves.toEqual({ content: 'A lighthouse', model: 'gemini-2.5-pro', tokens: 9 }) + + expect(mocks.secureFetchWithPinnedIP).toHaveBeenCalledWith( + 'https://images.example.com/a.png', + '203.0.113.10', + { + method: 'GET', + maxResponseBytes: MAX_BUFFERED_TRANSFER_BYTES, + signal: controller.signal, + } + ) + expect(mocks.generateContent).toHaveBeenCalledWith( + expect.objectContaining({ + model: 'gemini-2.5-pro', + config: { abortSignal: controller.signal }, + contents: [ + { + role: 'user', + parts: [ + { text: 'Describe it' }, + { inlineData: { mimeType: 'image/png', data: 'AQID' } }, + ], + }, + ], + }) + ) + }) + + it('preserves provider error status and message', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + Response.json({ error: { message: 'Invalid API key' } }, { status: 401 }) + ) + + await expect( + analyzeVision({ + apiKey: 'bad', + imageSource: 'https://images.example.com/a.png', + model: 'gpt-5.2', + prompt: 'Describe it', + }) + ).rejects.toMatchObject({ + status: 401, + body: { success: false, error: 'Invalid API key' }, + }) + }) + + it('rejects oversized Gemini images before buffering', async () => { + mocks.secureFetchWithPinnedIP.mockResolvedValue( + new Response(new Uint8Array([1]), { + status: 200, + headers: { 'content-length': String(MAX_BUFFERED_TRANSFER_BYTES + 1) }, + }) + ) + + await expect( + analyzeVision({ + apiKey: 'secret', + imageSource: 'https://images.example.com/a.png', + model: 'gemini-2.5-pro', + prompt: 'Describe it', + remoteImageResolvedIP: '203.0.113.10', + }) + ).rejects.toMatchObject({ name: 'PayloadSizeLimitError' }) + expect(mocks.generateContent).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/vision/client.ts b/apps/sim/lib/internal/vision/client.ts new file mode 100644 index 00000000000..7ff0a164d28 --- /dev/null +++ b/apps/sim/lib/internal/vision/client.ts @@ -0,0 +1,245 @@ +import { GoogleGenAI } from '@google/genai' +import { createLogger } from '@sim/logger' +import { isRecordLike } from '@sim/utils/object' +import { + MAX_JSON_API_RESPONSE_BYTES, + secureFetchWithPinnedIP, +} from '@/lib/core/security/input-validation.server' +import { + consumeOrCancelBody, + readResponseJsonWithLimit, + readResponseToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' +import { VisionOperationError } from '@/lib/internal/vision/errors' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { convertUsageMetadata, extractTextContent } from '@/providers/google/utils' + +const logger = createLogger('VisionClient') +const MAX_PROVIDER_ERROR_BYTES = 64 * 1024 + +export interface VisionClientInput { + apiKey: string + imageSource: string + imageContentType?: string + model: string + prompt: string + remoteImageResolvedIP?: string +} + +export interface VisionAnalysisResult { + content?: string + model?: string + tokens?: number + usage?: { + input_tokens?: number + output_tokens?: number + total_tokens?: number + } +} + +function record(value: unknown): Record { + return isRecordLike(value) ? value : {} +} + +function number(value: unknown): number | undefined { + return typeof value === 'number' ? value : undefined +} + +function string(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined +} + +function providerErrorMessage(value: unknown): string { + const data = record(value) + const nested = record(data.error) + return string(nested.message) || string(data.message) || 'Failed to analyze image' +} + +async function readProviderJson(response: Response, signal?: AbortSignal): Promise { + return readResponseJsonWithLimit(response, { + maxBytes: MAX_JSON_API_RESPONSE_BYTES, + label: 'Vision provider response', + signal, + }) +} + +async function readProviderError(response: Response, signal?: AbortSignal): Promise { + return readResponseJsonWithLimit(response, { + maxBytes: MAX_PROVIDER_ERROR_BYTES, + label: 'Vision provider error response', + signal, + }).catch(() => { + signal?.throwIfAborted() + return {} + }) +} + +function parseDataImage(imageSource: string): { mediaType: string; base64Data: string } { + const marker = ';base64,' + const markerIndex = imageSource.indexOf(marker) + if (!imageSource.startsWith('data:') || markerIndex === -1) { + throw new VisionOperationError('Invalid base64 image format', 400) + } + const rawMimeType = imageSource.slice('data:'.length, markerIndex) + const mediaType = rawMimeType.split(';')[0] || 'image/jpeg' + const base64Data = imageSource.slice(markerIndex + marker.length) + if (!base64Data) throw new VisionOperationError('Invalid base64 image format', 400) + return { mediaType, base64Data } +} + +async function fetchGeminiImage(input: VisionClientInput, signal?: AbortSignal): Promise { + if (input.imageSource.startsWith('data:')) return input.imageSource + if (!input.remoteImageResolvedIP) { + throw new VisionOperationError('Invalid image URL', 400) + } + + const response = await secureFetchWithPinnedIP(input.imageSource, input.remoteImageResolvedIP, { + method: 'GET', + maxResponseBytes: MAX_BUFFERED_TRANSFER_BYTES, + signal, + }) + if (!response.ok) { + await consumeOrCancelBody(response) + throw new VisionOperationError('Failed to fetch image for Gemini', 400) + } + const contentType = response.headers.get('content-type') || input.imageContentType || 'image/jpeg' + const buffer = await readResponseToBufferWithLimit(response, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + label: 'Gemini source image', + signal, + }) + return `data:${contentType};base64,${buffer.toString('base64')}` +} + +async function analyzeWithGemini( + input: VisionClientInput, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const base64Payload = await fetchGeminiImage(input, signal) + const { mediaType, base64Data } = parseDataImage(base64Payload) + const ai = new GoogleGenAI({ apiKey: input.apiKey }) + const response = await ai.models.generateContent({ + model: input.model, + contents: [ + { + role: 'user', + parts: [{ text: input.prompt }, { inlineData: { mimeType: mediaType, data: base64Data } }], + }, + ], + config: { abortSignal: signal }, + }) + signal?.throwIfAborted() + const usage = convertUsageMetadata(response.usageMetadata) + return { + content: extractTextContent(response.candidates?.[0]), + model: input.model, + tokens: usage.totalTokenCount || undefined, + } +} + +function anthropicRequest(input: VisionClientInput): Record { + const source = input.imageSource.startsWith('data:') + ? (() => { + const match = input.imageSource.match(/^data:([^;]+);base64,(.+)$/) + if (!match) throw new VisionOperationError('Invalid base64 image format', 400) + return { type: 'base64', media_type: match[1], data: match[2] } + })() + : { type: 'url', url: input.imageSource } + return { + model: input.model, + max_tokens: 1024, + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: input.prompt }, + { type: 'image', source }, + ], + }, + ], + } +} + +function openAiRequest(input: VisionClientInput): Record { + return { + model: input.model, + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: input.prompt }, + { type: 'image_url', image_url: { url: input.imageSource } }, + ], + }, + ], + max_completion_tokens: 1000, + } +} + +async function analyzeWithHttpProvider( + input: VisionClientInput, + signal?: AbortSignal +): Promise { + const isClaude = input.model.startsWith('claude-') + const apiUrl = isClaude + ? 'https://api.anthropic.com/v1/messages' + : 'https://api.openai.com/v1/chat/completions' + const headers: Record = { 'Content-Type': 'application/json' } + if (isClaude) { + headers['x-api-key'] = input.apiKey + headers['anthropic-version'] = '2023-06-01' + } else { + headers.Authorization = `Bearer ${input.apiKey}` + } + + signal?.throwIfAborted() + const response = await fetch(apiUrl, { + method: 'POST', + headers, + body: JSON.stringify(isClaude ? anthropicRequest(input) : openAiRequest(input)), + signal, + }) + signal?.throwIfAborted() + if (!response.ok) { + const error = await readProviderError(response, signal) + signal?.throwIfAborted() + logger.error('Vision provider request failed', { + model: input.model, + status: response.status, + error, + }) + throw new VisionOperationError(providerErrorMessage(error), response.status) + } + + const data = record(await readProviderJson(response, signal)) + const usage = record(data.usage) + const content = Array.isArray(data.content) ? record(data.content[0]) : {} + const choices = Array.isArray(data.choices) ? record(data.choices[0]) : {} + const message = record(choices.message) + const inputTokens = number(usage.input_tokens) + const outputTokens = number(usage.output_tokens) + const totalTokens = number(usage.total_tokens) + return { + content: string(content.text) || string(message.content), + model: string(data.model), + tokens: Array.isArray(data.content) ? (inputTokens || 0) + (outputTokens || 0) : totalTokens, + usage: + Object.keys(usage).length > 0 + ? { + input_tokens: inputTokens, + output_tokens: outputTokens, + total_tokens: totalTokens || (inputTokens || 0) + (outputTokens || 0), + } + : undefined, + } +} + +export async function analyzeVision( + input: VisionClientInput, + signal?: AbortSignal +): Promise { + return input.model.startsWith('gemini-') + ? analyzeWithGemini(input, signal) + : analyzeWithHttpProvider(input, signal) +} diff --git a/apps/sim/lib/internal/vision/errors.ts b/apps/sim/lib/internal/vision/errors.ts new file mode 100644 index 00000000000..cb703886077 --- /dev/null +++ b/apps/sim/lib/internal/vision/errors.ts @@ -0,0 +1,10 @@ +export class VisionOperationError extends Error { + constructor( + message: string, + readonly status: number, + readonly body: Record = { success: false, error: message } + ) { + super(message) + this.name = 'VisionOperationError' + } +} diff --git a/apps/sim/lib/internal/vision/execute-tool.test.ts b/apps/sim/lib/internal/vision/execute-tool.test.ts new file mode 100644 index 00000000000..302a19c9255 --- /dev/null +++ b/apps/sim/lib/internal/vision/execute-tool.test.ts @@ -0,0 +1,114 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' + +const executeVisionOperation = vi.hoisted(() => vi.fn()) + +vi.mock('@/lib/internal/vision/operations', () => ({ executeVisionOperation })) + +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' +import { VisionOperationError } from '@/lib/internal/vision/errors' +import { executeVisionTool } from '@/lib/internal/vision/execute-tool' + +function toolRequest(overrides: Partial = {}) { + return { + toolId: 'vision_tool', + input: { + apiKey: 'secret', + imageUrl: 'https://images.example.com/a.png', + imageFile: null, + model: 'gpt-5.2', + prompt: null, + }, + headers: new Headers(), + context: { ...createExecutionContext({ workflowId: 'workflow-1' }), userId: 'user-1' }, + requestId: 'request-1', + ...overrides, + } as InternalToolOperationCall +} + +describe('executeVisionTool', () => { + beforeEach(() => { + vi.clearAllMocks() + executeVisionOperation.mockResolvedValue({ content: 'A lighthouse', model: 'gpt-5.2' }) + }) + + it.each(['vision_tool', 'vision_tool_v2'])( + 'dispatches %s to the shared operation', + async (toolId) => { + const response = await executeVisionTool(toolRequest({ toolId })) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + success: true, + output: { content: 'A lighthouse', model: 'gpt-5.2' }, + }) + expect(executeVisionOperation).toHaveBeenCalledWith( + expect.objectContaining({ apiKey: 'secret', model: 'gpt-5.2' }), + expect.objectContaining({ userId: 'user-1', requestId: 'request-1' }) + ) + } + ) + + it('authenticates before parsing input', async () => { + const response = await executeVisionTool( + toolRequest({ + input: null, + context: createExecutionContext({ workflowId: 'workflow-1' }), + }) + ) + + expect(response.status).toBe(401) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Authentication required', + }) + expect(executeVisionOperation).not.toHaveBeenCalled() + }) + + it('preserves contract validation errors', async () => { + const response = await executeVisionTool(toolRequest({ input: {} })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Validation error', + details: expect.any(Array), + }) + }) + + it('preserves the route input byte ceiling', async () => { + const response = await executeVisionTool( + toolRequest({ input: { apiKey: 'secret', prompt: 'x'.repeat(DEFAULT_MAX_JSON_BODY_BYTES) } }) + ) + + expect(response.status).toBe(413) + expect(executeVisionOperation).not.toHaveBeenCalled() + }) + + it('stops before dispatch when execution is already aborted', async () => { + const controller = new AbortController() + controller.abort(new Error('Execution aborted')) + + await expect(executeVisionTool(toolRequest({ signal: controller.signal }))).rejects.toThrow( + 'Execution aborted' + ) + expect(executeVisionOperation).not.toHaveBeenCalled() + }) + + it('projects exact typed error envelopes', async () => { + executeVisionOperation.mockRejectedValueOnce( + new VisionOperationError('Either imageUrl or imageFile is required', 400) + ) + + const response = await executeVisionTool(toolRequest()) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Either imageUrl or imageFile is required', + }) + }) +}) diff --git a/apps/sim/lib/internal/vision/execute-tool.ts b/apps/sim/lib/internal/vision/execute-tool.ts new file mode 100644 index 00000000000..0356966003f --- /dev/null +++ b/apps/sim/lib/internal/vision/execute-tool.ts @@ -0,0 +1,70 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { VisionOperationError } from '@/lib/internal/vision/errors' +import { executeVisionOperation } from '@/lib/internal/vision/operations' +import { visionOperationInputSchema } from '@/lib/internal/vision/schema' + +const logger = createLogger('VisionToolExecution') +const VISION_TOOL_IDS = new Set(['vision_tool', 'vision_tool_v2']) + +export const executeVisionTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!VISION_TOOL_IDS.has(request.toolId)) { + return Response.json( + { success: false, error: `Unsupported Vision tool: ${request.toolId}` }, + { status: 500 } + ) + } + const userId = request.context.userId + if (!userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + + let serializedInput: string + try { + serializedInput = JSON.stringify(request.input) ?? '' + } catch { + return Response.json({ error: 'Request body must be valid JSON' }, { status: 400 }) + } + if (Buffer.byteLength(serializedInput, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + + const parsed = visionOperationInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { error: 'Validation error', details: parsed.error.issues }, + { status: 400 } + ) + } + + try { + const output = await executeVisionOperation(parsed.data, { + headers: request.headers, + requestId: request.requestId, + signal: request.signal, + userId, + }) + request.signal?.throwIfAborted() + return Response.json({ success: true, output }) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof VisionOperationError) { + return Response.json(error.body, { status: error.status }) + } + const message = getErrorMessage(error, 'Unknown error occurred') + logger.error('Vision operation failed', { + error: message, + requestId: request.requestId, + toolId: request.toolId, + }) + return Response.json({ success: false, error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/vision/operations.test.ts b/apps/sim/lib/internal/vision/operations.test.ts new file mode 100644 index 00000000000..8f8f6a9280e --- /dev/null +++ b/apps/sim/lib/internal/vision/operations.test.ts @@ -0,0 +1,252 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { PRIVATE_MODEL_INPUT_PROVENANCE_HEADER } from '@/lib/execution/model-input-provenance' +import { + RESOLVED_SECRET_PROVENANCE_FIELD, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, +} from '@/lib/execution/private-tool-metadata' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' + +const mocks = vi.hoisted(() => ({ + analyzeVision: vi.fn(), + assertToolFileAccess: vi.fn(), + downloadFileFromStorage: vi.fn(), + isModelSafeWorkspaceFileKey: vi.fn(), + resolveInternalFileUrl: vi.fn(), + validateUrlWithDNS: vi.fn(), +})) + +vi.mock('@/lib/internal/vision/client', () => ({ analyzeVision: mocks.analyzeVision })) +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertToolFileAccess, +})) +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadFileFromStorage: mocks.downloadFileFromStorage, + resolveInternalFileUrl: mocks.resolveInternalFileUrl, +})) +vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({ + isModelSafeWorkspaceFileKey: mocks.isModelSafeWorkspaceFileKey, + MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE: + 'File cannot be sent to a model because its secret provenance is unavailable', +})) +vi.mock('@/lib/core/security/input-validation.server', () => ({ + validateUrlWithDNS: mocks.validateUrlWithDNS, +})) + +import { executeVisionOperation } from '@/lib/internal/vision/operations' + +const imageFile = { + id: 'file-1', + key: 'workspace/workspace-1/image.png', + name: 'image.png', + size: 3, + type: 'image/png', + url: '/api/files/serve/s3/workspace/workspace-1/image.png', +} + +const context = { + headers: new Headers(), + requestId: 'request-1', + userId: 'user-1', +} + +describe('Vision operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.analyzeVision.mockResolvedValue({ content: 'A lighthouse', model: 'gpt-5.2' }) + mocks.assertToolFileAccess.mockResolvedValue(null) + mocks.downloadFileFromStorage.mockResolvedValue(Buffer.from([1, 2, 3])) + mocks.isModelSafeWorkspaceFileKey.mockResolvedValue(true) + mocks.resolveInternalFileUrl.mockResolvedValue({ + fileUrl: 'https://storage.example.com/image.png', + }) + mocks.validateUrlWithDNS.mockResolvedValue({ + isValid: true, + resolvedIP: '203.0.113.10', + }) + }) + + it('authorizes and bounds stored files before provider egress', async () => { + await executeVisionOperation( + { apiKey: 'secret', imageFile, model: 'gpt-5.2', prompt: null }, + context + ) + + expect(mocks.assertToolFileAccess).toHaveBeenCalledWith( + imageFile.key, + 'user-1', + 'request-1', + expect.anything() + ) + expect(mocks.downloadFileFromStorage).toHaveBeenCalledWith( + expect.objectContaining({ key: imageFile.key }), + 'request-1', + expect.anything(), + { maxBytes: MAX_BUFFERED_TRANSFER_BYTES } + ) + expect(mocks.analyzeVision).toHaveBeenCalledWith( + { + apiKey: 'secret', + imageSource: 'data:image/png;base64,AQID', + imageContentType: 'image/png', + model: 'gpt-5.2', + prompt: 'Please analyze this image and describe what you see in detail.', + remoteImageResolvedIP: undefined, + }, + undefined + ) + }) + + it('forwards cancellation through the provider operation', async () => { + const controller = new AbortController() + + await executeVisionOperation( + { apiKey: 'secret', imageFile, model: 'gpt-5.2', prompt: null }, + { ...context, signal: controller.signal } + ) + + expect(mocks.analyzeVision).toHaveBeenCalledWith(expect.anything(), controller.signal) + }) + + it('rejects incomplete private provenance before resolving the image', async () => { + const headers = new Headers({ + [PRIVATE_MODEL_INPUT_PROVENANCE_HEADER]: RESOLVED_SECRET_PROVENANCE_METADATA_V1, + }) + + await expect( + executeVisionOperation( + { + apiKey: 'secret', + imageFile, + model: 'gpt-5.2', + prompt: null, + [RESOLVED_SECRET_PROVENANCE_FIELD]: { + version: 1, + complete: false, + entries: [], + }, + }, + { ...context, headers } + ) + ).rejects.toMatchObject({ + status: 400, + body: { success: false, error: 'Model input provenance is unavailable' }, + }) + expect(mocks.assertToolFileAccess).not.toHaveBeenCalled() + expect(mocks.analyzeVision).not.toHaveBeenCalled() + }) + + it('rejects unsafe files before reading bytes', async () => { + mocks.isModelSafeWorkspaceFileKey.mockResolvedValue(false) + + await expect( + executeVisionOperation( + { apiKey: 'secret', imageFile, model: 'gpt-5.2', prompt: null }, + context + ) + ).rejects.toMatchObject({ + status: 400, + body: { + success: false, + error: 'File cannot be sent to a model because its secret provenance is unavailable', + }, + }) + expect(mocks.downloadFileFromStorage).not.toHaveBeenCalled() + expect(mocks.analyzeVision).not.toHaveBeenCalled() + }) + + it('uses the file over a simultaneous URL', async () => { + await executeVisionOperation( + { + apiKey: 'secret', + imageFile, + imageUrl: 'https://ignored.example.com/image.png', + model: 'gpt-5.2', + prompt: 'Describe it', + }, + context + ) + + expect(mocks.validateUrlWithDNS).not.toHaveBeenCalled() + expect(mocks.analyzeVision).toHaveBeenCalledWith( + expect.objectContaining({ imageSource: 'data:image/png;base64,AQID' }), + undefined + ) + }) + + it('preserves v1 data URL inputs without treating them as network destinations', async () => { + await executeVisionOperation( + { + apiKey: 'secret', + imageUrl: 'data:image/png;base64,AQID', + imageFile: null, + model: 'gpt-5.2', + prompt: 'Describe it', + }, + context + ) + + expect(mocks.validateUrlWithDNS).not.toHaveBeenCalled() + expect(mocks.analyzeVision).toHaveBeenCalledWith( + expect.objectContaining({ imageSource: 'data:image/png;base64,AQID' }), + undefined + ) + }) + + it('resolves internal URLs, checks model-safe provenance, then pins DNS', async () => { + await executeVisionOperation( + { + apiKey: 'secret', + imageUrl: '/api/files/serve/s3/workspace/workspace-1/image.png', + imageFile: null, + model: 'gemini-2.5-pro', + prompt: 'Describe it', + }, + context + ) + + expect(mocks.resolveInternalFileUrl).toHaveBeenCalledWith( + '/api/files/serve/s3/workspace/workspace-1/image.png', + 'user-1', + 'request-1', + expect.anything() + ) + expect(mocks.isModelSafeWorkspaceFileKey).toHaveBeenCalledWith( + 'workspace/workspace-1/image.png' + ) + expect(mocks.validateUrlWithDNS).toHaveBeenCalledWith( + 'https://storage.example.com/image.png', + 'imageUrl' + ) + expect(mocks.analyzeVision).toHaveBeenCalledWith( + expect.objectContaining({ + imageSource: 'https://storage.example.com/image.png', + remoteImageResolvedIP: '203.0.113.10', + }), + undefined + ) + }) + + it('rejects invalid external destinations before provider work', async () => { + mocks.validateUrlWithDNS.mockResolvedValue({ isValid: false, error: 'private address' }) + + await expect( + executeVisionOperation( + { + apiKey: 'secret', + imageUrl: 'http://127.0.0.1/image.png', + imageFile: null, + model: 'gpt-5.2', + prompt: null, + }, + context + ) + ).rejects.toMatchObject({ + status: 400, + body: { success: false, error: 'private address' }, + }) + expect(mocks.analyzeVision).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/vision/operations.ts b/apps/sim/lib/internal/vision/operations.ts new file mode 100644 index 00000000000..7c4c219d0d3 --- /dev/null +++ b/apps/sim/lib/internal/vision/operations.ts @@ -0,0 +1,143 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { validateUrlWithDNS } from '@/lib/core/security/input-validation.server' +import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' +import { analyzeVision, type VisionAnalysisResult } from '@/lib/internal/vision/client' +import { VisionOperationError } from '@/lib/internal/vision/errors' +import type { VisionOperationInput } from '@/lib/internal/vision/schema' +import { + isModelSafeWorkspaceFileKey, + MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, +} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { + extractStorageKey, + isInternalFileUrl, + processSingleFileToUserFile, +} from '@/lib/uploads/utils/file-utils' +import { + downloadFileFromStorage, + resolveInternalFileUrl, +} from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' + +const logger = createLogger('VisionOperations') +const DEFAULT_PROMPT = 'Please analyze this image and describe what you see in detail.' + +export interface VisionOperationContext { + headers: Headers + requestId: string + signal?: AbortSignal + userId: string +} + +interface ResolvedImage { + source: string + contentType?: string + resolvedIP?: string +} + +function fail(message: string, status: number, body?: Record): never { + throw new VisionOperationError(message, status, body) +} + +async function resolveFileImage( + input: VisionOperationInput, + context: VisionOperationContext +): Promise { + if (!input.imageFile) return null + let file + try { + file = processSingleFileToUserFile(input.imageFile, context.requestId, logger) + } catch (error) { + fail(getErrorMessage(error, 'Failed to process image file'), 400) + } + + let base64 = file.base64 + if (!base64) { + context.signal?.throwIfAborted() + const denied = await assertToolFileAccess(file.key, context.userId, context.requestId, logger) + context.signal?.throwIfAborted() + if (denied) { + fail('File not found', denied.status, (await denied.json()) as Record) + } + if (!(await isModelSafeWorkspaceFileKey(file.key))) { + fail(MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, 400) + } + context.signal?.throwIfAborted() + const buffer = await downloadFileFromStorage(file, context.requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) + context.signal?.throwIfAborted() + base64 = buffer.toString('base64') + } + const contentType = file.type || 'image/jpeg' + return { source: `data:${contentType};base64,${base64}`, contentType } +} + +async function resolveUrlImage( + input: VisionOperationInput, + context: VisionOperationContext +): Promise { + let source = input.imageUrl || '' + if (source.startsWith('data:')) return { source } + if (source.startsWith('/') && !isInternalFileUrl(source)) { + fail('Invalid file path. Only uploaded files are supported for internal paths.', 400) + } + if (isInternalFileUrl(source)) { + context.signal?.throwIfAborted() + const resolution = await resolveInternalFileUrl( + source, + context.userId, + context.requestId, + logger + ) + context.signal?.throwIfAborted() + if (resolution.error) fail(resolution.error.message, resolution.error.status) + source = resolution.fileUrl || source + if (!(await isModelSafeWorkspaceFileKey(extractStorageKey(input.imageUrl!)))) { + fail(MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE, 400) + } + } + + context.signal?.throwIfAborted() + const validation = await validateUrlWithDNS(source, 'imageUrl') + context.signal?.throwIfAborted() + if (!validation.isValid) { + fail(validation.error || 'Invalid image URL', 400, { + success: false, + error: validation.error, + }) + } + return { source, resolvedIP: validation.resolvedIP } +} + +export async function executeVisionOperation( + input: VisionOperationInput, + context: VisionOperationContext +): Promise { + context.signal?.throwIfAborted() + const provenance = validateOpaqueModelInputProvenance({ + headers: context.headers, + payload: input, + isInternalRequest: true, + }) + if (!provenance.success) fail(provenance.error, provenance.status) + if (!input.imageUrl && !input.imageFile) { + fail('Either imageUrl or imageFile is required', 400) + } + + const image = (await resolveFileImage(input, context)) ?? (await resolveUrlImage(input, context)) + context.signal?.throwIfAborted() + return analyzeVision( + { + apiKey: input.apiKey, + imageSource: image.source, + imageContentType: image.contentType, + model: input.model, + prompt: input.prompt || DEFAULT_PROMPT, + remoteImageResolvedIP: image.resolvedIP, + }, + context.signal + ) +} diff --git a/apps/sim/lib/internal/vision/schema.ts b/apps/sim/lib/internal/vision/schema.ts new file mode 100644 index 00000000000..764c356cd93 --- /dev/null +++ b/apps/sim/lib/internal/vision/schema.ts @@ -0,0 +1,15 @@ +import { z } from 'zod' +import { resolvedSecretTraceProvenanceSchema } from '@/lib/api/contracts/primitives' +import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' + +export const visionOperationInputSchema = z.object({ + apiKey: z.string().min(1, 'API key is required'), + imageUrl: z.string().optional().nullable(), + imageFile: RawFileInputSchema.optional().nullable(), + model: z.string().optional().default('gpt-5.2'), + prompt: z.string().optional().nullable(), + [RESOLVED_SECRET_PROVENANCE_FIELD]: resolvedSecretTraceProvenanceSchema.optional(), +}) + +export type VisionOperationInput = z.output diff --git a/apps/sim/lib/internal/whatsapp/client.ts b/apps/sim/lib/internal/whatsapp/client.ts new file mode 100644 index 00000000000..ff041fe0219 --- /dev/null +++ b/apps/sim/lib/internal/whatsapp/client.ts @@ -0,0 +1,18 @@ +import { isRecordLike } from '@sim/utils/object' +import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' + +export const MAX_WHATSAPP_GRAPH_RESPONSE_BYTES = 256 * 1024 + +export async function readWhatsAppGraphResponse( + response: Response, + label: string, + signal?: AbortSignal +): Promise> { + const text = await readResponseTextWithLimit(response, { + maxBytes: MAX_WHATSAPP_GRAPH_RESPONSE_BYTES, + label, + signal, + }) + const parsed = text ? (JSON.parse(text) as unknown) : {} + return isRecordLike(parsed) ? parsed : {} +} diff --git a/apps/sim/lib/internal/whatsapp/execute-tool.test.ts b/apps/sim/lib/internal/whatsapp/execute-tool.test.ts new file mode 100644 index 00000000000..67d72783f79 --- /dev/null +++ b/apps/sim/lib/internal/whatsapp/execute-tool.test.ts @@ -0,0 +1,91 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + getMedia: vi.fn(), + sendMedia: vi.fn(), + uploadMedia: vi.fn(), +})) + +vi.mock('@/lib/internal/whatsapp/operations', () => ({ + executeWhatsAppGetMedia: mocks.getMedia, + executeWhatsAppSendMedia: mocks.sendMedia, + executeWhatsAppUploadMedia: mocks.uploadMedia, +})) + +import { executeWhatsAppTool } from '@/lib/internal/whatsapp/execute-tool' +import { getMediaTool } from '@/tools/whatsapp/get_media' +import { sendMediaTool } from '@/tools/whatsapp/send_media' +import { uploadMediaTool } from '@/tools/whatsapp/upload_media' + +const auth = { accessToken: 'token', phoneNumberId: 'phone-id' } + +describe('WhatsApp internal tool execution', () => { + beforeEach(() => { + vi.clearAllMocks() + for (const execute of Object.values(mocks)) { + execute.mockResolvedValue(Response.json({ success: true, output: {} })) + } + }) + + it.each([ + ['whatsapp_get_media', { ...auth, mediaId: 'media-id' }, mocks.getMedia], + [ + 'whatsapp_send_media', + { ...auth, phoneNumber: '+14155550100', mediaType: 'image', mediaId: 'media-id' }, + mocks.sendMedia, + ], + [ + 'whatsapp_upload_media', + { + ...auth, + file: { key: 'workspace/file', name: 'image.png', size: 4, type: 'image/png' }, + }, + mocks.uploadMedia, + ], + ])('dispatches %s with trusted execution scope', async (toolId, input, execute) => { + await executeWhatsAppTool({ + toolId, + input, + headers: new Headers(), + context: { + userId: 'user-1', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }, + requestId: 'request-1', + }) + + expect(execute).toHaveBeenCalledOnce() + expect(execute.mock.calls[0][1]).toMatchObject({ + userId: 'user-1', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + requestId: 'request-1', + }) + }) + + it('rejects unauthenticated direct execution', async () => { + const response = await executeWhatsAppTool({ + toolId: 'whatsapp_get_media', + input: { ...auth, mediaId: 'media-id' }, + headers: new Headers(), + context: {}, + requestId: 'request-1', + }) + + expect(response.status).toBe(401) + expect(mocks.getMedia).not.toHaveBeenCalled() + }) + + it('uses operation-only declarations', () => { + for (const tool of [getMediaTool, sendMediaTool, uploadMediaTool]) { + expect(tool.operation).toBeDefined() + expect('request' in tool).toBe(false) + } + }) +}) diff --git a/apps/sim/lib/internal/whatsapp/execute-tool.ts b/apps/sim/lib/internal/whatsapp/execute-tool.ts new file mode 100644 index 00000000000..af9ec4b7854 --- /dev/null +++ b/apps/sim/lib/internal/whatsapp/execute-tool.ts @@ -0,0 +1,95 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { z } from 'zod' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import type { + InternalToolOperationCall, + InternalToolOperationHandler, +} from '@/lib/internal/tool-operations/types' +import { + executeWhatsAppGetMedia, + executeWhatsAppSendMedia, + executeWhatsAppUploadMedia, + type WhatsAppOperationContext, +} from '@/lib/internal/whatsapp/operations' +import { + whatsappGetMediaInputSchema, + whatsappSendMediaInputSchema, + whatsappUploadMediaInputSchema, +} from '@/lib/internal/whatsapp/schema' + +const logger = createLogger('WhatsAppToolExecution') + +async function executeParsed( + request: InternalToolOperationCall, + schema: S, + execute: (input: z.output, context: WhatsAppOperationContext) => Promise +): Promise { + const parsed = schema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { + success: false, + error: getValidationErrorMessage(parsed.error, 'Invalid request data'), + }, + { status: 400 } + ) + } + const userId = request.context.userId + if (!userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + return execute(parsed.data, { + userId, + requestId: request.requestId, + workspaceId: request.context.workspaceId, + workflowId: request.context.workflowId, + executionId: request.context.executionId, + signal: request.signal, + }) +} + +export const executeWhatsAppTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + let serializedInput: string + try { + serializedInput = JSON.stringify(request.input) ?? '' + } catch { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + if (Buffer.byteLength(serializedInput, 'utf8') > DEFAULT_MAX_JSON_BODY_BYTES) { + return Response.json( + { + success: false, + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + + try { + switch (request.toolId) { + case 'whatsapp_get_media': + return executeParsed(request, whatsappGetMediaInputSchema, executeWhatsAppGetMedia) + case 'whatsapp_send_media': + return executeParsed(request, whatsappSendMediaInputSchema, executeWhatsAppSendMedia) + case 'whatsapp_upload_media': + return executeParsed(request, whatsappUploadMediaInputSchema, executeWhatsAppUploadMedia) + default: + return Response.json( + { success: false, error: `Unsupported WhatsApp tool: ${request.toolId}` }, + { status: 500 } + ) + } + } catch (error) { + request.signal?.throwIfAborted() + const message = getErrorMessage(error, 'Unknown error') + logger.error('WhatsApp operation dispatch failed', { + error: message, + requestId: request.requestId, + toolId: request.toolId, + }) + return Response.json({ success: false, error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/whatsapp/operations.test.ts b/apps/sim/lib/internal/whatsapp/operations.test.ts new file mode 100644 index 00000000000..6d9fe5abda7 --- /dev/null +++ b/apps/sim/lib/internal/whatsapp/operations.test.ts @@ -0,0 +1,137 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + readGraph: vi.fn(), + validateUrl: vi.fn(), + secureFetch: vi.fn(), + uploadExecution: vi.fn(), + uploadCopilot: vi.fn(), + uploadMedia: vi.fn(), +})) + +vi.mock('@/lib/internal/whatsapp/client', () => ({ + readWhatsAppGraphResponse: mocks.readGraph, +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + validateUrlWithDNS: mocks.validateUrl, + secureFetchWithPinnedIP: mocks.secureFetch, +})) + +vi.mock('@/lib/uploads/contexts/execution', () => ({ + uploadExecutionFile: mocks.uploadExecution, +})) + +vi.mock('@/lib/uploads/contexts/copilot', () => ({ + uploadCopilotFile: mocks.uploadCopilot, +})) + +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + getExtensionFromMimeType: () => 'jpg', +})) + +vi.mock('@/lib/internal/whatsapp/upload', () => ({ + uploadWhatsAppMedia: mocks.uploadMedia, +})) + +import { executeWhatsAppGetMedia } from '@/lib/internal/whatsapp/operations' + +const input = { accessToken: ' token ', mediaId: 'media-id', phoneNumberId: 'phone-id' } +const storedFile = { + key: 'workspace/file', + name: 'whatsapp-media-id.jpg', + size: 3, + type: 'image/jpeg', +} + +describe('WhatsApp media operations', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('{}'))) + mocks.readGraph.mockResolvedValue({ + url: 'https://cdn.example.com/media', + mime_type: 'image/jpeg', + file_size: '3', + sha256: 'hash', + id: 'media-id', + }) + mocks.validateUrl.mockResolvedValue({ isValid: true, resolvedIP: '203.0.113.10' }) + mocks.secureFetch.mockResolvedValue(new Response('abc')) + mocks.uploadExecution.mockResolvedValue(storedFile) + mocks.uploadCopilot.mockResolvedValue(storedFile) + }) + + it('stores downloads under the trusted execution scope, not serialized input', async () => { + const controller = new AbortController() + const response = await executeWhatsAppGetMedia(input, { + userId: 'user-1', + requestId: 'request-1', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + signal: controller.signal, + }) + + expect(response.status).toBe(200) + expect(mocks.uploadExecution).toHaveBeenCalledWith( + { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }, + Buffer.from('abc'), + 'whatsapp-media-id.jpg', + 'image/jpeg', + 'user-1' + ) + expect(mocks.uploadCopilot).not.toHaveBeenCalled() + expect(mocks.secureFetch).toHaveBeenCalledWith( + 'https://cdn.example.com/media', + '203.0.113.10', + expect.objectContaining({ + headers: { + Authorization: 'Bearer token', + 'User-Agent': 'SimWhatsAppMedia/1.0', + }, + maxResponseBytes: 100 * 1024 * 1024, + signal: controller.signal, + stripAuthOnRedirect: true, + }) + ) + }) + + it('rejects declared media over 100MB before contacting the CDN', async () => { + mocks.readGraph.mockResolvedValue({ + url: 'https://cdn.example.com/media', + mime_type: 'video/mp4', + file_size: 100 * 1024 * 1024 + 1, + id: 'media-id', + }) + + const response = await executeWhatsAppGetMedia(input, { + userId: 'user-1', + requestId: 'request-1', + }) + + expect(response.status).toBe(413) + expect(mocks.secureFetch).not.toHaveBeenCalled() + expect(mocks.uploadExecution).not.toHaveBeenCalled() + }) + + it('does no work when the execution is already canceled', async () => { + const controller = new AbortController() + controller.abort(new Error('execution canceled')) + + await expect( + executeWhatsAppGetMedia(input, { + userId: 'user-1', + requestId: 'request-1', + signal: controller.signal, + }) + ).rejects.toThrow('execution canceled') + expect(fetch).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/whatsapp/operations.ts b/apps/sim/lib/internal/whatsapp/operations.ts new file mode 100644 index 00000000000..1af0d6f6042 --- /dev/null +++ b/apps/sim/lib/internal/whatsapp/operations.ts @@ -0,0 +1,293 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { isRecordLike } from '@sim/utils/object' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { + isPayloadSizeLimitError, + readResponseToBufferWithLimit, +} from '@/lib/core/utils/stream-limits' +import { readWhatsAppGraphResponse } from '@/lib/internal/whatsapp/client' +import type { + WhatsAppGetMediaInput, + WhatsAppSendMediaInput, + WhatsAppUploadMediaInput, +} from '@/lib/internal/whatsapp/schema' +import { + whatsappGetMediaOutputSchema, + whatsappSendMediaOutputSchema, + whatsappUploadMediaOutputSchema, +} from '@/lib/internal/whatsapp/schema' +import { uploadWhatsAppMedia } from '@/lib/internal/whatsapp/upload' +import { uploadCopilotFile } from '@/lib/uploads/contexts/copilot' +import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' +import { getExtensionFromMimeType } from '@/lib/uploads/utils/file-utils' +import { sanitizeFileName } from '@/executor/constants' +import type { UserFile } from '@/executor/types' +import { + buildAuthHeaders, + buildMediaMessageBody, + buildMediaUrl, + buildMessagesUrl, + extractWhatsAppErrorMessage, + WHATSAPP_MEDIA_MAX_BYTES, +} from '@/tools/whatsapp/utils' + +const logger = createLogger('WhatsAppOperations') +const DOWNLOAD_USER_AGENT = 'SimWhatsAppMedia/1.0' + +export interface WhatsAppOperationContext { + userId: string + requestId: string + workspaceId?: string + workflowId?: string + executionId?: string + signal?: AbortSignal +} + +interface WhatsAppMediaMetadata { + url: string + mimeType: string + fileSize: number | null + sha256: string | null + id: string +} + +function failureResponse(error: string, status: number): Response { + return Response.json({ success: false, error }, { status }) +} + +function normalizeSendOutput(data: Record) { + const contacts = Array.isArray(data.contacts) + ? data.contacts.filter(isRecordLike).map((contact) => ({ + input: typeof contact.input === 'string' ? contact.input : '', + wa_id: typeof contact.wa_id === 'string' ? contact.wa_id : null, + })) + : [] + const firstMessage = + Array.isArray(data.messages) && isRecordLike(data.messages[0]) ? data.messages[0] : undefined + const messageId = typeof firstMessage?.id === 'string' ? firstMessage.id : undefined + if (!messageId) throw new Error('WhatsApp API response did not include a message ID') + + return { + success: true as const, + messageId, + ...(typeof firstMessage?.message_status === 'string' + ? { messageStatus: firstMessage.message_status } + : {}), + ...(typeof data.messaging_product === 'string' + ? { messagingProduct: data.messaging_product } + : {}), + inputPhoneNumber: contacts[0]?.input ?? null, + whatsappUserId: contacts[0]?.wa_id ?? null, + contacts, + } +} + +export async function executeWhatsAppUploadMedia( + input: WhatsAppUploadMediaInput, + context: WhatsAppOperationContext +): Promise { + context.signal?.throwIfAborted() + try { + const result = await uploadWhatsAppMedia({ ...input, ...context }) + if (!result.ok) { + return 'response' in result ? result.response : failureResponse(result.error, result.status) + } + return Response.json({ + success: true, + output: whatsappUploadMediaOutputSchema.parse(result.media), + }) + } catch (error) { + context.signal?.throwIfAborted() + return failureResponse( + getErrorMessage(error, 'Failed to upload media to WhatsApp'), + isPayloadSizeLimitError(error) ? 413 : 500 + ) + } +} + +export async function executeWhatsAppSendMedia( + input: WhatsAppSendMediaInput, + context: WhatsAppOperationContext +): Promise { + context.signal?.throwIfAborted() + const sourceCount = [input.file, input.mediaId, input.mediaLink].filter(Boolean).length + if (sourceCount === 0) return failureResponse('Provide a file, a media ID, or a media link', 400) + if (sourceCount > 1) { + return failureResponse('Provide only one of file, media ID, or media link', 400) + } + + try { + let uploadedMediaId: string | undefined + let filename = input.filename ?? undefined + if (input.file) { + const uploaded = await uploadWhatsAppMedia({ + file: input.file, + accessToken: input.accessToken, + phoneNumberId: input.phoneNumberId, + userId: context.userId, + requestId: context.requestId, + signal: context.signal, + }) + if (!uploaded.ok) { + return 'response' in uploaded + ? uploaded.response + : failureResponse(uploaded.error, uploaded.status) + } + uploadedMediaId = uploaded.media.mediaId + filename = filename ?? uploaded.media.fileName + } + + const messageBody = buildMediaMessageBody({ + phoneNumber: input.phoneNumber, + mediaType: input.mediaType, + mediaId: uploadedMediaId ?? input.mediaId ?? undefined, + mediaLink: input.mediaLink ?? undefined, + caption: input.caption ?? undefined, + filename, + }) + const response = await fetch(buildMessagesUrl(input.phoneNumberId), { + method: 'POST', + headers: buildAuthHeaders(input.accessToken), + body: JSON.stringify(messageBody), + signal: context.signal, + }) + const data = await readWhatsAppGraphResponse( + response, + 'WhatsApp media send response', + context.signal + ) + if (!response.ok) throw new Error(extractWhatsAppErrorMessage(data, response.status)) + + const output = whatsappSendMediaOutputSchema.parse({ + ...normalizeSendOutput(data), + ...(uploadedMediaId ? { mediaId: uploadedMediaId } : {}), + }) + return Response.json({ success: true, output }) + } catch (error) { + context.signal?.throwIfAborted() + return failureResponse( + getErrorMessage(error, 'Failed to send WhatsApp media'), + isPayloadSizeLimitError(error) ? 413 : 500 + ) + } +} + +export async function executeWhatsAppGetMedia( + input: WhatsAppGetMediaInput, + context: WhatsAppOperationContext +): Promise { + context.signal?.throwIfAborted() + const authorization = `Bearer ${input.accessToken.trim()}` + try { + const metadataResponse = await fetch(buildMediaUrl(input.mediaId, input.phoneNumberId), { + headers: { Authorization: authorization }, + signal: context.signal, + }) + const metadataBody = await readWhatsAppGraphResponse( + metadataResponse, + `WhatsApp media ${input.mediaId} metadata`, + context.signal + ) + if (!metadataResponse.ok) { + return failureResponse( + extractWhatsAppErrorMessage(metadataBody, metadataResponse.status), + metadataResponse.status >= 400 && metadataResponse.status < 500 + ? metadataResponse.status + : 502 + ) + } + + const url = typeof metadataBody.url === 'string' ? metadataBody.url : undefined + if (!url) return failureResponse('WhatsApp media metadata did not include a download URL', 502) + const parsedSize = Number(metadataBody.file_size) + const metadata: WhatsAppMediaMetadata = { + url, + mimeType: + typeof metadataBody.mime_type === 'string' && metadataBody.mime_type.length > 0 + ? metadataBody.mime_type + : 'application/octet-stream', + fileSize: Number.isFinite(parsedSize) ? parsedSize : null, + sha256: typeof metadataBody.sha256 === 'string' ? metadataBody.sha256 : null, + id: typeof metadataBody.id === 'string' ? metadataBody.id : input.mediaId, + } + if (metadata.fileSize !== null && metadata.fileSize > WHATSAPP_MEDIA_MAX_BYTES) { + return failureResponse( + `WhatsApp media is ${(metadata.fileSize / (1024 * 1024)).toFixed(2)} MB, which exceeds the 100 MB download limit`, + 413 + ) + } + + const urlValidation = await validateUrlWithDNS(metadata.url, 'mediaUrl') + if (!urlValidation.isValid) { + return failureResponse(`Invalid WhatsApp media URL: ${urlValidation.error}`, 502) + } + const mediaResponse = await secureFetchWithPinnedIP(metadata.url, urlValidation.resolvedIP!, { + method: 'GET', + headers: { Authorization: authorization, 'User-Agent': DOWNLOAD_USER_AGENT }, + maxResponseBytes: WHATSAPP_MEDIA_MAX_BYTES, + stripAuthOnRedirect: true, + signal: context.signal, + }) + if (!mediaResponse.ok) { + return failureResponse( + mediaResponse.status === 404 + ? 'WhatsApp media not found or its download URL expired (URLs are valid for 5 minutes)' + : `Failed to download WhatsApp media (${mediaResponse.status})`, + mediaResponse.status >= 400 && mediaResponse.status < 500 ? mediaResponse.status : 502 + ) + } + const buffer = await readResponseToBufferWithLimit(mediaResponse, { + maxBytes: WHATSAPP_MEDIA_MAX_BYTES, + label: 'WhatsApp media download', + signal: context.signal, + }) + const extension = getExtensionFromMimeType(metadata.mimeType) ?? 'bin' + const fileName = sanitizeFileName(`whatsapp-${metadata.id}.${extension}`) + const executionScope = + context.workspaceId && context.workflowId && context.executionId + ? { + workspaceId: context.workspaceId, + workflowId: context.workflowId, + executionId: context.executionId, + } + : undefined + const file: UserFile = executionScope + ? await uploadExecutionFile( + executionScope, + buffer, + fileName, + metadata.mimeType, + context.userId + ) + : await uploadCopilotFile({ + buffer, + fileName, + contentType: metadata.mimeType, + userId: context.userId, + }) + context.signal?.throwIfAborted() + + const output = whatsappGetMediaOutputSchema.parse({ + file, + mediaId: metadata.id, + mimeType: metadata.mimeType, + fileSize: buffer.length, + sha256: metadata.sha256, + }) + return Response.json({ success: true, output }) + } catch (error) { + context.signal?.throwIfAborted() + if (isPayloadSizeLimitError(error)) { + return failureResponse('WhatsApp media exceeds the 100 MB download limit', 413) + } + logger.error('WhatsApp media download failed', { + error: getErrorMessage(error), + requestId: context.requestId, + }) + return failureResponse(getErrorMessage(error, 'Failed to download WhatsApp media'), 500) + } +} diff --git a/apps/sim/lib/internal/whatsapp/schema.ts b/apps/sim/lib/internal/whatsapp/schema.ts new file mode 100644 index 00000000000..03898eb057e --- /dev/null +++ b/apps/sim/lib/internal/whatsapp/schema.ts @@ -0,0 +1,77 @@ +import { z } from 'zod' +import { userFileSchema } from '@/lib/api/contracts/primitives' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' + +const MAX_ACCESS_TOKEN_LENGTH = 8192 +const MAX_GRAPH_ID_LENGTH = 256 + +const accessTokenSchema = z + .string() + .min(1, 'Access token is required') + .max(MAX_ACCESS_TOKEN_LENGTH, 'Access token is too long') + +const phoneNumberIdSchema = z + .string() + .trim() + .min(1, 'Phone Number ID is required') + .max(MAX_GRAPH_ID_LENGTH, 'Phone Number ID is too long') + +const mediaIdSchema = z + .string() + .trim() + .min(1, 'Media ID is required') + .max(MAX_GRAPH_ID_LENGTH, 'Media ID is too long') + +export const whatsappUploadMediaInputSchema = z.object({ + accessToken: accessTokenSchema, + phoneNumberId: phoneNumberIdSchema, + file: RawFileInputSchema, +}) + +export const whatsappUploadMediaOutputSchema = z.object({ + mediaId: z.string().min(1).max(MAX_GRAPH_ID_LENGTH), + fileName: z.string().min(1), + mimeType: z.string().min(1), + size: z.number().int().nonnegative(), +}) + +export const whatsappSendMediaInputSchema = z.object({ + accessToken: accessTokenSchema, + phoneNumberId: phoneNumberIdSchema, + phoneNumber: z.string().trim().min(1, 'Recipient phone number is required').max(64), + mediaType: z.enum(['image', 'document', 'video', 'audio', 'sticker']), + file: RawFileInputSchema.optional().nullable(), + mediaId: mediaIdSchema.optional().nullable(), + mediaLink: z.string().trim().max(8192).optional().nullable(), + caption: z.string().max(1024, 'Caption cannot exceed 1024 characters').optional().nullable(), + filename: z.string().max(1024).optional().nullable(), +}) + +export const whatsappSendMediaOutputSchema = z.object({ + success: z.literal(true), + messageId: z.string().min(1), + messageStatus: z.string().optional(), + messagingProduct: z.string().optional(), + inputPhoneNumber: z.string().nullable(), + whatsappUserId: z.string().nullable(), + contacts: z.array(z.object({ input: z.string(), wa_id: z.string().nullable() })), + mediaId: z.string().optional(), +}) + +export const whatsappGetMediaInputSchema = z.object({ + accessToken: accessTokenSchema, + mediaId: mediaIdSchema, + phoneNumberId: phoneNumberIdSchema.optional(), +}) + +export const whatsappGetMediaOutputSchema = z.object({ + file: userFileSchema, + mediaId: z.string().min(1).max(MAX_GRAPH_ID_LENGTH), + mimeType: z.string().min(1), + fileSize: z.number().int().nonnegative(), + sha256: z.string().nullable(), +}) + +export type WhatsAppUploadMediaInput = z.output +export type WhatsAppSendMediaInput = z.output +export type WhatsAppGetMediaInput = z.output diff --git a/apps/sim/lib/internal/whatsapp/upload.test.ts b/apps/sim/lib/internal/whatsapp/upload.test.ts new file mode 100644 index 00000000000..ed23313574b --- /dev/null +++ b/apps/sim/lib/internal/whatsapp/upload.test.ts @@ -0,0 +1,122 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + processFile: vi.fn(), + assertAccess: vi.fn(), + downloadFile: vi.fn(), + docNotReady: vi.fn(), + readGraph: vi.fn(), +})) + +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + processSingleFileToUserFile: mocks.processFile, +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertAccess, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mocks.downloadFile, +})) + +vi.mock('@/lib/uploads/utils/servable-file-response', () => ({ + docNotReadyResponse: mocks.docNotReady, +})) + +vi.mock('@/lib/internal/whatsapp/client', () => ({ + readWhatsAppGraphResponse: mocks.readGraph, +})) + +import { uploadWhatsAppMedia } from '@/lib/internal/whatsapp/upload' + +const file = { key: 'workspace/file', name: 'image.png', size: 4, type: 'image/png' } + +describe('WhatsApp media upload', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.processFile.mockReturnValue(file) + mocks.assertAccess.mockResolvedValue(null) + mocks.downloadFile.mockResolvedValue({ buffer: Buffer.from('data'), contentType: 'image/png' }) + mocks.readGraph.mockResolvedValue({ id: 'media-id' }) + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('{}'))) + }) + + it('authorizes a Sim file before reading or sending it', async () => { + const denied = Response.json({ success: false, error: 'File not found' }, { status: 404 }) + mocks.assertAccess.mockResolvedValue(denied) + + const result = await uploadWhatsAppMedia({ + file, + accessToken: 'token', + phoneNumberId: 'phone-id', + userId: 'user-1', + requestId: 'request-1', + }) + + expect(result).toEqual({ ok: false, response: denied }) + expect(mocks.assertAccess).toHaveBeenCalledWith( + 'workspace/file', + 'user-1', + 'request-1', + expect.anything() + ) + expect(mocks.downloadFile).not.toHaveBeenCalled() + expect(fetch).not.toHaveBeenCalled() + }) + + it('enforces the media-type cap before downloading declared oversized files', async () => { + mocks.processFile.mockReturnValue({ ...file, size: 5 * 1024 * 1024 + 1 }) + + const result = await uploadWhatsAppMedia({ + file, + accessToken: 'token', + phoneNumberId: 'phone-id', + userId: 'user-1', + requestId: 'request-1', + }) + + expect(result).toMatchObject({ ok: false, status: 413 }) + expect(mocks.downloadFile).not.toHaveBeenCalled() + }) + + it('forwards cancellation to storage and Meta transfers', async () => { + const controller = new AbortController() + + const result = await uploadWhatsAppMedia({ + file, + accessToken: ' token ', + phoneNumberId: 'phone-id', + userId: 'user-1', + requestId: 'request-1', + signal: controller.signal, + }) + + expect(result).toEqual({ + ok: true, + media: { mediaId: 'media-id', fileName: 'image.png', mimeType: 'image/png', size: 4 }, + }) + expect(mocks.downloadFile).toHaveBeenCalledWith( + file, + 'request-1', + expect.anything(), + expect.objectContaining({ maxBytes: 5 * 1024 * 1024, signal: controller.signal }) + ) + expect(fetch).toHaveBeenCalledWith( + expect.stringContaining('/phone-id/media'), + expect.objectContaining({ + method: 'POST', + headers: { Authorization: 'Bearer token' }, + signal: controller.signal, + }) + ) + expect(mocks.readGraph).toHaveBeenCalledWith( + expect.any(Response), + 'WhatsApp media upload response', + controller.signal + ) + }) +}) diff --git a/apps/sim/lib/internal/whatsapp/upload.ts b/apps/sim/lib/internal/whatsapp/upload.ts new file mode 100644 index 00000000000..8aa0d6452bb --- /dev/null +++ b/apps/sim/lib/internal/whatsapp/upload.ts @@ -0,0 +1,120 @@ +import { createLogger } from '@sim/logger' +import { readWhatsAppGraphResponse } from '@/lib/internal/whatsapp/client' +import type { RawFileInput } from '@/lib/uploads/utils/file-utils' +import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' +import { assertToolFileAccess } from '@/app/api/files/authorization' +import { + buildMediaUploadUrl, + extractWhatsAppErrorMessage, + whatsappMediaLimitFor, +} from '@/tools/whatsapp/utils' + +const logger = createLogger('WhatsAppMediaUpload') + +export interface UploadedWhatsAppMedia { + mediaId: string + fileName: string + mimeType: string + size: number +} + +export type UploadWhatsAppMediaResult = + | { ok: true; media: UploadedWhatsAppMedia } + | { ok: false; error: string; status: number } + | { ok: false; response: Response } + +export async function uploadWhatsAppMedia({ + file, + accessToken, + phoneNumberId, + userId, + requestId, + signal, +}: { + file: RawFileInput + accessToken: string + phoneNumberId: string + userId: string + requestId: string + signal?: AbortSignal +}): Promise { + signal?.throwIfAborted() + const userFile = processSingleFileToUserFile(file, requestId, logger) + if (!userFile) return { ok: false, error: 'No valid file provided for upload', status: 400 } + + const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) + if (denied) return { ok: false, response: denied } + signal?.throwIfAborted() + + const declaredMimeType = userFile.type || 'application/octet-stream' + const declaredLimit = whatsappMediaLimitFor(declaredMimeType) + if (userFile.size > declaredLimit.maxBytes) { + return { + ok: false, + error: `${userFile.name} is ${(userFile.size / (1024 * 1024)).toFixed(2)} MB, which exceeds WhatsApp's limit for ${declaredLimit.label}`, + status: 413, + } + } + + let buffer: Buffer + let contentType: string + try { + const downloaded = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: declaredLimit.maxBytes, + signal, + }) + buffer = downloaded.buffer + contentType = downloaded.contentType + } catch (error) { + signal?.throwIfAborted() + const notReady = docNotReadyResponse(error) + if (notReady) return { ok: false, response: notReady } + throw error + } + + const resolvedMimeType = contentType || declaredMimeType + const resolvedLimit = whatsappMediaLimitFor(resolvedMimeType) + if (buffer.length > resolvedLimit.maxBytes) { + return { + ok: false, + error: `${userFile.name} is ${(buffer.length / (1024 * 1024)).toFixed(2)} MB, which exceeds WhatsApp's limit for ${resolvedLimit.label}`, + status: 413, + } + } + + const formData = new FormData() + formData.append('messaging_product', 'whatsapp') + formData.append('type', resolvedMimeType) + formData.append( + 'file', + new Blob([new Uint8Array(buffer)], { type: resolvedMimeType }), + userFile.name + ) + + const response = await fetch(buildMediaUploadUrl(phoneNumberId), { + method: 'POST', + headers: { Authorization: `Bearer ${accessToken.trim()}` }, + body: formData, + signal, + }) + const data = await readWhatsAppGraphResponse(response, 'WhatsApp media upload response', signal) + + if (!response.ok) { + return { + ok: false, + error: extractWhatsAppErrorMessage(data, response.status), + status: response.status >= 400 && response.status < 500 ? response.status : 502, + } + } + const mediaId = typeof data.id === 'string' ? data.id : undefined + if (!mediaId) { + return { ok: false, error: 'WhatsApp upload response did not include a media ID', status: 502 } + } + + return { + ok: true, + media: { mediaId, fileName: userFile.name, mimeType: resolvedMimeType, size: buffer.length }, + } +} diff --git a/apps/sim/lib/internal/windchill/client.ts b/apps/sim/lib/internal/windchill/client.ts new file mode 100644 index 00000000000..dc5f43bf60b --- /dev/null +++ b/apps/sim/lib/internal/windchill/client.ts @@ -0,0 +1,453 @@ +import { generateShortId } from '@sim/utils/id' +import { isRecordLike } from '@sim/utils/object' +import { + MAX_JSON_API_RESPONSE_BYTES, + type SecureFetchResponse, + secureFetchWithValidation, +} from '@/lib/core/security/input-validation.server' +import { + createBasicAuthHeader, + encodeWindchillOid, + normalizeServiceRoot, + sanitizeWindchillError, +} from '@/tools/windchill/utils' + +const WINDCHILL_CONTROL_RESPONSE_BYTES = 2 * 1024 * 1024 +const WINDCHILL_TIMEOUT_MS = 60_000 + +interface WindchillSession { + nonceHeader: string + nonceValue: string + cookie: string | null +} + +interface WindchillCredentials { + baseUrl: string + username: string + password: string +} + +export interface WindchillUploadFile { + name: string + mimeType: string + size: number + buffer: Buffer +} + +export class WindchillProviderError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'WindchillProviderError' + } +} + +function cookieHeader(response: SecureFetchResponse): string | null { + const cookies = response.headers + .getSetCookie() + .map((cookie) => cookie.split(';', 1)[0]?.trim()) + .filter((cookie): cookie is string => Boolean(cookie)) + return cookies.length > 0 ? cookies.join('; ') : null +} + +async function responseBody( + response: SecureFetchResponse +): Promise<{ data: unknown; invalidJson: boolean }> { + const text = await response.text() + if (!text.trim()) return { data: null, invalidJson: false } + try { + return { data: JSON.parse(text) as unknown, invalidJson: false } + } catch { + return { data: text, invalidJson: true } + } +} + +function providerMessage(data: unknown, response: SecureFetchResponse): string { + if (typeof data === 'string' && data.trim()) return data.trim() + if (isRecordLike(data)) { + if (typeof data.message === 'string' && data.message.trim()) return data.message.trim() + if (isRecordLike(data.error)) { + if (typeof data.error.message === 'string' && data.error.message.trim()) { + return data.error.message.trim() + } + if ( + isRecordLike(data.error.message) && + typeof data.error.message.value === 'string' && + data.error.message.value.trim() + ) { + return data.error.message.value.trim() + } + } + } + return `Windchill request failed with status ${response.status}` +} + +async function checkedBody(response: SecureFetchResponse): Promise { + const { data, invalidJson } = await responseBody(response) + if (!response.ok) { + throw new WindchillProviderError( + sanitizeWindchillError(providerMessage(data, response)), + response.status + ) + } + if (invalidJson) { + throw new WindchillProviderError( + `Windchill returned invalid JSON with status ${response.status}`, + 502 + ) + } + return data +} + +function ptcRoot(baseUrl: string): string { + return normalizeServiceRoot(baseUrl).replace(/\/v\d+$/i, '') +} + +export function windchillDocumentUrl(baseUrl: string, documentOid: string): string { + return `${normalizeServiceRoot(baseUrl)}/DocMgmt/Documents('${encodeWindchillOid(documentOid)}')` +} + +export async function createWindchillSession( + params: WindchillCredentials, + signal?: AbortSignal +): Promise { + const response = await secureFetchWithValidation( + `${ptcRoot(params.baseUrl)}/PTC/GetCSRFToken()`, + { + method: 'GET', + headers: { + Authorization: createBasicAuthHeader(params.username, params.password), + Accept: 'application/json', + }, + maxRedirects: 0, + maxResponseBytes: WINDCHILL_CONTROL_RESPONSE_BYTES, + timeout: WINDCHILL_TIMEOUT_MS, + signal, + }, + 'baseUrl' + ) + const data = await checkedBody(response) + if (!isRecordLike(data)) { + throw new WindchillProviderError('Windchill returned an invalid CSRF response', 502) + } + const nonceHeader = data.NonceKey + const nonceValue = data.NonceValue + if ( + typeof nonceHeader !== 'string' || + !/^[A-Za-z0-9_-]{1,128}$/.test(nonceHeader) || + typeof nonceValue !== 'string' || + nonceValue.length === 0 || + nonceValue.length > 8192 + ) { + throw new WindchillProviderError('Windchill returned an invalid CSRF token', 502) + } + return { nonceHeader, nonceValue, cookie: cookieHeader(response) } +} + +export async function windchillMutationRequest({ + params, + session, + url, + method, + body, + signal, +}: { + params: Pick + session: WindchillSession + url: string + method: 'POST' | 'PUT' | 'PATCH' | 'DELETE' + body?: unknown + signal?: AbortSignal +}): Promise { + const headers: Record = { + Authorization: createBasicAuthHeader(params.username, params.password), + Accept: 'application/json', + [session.nonceHeader]: session.nonceValue, + } + if (body !== undefined) headers['Content-Type'] = 'application/json' + if (session.cookie) headers.Cookie = session.cookie + + const response = await secureFetchWithValidation( + url, + { + method, + headers, + body: body === undefined ? undefined : JSON.stringify(body), + maxRedirects: 0, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + timeout: WINDCHILL_TIMEOUT_MS, + signal, + }, + 'baseUrl' + ) + return checkedBody(response) +} + +function stageOneDescriptor(data: unknown): { + replicaUrl: string + masterUrl: string + streamIds: Array + fileNames: Array +} { + const descriptor = isRecordLike(data) && Array.isArray(data.value) ? data.value[0] : null + if (!isRecordLike(descriptor)) { + throw new WindchillProviderError('Windchill upload Stage 1 returned no cache descriptor', 502) + } + if ( + typeof descriptor.ReplicaUrl !== 'string' || + typeof descriptor.MasterUrl !== 'string' || + !Array.isArray(descriptor.StreamIds) || + !Array.isArray(descriptor.FileNames) + ) { + throw new WindchillProviderError( + 'Windchill upload Stage 1 returned an invalid cache descriptor', + 502 + ) + } + if (descriptor.StreamIds.length !== descriptor.FileNames.length) { + throw new WindchillProviderError( + 'Windchill upload Stage 1 returned mismatched file identifiers', + 502 + ) + } + return { + replicaUrl: descriptor.ReplicaUrl, + masterUrl: descriptor.MasterUrl, + streamIds: descriptor.StreamIds.filter( + (value): value is string | number => typeof value === 'string' || typeof value === 'number' + ), + fileNames: descriptor.FileNames.filter( + (value): value is string | number => typeof value === 'string' || typeof value === 'number' + ), + } +} + +function multipartBody( + descriptor: ReturnType, + files: WindchillUploadFile[] +): { contentType: string; body: Buffer } { + if ( + descriptor.streamIds.length !== files.length || + descriptor.fileNames.length !== files.length + ) { + throw new WindchillProviderError('Windchill upload Stage 1 returned the wrong file count', 502) + } + const boundary = `sim-windchill-${generateShortId()}` + const chunks: Buffer[] = [] + const appendField = (name: string, value: string) => { + chunks.push( + Buffer.from( + `--${boundary}\r\nContent-Disposition: form-data; name="${name}"\r\n\r\n${value}\r\n` + ) + ) + } + appendField('Master_URL', descriptor.masterUrl) + appendField( + 'CacheDescriptor_array', + descriptor.streamIds + .map( + (streamId, index) => + `${streamId}:${descriptor.fileNames[index]}:${streamId}:${files[index].size};` + ) + .join(' ') + ) + files.forEach((file, index) => { + chunks.push( + Buffer.from( + `--${boundary}\r\nContent-Disposition: form-data; name="${descriptor.streamIds[index]}"; filename="${file.name.replace(/["\r\n]/g, '_')}"\r\nContent-Type: ${file.mimeType}\r\n\r\n` + ), + file.buffer, + Buffer.from('\r\n') + ) + }) + chunks.push(Buffer.from(`--${boundary}--\r\n`)) + return { contentType: `multipart/form-data; boundary=${boundary}`, body: Buffer.concat(chunks) } +} + +function stageTwoContent(data: unknown): Array<{ + streamId: string | number + fileSize: number + encodedInfo: string +}> { + if (!isRecordLike(data) || !Array.isArray(data.contentInfos)) { + throw new WindchillProviderError('Windchill upload Stage 2 returned invalid content info', 502) + } + return data.contentInfos.map((item) => { + if ( + !isRecordLike(item) || + (typeof item.streamId !== 'string' && typeof item.streamId !== 'number') || + typeof item.fileSize !== 'number' || + typeof item.encodedInfo !== 'string' + ) { + throw new WindchillProviderError( + 'Windchill upload Stage 2 returned invalid content info', + 502 + ) + } + return { streamId: item.streamId, fileSize: item.fileSize, encodedInfo: item.encodedInfo } + }) +} + +export async function uploadWindchillContent({ + params, + documentOid, + files, + primaryContent, + signal, +}: { + params: WindchillCredentials + documentOid: string + files: WindchillUploadFile[] + primaryContent: boolean + signal?: AbortSignal +}): Promise { + const session = await createWindchillSession(params, signal) + const documentUrl = windchillDocumentUrl(params.baseUrl, documentOid) + const stageOne = await windchillMutationRequest({ + params, + session, + url: `${documentUrl}/PTC.DocMgmt.UploadStage1Action`, + method: 'POST', + body: { NoOfFiles: files.length }, + signal, + }) + const descriptor = stageOneDescriptor(stageOne) + const multipart = multipartBody(descriptor, files) + const stageTwoResponse = await secureFetchWithValidation( + descriptor.replicaUrl, + { + method: 'POST', + headers: { 'Content-Type': multipart.contentType, Accept: 'application/json' }, + body: multipart.body, + maxRedirects: 0, + maxResponseBytes: WINDCHILL_CONTROL_RESPONSE_BYTES, + timeout: WINDCHILL_TIMEOUT_MS, + signal, + }, + 'ReplicaUrl' + ) + const uploaded = stageTwoContent(await checkedBody(stageTwoResponse)) + if (uploaded.length !== files.length) { + throw new WindchillProviderError('Windchill upload Stage 2 returned the wrong file count', 502) + } + const byStreamId = new Map(uploaded.map((item) => [String(item.streamId), item])) + const contentInfo = descriptor.streamIds.map((streamId, index) => { + const uploadedItem = byStreamId.get(String(streamId)) + if (!uploadedItem) { + throw new WindchillProviderError('Windchill upload Stage 2 omitted a file', 502) + } + return { + StreamId: streamId, + EncodedInfo: uploadedItem.encodedInfo, + FileName: files[index].name, + PrimaryContent: primaryContent, + MimeType: files[index].mimeType, + FileSize: uploadedItem.fileSize, + } + }) + await windchillMutationRequest({ + params, + session, + url: `${documentUrl}/PTC.DocMgmt.UploadStage3Action`, + method: 'POST', + body: { ContentInfo: contentInfo }, + signal, + }) + return files.map((file) => file.name) +} + +/** + * Resolves the signed vault URL that serves a content item's bytes. + * + * Windchill has no OData media-stream segment for content. The documented path is a typed + * navigation to `Content/URL`, which returns a short-lived signed WindchillGW/WindchillAuthGW + * download URL on the same origin as the service root. + */ +export async function resolveWindchillContentUrl({ + params, + contentPath, + signal, +}: { + params: WindchillCredentials + contentPath: string + signal?: AbortSignal +}): Promise { + const response = await secureFetchWithValidation( + `${contentPath}/PTC.ApplicationData/Content/URL`, + { + method: 'GET', + headers: { + Authorization: createBasicAuthHeader(params.username, params.password), + Accept: 'application/json', + }, + maxRedirects: 0, + maxResponseBytes: WINDCHILL_CONTROL_RESPONSE_BYTES, + timeout: WINDCHILL_TIMEOUT_MS, + signal, + }, + 'baseUrl' + ) + const data = await checkedBody(response) + const value = isRecordLike(data) ? data.value : null + if (typeof value !== 'string' || value.length === 0) { + throw new WindchillProviderError('Windchill did not return a content download URL', 502) + } + + const serviceRoot = new URL(normalizeServiceRoot(params.baseUrl)) + let resolved: URL + try { + resolved = new URL(value, `${serviceRoot.toString()}/`) + } catch { + throw new WindchillProviderError('Windchill returned an invalid content download URL', 502) + } + if ( + resolved.protocol !== 'https:' || + resolved.origin !== serviceRoot.origin || + resolved.username || + resolved.password || + resolved.hash + ) { + throw new WindchillProviderError( + 'Windchill content download URL must remain on the configured HTTPS origin', + 502 + ) + } + return resolved.toString() +} + +export async function downloadWindchillContent({ + params, + url, + maxBytes, + signal, +}: { + params: Pick + url: string + maxBytes: number + signal?: AbortSignal +}): Promise<{ + buffer: Buffer + contentType: string + contentDisposition: string | null +}> { + const response = await secureFetchWithValidation( + url, + { + method: 'GET', + headers: { Authorization: createBasicAuthHeader(params.username, params.password) }, + stripAuthOnRedirect: true, + maxResponseBytes: maxBytes, + timeout: WINDCHILL_TIMEOUT_MS, + signal, + }, + 'contentUrl' + ) + if (!response.ok) await checkedBody(response) + return { + buffer: Buffer.from(await response.arrayBuffer()), + contentType: response.headers.get('content-type') || 'application/octet-stream', + contentDisposition: response.headers.get('content-disposition'), + } +} diff --git a/apps/sim/lib/internal/windchill/errors.ts b/apps/sim/lib/internal/windchill/errors.ts new file mode 100644 index 00000000000..f4350b15b33 --- /dev/null +++ b/apps/sim/lib/internal/windchill/errors.ts @@ -0,0 +1,9 @@ +export class WindchillOperationError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'WindchillOperationError' + } +} diff --git a/apps/sim/lib/internal/windchill/execute-tool.test.ts b/apps/sim/lib/internal/windchill/execute-tool.test.ts new file mode 100644 index 00000000000..2f1e3780de6 --- /dev/null +++ b/apps/sim/lib/internal/windchill/execute-tool.test.ts @@ -0,0 +1,197 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + createExecutorPrincipalFromExecutionContext: vi.fn(), + executeWindchillOperation: vi.fn(), +})) + +vi.mock('@/lib/internal/principals/executor', () => ({ + createExecutorPrincipalFromExecutionContext: mocks.createExecutorPrincipalFromExecutionContext, +})) + +vi.mock('@/lib/internal/windchill/operations', () => ({ + executeWindchillOperation: mocks.executeWindchillOperation, +})) + +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' +import { WindchillProviderError } from '@/lib/internal/windchill/client' +import { executeWindchillTool } from '@/lib/internal/windchill/execute-tool' + +const BASE = { + baseUrl: 'https://windchill.example.com/Windchill/servlet/odata/v6', + username: 'windchill-user', + password: 'not-a-real-password', +} +const DOCUMENT_OID = 'OR:wt.doc.WTDocument:1' +const ATTACHMENT_OID = 'OR:wt.content.ApplicationData:1' +const FILE = { key: 'uploads/specification.pdf', name: 'specification.pdf', size: 3 } +const PRINCIPAL = { + kind: 'delegated' as const, + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'workspace-1', + delegationId: 'delegation-1', + audience: 'sim:windchill', + issuedAt: new Date('2026-01-01T00:00:00.000Z'), + expiresAt: new Date('2026-01-01T01:00:00.000Z'), + delegationContext: { + kind: 'workflow_execution' as const, + workflowId: 'workflow-1', + executionId: 'execution-1', + }, +} + +const TOOL_CASES = [ + ['windchill_create_document', { name: 'Specification', containerOid: DOCUMENT_OID }], + [ + 'windchill_create_documents', + { documents: [{ name: 'Specification', containerOid: DOCUMENT_OID }] }, + ], + ['windchill_update_document', { documentOid: DOCUMENT_OID, attributes: { Custom: 'value' } }], + [ + 'windchill_update_common_properties', + { documentOid: DOCUMENT_OID, commonProperties: { Name: 'Renamed' } }, + ], + [ + 'windchill_update_documents', + { documents: [{ id: DOCUMENT_OID, attributes: { Custom: 'value' } }] }, + ], + ['windchill_delete_document', { documentOid: DOCUMENT_OID }], + ['windchill_delete_documents', { documentOids: [DOCUMENT_OID] }], + ['windchill_check_out_document', { documentOid: DOCUMENT_OID }], + ['windchill_check_out_documents', { documentOids: [DOCUMENT_OID] }], + ['windchill_check_in_document', { documentOid: DOCUMENT_OID }], + ['windchill_check_in_documents', { documentOids: [DOCUMENT_OID] }], + ['windchill_undo_check_out_document', { documentOid: DOCUMENT_OID }], + ['windchill_undo_check_out_documents', { documentOids: [DOCUMENT_OID] }], + ['windchill_revise_document', { documentOid: DOCUMENT_OID }], + ['windchill_revise_documents', { documentOids: [DOCUMENT_OID] }], + [ + 'windchill_set_lifecycle_state', + { documentOid: DOCUMENT_OID, stateValue: 'RELEASED', stateDisplay: 'Released' }, + ], + [ + 'windchill_update_document_security_labels', + { securityLabelUpdates: [{ id: DOCUMENT_OID, labels: { EXPORT_CONTROL: 'EAR99' } }] }, + ], + ['windchill_download_primary_content', { documentOid: DOCUMENT_OID }], + ['windchill_upload_primary_content', { documentOid: DOCUMENT_OID, primaryFile: FILE }], + ['windchill_download_attachment', { documentOid: DOCUMENT_OID, attachmentOid: ATTACHMENT_OID }], + ['windchill_upload_attachments', { documentOid: DOCUMENT_OID, attachmentFiles: [FILE] }], +] as const + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + const defaultBody = { ...BASE, operation: 'windchill_delete_document', documentOid: DOCUMENT_OID } + return { + toolId: 'windchill_delete_document', + input: defaultBody, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + executionId: 'execution-1', + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeWindchillTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.createExecutorPrincipalFromExecutionContext.mockResolvedValue(PRINCIPAL) + }) + + it.each(TOOL_CASES)('authorizes, validates, and dispatches %s', async (toolId, input) => { + const controller = new AbortController() + const operationInput = { ...BASE, operation: toolId, ...input } + const output = { operation: toolId, affectedIds: [DOCUMENT_OID] } + mocks.executeWindchillOperation.mockResolvedValue(output) + + const response = await executeWindchillTool( + createRequest({ toolId, input: operationInput, signal: controller.signal }) + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ success: true, output }) + expect(mocks.createExecutorPrincipalFromExecutionContext).toHaveBeenCalledWith({ + context: expect.objectContaining({ + workflowId: 'workflow-1', + executionId: 'execution-1', + userId: 'user-1', + }), + audience: 'sim:windchill', + }) + expect(mocks.executeWindchillOperation).toHaveBeenCalledWith(operationInput, { + principal: PRINCIPAL, + requestId: 'request-1', + signal: controller.signal, + }) + }) + + it('authenticates before validating non-object input', async () => { + const response = await executeWindchillTool(createRequest({ input: '{' })) + + expect(mocks.createExecutorPrincipalFromExecutionContext).toHaveBeenCalledOnce() + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Invalid input: expected object, received string', + }) + expect(mocks.executeWindchillOperation).not.toHaveBeenCalled() + }) + + it('rejects a prepared body for a different operation before provider work', async () => { + const response = await executeWindchillTool( + createRequest({ + toolId: 'windchill_delete_document', + input: { + ...BASE, + operation: 'windchill_revise_document', + documentOid: DOCUMENT_OID, + }, + }) + ) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Windchill request operation does not match the selected tool', + }) + expect(mocks.executeWindchillOperation).not.toHaveBeenCalled() + }) + + it('preserves provider status and sanitizes sensitive error material', async () => { + mocks.executeWindchillOperation.mockRejectedValue( + new WindchillProviderError( + 'Request https://windchill.example.com/file?token=secret with Basic dXNlcjpwYXNz failed', + 409 + ) + ) + + const response = await executeWindchillTool(createRequest()) + + expect(response.status).toBe(409) + const result = await response.json() + expect(result.success).toBe(false) + expect(result.error).not.toContain('token=secret') + expect(result.error).not.toContain('dXNlcjpwYXNz') + }) + + it('propagates cancellation before authorization', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeWindchillTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(mocks.createExecutorPrincipalFromExecutionContext).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/windchill/execute-tool.ts b/apps/sim/lib/internal/windchill/execute-tool.ts new file mode 100644 index 00000000000..689e140e6ad --- /dev/null +++ b/apps/sim/lib/internal/windchill/execute-tool.ts @@ -0,0 +1,115 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { + WindchillOperationBody, + WindchillOperationResponse, +} from '@/lib/api/contracts/tools/windchill' +import { windchillOperationBodySchema } from '@/lib/api/contracts/tools/windchill' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import { InvalidInternalDelegationBindingError } from '@/lib/auth/internal-delegation' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { createExecutorPrincipalFromExecutionContext } from '@/lib/internal/principals/executor' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { WindchillProviderError } from '@/lib/internal/windchill/client' +import { WindchillOperationError } from '@/lib/internal/windchill/errors' +import { executeWindchillOperation } from '@/lib/internal/windchill/operations' +import { sanitizeWindchillError } from '@/tools/windchill/utils' + +const logger = createLogger('WindchillInternalOperation') +const WINDCHILL_DELEGATION_AUDIENCE = 'sim:windchill' + +const WINDCHILL_INTERNAL_TOOL_IDS = new Set([ + 'windchill_create_document', + 'windchill_create_documents', + 'windchill_update_document', + 'windchill_update_common_properties', + 'windchill_update_documents', + 'windchill_delete_document', + 'windchill_delete_documents', + 'windchill_check_out_document', + 'windchill_check_out_documents', + 'windchill_check_in_document', + 'windchill_check_in_documents', + 'windchill_undo_check_out_document', + 'windchill_undo_check_out_documents', + 'windchill_revise_document', + 'windchill_revise_documents', + 'windchill_set_lifecycle_state', + 'windchill_update_document_security_labels', + 'windchill_download_primary_content', + 'windchill_upload_primary_content', + 'windchill_download_attachment', + 'windchill_upload_attachments', +]) + +function failureResponse(error: string, status: number): Response { + const body = { + success: false, + error: sanitizeWindchillError(error), + } satisfies WindchillOperationResponse + return Response.json(body, { status }) +} + +function parseInput(input: unknown): WindchillOperationBody | Response { + if (Buffer.byteLength(JSON.stringify(input) ?? '') > DEFAULT_MAX_JSON_BODY_BYTES) { + return failureResponse('Windchill request body is too large', 413) + } + + const parsed = windchillOperationBodySchema.safeParse(input) + if (!parsed.success) { + return failureResponse( + getValidationErrorMessage(parsed.error, 'Invalid Windchill request'), + 400 + ) + } + return parsed.data +} + +export const executeWindchillTool: InternalToolOperationHandler = async (request) => { + const { context, requestId, signal, toolId } = request + signal?.throwIfAborted() + if (!WINDCHILL_INTERNAL_TOOL_IDS.has(toolId)) { + return failureResponse(`Unsupported Windchill tool: ${toolId}`, 500) + } + + try { + const principal = await createExecutorPrincipalFromExecutionContext({ + context, + audience: WINDCHILL_DELEGATION_AUDIENCE, + }) + signal?.throwIfAborted() + const input = parseInput(request.input) + if (input instanceof Response) return input + if (input.operation !== toolId) { + return failureResponse('Windchill request operation does not match the selected tool', 400) + } + + const output = await executeWindchillOperation(input, { principal, requestId, signal }) + signal?.throwIfAborted() + return Response.json({ success: true, output } satisfies WindchillOperationResponse) + } catch (error) { + signal?.throwIfAborted() + if ( + error instanceof InvalidInternalDelegationBindingError || + (error instanceof Error && error.message === 'Authentication required') + ) { + return failureResponse('Authentication required', 401) + } + logger.error(`[${requestId}] Windchill operation failed`, { + operation: toolId, + error: sanitizeWindchillError(getErrorMessage(error, 'Windchill operation failed')), + }) + if (error instanceof WindchillOperationError) { + return failureResponse(error.message, error.status) + } + if (error instanceof WindchillProviderError) { + const status = error.status >= 400 && error.status <= 599 ? error.status : 502 + return failureResponse(error.message, status) + } + return failureResponse( + getErrorMessage(error, 'Windchill operation failed'), + isPayloadSizeLimitError(error) ? 413 : 500 + ) + } +} diff --git a/apps/sim/lib/internal/windchill/operations.test.ts b/apps/sim/lib/internal/windchill/operations.test.ts new file mode 100644 index 00000000000..95b2b1bff95 --- /dev/null +++ b/apps/sim/lib/internal/windchill/operations.test.ts @@ -0,0 +1,416 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { WindchillOperationBody } from '@/lib/api/contracts/tools/windchill' +import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' + +const mocks = vi.hoisted(() => ({ + assertToolFileAccess: vi.fn(), + createWindchillSession: vi.fn(), + downloadServableFileFromStorage: vi.fn(), + downloadWindchillContent: vi.fn(), + processFilesToUserFiles: vi.fn(), + resolveWindchillContentUrl: vi.fn(), + uploadCopilotFile: vi.fn(), + uploadExecutionFile: vi.fn(), + uploadWindchillContent: vi.fn(), + windchillMutationRequest: vi.fn(), +})) + +vi.mock('@/lib/internal/windchill/client', () => ({ + createWindchillSession: mocks.createWindchillSession, + downloadWindchillContent: mocks.downloadWindchillContent, + resolveWindchillContentUrl: mocks.resolveWindchillContentUrl, + uploadWindchillContent: mocks.uploadWindchillContent, + windchillDocumentUrl: (baseUrl: string, documentOid: string) => + `${baseUrl}/DocMgmt/Documents('${encodeURIComponent(documentOid)}')`, + windchillMutationRequest: mocks.windchillMutationRequest, +})) + +vi.mock('@/app/api/files/authorization', () => ({ + assertToolFileAccess: mocks.assertToolFileAccess, +})) + +vi.mock('@/lib/uploads/utils/file-utils', () => ({ + processFilesToUserFiles: mocks.processFilesToUserFiles, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadServableFileFromStorage: mocks.downloadServableFileFromStorage, +})) + +vi.mock('@/lib/uploads/contexts/copilot', () => ({ + uploadCopilotFile: mocks.uploadCopilotFile, +})) + +vi.mock('@/lib/uploads/contexts/execution', () => ({ + uploadExecutionFile: mocks.uploadExecutionFile, +})) + +import { WindchillOperationError } from '@/lib/internal/windchill/errors' +import { executeWindchillOperation } from '@/lib/internal/windchill/operations' + +const BASE = { + baseUrl: 'https://windchill.example.com/Windchill/servlet/odata/v6', + username: 'windchill-user', + password: 'not-a-real-password', +} +const DOCUMENT_OID = 'OR:wt.doc.WTDocument:1' +const SECOND_DOCUMENT_OID = 'OR:wt.doc.WTDocument:2' +const PRINCIPAL = { + kind: 'delegated' as const, + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'workspace-1', + delegationId: 'delegation-1', + audience: 'sim:windchill', + issuedAt: new Date('2026-01-01T00:00:00.000Z'), + expiresAt: new Date('2026-01-01T01:00:00.000Z'), + delegationContext: { + kind: 'workflow_execution' as const, + workflowId: 'workflow-1', + executionId: 'execution-1', + }, +} + +const MUTATION_CASES = [ + { + operation: 'windchill_create_document', + input: { name: 'Specification', containerOid: 'OR:wt.pdmlink.PDMLinkProduct:1' }, + url: '/DocMgmt/Documents', + method: 'POST', + }, + { + operation: 'windchill_create_documents', + input: { + documents: [{ name: 'Specification', containerOid: 'OR:wt.pdmlink.PDMLinkProduct:1' }], + }, + url: '/DocMgmt/CreateDocuments', + method: 'POST', + }, + { + operation: 'windchill_update_document', + input: { documentOid: DOCUMENT_OID, attributes: { Title: 'Updated' } }, + url: '/DocMgmt/Documents(', + method: 'PATCH', + }, + { + operation: 'windchill_update_common_properties', + input: { documentOid: DOCUMENT_OID, commonProperties: { Name: 'Renamed' } }, + url: '/PTC.DocMgmt.UpdateCommonProperties', + method: 'POST', + }, + { + operation: 'windchill_update_documents', + input: { documents: [{ id: DOCUMENT_OID, attributes: { Title: 'Updated' } }] }, + url: '/DocMgmt/UpdateDocuments', + method: 'POST', + }, + { + operation: 'windchill_delete_document', + input: { documentOid: DOCUMENT_OID }, + url: '/DocMgmt/Documents(', + method: 'DELETE', + }, + { + operation: 'windchill_delete_documents', + input: { documentOids: [DOCUMENT_OID, SECOND_DOCUMENT_OID] }, + url: '/DocMgmt/DeleteDocuments', + method: 'POST', + }, + { + operation: 'windchill_check_out_document', + input: { documentOid: DOCUMENT_OID, checkOutNote: 'Editing' }, + url: '/PTC.DocMgmt.CheckOut', + method: 'POST', + }, + { + operation: 'windchill_check_out_documents', + input: { documentOids: [DOCUMENT_OID], checkOutNote: 'Editing' }, + url: '/DocMgmt/CheckOutDocuments', + method: 'POST', + }, + { + operation: 'windchill_check_in_document', + input: { documentOid: DOCUMENT_OID, checkInNote: 'Done', keepCheckedOut: false }, + url: '/PTC.DocMgmt.CheckIn', + method: 'POST', + }, + { + operation: 'windchill_check_in_documents', + input: { documentOids: [DOCUMENT_OID], checkInNote: 'Done' }, + url: '/DocMgmt/CheckInDocuments', + method: 'POST', + }, + { + operation: 'windchill_undo_check_out_document', + input: { documentOid: DOCUMENT_OID }, + url: '/PTC.DocMgmt.UndoCheckOut', + method: 'POST', + }, + { + operation: 'windchill_undo_check_out_documents', + input: { documentOids: [DOCUMENT_OID] }, + url: '/DocMgmt/UndoCheckOutDocuments', + method: 'POST', + }, + { + operation: 'windchill_revise_document', + input: { documentOid: DOCUMENT_OID, versionId: 'B' }, + url: '/PTC.DocMgmt.Revise', + method: 'POST', + }, + { + operation: 'windchill_revise_documents', + input: { documentOids: [DOCUMENT_OID] }, + url: '/DocMgmt/ReviseDocuments', + method: 'POST', + }, + { + operation: 'windchill_set_lifecycle_state', + input: { documentOid: DOCUMENT_OID, stateValue: 'RELEASED', stateDisplay: 'Released' }, + url: '/PTC.DocMgmt.SetState', + method: 'POST', + }, + { + operation: 'windchill_update_document_security_labels', + input: { + securityLabelUpdates: [{ id: DOCUMENT_OID, labels: { EXPORT_CONTROL: 'EAR99' } }], + }, + url: '/DocMgmt/EditDocumentsSecurityLabels', + method: 'POST', + }, +] as const + +describe('Windchill operations', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.createWindchillSession.mockResolvedValue({ + nonceHeader: 'CSRF_NONCE', + nonceValue: 'nonce', + cookie: null, + }) + mocks.windchillMutationRequest.mockResolvedValue({ + value: [{ ID: DOCUMENT_OID, Name: 'Specification' }], + }) + mocks.assertToolFileAccess.mockResolvedValue(null) + mocks.uploadWindchillContent.mockResolvedValue(['specification.pdf']) + mocks.resolveWindchillContentUrl.mockResolvedValue( + 'https://windchill.example.com/WindchillGW/download?token=opaque' + ) + mocks.downloadWindchillContent.mockResolvedValue({ + buffer: Buffer.from('pdf'), + contentType: 'application/pdf; charset=binary', + contentDisposition: 'attachment; filename="specification.pdf"', + }) + mocks.uploadExecutionFile.mockResolvedValue({ + id: 'file-1', + name: 'specification.pdf', + url: '/api/files/serve?key=execution/specification.pdf', + size: 3, + type: 'application/pdf', + key: 'execution/specification.pdf', + }) + }) + + it.each(MUTATION_CASES)( + 'executes $operation through one CSRF session with cancellation', + async ({ operation, input, url, method }) => { + const controller = new AbortController() + const body = { ...BASE, operation, ...input } as WindchillOperationBody + + const result = await executeWindchillOperation(body, { + principal: PRINCIPAL, + requestId: 'request-1', + signal: controller.signal, + }) + + expect(mocks.createWindchillSession).toHaveBeenCalledWith(body, controller.signal) + expect(mocks.windchillMutationRequest).toHaveBeenCalledOnce() + expect(mocks.windchillMutationRequest.mock.calls[0][0]).toMatchObject({ + method, + signal: controller.signal, + }) + expect(mocks.windchillMutationRequest.mock.calls[0][0].url).toContain(url) + expect(result.operation).toBe(operation) + } + ) + + it.each([ + { + operation: 'windchill_check_out_documents', + input: { documentOids: [DOCUMENT_OID], checkOutNote: 'Editing' }, + payload: { Documents: [{ ID: DOCUMENT_OID }], CheckOutNote: 'Editing' }, + }, + { + operation: 'windchill_check_in_document', + input: { + documentOid: DOCUMENT_OID, + checkInNote: 'Done', + keepCheckedOut: false, + checkOutNote: 'Continue editing', + }, + payload: { + CheckInNote: 'Done', + KeepCheckedOut: false, + CheckOutNote: 'Continue editing', + }, + }, + { + operation: 'windchill_set_lifecycle_state', + input: { documentOid: DOCUMENT_OID, stateValue: 'RELEASED', stateDisplay: 'Released' }, + payload: { State: { Display: 'Released', Value: 'RELEASED' } }, + }, + { + operation: 'windchill_update_document_security_labels', + input: { + securityLabelUpdates: [{ id: DOCUMENT_OID, labels: { EXPORT_CONTROL: 'EAR99' } }], + }, + payload: { Documents: [{ EXPORT_CONTROL: 'EAR99', ID: DOCUMENT_OID }] }, + }, + ] as const)( + 'encodes the exact $operation action payload', + async ({ operation, input, payload }) => { + await executeWindchillOperation({ ...BASE, operation, ...input } as WindchillOperationBody, { + principal: PRINCIPAL, + requestId: 'request-1', + }) + + expect(mocks.windchillMutationRequest.mock.calls[0][0].body).toEqual(payload) + } + ) + + it('authorizes, bounds, and downloads stored files before provider upload', async () => { + const controller = new AbortController() + const rawFile = { + key: 'workspace/specification.pdf', + name: 'specification.pdf', + size: 3, + type: 'application/pdf', + } + mocks.processFilesToUserFiles.mockReturnValue([rawFile]) + mocks.downloadServableFileFromStorage.mockResolvedValue({ + buffer: Buffer.from('pdf'), + contentType: 'application/pdf', + }) + + const result = await executeWindchillOperation( + { + ...BASE, + operation: 'windchill_upload_primary_content', + documentOid: DOCUMENT_OID, + primaryFile: rawFile, + }, + { principal: PRINCIPAL, requestId: 'request-1', signal: controller.signal } + ) + + expect(mocks.assertToolFileAccess).toHaveBeenCalledWith( + 'workspace/specification.pdf', + 'user-1', + 'request-1', + expect.anything() + ) + expect(mocks.downloadServableFileFromStorage).toHaveBeenCalledWith( + rawFile, + 'request-1', + expect.anything(), + { maxBytes: MAX_FILE_SIZE, signal: controller.signal } + ) + expect(mocks.uploadWindchillContent).toHaveBeenCalledWith( + expect.objectContaining({ + documentOid: DOCUMENT_OID, + primaryContent: true, + signal: controller.signal, + }) + ) + expect(result).toEqual({ + operation: 'windchill_upload_primary_content', + affectedIds: [DOCUMENT_OID], + uploadedFileNames: ['specification.pdf'], + }) + }) + + it('fails closed before storage or provider work when file access is denied', async () => { + const rawFile = { key: 'other/file.pdf', name: 'file.pdf', size: 3, type: 'application/pdf' } + mocks.processFilesToUserFiles.mockReturnValue([rawFile]) + mocks.assertToolFileAccess.mockResolvedValue(new Response(null, { status: 404 })) + + await expect( + executeWindchillOperation( + { + ...BASE, + operation: 'windchill_upload_primary_content', + documentOid: DOCUMENT_OID, + primaryFile: rawFile, + }, + { principal: PRINCIPAL, requestId: 'request-1' } + ) + ).rejects.toEqual(new WindchillOperationError('File not found', 404)) + expect(mocks.downloadServableFileFromStorage).not.toHaveBeenCalled() + expect(mocks.uploadWindchillContent).not.toHaveBeenCalled() + }) + + it('rejects declared aggregate upload size before authorization or download', async () => { + const rawFile = { key: 'file.bin', name: 'file.bin', size: MAX_FILE_SIZE + 1 } + mocks.processFilesToUserFiles.mockReturnValue([rawFile]) + + await expect( + executeWindchillOperation( + { + ...BASE, + operation: 'windchill_upload_primary_content', + documentOid: DOCUMENT_OID, + primaryFile: rawFile, + }, + { principal: PRINCIPAL, requestId: 'request-1' } + ) + ).rejects.toEqual( + new WindchillOperationError('Combined Windchill upload exceeds the maximum file size', 413) + ) + expect(mocks.assertToolFileAccess).not.toHaveBeenCalled() + expect(mocks.downloadServableFileFromStorage).not.toHaveBeenCalled() + }) + + it('stores provider downloads in the bound execution scope without returning inline bytes', async () => { + const controller = new AbortController() + + const result = await executeWindchillOperation( + { + ...BASE, + operation: 'windchill_download_primary_content', + documentOid: DOCUMENT_OID, + }, + { principal: PRINCIPAL, requestId: 'request-1', signal: controller.signal } + ) + + expect(mocks.resolveWindchillContentUrl).toHaveBeenCalledWith( + expect.objectContaining({ + contentPath: expect.stringContaining('/PrimaryContent'), + signal: controller.signal, + }) + ) + expect(mocks.downloadWindchillContent).toHaveBeenCalledWith( + expect.objectContaining({ maxBytes: MAX_FILE_SIZE, signal: controller.signal }) + ) + expect(mocks.uploadExecutionFile).toHaveBeenCalledWith( + { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }, + Buffer.from('pdf'), + 'specification.pdf', + 'application/pdf', + 'user-1' + ) + expect(mocks.uploadCopilotFile).not.toHaveBeenCalled() + expect(result).toMatchObject({ + operation: 'windchill_download_primary_content', + file: { key: 'execution/specification.pdf' }, + fileName: 'specification.pdf', + mimeType: 'application/pdf', + }) + expect(result).not.toHaveProperty('content') + }) +}) diff --git a/apps/sim/lib/internal/windchill/operations.ts b/apps/sim/lib/internal/windchill/operations.ts new file mode 100644 index 00000000000..5bd72c664d1 --- /dev/null +++ b/apps/sim/lib/internal/windchill/operations.ts @@ -0,0 +1,597 @@ +import { + type BoundWorkflowExecutionDelegatedPrincipal, + requirePrincipalSubjectUserId, +} from '@sim/auth/principal' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import type { + WindchillOperationBody, + WindchillOperationResponse, +} from '@/lib/api/contracts/tools/windchill' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { + createWindchillSession, + downloadWindchillContent, + resolveWindchillContentUrl, + uploadWindchillContent, + type WindchillUploadFile, + windchillDocumentUrl, + windchillMutationRequest, +} from '@/lib/internal/windchill/client' +import { WindchillOperationError } from '@/lib/internal/windchill/errors' +import { uploadCopilotFile } from '@/lib/uploads/contexts/copilot' +import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' +import { docNotReadyMessage, isDocNotReadyError } from '@/lib/uploads/utils/doc-not-ready' +import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' +import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' +import { assertToolFileAccess } from '@/app/api/files/authorization' +import { sanitizeFileName } from '@/executor/constants' +import type { UserFile } from '@/executor/types' +import { + encodeWindchillOid, + normalizeServiceRoot, + normalizeWindchillDocument, + normalizeWindchillDocuments, +} from '@/tools/windchill/utils' + +const logger = createLogger('WindchillOperations') + +export type WindchillOperationOutput = Extract< + WindchillOperationResponse, + { success: true } +>['output'] +type MutationOperation = Exclude< + WindchillOperationOutput['operation'], + | 'windchill_download_attachment' + | 'windchill_download_primary_content' + | 'windchill_upload_attachments' + | 'windchill_upload_primary_content' +> + +const BULK_RESULT_OPERATIONS = [ + 'windchill_create_documents', + 'windchill_update_documents', + 'windchill_check_out_documents', + 'windchill_check_in_documents', + 'windchill_undo_check_out_documents', + 'windchill_revise_documents', + 'windchill_update_document_security_labels', +] as const satisfies readonly MutationOperation[] + +const DELETE_OPERATIONS = [ + 'windchill_delete_document', + 'windchill_delete_documents', +] as const satisfies readonly MutationOperation[] + +type BulkResultOperation = (typeof BULK_RESULT_OPERATIONS)[number] +type DeleteOperation = (typeof DELETE_OPERATIONS)[number] + +function isBulkResultOperation(operation: MutationOperation): operation is BulkResultOperation { + return BULK_RESULT_OPERATIONS.includes(operation as BulkResultOperation) +} + +function isDeleteOperation(operation: MutationOperation): operation is DeleteOperation { + return DELETE_OPERATIONS.includes(operation as DeleteOperation) +} + +function documentsById(documentOids: string[]) { + return documentOids.map((ID) => ({ ID })) +} + +/** Keeps the media type and drops any `; charset=...` parameters Windchill cannot use. */ +function safeMimeType(value: string | undefined): string { + const mediaType = value?.split(';', 1)[0]?.trim() + if (mediaType && /^[A-Za-z0-9!#$&^_.+-]+\/[A-Za-z0-9!#$&^_.+-]+$/.test(mediaType)) { + return mediaType + } + return 'application/octet-stream' +} + +function mutationOutput( + operation: MutationOperation, + data: unknown, + fallbackIds: string[] +): WindchillOperationOutput { + const documents = normalizeWindchillDocuments(data) + const document = documents[0] ?? normalizeWindchillDocument(data) + const collectionIds = documents + .map((item) => item.id) + .filter((id): id is string => typeof id === 'string') + const returnedIds = + document?.id && !collectionIds.includes(document.id) + ? [document.id, ...collectionIds] + : collectionIds + const affectedIds = returnedIds.length > 0 ? returnedIds : fallbackIds + if (isDeleteOperation(operation)) return { operation, affectedIds: fallbackIds } + if (isBulkResultOperation(operation)) { + return { + operation, + affectedIds, + ...(documents.length > 0 ? { documents } : {}), + } + } + return { + operation, + affectedIds, + ...(document ? { document } : {}), + } +} + +async function executeMutation( + body: Exclude< + WindchillOperationBody, + | { operation: 'windchill_download_primary_content' } + | { operation: 'windchill_upload_primary_content' } + | { operation: 'windchill_download_attachment' } + | { operation: 'windchill_upload_attachments' } + >, + signal?: AbortSignal +): Promise { + const session = await createWindchillSession(body, signal) + const root = normalizeServiceRoot(body.baseUrl) + + switch (body.operation) { + case 'windchill_create_document': { + const data = await windchillMutationRequest({ + params: body, + session, + url: `${root}/DocMgmt/Documents`, + method: 'POST', + body: { + ...(body.attributes ?? {}), + Name: body.name, + ...(body.number ? { Number: body.number } : {}), + ...(body.title ? { Title: body.title } : {}), + ...(body.description ? { Description: body.description } : {}), + 'Context@odata.bind': `Containers('${encodeWindchillOid(body.containerOid)}')`, + ...(body.folderOid + ? { 'Folder@odata.bind': `Folders('${encodeWindchillOid(body.folderOid)}')` } + : {}), + }, + signal, + }) + return mutationOutput(body.operation, data, []) + } + case 'windchill_create_documents': { + const data = await windchillMutationRequest({ + params: body, + session, + url: `${root}/DocMgmt/CreateDocuments`, + method: 'POST', + body: { + Documents: body.documents.map((document) => ({ + ...(document.attributes ?? {}), + Name: document.name, + ...(document.number ? { Number: document.number } : {}), + ...(document.title ? { Title: document.title } : {}), + ...(document.description ? { Description: document.description } : {}), + 'Context@odata.bind': `Containers('${encodeWindchillOid(document.containerOid)}')`, + ...(document.folderOid + ? { 'Folder@odata.bind': `Folders('${encodeWindchillOid(document.folderOid)}')` } + : {}), + })), + }, + signal, + }) + return mutationOutput(body.operation, data, []) + } + case 'windchill_update_document': { + const data = await windchillMutationRequest({ + params: body, + session, + url: windchillDocumentUrl(root, body.documentOid), + method: 'PATCH', + body: body.attributes, + signal, + }) + return mutationOutput(body.operation, data, [body.documentOid]) + } + case 'windchill_update_common_properties': { + const data = await windchillMutationRequest({ + params: body, + session, + url: `${windchillDocumentUrl(root, body.documentOid)}/PTC.DocMgmt.UpdateCommonProperties`, + method: 'POST', + body: { Updates: body.commonProperties }, + signal, + }) + return mutationOutput(body.operation, data, [body.documentOid]) + } + case 'windchill_update_documents': { + const data = await windchillMutationRequest({ + params: body, + session, + url: `${root}/DocMgmt/UpdateDocuments`, + method: 'POST', + body: { + Documents: body.documents.map((document) => ({ + ...document.attributes, + ID: document.id, + })), + }, + signal, + }) + return mutationOutput( + body.operation, + data, + body.documents.map((document) => document.id) + ) + } + case 'windchill_delete_document': { + const data = await windchillMutationRequest({ + params: body, + session, + url: windchillDocumentUrl(root, body.documentOid), + method: 'DELETE', + signal, + }) + return mutationOutput(body.operation, data, [body.documentOid]) + } + case 'windchill_delete_documents': { + const data = await windchillMutationRequest({ + params: body, + session, + url: `${root}/DocMgmt/DeleteDocuments`, + method: 'POST', + body: { Documents: documentsById(body.documentOids) }, + signal, + }) + return mutationOutput(body.operation, data, body.documentOids) + } + case 'windchill_check_out_document': { + const data = await windchillMutationRequest({ + params: body, + session, + url: `${windchillDocumentUrl(root, body.documentOid)}/PTC.DocMgmt.CheckOut`, + method: 'POST', + body: { ...(body.checkOutNote ? { CheckOutNote: body.checkOutNote } : {}) }, + signal, + }) + return mutationOutput(body.operation, data, [body.documentOid]) + } + case 'windchill_check_out_documents': { + const data = await windchillMutationRequest({ + params: body, + session, + url: `${root}/DocMgmt/CheckOutDocuments`, + method: 'POST', + body: { + Documents: documentsById(body.documentOids), + ...(body.checkOutNote ? { CheckOutNote: body.checkOutNote } : {}), + }, + signal, + }) + return mutationOutput(body.operation, data, body.documentOids) + } + case 'windchill_check_in_document': { + const data = await windchillMutationRequest({ + params: body, + session, + url: `${windchillDocumentUrl(root, body.documentOid)}/PTC.DocMgmt.CheckIn`, + method: 'POST', + body: { + ...(body.checkInNote ? { CheckInNote: body.checkInNote } : {}), + ...(body.keepCheckedOut !== undefined ? { KeepCheckedOut: body.keepCheckedOut } : {}), + ...(body.checkOutNote ? { CheckOutNote: body.checkOutNote } : {}), + }, + signal, + }) + return mutationOutput(body.operation, data, [body.documentOid]) + } + case 'windchill_check_in_documents': { + const data = await windchillMutationRequest({ + params: body, + session, + url: `${root}/DocMgmt/CheckInDocuments`, + method: 'POST', + body: { + Documents: documentsById(body.documentOids), + ...(body.checkInNote ? { CheckInNote: body.checkInNote } : {}), + ...(body.keepCheckedOut !== undefined ? { KeepCheckedOut: body.keepCheckedOut } : {}), + ...(body.checkOutNote ? { CheckOutNote: body.checkOutNote } : {}), + }, + signal, + }) + return mutationOutput(body.operation, data, body.documentOids) + } + case 'windchill_undo_check_out_document': { + const data = await windchillMutationRequest({ + params: body, + session, + url: `${windchillDocumentUrl(root, body.documentOid)}/PTC.DocMgmt.UndoCheckOut`, + method: 'POST', + body: {}, + signal, + }) + return mutationOutput(body.operation, data, [body.documentOid]) + } + case 'windchill_undo_check_out_documents': { + const data = await windchillMutationRequest({ + params: body, + session, + url: `${root}/DocMgmt/UndoCheckOutDocuments`, + method: 'POST', + body: { Documents: documentsById(body.documentOids) }, + signal, + }) + return mutationOutput(body.operation, data, body.documentOids) + } + case 'windchill_revise_document': { + const data = await windchillMutationRequest({ + params: body, + session, + url: `${windchillDocumentUrl(root, body.documentOid)}/PTC.DocMgmt.Revise`, + method: 'POST', + body: { ...(body.versionId ? { VersionId: body.versionId } : {}) }, + signal, + }) + return mutationOutput(body.operation, data, [body.documentOid]) + } + case 'windchill_revise_documents': { + const data = await windchillMutationRequest({ + params: body, + session, + url: `${root}/DocMgmt/ReviseDocuments`, + method: 'POST', + body: { + Documents: documentsById(body.documentOids), + }, + signal, + }) + return mutationOutput(body.operation, data, body.documentOids) + } + case 'windchill_set_lifecycle_state': { + const data = await windchillMutationRequest({ + params: body, + session, + url: `${windchillDocumentUrl(root, body.documentOid)}/PTC.DocMgmt.SetState`, + method: 'POST', + body: { State: { Display: body.stateDisplay, Value: body.stateValue } }, + signal, + }) + return mutationOutput(body.operation, data, [body.documentOid]) + } + case 'windchill_update_document_security_labels': { + const data = await windchillMutationRequest({ + params: body, + session, + url: `${root}/DocMgmt/EditDocumentsSecurityLabels`, + method: 'POST', + body: { + Documents: body.securityLabelUpdates.map((update) => ({ + ...update.labels, + ID: update.id, + })), + }, + signal, + }) + return mutationOutput( + body.operation, + data, + body.securityLabelUpdates.map((update) => update.id) + ) + } + } +} + +async function loadUploadFiles( + inputs: RawFileInput[], + userId: string, + requestId: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + let userFiles: UserFile[] + try { + userFiles = processFilesToUserFiles(inputs, requestId, logger) + } catch (error) { + throw new WindchillOperationError(getErrorMessage(error, 'Invalid file input'), 400) + } + if (userFiles.length !== inputs.length) { + throw new WindchillOperationError('Invalid file input', 400) + } + + const declaredTotal = userFiles.reduce((total, file) => total + file.size, 0) + if (declaredTotal > MAX_FILE_SIZE) { + throw new WindchillOperationError( + 'Combined Windchill upload exceeds the maximum file size', + 413 + ) + } + + const files: WindchillUploadFile[] = [] + let actualTotal = 0 + for (const userFile of userFiles) { + signal?.throwIfAborted() + const denied = await assertToolFileAccess(userFile.key, userId, requestId, logger) + if (denied) throw new WindchillOperationError('File not found', denied.status) + try { + const servable = await downloadServableFileFromStorage(userFile, requestId, logger, { + maxBytes: MAX_FILE_SIZE - actualTotal, + signal, + }) + actualTotal += servable.buffer.length + if (actualTotal > MAX_FILE_SIZE) { + throw new WindchillOperationError( + 'Combined Windchill upload exceeds the maximum file size', + 413 + ) + } + files.push({ + name: sanitizeFileName(userFile.name), + mimeType: safeMimeType(servable.contentType || userFile.type), + size: servable.buffer.length, + buffer: servable.buffer, + }) + } catch (error) { + signal?.throwIfAborted() + if (error instanceof WindchillOperationError) throw error + if (isDocNotReadyError(error)) { + throw new WindchillOperationError(docNotReadyMessage(), 409) + } + throw new WindchillOperationError( + getErrorMessage(error, 'Failed to read uploaded file'), + isPayloadSizeLimitError(error) ? 413 : 400 + ) + } + } + return files +} + +function contentDispositionFileName(value: string | null): string | null { + if (!value) return null + const encoded = value.match(/filename\*=UTF-8''([^;]+)/i)?.[1] + if (encoded) { + try { + return decodeURIComponent(encoded) + } catch { + return encoded + } + } + return ( + value.match(/filename\s*=\s*"([^"]+)"/i)?.[1] ?? + value.match(/filename\s*=\s*([^;]+)/i)?.[1]?.trim() ?? + null + ) +} + +async function storeDownloadedFile({ + principal, + buffer, + fileName, + contentType, + signal, +}: { + principal: BoundWorkflowExecutionDelegatedPrincipal + buffer: Buffer + fileName: string + contentType: string + signal?: AbortSignal +}): Promise { + signal?.throwIfAborted() + const { workflowId, executionId } = principal.delegationContext + if (executionId) { + const file = await uploadExecutionFile( + { + workspaceId: principal.workspaceId, + workflowId, + executionId, + }, + buffer, + fileName, + contentType, + requirePrincipalSubjectUserId(principal) + ) + signal?.throwIfAborted() + return file + } + const file = await uploadCopilotFile({ + buffer, + fileName, + contentType, + userId: requirePrincipalSubjectUserId(principal), + }) + signal?.throwIfAborted() + return file +} + +async function executeDownload( + body: Extract< + WindchillOperationBody, + | { operation: 'windchill_download_primary_content' } + | { operation: 'windchill_download_attachment' } + >, + principal: BoundWorkflowExecutionDelegatedPrincipal, + signal?: AbortSignal +): Promise { + const documentUrl = windchillDocumentUrl(body.baseUrl, body.documentOid) + const contentPath = + body.operation === 'windchill_download_primary_content' + ? `${documentUrl}/PrimaryContent` + : `${documentUrl}/Attachments('${encodeWindchillOid(body.attachmentOid)}')` + const contentUrl = await resolveWindchillContentUrl({ + params: body, + contentPath, + signal, + }) + const downloaded = await downloadWindchillContent({ + params: body, + url: contentUrl, + maxBytes: MAX_FILE_SIZE, + signal, + }) + const fallback = + body.operation === 'windchill_download_primary_content' + ? 'windchill-primary-content.bin' + : 'windchill-attachment.bin' + const fileName = sanitizeFileName( + body.fileName || contentDispositionFileName(downloaded.contentDisposition) || fallback + ) + const mimeType = safeMimeType(downloaded.contentType) + const file = await storeDownloadedFile({ + principal, + buffer: downloaded.buffer, + fileName, + contentType: mimeType, + signal, + }) + return { + operation: body.operation, + file: { ...file }, + fileName, + mimeType, + } +} + +export interface WindchillOperationContext { + principal: BoundWorkflowExecutionDelegatedPrincipal + requestId: string + signal?: AbortSignal +} + +export async function executeWindchillOperation( + body: WindchillOperationBody, + context: WindchillOperationContext +): Promise { + const { principal, requestId, signal } = context + signal?.throwIfAborted() + + if ( + body.operation === 'windchill_download_primary_content' || + body.operation === 'windchill_download_attachment' + ) { + return executeDownload(body, principal, signal) + } + + if ( + body.operation === 'windchill_upload_primary_content' || + body.operation === 'windchill_upload_attachments' + ) { + const inputs = + body.operation === 'windchill_upload_primary_content' + ? [body.primaryFile] + : body.attachmentFiles + const files = await loadUploadFiles( + inputs, + requirePrincipalSubjectUserId(principal), + requestId, + signal + ) + const uploadedFileNames = await uploadWindchillContent({ + params: body, + documentOid: body.documentOid, + files, + primaryContent: body.operation === 'windchill_upload_primary_content', + signal, + }) + return { + operation: body.operation, + affectedIds: [body.documentOid], + uploadedFileNames, + } + } + + return executeMutation(body, signal) +} diff --git a/apps/sim/lib/internal/wordpress/errors.ts b/apps/sim/lib/internal/wordpress/errors.ts new file mode 100644 index 00000000000..ff8df3444e6 --- /dev/null +++ b/apps/sim/lib/internal/wordpress/errors.ts @@ -0,0 +1,9 @@ +export class WordPressOperationError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'WordPressOperationError' + } +} diff --git a/apps/sim/lib/internal/wordpress/execute-tool.test.ts b/apps/sim/lib/internal/wordpress/execute-tool.test.ts new file mode 100644 index 00000000000..d8aaeb27993 --- /dev/null +++ b/apps/sim/lib/internal/wordpress/execute-tool.test.ts @@ -0,0 +1,62 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ uploadWordPressMedia: vi.fn() })) + +vi.mock('@/lib/internal/wordpress/operations', () => ({ + uploadWordPressMedia: mocks.uploadWordPressMedia, +})) + +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' +import { executeWordPressTool } from '@/lib/internal/wordpress/execute-tool' + +function request(overrides: Partial = {}): InternalToolOperationCall { + return { + toolId: 'wordpress_upload_media', + input: { + accessToken: 'token', + siteId: 'site-1', + file: { key: 'workspace/ws/file-1', name: 'image.png', size: 3, type: 'image/png' }, + }, + headers: new Headers(), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +describe('executeWordPressTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.uploadWordPressMedia.mockResolvedValue({ + success: true, + output: { media: { id: 1 } }, + }) + }) + + it('uses trusted user identity and operation cancellation', async () => { + const controller = new AbortController() + const response = await executeWordPressTool(request({ signal: controller.signal })) + + expect(response.status).toBe(200) + expect(mocks.uploadWordPressMedia).toHaveBeenCalledWith(expect.any(Object), { + userId: 'user-1', + requestId: 'request-1', + signal: controller.signal, + }) + }) + + it('rejects missing trusted identity before file work', async () => { + const response = await executeWordPressTool( + request({ context: createExecutionContext({ workflowId: 'workflow-1' }) }) + ) + expect(response.status).toBe(401) + expect(mocks.uploadWordPressMedia).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/wordpress/execute-tool.ts b/apps/sim/lib/internal/wordpress/execute-tool.ts new file mode 100644 index 00000000000..abafba0d082 --- /dev/null +++ b/apps/sim/lib/internal/wordpress/execute-tool.ts @@ -0,0 +1,65 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { z } from 'zod' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { WordPressOperationError } from '@/lib/internal/wordpress/errors' +import { uploadWordPressMedia } from '@/lib/internal/wordpress/operations' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { RawFileInputSchema } from '@/lib/uploads/utils/file-schemas' +import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' + +const inputSchema = z.object({ + accessToken: z.string().min(1), + siteId: z.string().min(1), + file: RawFileInputSchema.optional().nullable(), + filename: z.string().optional().nullable(), + title: z.string().optional().nullable(), + caption: z.string().optional().nullable(), + altText: z.string().optional().nullable(), + description: z.string().optional().nullable(), +}) + +export const executeWordPressTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'wordpress_upload_media') { + return Response.json( + { success: false, error: `Unsupported WordPress tool: ${request.toolId}` }, + { status: 500 } + ) + } + const userId = request.context.userId + if (!userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + const parsed = inputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + try { + return Response.json( + await uploadWordPressMedia(parsed.data, { + userId, + requestId: request.requestId, + signal: request.signal, + }) + ) + } catch (error) { + request.signal?.throwIfAborted() + const notReady = docNotReadyResponse(error) + if (notReady) return notReady + if (isPayloadSizeLimitError(error)) { + return Response.json( + { + success: false, + error: `Failed to download file: file exceeds maximum size of ${MAX_BUFFERED_TRANSFER_BYTES} bytes`, + }, + { status: 413 } + ) + } + const status = error instanceof WordPressOperationError ? error.status : 500 + return Response.json( + { success: false, error: getErrorMessage(error, 'Internal server error') }, + { status } + ) + } +} diff --git a/apps/sim/lib/internal/wordpress/operations.ts b/apps/sim/lib/internal/wordpress/operations.ts new file mode 100644 index 00000000000..33d5f1318f2 --- /dev/null +++ b/apps/sim/lib/internal/wordpress/operations.ts @@ -0,0 +1,124 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { + readResponseJsonWithLimit, + readResponseTextWithLimit, +} from '@/lib/core/utils/stream-limits' +import { WordPressOperationError } from '@/lib/internal/wordpress/errors' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import type { RawFileInput } from '@/lib/uploads/utils/file-schemas' +import { + getFileExtension, + getMimeTypeFromExtension, + processSingleFileToUserFile, +} from '@/lib/uploads/utils/file-utils' +import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' +import type { WordPressUploadMediaResponse } from '@/tools/wordpress/types' + +const logger = createLogger('WordPressOperations') +const WORDPRESS_COM_API_BASE = 'https://public-api.wordpress.com/wp/v2/sites' +const MAX_WORDPRESS_RESPONSE_BYTES = 2 * 1024 * 1024 + +interface WordPressMediaPayload { + id: number + date: string + slug: string + type: string + link: string + title: { rendered: string } + caption: { rendered: string } + alt_text: string + media_type: string + mime_type: string + source_url: string + media_details?: { width?: number; height?: number; file?: string } +} + +export interface WordPressOperationContext { + userId: string + requestId: string + signal?: AbortSignal +} + +export interface WordPressUploadMediaInput { + accessToken: string + siteId: string + file?: RawFileInput | null + filename?: string | null + title?: string | null + caption?: string | null + altText?: string | null + description?: string | null +} + +export async function uploadWordPressMedia( + input: WordPressUploadMediaInput, + context: WordPressOperationContext +): Promise { + context.signal?.throwIfAborted() + if (!input.file) { + throw new WordPressOperationError('No file provided. Please upload a file.', 400) + } + let userFile + try { + userFile = processSingleFileToUserFile(input.file, context.requestId, logger) + } catch (error) { + throw new WordPressOperationError(getErrorMessage(error, 'Failed to process file'), 400) + } + const denied = await assertToolFileAccess(userFile.key, context.userId, context.requestId, logger) + if (denied) throw new WordPressOperationError('File not found', denied.status) + context.signal?.throwIfAborted() + + let fileBuffer: Buffer + let resolvedContentType: string + try { + const servable = await downloadServableFileFromStorage(userFile, context.requestId, logger, { + maxBytes: MAX_BUFFERED_TRANSFER_BYTES, + }) + fileBuffer = servable.buffer + resolvedContentType = servable.contentType + } catch (error) { + throw new WordPressOperationError( + `Failed to download file: ${getErrorMessage(error, 'Unknown error')}`, + 500 + ) + } + context.signal?.throwIfAborted() + + const filename = input.filename || userFile.name + const mimeType = + resolvedContentType || userFile.type || getMimeTypeFromExtension(getFileExtension(filename)) + const formData = new FormData() + formData.append('file', new Blob([new Uint8Array(fileBuffer)], { type: mimeType }), filename) + if (input.title) formData.append('title', input.title) + if (input.caption) formData.append('caption', input.caption) + if (input.altText) formData.append('alt_text', input.altText) + if (input.description) formData.append('description', input.description) + + const response = await fetch(`${WORDPRESS_COM_API_BASE}/${input.siteId}/media`, { + method: 'POST', + headers: { Authorization: `Bearer ${input.accessToken}` }, + body: formData, + signal: context.signal, + }) + if (!response.ok) { + const errorText = await readResponseTextWithLimit(response, { + maxBytes: MAX_WORDPRESS_RESPONSE_BYTES, + label: 'WordPress error response', + signal: context.signal, + }) + let message = `WordPress API error: ${response.statusText}` + try { + const parsed = JSON.parse(errorText) as { message?: string; error?: string } + message = parsed.message || parsed.error || message + } catch {} + throw new WordPressOperationError(message, response.status) + } + const media = await readResponseJsonWithLimit(response, { + maxBytes: MAX_WORDPRESS_RESPONSE_BYTES, + label: 'WordPress upload response', + signal: context.signal, + }) + return { success: true, output: { media } } +} diff --git a/apps/sim/lib/internal/workday/client.test.ts b/apps/sim/lib/internal/workday/client.test.ts new file mode 100644 index 00000000000..40b5f15d5e5 --- /dev/null +++ b/apps/sim/lib/internal/workday/client.test.ts @@ -0,0 +1,146 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { + buildServiceUrl, + createWorkdaySoapClient, + extractRefId, +} from '@/lib/internal/workday/client' + +const SUCCESS_RESPONSE = ` + + + + + + + worker-1 + + + + + 1 + + + +` + +const FAULT_RESPONSE = ` + + + env:ClientInvalid worker + +` + +describe('Workday SOAP client', () => { + const fetchMock = vi.fn() + + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', fetchMock) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('builds an allowlisted Workday service URL and rejects untrusted hosts', () => { + expect( + buildServiceUrl('https://wd2-impl-services1.workday.com/', 'example', 'humanResources') + ).toBe('https://wd2-impl-services1.workday.com/ccx/service/example/Human_Resources/v45.2') + expect(() => buildServiceUrl('https://127.0.0.1', 'example', 'staffing')).toThrow( + 'tenantUrl must be a Workday-hosted domain' + ) + }) + + it('passes cancellation to fetch, escapes XML, and parses SOAP responses', async () => { + fetchMock.mockResolvedValue(new Response(SUCCESS_RESPONSE, { status: 200 })) + const controller = new AbortController() + const client = await createWorkdaySoapClient( + 'https://wd2-impl-services1.workday.com', + 'example', + 'humanResources', + 'user<&', + 'password<&', + controller.signal + ) + + const [result] = await client.Get_WorkersAsync({ + Request_References: { + Worker_Reference: { ID: { $value: 'worker<&', attributes: { 'wd:type': 'Employee_ID' } } }, + }, + }) + + expect(fetchMock).toHaveBeenCalledOnce() + const [url, init] = fetchMock.mock.calls[0] + expect(url).toBe( + 'https://wd2-impl-services1.workday.com/ccx/service/example/Human_Resources/v45.2' + ) + expect(init).toMatchObject({ method: 'POST', signal: controller.signal }) + expect(init?.body).toContain('user<&') + expect(init?.body).toContain('') + expect(init?.body).toContain('worker<&') + const worker = result.Response_Data?.Worker as { + Worker_Reference?: { ID?: { $value?: string } } + } + expect(extractRefId(worker.Worker_Reference)).toBe('worker-1') + expect(result.Response_Results?.Total_Results).toBe('1') + }) + + it('preserves Workday SOAP fault messages', async () => { + fetchMock.mockResolvedValue(new Response(FAULT_RESPONSE, { status: 500 })) + const client = await createWorkdaySoapClient( + 'https://wd2-impl-services1.workday.com', + 'example', + 'humanResources', + 'user', + 'not-a-real-password' + ) + + await expect(client.Get_WorkersAsync({})).rejects.toThrow('Invalid worker') + }) + + it('caps provider responses before materializing oversized bodies', async () => { + fetchMock.mockResolvedValue( + new Response('oversized', { + status: 200, + headers: { 'content-length': String(10 * 1024 * 1024 + 1) }, + }) + ) + const client = await createWorkdaySoapClient( + 'https://wd2-impl-services1.workday.com', + 'example', + 'humanResources', + 'user', + 'not-a-real-password' + ) + + await expect(client.Get_WorkersAsync({})).rejects.toEqual( + new PayloadSizeLimitError({ + label: 'Workday SOAP response', + maxBytes: 10 * 1024 * 1024, + observedBytes: 10 * 1024 * 1024 + 1, + }) + ) + }) + + it('stops before provider work when already cancelled', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + createWorkdaySoapClient( + 'https://wd2-impl-services1.workday.com', + 'example', + 'humanResources', + 'user', + 'not-a-real-password', + controller.signal + ) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(fetchMock).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/workday/client.ts b/apps/sim/lib/internal/workday/client.ts new file mode 100644 index 00000000000..faa15c6499e --- /dev/null +++ b/apps/sim/lib/internal/workday/client.ts @@ -0,0 +1,672 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { validateWorkdayTenantUrl } from '@/lib/core/security/input-validation' +import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' + +const logger = createLogger('WorkdaySoapClient') +const WORKDAY_SOAP_RESPONSE_MAX_BYTES = 10 * 1024 * 1024 + +const WORKDAY_SERVICES = { + staffing: { name: 'Staffing', version: 'v45.1' }, + humanResources: { name: 'Human_Resources', version: 'v45.2' }, + compensation: { name: 'Compensation', version: 'v45.0' }, + recruiting: { name: 'Recruiting', version: 'v45.0' }, +} as const + +export type WorkdayServiceKey = keyof typeof WORKDAY_SERVICES + +interface WorkdaySoapResult { + Response_Data?: Record + Response_Results?: { + Total_Results?: number | string + Total_Pages?: number | string + Page_Results?: number | string + Page?: number | string + } + Event_Reference?: WorkdayReference + Employee_Reference?: WorkdayReference + Position_Reference?: WorkdayReference + Applicant_Reference?: WorkdayReference & { attributes?: { Descriptor?: string } } + Onboarding_Plan_Assignment_Reference?: WorkdayReference + Personal_Information_Change_Event_Reference?: WorkdayReference + Exceptions_Response_Data?: unknown +} + +export interface WorkdayReference { + ID?: WorkdayIdEntry[] | WorkdayIdEntry + attributes?: Record +} + +export interface WorkdayIdEntry { + $value?: string + _?: string + attributes?: Record +} + +/** + * Raw SOAP response shape for a single Worker returned by Get_Workers. + * Fields are optional since the Response_Group controls what gets included. + */ +export interface WorkdayWorkerSoap { + Worker_Reference?: WorkdayReference + Worker_Descriptor?: string + Worker_Data?: WorkdayWorkerDataSoap +} + +interface WorkdayWorkerDataSoap { + Personal_Data?: Record + Employment_Data?: Record + Compensation_Data?: WorkdayCompensationDataSoap + Organization_Data?: Record +} + +export interface WorkdayCompensationDataSoap { + Employee_Base_Pay_Plan_Assignment_Data?: + | WorkdayCompensationPlanSoap + | WorkdayCompensationPlanSoap[] + Employee_Salary_Unit_Plan_Assignment_Data?: + | WorkdayCompensationPlanSoap + | WorkdayCompensationPlanSoap[] + Employee_Bonus_Plan_Assignment_Data?: WorkdayCompensationPlanSoap | WorkdayCompensationPlanSoap[] + Employee_Allowance_Plan_Assignment_Data?: + | WorkdayCompensationPlanSoap + | WorkdayCompensationPlanSoap[] + Employee_Commission_Plan_Assignment_Data?: + | WorkdayCompensationPlanSoap + | WorkdayCompensationPlanSoap[] + Employee_Stock_Plan_Assignment_Data?: WorkdayCompensationPlanSoap | WorkdayCompensationPlanSoap[] + Employee_Period_Salary_Plan_Assignment_Data?: + | WorkdayCompensationPlanSoap + | WorkdayCompensationPlanSoap[] +} + +export interface WorkdayCompensationPlanSoap { + Compensation_Plan_Reference?: WorkdayReference + Amount?: number | string + Per_Unit_Amount?: number | string + Individual_Target_Amount?: number | string + Currency_Reference?: WorkdayReference + Frequency_Reference?: WorkdayReference +} + +/** + * Raw SOAP response shape for a single Organization returned by Get_Organizations. + */ +export interface WorkdayOrganizationSoap { + Organization_Reference?: WorkdayReference + Organization_Descriptor?: string + Organization_Data?: WorkdayOrganizationDataSoap +} + +interface WorkdayOrganizationDataSoap { + Organization_Type_Reference?: WorkdayReference + Organization_Subtype_Reference?: WorkdayReference + Inactive?: boolean | string +} + +/** + * Normalizes a SOAP response field that may be a single object, an array, or undefined + * into a consistently typed array. + */ +export function normalizeSoapArray(value: T | T[] | undefined): T[] { + if (!value) return [] + return Array.isArray(value) ? value : [value] +} + +/** + * Coerces a SOAP scalar to a boolean. The XML parser returns leaf text as strings, + * so `"true"`/`"false"` must be normalized before boolean operations like negation. + * Returns null when the value is null/undefined or unrecognized. + */ +export function parseSoapBoolean(value: unknown): boolean | null { + if (value == null) return null + if (typeof value === 'boolean') return value + if (typeof value === 'string') { + const trimmed = value.trim().toLowerCase() + if (trimmed === 'true' || trimmed === '1') return true + if (trimmed === 'false' || trimmed === '0') return false + } + return null +} + +/** + * Coerces a SOAP scalar to a number. The XML parser returns leaf text as strings, + * so numeric fields like `Total_Results` must be normalized before arithmetic. + * Returns null when the value is null/undefined or not a finite number. + */ +export function parseSoapNumber(value: unknown): number | null { + if (value == null) return null + if (typeof value === 'number') return Number.isFinite(value) ? value : null + if (typeof value === 'string') { + const trimmed = value.trim() + if (trimmed === '') return null + const n = Number(trimmed) + return Number.isFinite(n) ? n : null + } + return null +} + +const WD_OPERATIONS = [ + 'Get_Workers', + 'Get_Organizations', + 'Put_Applicant', + 'Hire_Employee', + 'Change_Job', + 'Terminate_Employee', + 'Change_Personal_Information', + 'Put_Onboarding_Plan_Assignment', +] as const + +type WorkdayOperation = (typeof WD_OPERATIONS)[number] + +type SoapOperationFn = ( + args: Record +) => Promise<[WorkdaySoapResult, string, Record, string]> + +export interface WorkdayClient { + Get_WorkersAsync: SoapOperationFn + Get_OrganizationsAsync: SoapOperationFn + Put_ApplicantAsync: SoapOperationFn + Hire_EmployeeAsync: SoapOperationFn + Change_JobAsync: SoapOperationFn + Terminate_EmployeeAsync: SoapOperationFn + Change_Personal_InformationAsync: SoapOperationFn + Put_Onboarding_Plan_AssignmentAsync: SoapOperationFn +} + +/** + * Builds the service endpoint URL for a Workday SOAP service. + * Pattern: {tenantUrl}/ccx/service/{tenant}/{serviceName}/{version} + * + * @throws Error if tenantUrl is not a trusted Workday-hosted URL (SSRF guard) + */ +export function buildServiceUrl( + tenantUrl: string, + tenant: string, + service: WorkdayServiceKey +): string { + const validation = validateWorkdayTenantUrl(tenantUrl) + if (!validation.isValid) { + throw new Error(validation.error ?? 'Invalid tenantUrl') + } + const svc = WORKDAY_SERVICES[service] + const baseUrl = (validation.sanitized ?? tenantUrl).replace(/\/$/, '') + return `${baseUrl}/ccx/service/${tenant}/${svc.name}/${svc.version}` +} + +const XML_ENTITIES: Record = { + '&': '&', + '<': '<', + '>': '>', + '"': '"', + "'": ''', +} + +function escapeXml(value: string): string { + return value.replace(/[&<>"']/g, (c) => XML_ENTITIES[c] ?? c) +} + +function serializeAttributes(attrs?: Record): string { + if (!attrs) return '' + let out = '' + for (const [k, v] of Object.entries(attrs)) { + if (v === undefined || v === null) continue + out += ` ${k}="${escapeXml(String(v))}"` + } + return out +} + +/** + * Marshals a JS value into XML under the `wd:` namespace. + * Conventions: + * - Plain objects become elements with named children + * - `attributes` becomes element attributes + * - `$value` (or `_`) provides the element text content + * - Arrays produce repeated elements with the same name + * - Booleans render as "true"/"false", numbers via String() + */ +function marshal(name: string, value: unknown): string { + if (value === undefined || value === null) return '' + const tag = `wd:${name}` + + if (Array.isArray(value)) { + let out = '' + for (const item of value) { + out += marshal(name, item) + } + return out + } + + if (value instanceof Date) { + return `<${tag}>${value.toISOString()}` + } + + if (typeof value === 'object') { + const obj = value as Record + const attrs = obj.attributes as Record | undefined + const text = (obj.$value ?? obj._) as string | number | boolean | undefined + + if (text !== undefined) { + const childKeys = Object.keys(obj).filter( + (k) => k !== 'attributes' && k !== '$value' && k !== '_' + ) + if (childKeys.length === 0) { + return `<${tag}${serializeAttributes(attrs)}>${escapeXml(String(text))}` + } + } + + let inner = '' + for (const [k, v] of Object.entries(obj)) { + if (k === 'attributes' || k === '$value' || k === '_') continue + inner += marshal(k, v) + } + if (text !== undefined) inner = escapeXml(String(text)) + inner + return `<${tag}${serializeAttributes(attrs)}>${inner}` + } + + if (typeof value === 'boolean') { + return `<${tag}>${value ? 'true' : 'false'}` + } + + return `<${tag}>${escapeXml(String(value))}` +} + +function buildEnvelope( + operation: string, + args: Record, + username: string, + password: string +): string { + let body = '' + for (const [k, v] of Object.entries(args)) { + body += marshal(k, v) + } + + const wsseNs = 'http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd' + const wssePwdType = + 'http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-username-token-profile-1.0#PasswordText' + + return ( + `` + + `` + + `` + + `` + + `` + + `${escapeXml(username)}` + + `${escapeXml(password)}` + + `` + + `` + + `` + + `` + + `` + + body + + `` + + `` + + `` + ) +} + +interface XmlNode { + name: string + localName: string + attributes: Record + children: XmlNode[] + text: string +} + +/** + * Minimal XML parser tuned for Workday SOAP responses: namespaced tags, + * attributes, mixed text, self-closing tags, and CDATA sections. + * Not a general-purpose parser — it does not expand entities beyond the + * standard five and ignores processing instructions and DOCTYPE. + */ +function parseXml(xml: string): XmlNode { + let i = 0 + const len = xml.length + + function skipWhitespace() { + while (i < len && xml.charCodeAt(i) <= 32) i++ + } + + function readName(): string { + const start = i + while (i < len) { + const c = xml[i] + if ( + c === ' ' || + c === '\t' || + c === '\n' || + c === '\r' || + c === '>' || + c === '/' || + c === '=' + ) + break + i++ + } + return xml.slice(start, i) + } + + function readAttributes(): Record { + const attrs: Record = {} + while (i < len) { + skipWhitespace() + const c = xml[i] + if (c === '>' || c === '/' || c === '?') return attrs + const name = readName() + skipWhitespace() + if (xml[i] !== '=') { + attrs[name] = '' + continue + } + i++ + skipWhitespace() + const quote = xml[i] + if (quote !== '"' && quote !== "'") { + attrs[name] = '' + continue + } + i++ + const start = i + while (i < len && xml[i] !== quote) i++ + attrs[name] = decodeEntities(xml.slice(start, i)) + if (i < len) i++ + } + return attrs + } + + function decodeEntities(s: string): string { + return s.replace(/&(amp|lt|gt|quot|apos|#\d+|#x[0-9a-fA-F]+);/g, (_, ent) => { + switch (ent) { + case 'amp': + return '&' + case 'lt': + return '<' + case 'gt': + return '>' + case 'quot': + return '"' + case 'apos': + return "'" + default: + if (ent.startsWith('#x')) return String.fromCodePoint(Number.parseInt(ent.slice(2), 16)) + if (ent.startsWith('#')) return String.fromCodePoint(Number.parseInt(ent.slice(1), 10)) + return `&${ent};` + } + }) + } + + function localOf(name: string): string { + const idx = name.indexOf(':') + return idx === -1 ? name : name.slice(idx + 1) + } + + function parseNode(): XmlNode { + if (xml[i] !== '<') throw new Error(`Expected '<' at ${i}`) + i++ + const name = readName() + const attrs = readAttributes() + skipWhitespace() + const node: XmlNode = { + name, + localName: localOf(name), + attributes: attrs, + children: [], + text: '', + } + if (xml[i] === '/') { + i += 2 + return node + } + if (xml[i] !== '>') throw new Error(`Expected '>' at ${i}`) + i++ + + while (i < len) { + if (xml[i] === '<') { + if (xml.startsWith('', i) + i = end === -1 ? len : end + 3 + continue + } + if (xml.startsWith('', i + 9) + const data = xml.slice(i + 9, end === -1 ? len : end) + node.text += data + i = end === -1 ? len : end + 3 + continue + } + if (xml[i + 1] === '/') { + i += 2 + while (i < len && xml[i] !== '>') i++ + if (i < len) i++ + return node + } + node.children.push(parseNode()) + } else { + const start = i + while (i < len && xml[i] !== '<') i++ + node.text += decodeEntities(xml.slice(start, i)) + } + } + return node + } + + while (i < len) { + skipWhitespace() + if (xml.startsWith('', i) + i = end === -1 ? len : end + 2 + continue + } + if (xml.startsWith('', i) + i = end === -1 ? len : end + 3 + continue + } + if (xml.startsWith('', i) + i = end === -1 ? len : end + 1 + continue + } + if (xml[i] === '<') break + i++ + } + return parseNode() +} + +/** + * Converts a parsed XML node tree into the JS object shape that the previous + * `soap` library produced: nested objects keyed by local element name, + * attributes under `attributes`, repeated elements collapsed into arrays, + * and pure text nodes returned as strings. + */ +function nodeToValue(node: XmlNode): unknown { + const hasChildren = node.children.length > 0 + const trimmedText = node.text.trim() + const attrKeys = Object.keys(node.attributes).filter( + (k) => k !== 'xmlns' && !k.startsWith('xmlns:') + ) + + if (!hasChildren && attrKeys.length === 0) { + return trimmedText + } + + const obj: Record = {} + if (attrKeys.length > 0) { + const attrs: Record = {} + for (const k of attrKeys) { + const localKey = k.includes(':') ? k.slice(k.indexOf(':') + 1) : k + attrs[localKey] = node.attributes[k] + } + obj.attributes = attrs + } + + if (!hasChildren && trimmedText !== '') { + obj.$value = trimmedText + return obj + } + + for (const child of node.children) { + const key = child.localName + const value = nodeToValue(child) + if (key in obj) { + const existing = obj[key] + if (Array.isArray(existing)) { + existing.push(value) + } else { + obj[key] = [existing, value] + } + } else { + obj[key] = value + } + } + return obj +} + +function findFirst(node: XmlNode, localName: string): XmlNode | null { + if (node.localName === localName) return node + for (const child of node.children) { + const found = findFirst(child, localName) + if (found) return found + } + return null +} + +function extractFaultMessage(envelope: XmlNode): string | null { + const fault = findFirst(envelope, 'Fault') + if (!fault) return null + const faultstring = findFirst(fault, 'faultstring') + if (faultstring?.text.trim()) return faultstring.text.trim() + const reason = findFirst(fault, 'Reason') + if (reason) { + const text = findFirst(reason, 'Text') + if (text?.text.trim()) return text.text.trim() + } + const detail = findFirst(fault, 'detail') ?? findFirst(fault, 'Detail') + if (detail) { + const msg = findFirst(detail, 'Validation_Error') ?? findFirst(detail, 'Detail_Message') + if (msg?.text.trim()) return msg.text.trim() + } + return 'SOAP fault returned by Workday' +} + +async function callOperation( + operation: WorkdayOperation, + args: Record, + endpoint: string, + username: string, + password: string, + signal?: AbortSignal +): Promise<[WorkdaySoapResult, string, Record, string]> { + signal?.throwIfAborted() + const envelope = buildEnvelope(operation, args, username, password) + + const response = await fetch(endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'text/xml; charset=utf-8', + SOAPAction: `""`, + }, + body: envelope, + signal, + }) + + const responseText = await readResponseTextWithLimit(response, { + maxBytes: WORKDAY_SOAP_RESPONSE_MAX_BYTES, + label: 'Workday SOAP response', + signal, + }) + signal?.throwIfAborted() + + let root: XmlNode + try { + root = parseXml(responseText) + } catch (err) { + logger.error('Failed to parse Workday SOAP response', { + operation, + status: response.status, + error: getErrorMessage(err), + }) + throw new Error( + `Workday returned an unparseable response (HTTP ${response.status}): ${responseText.slice(0, 500)}` + ) + } + + const fault = extractFaultMessage(root) + if (fault) { + throw new Error(fault) + } + + if (!response.ok) { + throw new Error(`Workday SOAP request failed (HTTP ${response.status})`) + } + + const responseElement = findFirst(root, `${operation}_Response`) + const value = (responseElement ? nodeToValue(responseElement) : {}) as WorkdaySoapResult + + return [value, responseText, {}, envelope] +} + +/** + * Creates a typed SOAP client for a Workday service. The returned object + * exposes the same `Async` methods the previous `soap`-library + * client did, so existing call sites do not change. Internally this issues + * SOAP-over-HTTP requests directly with hand-built envelopes and an XML + * response parser — no WSDL fetch. + */ +export async function createWorkdaySoapClient( + tenantUrl: string, + tenant: string, + service: WorkdayServiceKey, + username: string, + password: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const endpoint = buildServiceUrl(tenantUrl, tenant, service) + logger.info('Creating Workday SOAP client', { service, endpoint }) + + function bind(operation: WorkdayOperation): SoapOperationFn { + return (args) => callOperation(operation, args, endpoint, username, password, signal) + } + + return { + Get_WorkersAsync: bind('Get_Workers'), + Get_OrganizationsAsync: bind('Get_Organizations'), + Put_ApplicantAsync: bind('Put_Applicant'), + Hire_EmployeeAsync: bind('Hire_Employee'), + Change_JobAsync: bind('Change_Job'), + Terminate_EmployeeAsync: bind('Terminate_Employee'), + Change_Personal_InformationAsync: bind('Change_Personal_Information'), + Put_Onboarding_Plan_AssignmentAsync: bind('Put_Onboarding_Plan_Assignment'), + } +} + +/** + * Builds a Workday object reference in the format the SOAP API expects. + * Generates: { ID: { attributes: { 'wd:type': idType }, $value: idValue } } + */ +export function wdRef(idType: string, idValue: string): { ID: WorkdayIdEntry } { + return { + ID: { + attributes: { 'wd:type': idType }, + $value: idValue, + }, + } +} + +/** + * Extracts a reference ID from a SOAP response object. + * Handles the nested ID structure that Workday returns. + */ +export function extractRefId(ref: WorkdayReference | undefined): string | null { + if (!ref) return null + const id = ref.ID + if (Array.isArray(id)) { + return id[0]?.$value ?? id[0]?._ ?? null + } + if (id && typeof id === 'object') { + return id.$value ?? id._ ?? null + } + return null +} diff --git a/apps/sim/lib/internal/workday/errors.ts b/apps/sim/lib/internal/workday/errors.ts new file mode 100644 index 00000000000..b05aa39504f --- /dev/null +++ b/apps/sim/lib/internal/workday/errors.ts @@ -0,0 +1,9 @@ +export class WorkdayOperationError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'WorkdayOperationError' + } +} diff --git a/apps/sim/lib/internal/workday/execute-tool.test.ts b/apps/sim/lib/internal/workday/execute-tool.test.ts new file mode 100644 index 00000000000..e34c4237d29 --- /dev/null +++ b/apps/sim/lib/internal/workday/execute-tool.test.ts @@ -0,0 +1,203 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const operationMocks = vi.hoisted(() => ({ + executeWorkdayAssignOnboarding: vi.fn(), + executeWorkdayChangeJob: vi.fn(), + executeWorkdayCreatePrehire: vi.fn(), + executeWorkdayGetCompensation: vi.fn(), + executeWorkdayGetOrganizations: vi.fn(), + executeWorkdayGetWorker: vi.fn(), + executeWorkdayHire: vi.fn(), + executeWorkdayListWorkers: vi.fn(), + executeWorkdayTerminate: vi.fn(), + executeWorkdayUpdateWorker: vi.fn(), +})) + +vi.mock('@/lib/internal/workday/operations', () => operationMocks) + +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' +import { WorkdayOperationError } from '@/lib/internal/workday/errors' +import { executeWorkdayTool } from '@/lib/internal/workday/execute-tool' + +const CREDENTIALS = { + tenantUrl: 'https://wd2-impl-services1.workday.com', + tenant: 'example', + username: 'user', + password: 'not-a-real-password', +} + +function createRequest( + overrides: Partial = {} +): InternalToolOperationCall { + return { + toolId: 'workday_get_worker', + input: { ...CREDENTIALS, workerId: 'worker-1' }, + headers: new Headers({ 'content-type': 'application/json' }), + context: { + ...createExecutionContext({ workflowId: 'workflow-1' }), + workspaceId: 'workspace-1', + userId: 'user-1', + }, + requestId: 'request-1', + ...overrides, + } +} + +const TOOL_CASES = [ + [ + 'workday_assign_onboarding', + { ...CREDENTIALS, workerId: 'worker-1', onboardingPlanId: 'plan-1', actionEventId: 'event-1' }, + operationMocks.executeWorkdayAssignOnboarding, + ], + [ + 'workday_change_job', + { + ...CREDENTIALS, + workerId: 'worker-1', + effectiveDate: '2026-08-27', + reason: 'promotion', + }, + operationMocks.executeWorkdayChangeJob, + ], + [ + 'workday_create_prehire', + { ...CREDENTIALS, legalName: 'Ada Lovelace', email: 'ada@example.com' }, + operationMocks.executeWorkdayCreatePrehire, + ], + [ + 'workday_get_compensation', + { ...CREDENTIALS, workerId: 'worker-1' }, + operationMocks.executeWorkdayGetCompensation, + ], + ['workday_get_organizations', CREDENTIALS, operationMocks.executeWorkdayGetOrganizations], + [ + 'workday_get_worker', + { ...CREDENTIALS, workerId: 'worker-1' }, + operationMocks.executeWorkdayGetWorker, + ], + [ + 'workday_hire_employee', + { + ...CREDENTIALS, + preHireId: 'prehire-1', + positionId: 'position-1', + hireDate: '2026-08-27', + }, + operationMocks.executeWorkdayHire, + ], + ['workday_list_workers', CREDENTIALS, operationMocks.executeWorkdayListWorkers], + [ + 'workday_terminate_worker', + { + ...CREDENTIALS, + workerId: 'worker-1', + terminationDate: '2026-08-27', + reason: 'voluntary', + }, + operationMocks.executeWorkdayTerminate, + ], + [ + 'workday_update_worker', + { ...CREDENTIALS, workerId: 'worker-1', fields: { Preferred_Name: 'Ada' } }, + operationMocks.executeWorkdayUpdateWorker, + ], +] as const + +describe('executeWorkdayTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it.each(TOOL_CASES)('validates and dispatches %s', async (toolId, input, operation) => { + const controller = new AbortController() + operation.mockResolvedValue({ success: true, output: { toolId } }) + + const response = await executeWorkdayTool( + createRequest({ toolId, input, signal: controller.signal }) + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ success: true, output: { toolId } }) + expect(operation).toHaveBeenCalledWith(input, controller.signal) + }) + + it('returns the canonical validation envelope before provider work', async () => { + const response = await executeWorkdayTool( + createRequest({ input: { ...CREDENTIALS, workerId: '' } }) + ) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(operationMocks.executeWorkdayGetWorker).not.toHaveBeenCalled() + }) + + it('rejects non-object operation input', async () => { + const response = await executeWorkdayTool(createRequest({ input: '{' })) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Invalid request data', + details: expect.any(Array), + }) + expect(operationMocks.executeWorkdayGetWorker).not.toHaveBeenCalled() + }) + + it('preserves operation status and error envelopes', async () => { + operationMocks.executeWorkdayCreatePrehire.mockRejectedValue( + new WorkdayOperationError('Legal name must include both a first name and last name', 400) + ) + + const response = await executeWorkdayTool( + createRequest({ + toolId: 'workday_create_prehire', + input: { ...CREDENTIALS, legalName: 'Ada', email: 'ada@example.com' }, + }) + ) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Legal name must include both a first name and last name', + }) + }) + + it('preserves unexpected provider errors', async () => { + operationMocks.executeWorkdayGetWorker.mockRejectedValue(new Error('Workday unavailable')) + + const response = await executeWorkdayTool(createRequest()) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Workday unavailable', + }) + }) + + it('rejects unsupported Workday IDs without provider work', async () => { + const response = await executeWorkdayTool(createRequest({ toolId: 'workday_unknown' })) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Unsupported Workday tool: workday_unknown', + }) + expect(operationMocks.executeWorkdayGetWorker).not.toHaveBeenCalled() + }) + + it('propagates cancellation without starting provider work', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeWorkdayTool(createRequest({ signal: controller.signal })) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(operationMocks.executeWorkdayGetWorker).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/workday/execute-tool.ts b/apps/sim/lib/internal/workday/execute-tool.ts new file mode 100644 index 00000000000..20dcf3b7124 --- /dev/null +++ b/apps/sim/lib/internal/workday/execute-tool.ts @@ -0,0 +1,115 @@ +import { getErrorMessage } from '@sim/utils/errors' +import type { AnyApiRouteContract, ContractBody } from '@/lib/api/contracts' +import { + workdayAssignOnboardingContract, + workdayChangeJobContract, + workdayCreatePrehireContract, + workdayGetCompensationContract, + workdayGetOrganizationsContract, + workdayGetWorkerContract, + workdayHireContract, + workdayListWorkersContract, + workdayTerminateContract, + workdayUpdateWorkerContract, +} from '@/lib/api/contracts/tools/workday' +import { parseInternalToolInput } from '@/lib/internal/tool-operations/parse-input' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { WorkdayOperationError } from '@/lib/internal/workday/errors' +import { + executeWorkdayAssignOnboarding, + executeWorkdayChangeJob, + executeWorkdayCreatePrehire, + executeWorkdayGetCompensation, + executeWorkdayGetOrganizations, + executeWorkdayGetWorker, + executeWorkdayHire, + executeWorkdayListWorkers, + executeWorkdayTerminate, + executeWorkdayUpdateWorker, +} from '@/lib/internal/workday/operations' + +async function executeOperation( + contract: C, + input: unknown, + execute: (input: ContractBody, signal?: AbortSignal) => Promise, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const parsed = parseInternalToolInput(contract, input) + if (!parsed.success) return parsed.response + + try { + const result = await execute(parsed.data, signal) + signal?.throwIfAborted() + return Response.json(result) + } catch (error) { + signal?.throwIfAborted() + if (error instanceof WorkdayOperationError) { + return Response.json({ success: false, error: error.message }, { status: error.status }) + } + return Response.json( + { success: false, error: getErrorMessage(error, 'Unknown error') }, + { status: 500 } + ) + } +} + +export const executeWorkdayTool: InternalToolOperationHandler = async ({ + toolId, + input, + signal, +}) => { + signal?.throwIfAborted() + switch (toolId) { + case 'workday_assign_onboarding': + return executeOperation( + workdayAssignOnboardingContract, + input, + executeWorkdayAssignOnboarding, + signal + ) + case 'workday_change_job': + return executeOperation(workdayChangeJobContract, input, executeWorkdayChangeJob, signal) + case 'workday_create_prehire': + return executeOperation( + workdayCreatePrehireContract, + input, + executeWorkdayCreatePrehire, + signal + ) + case 'workday_get_compensation': + return executeOperation( + workdayGetCompensationContract, + input, + executeWorkdayGetCompensation, + signal + ) + case 'workday_get_organizations': + return executeOperation( + workdayGetOrganizationsContract, + input, + executeWorkdayGetOrganizations, + signal + ) + case 'workday_get_worker': + return executeOperation(workdayGetWorkerContract, input, executeWorkdayGetWorker, signal) + case 'workday_hire_employee': + return executeOperation(workdayHireContract, input, executeWorkdayHire, signal) + case 'workday_list_workers': + return executeOperation(workdayListWorkersContract, input, executeWorkdayListWorkers, signal) + case 'workday_terminate_worker': + return executeOperation(workdayTerminateContract, input, executeWorkdayTerminate, signal) + case 'workday_update_worker': + return executeOperation( + workdayUpdateWorkerContract, + input, + executeWorkdayUpdateWorker, + signal + ) + default: + return Response.json( + { success: false, error: `Unsupported Workday tool: ${toolId}` }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/workday/operations.test.ts b/apps/sim/lib/internal/workday/operations.test.ts new file mode 100644 index 00000000000..6e4fa779fe6 --- /dev/null +++ b/apps/sim/lib/internal/workday/operations.test.ts @@ -0,0 +1,355 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const clientMocks = vi.hoisted(() => ({ + Change_JobAsync: vi.fn(), + Change_Personal_InformationAsync: vi.fn(), + Get_OrganizationsAsync: vi.fn(), + Get_WorkersAsync: vi.fn(), + Hire_EmployeeAsync: vi.fn(), + Put_ApplicantAsync: vi.fn(), + Put_Onboarding_Plan_AssignmentAsync: vi.fn(), + Terminate_EmployeeAsync: vi.fn(), + createWorkdaySoapClient: vi.fn(), +})) + +vi.mock('@/lib/internal/workday/client', () => ({ + createWorkdaySoapClient: clientMocks.createWorkdaySoapClient, + extractRefId: (reference: { ID?: { $value?: string; _?: string } } | undefined) => + reference?.ID?.$value ?? reference?.ID?._ ?? null, + normalizeSoapArray: (value: T | T[] | undefined) => + value === undefined ? [] : Array.isArray(value) ? value : [value], + parseSoapBoolean: (value: unknown) => { + if (typeof value === 'boolean') return value + if (value === 'true' || value === '1') return true + if (value === 'false' || value === '0') return false + return null + }, + parseSoapNumber: (value: unknown) => { + if (value === null || value === undefined || value === '') return null + const number = Number(value) + return Number.isFinite(number) ? number : null + }, + wdRef: (idType: string, idValue: string) => ({ + ID: { attributes: { 'wd:type': idType }, $value: idValue }, + }), +})) + +import { WorkdayOperationError } from '@/lib/internal/workday/errors' +import { + executeWorkdayAssignOnboarding, + executeWorkdayChangeJob, + executeWorkdayCreatePrehire, + executeWorkdayGetCompensation, + executeWorkdayGetOrganizations, + executeWorkdayGetWorker, + executeWorkdayHire, + executeWorkdayListWorkers, + executeWorkdayTerminate, + executeWorkdayUpdateWorker, +} from '@/lib/internal/workday/operations' + +const CREDENTIALS = { + tenantUrl: 'https://wd2-impl-services1.workday.com', + tenant: 'example', + username: 'user', + password: 'not-a-real-password', +} + +const CLIENT = { + Change_JobAsync: clientMocks.Change_JobAsync, + Change_Personal_InformationAsync: clientMocks.Change_Personal_InformationAsync, + Get_OrganizationsAsync: clientMocks.Get_OrganizationsAsync, + Get_WorkersAsync: clientMocks.Get_WorkersAsync, + Hire_EmployeeAsync: clientMocks.Hire_EmployeeAsync, + Put_ApplicantAsync: clientMocks.Put_ApplicantAsync, + Put_Onboarding_Plan_AssignmentAsync: clientMocks.Put_Onboarding_Plan_AssignmentAsync, + Terminate_EmployeeAsync: clientMocks.Terminate_EmployeeAsync, +} + +describe('Workday operations', () => { + beforeEach(() => { + vi.clearAllMocks() + clientMocks.createWorkdaySoapClient.mockResolvedValue(CLIENT) + for (const operation of Object.values(CLIENT)) operation.mockResolvedValue([{}]) + }) + + it.each([ + { + name: 'get worker', + service: 'humanResources', + operation: clientMocks.Get_WorkersAsync, + execute: (signal: AbortSignal) => + executeWorkdayGetWorker({ ...CREDENTIALS, workerId: 'worker-1' }, signal), + }, + { + name: 'list workers', + service: 'humanResources', + operation: clientMocks.Get_WorkersAsync, + execute: (signal: AbortSignal) => executeWorkdayListWorkers(CREDENTIALS, signal), + }, + { + name: 'create prehire', + service: 'recruiting', + operation: clientMocks.Put_ApplicantAsync, + execute: (signal: AbortSignal) => + executeWorkdayCreatePrehire( + { ...CREDENTIALS, legalName: 'Ada Lovelace', email: 'ada@example.com' }, + signal + ), + }, + { + name: 'hire', + service: 'staffing', + operation: clientMocks.Hire_EmployeeAsync, + execute: (signal: AbortSignal) => + executeWorkdayHire( + { + ...CREDENTIALS, + preHireId: 'prehire-1', + positionId: 'position-1', + hireDate: '2026-08-27', + }, + signal + ), + }, + { + name: 'update worker', + service: 'humanResources', + operation: clientMocks.Change_Personal_InformationAsync, + execute: (signal: AbortSignal) => + executeWorkdayUpdateWorker( + { ...CREDENTIALS, workerId: 'worker-1', fields: { Preferred_Name: 'Ada' } }, + signal + ), + }, + { + name: 'assign onboarding', + service: 'humanResources', + operation: clientMocks.Put_Onboarding_Plan_AssignmentAsync, + execute: (signal: AbortSignal) => + executeWorkdayAssignOnboarding( + { + ...CREDENTIALS, + workerId: 'worker-1', + onboardingPlanId: 'plan-1', + actionEventId: 'event-1', + }, + signal + ), + }, + { + name: 'get organizations', + service: 'humanResources', + operation: clientMocks.Get_OrganizationsAsync, + execute: (signal: AbortSignal) => executeWorkdayGetOrganizations(CREDENTIALS, signal), + }, + { + name: 'change job', + service: 'staffing', + operation: clientMocks.Change_JobAsync, + execute: (signal: AbortSignal) => + executeWorkdayChangeJob( + { + ...CREDENTIALS, + workerId: 'worker-1', + effectiveDate: '2026-08-27', + reason: 'promotion', + }, + signal + ), + }, + { + name: 'get compensation', + service: 'humanResources', + operation: clientMocks.Get_WorkersAsync, + execute: (signal: AbortSignal) => + executeWorkdayGetCompensation({ ...CREDENTIALS, workerId: 'worker-1' }, signal), + }, + { + name: 'terminate', + service: 'staffing', + operation: clientMocks.Terminate_EmployeeAsync, + execute: (signal: AbortSignal) => + executeWorkdayTerminate( + { + ...CREDENTIALS, + workerId: 'worker-1', + terminationDate: '2026-08-27', + reason: 'voluntary', + }, + signal + ), + }, + ])('dispatches $name through the typed SOAP client', async ({ service, operation, execute }) => { + const controller = new AbortController() + + await execute(controller.signal) + + expect(clientMocks.createWorkdaySoapClient).toHaveBeenCalledWith( + CREDENTIALS.tenantUrl, + CREDENTIALS.tenant, + service, + CREDENTIALS.username, + CREDENTIALS.password, + controller.signal + ) + expect(operation).toHaveBeenCalledOnce() + }) + + it('maps singleton workers and preserves pagination semantics', async () => { + clientMocks.Get_WorkersAsync.mockResolvedValue([ + { + Response_Data: { + Worker: { + Worker_Reference: { ID: { $value: 'worker-1' } }, + Worker_Descriptor: 'Ada Lovelace', + Worker_Data: { + Personal_Data: { name: 'Ada' }, + Employment_Data: { status: 'active' }, + }, + }, + }, + Response_Results: { Total_Results: '42' }, + }, + ]) + + const result = await executeWorkdayListWorkers({ ...CREDENTIALS, limit: 10, offset: 20 }) + + expect(clientMocks.Get_WorkersAsync).toHaveBeenCalledWith({ + Response_Filter: { Page: 3, Count: 10 }, + Response_Group: { + Include_Reference: true, + Include_Personal_Information: true, + Include_Employment_Information: true, + }, + }) + expect(result).toEqual({ + success: true, + output: { + workers: [ + { + id: 'worker-1', + descriptor: 'Ada Lovelace', + personalData: { name: 'Ada' }, + employmentData: { status: 'active' }, + }, + ], + total: 42, + }, + }) + }) + + it('flattens and normalizes compensation plans', async () => { + clientMocks.Get_WorkersAsync.mockResolvedValue([ + { + Response_Data: { + Worker: { + Worker_Data: { + Compensation_Data: { + Employee_Base_Pay_Plan_Assignment_Data: { + Compensation_Plan_Reference: { + ID: { $value: 'base-plan' }, + attributes: { Descriptor: 'Base Pay' }, + }, + Amount: '125000', + Currency_Reference: { ID: { $value: 'USD' } }, + Frequency_Reference: { ID: { $value: 'Annual' } }, + }, + Employee_Bonus_Plan_Assignment_Data: [ + { + Compensation_Plan_Reference: { ID: { $value: 'bonus-plan' } }, + Individual_Target_Amount: '15000', + }, + ], + }, + }, + }, + }, + }, + ]) + + const result = await executeWorkdayGetCompensation({ + ...CREDENTIALS, + workerId: 'worker-1', + }) + + expect(result.output.compensationPlans).toEqual([ + { + id: 'base-plan', + planName: 'Base Pay', + amount: 125000, + currency: 'USD', + frequency: 'Annual', + }, + { + id: 'bonus-plan', + planName: null, + amount: 15000, + currency: null, + frequency: null, + }, + ]) + }) + + it('maps organization activity and total values', async () => { + clientMocks.Get_OrganizationsAsync.mockResolvedValue([ + { + Response_Data: { + Organization: { + Organization_Reference: { ID: { $value: 'org-1' } }, + Organization_Descriptor: 'Engineering', + Organization_Data: { + Organization_Type_Reference: { ID: { $value: 'Department' } }, + Organization_Subtype_Reference: { ID: { $value: 'Product' } }, + Inactive: 'false', + }, + }, + }, + Response_Results: { Total_Results: '1' }, + }, + ]) + + const result = await executeWorkdayGetOrganizations({ + ...CREDENTIALS, + type: 'Department', + }) + + expect(result.output).toEqual({ + organizations: [ + { + id: 'org-1', + descriptor: 'Engineering', + type: 'Department', + subtype: 'Product', + isActive: true, + }, + ], + total: 1, + }) + }) + + it('rejects invalid prehire contact data before creating a client', async () => { + await expect( + executeWorkdayCreatePrehire({ ...CREDENTIALS, legalName: 'Ada Lovelace' }) + ).rejects.toEqual( + new WorkdayOperationError( + 'At least one contact method (email, phone, or address) is required', + 400 + ) + ) + expect(clientMocks.createWorkdaySoapClient).not.toHaveBeenCalled() + }) + + it('propagates cancellation before client creation', async () => { + const controller = new AbortController() + controller.abort(new DOMException('cancelled', 'AbortError')) + + await expect( + executeWorkdayGetWorker({ ...CREDENTIALS, workerId: 'worker-1' }, controller.signal) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(clientMocks.createWorkdaySoapClient).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/workday/operations.ts b/apps/sim/lib/internal/workday/operations.ts new file mode 100644 index 00000000000..efa1487994a --- /dev/null +++ b/apps/sim/lib/internal/workday/operations.ts @@ -0,0 +1,424 @@ +import type { + WorkdayAssignOnboardingBody, + WorkdayChangeJobBody, + WorkdayCreatePrehireBody, + WorkdayGetCompensationBody, + WorkdayGetOrganizationsBody, + WorkdayGetWorkerBody, + WorkdayHireBody, + WorkdayListWorkersBody, + WorkdayTerminateBody, + WorkdayUpdateWorkerBody, +} from '@/lib/api/contracts/tools/workday' +import { + createWorkdaySoapClient, + extractRefId, + normalizeSoapArray, + parseSoapBoolean, + parseSoapNumber, + type WorkdayCompensationDataSoap, + type WorkdayCompensationPlanSoap, + type WorkdayOrganizationSoap, + type WorkdayWorkerSoap, + wdRef, +} from '@/lib/internal/workday/client' +import { WorkdayOperationError } from '@/lib/internal/workday/errors' + +interface WorkdayCredentials { + tenantUrl: string + tenant: string + username: string + password: string +} + +function createClient( + input: WorkdayCredentials, + service: Parameters[2], + signal?: AbortSignal +) { + signal?.throwIfAborted() + return createWorkdaySoapClient( + input.tenantUrl, + input.tenant, + service, + input.username, + input.password, + signal + ) +} + +function workerSummary(worker: WorkdayWorkerSoap) { + return { + id: extractRefId(worker.Worker_Reference) ?? null, + descriptor: worker.Worker_Descriptor ?? null, + personalData: worker.Worker_Data?.Personal_Data ?? null, + employmentData: worker.Worker_Data?.Employment_Data ?? null, + } +} + +export async function executeWorkdayGetWorker(input: WorkdayGetWorkerBody, signal?: AbortSignal) { + const client = await createClient(input, 'humanResources', signal) + const [result] = await client.Get_WorkersAsync({ + Request_References: { + Worker_Reference: { + ID: { attributes: { 'wd:type': 'Employee_ID' }, $value: input.workerId }, + }, + }, + Response_Group: { + Include_Reference: true, + Include_Personal_Information: true, + Include_Employment_Information: true, + Include_Compensation: true, + Include_Organizations: true, + }, + }) + signal?.throwIfAborted() + const worker = + normalizeSoapArray( + result?.Response_Data?.Worker as WorkdayWorkerSoap | WorkdayWorkerSoap[] | undefined + )[0] ?? null + return { + success: true as const, + output: { + worker: worker + ? { + ...workerSummary(worker), + compensationData: worker.Worker_Data?.Compensation_Data ?? null, + organizationData: worker.Worker_Data?.Organization_Data ?? null, + } + : null, + }, + } +} + +export async function executeWorkdayListWorkers( + input: WorkdayListWorkersBody, + signal?: AbortSignal +) { + const client = await createClient(input, 'humanResources', signal) + const limit = input.limit ?? 20 + const offset = input.offset ?? 0 + const page = offset > 0 ? Math.floor(offset / limit) + 1 : 1 + const [result] = await client.Get_WorkersAsync({ + Response_Filter: { Page: page, Count: limit }, + Response_Group: { + Include_Reference: true, + Include_Personal_Information: true, + Include_Employment_Information: true, + }, + }) + signal?.throwIfAborted() + const workers = normalizeSoapArray( + result?.Response_Data?.Worker as WorkdayWorkerSoap | WorkdayWorkerSoap[] | undefined + ).map(workerSummary) + const total = parseSoapNumber(result?.Response_Results?.Total_Results) ?? workers.length + return { success: true as const, output: { workers, total } } +} + +export async function executeWorkdayCreatePrehire( + input: WorkdayCreatePrehireBody, + signal?: AbortSignal +) { + if (!input.email && !input.phoneNumber && !input.address) { + throw new WorkdayOperationError( + 'At least one contact method (email, phone, or address) is required', + 400 + ) + } + const parts = input.legalName.trim().split(/\s+/) + const firstName = parts[0] ?? '' + const lastName = parts.length > 1 ? parts.slice(1).join(' ') : '' + if (!lastName) { + throw new WorkdayOperationError('Legal name must include both a first name and last name', 400) + } + + const contactData: Record = {} + if (input.email) { + contactData.Email_Address_Data = [ + { + Email_Address: input.email, + Usage_Data: { + Type_Data: { Type_Reference: wdRef('Communication_Usage_Type_ID', 'WORK') }, + Public: true, + }, + }, + ] + } + if (input.phoneNumber) { + contactData.Phone_Data = [ + { + Phone_Number: input.phoneNumber, + Phone_Device_Type_Reference: wdRef('Phone_Device_Type_ID', 'Landline'), + Usage_Data: { + Type_Data: { Type_Reference: wdRef('Communication_Usage_Type_ID', 'WORK') }, + Public: true, + }, + }, + ] + } + if (input.address) { + contactData.Address_Data = [ + { + Formatted_Address: input.address, + Usage_Data: { + Type_Data: { Type_Reference: wdRef('Communication_Usage_Type_ID', 'WORK') }, + Public: true, + }, + }, + ] + } + + const client = await createClient(input, 'recruiting', signal) + const [result] = await client.Put_ApplicantAsync({ + Applicant_Data: { + Personal_Data: { + Name_Data: { + Legal_Name_Data: { + Name_Detail_Data: { + Country_Reference: wdRef('ISO_3166-1_Alpha-2_Code', input.countryCode ?? 'US'), + First_Name: firstName, + Last_Name: lastName, + }, + }, + }, + Contact_Information_Data: contactData, + }, + }, + }) + signal?.throwIfAborted() + const applicantRef = result?.Applicant_Reference + return { + success: true as const, + output: { + preHireId: extractRefId(applicantRef), + descriptor: applicantRef?.attributes?.Descriptor ?? null, + }, + } +} + +export async function executeWorkdayHire(input: WorkdayHireBody, signal?: AbortSignal) { + const client = await createClient(input, 'staffing', signal) + const [result] = await client.Hire_EmployeeAsync({ + Business_Process_Parameters: { Auto_Complete: true, Run_Now: true }, + Hire_Employee_Data: { + Applicant_Reference: wdRef('Applicant_ID', input.preHireId), + Position_Reference: wdRef('Position_ID', input.positionId), + Hire_Date: input.hireDate, + Hire_Employee_Event_Data: { + Employee_Type_Reference: wdRef('Employee_Type_ID', input.employeeType ?? 'Regular'), + First_Day_of_Work: input.hireDate, + }, + }, + }) + signal?.throwIfAborted() + const employeeRef = result?.Employee_Reference + return { + success: true as const, + output: { + workerId: extractRefId(employeeRef), + employeeId: extractRefId(employeeRef), + eventId: extractRefId(result?.Event_Reference), + hireDate: input.hireDate, + }, + } +} + +export async function executeWorkdayUpdateWorker( + input: WorkdayUpdateWorkerBody, + signal?: AbortSignal +) { + const client = await createClient(input, 'humanResources', signal) + const [result] = await client.Change_Personal_InformationAsync({ + Business_Process_Parameters: { Auto_Complete: true, Run_Now: true }, + Change_Personal_Information_Business_Process_Data: { + Person_Reference: wdRef('Employee_ID', input.workerId), + Personal_Information_Data: input.fields, + }, + }) + signal?.throwIfAborted() + return { + success: true as const, + output: { + eventId: extractRefId(result?.Personal_Information_Change_Event_Reference), + workerId: input.workerId, + }, + } +} + +export async function executeWorkdayAssignOnboarding( + input: WorkdayAssignOnboardingBody, + signal?: AbortSignal +) { + const client = await createClient(input, 'humanResources', signal) + const [result] = await client.Put_Onboarding_Plan_AssignmentAsync({ + Onboarding_Plan_Assignment_Data: { + Onboarding_Plan_Reference: wdRef('Onboarding_Plan_ID', input.onboardingPlanId), + Person_Reference: wdRef('WID', input.workerId), + Action_Event_Reference: wdRef('WID', input.actionEventId), + Assignment_Effective_Moment: new Date().toISOString(), + Active: true, + }, + }) + signal?.throwIfAborted() + return { + success: true as const, + output: { + assignmentId: extractRefId(result?.Onboarding_Plan_Assignment_Reference), + workerId: input.workerId, + planId: input.onboardingPlanId, + }, + } +} + +export async function executeWorkdayGetOrganizations( + input: WorkdayGetOrganizationsBody, + signal?: AbortSignal +) { + const client = await createClient(input, 'humanResources', signal) + const limit = input.limit ?? 20 + const offset = input.offset ?? 0 + const page = offset > 0 ? Math.floor(offset / limit) + 1 : 1 + const [result] = await client.Get_OrganizationsAsync({ + Response_Filter: { Page: page, Count: limit }, + Request_Criteria: input.type + ? { + Organization_Type_Reference: { + ID: { attributes: { 'wd:type': 'Organization_Type_ID' }, $value: input.type }, + }, + } + : undefined, + Response_Group: { Include_Hierarchy_Data: true }, + }) + signal?.throwIfAborted() + const organizations = normalizeSoapArray( + result?.Response_Data?.Organization as + | WorkdayOrganizationSoap + | WorkdayOrganizationSoap[] + | undefined + ).map((organization) => { + const inactive = parseSoapBoolean(organization.Organization_Data?.Inactive) + return { + id: extractRefId(organization.Organization_Reference) ?? null, + descriptor: organization.Organization_Descriptor ?? null, + type: extractRefId(organization.Organization_Data?.Organization_Type_Reference) ?? null, + subtype: extractRefId(organization.Organization_Data?.Organization_Subtype_Reference) ?? null, + isActive: inactive == null ? null : !inactive, + } + }) + const total = parseSoapNumber(result?.Response_Results?.Total_Results) ?? organizations.length + return { success: true as const, output: { organizations, total } } +} + +export async function executeWorkdayChangeJob(input: WorkdayChangeJobBody, signal?: AbortSignal) { + const changeJobDetailData: Record = { + Reason_Reference: wdRef('Change_Job_Subcategory_ID', input.reason), + } + if (input.newSupervisoryOrgId) { + changeJobDetailData.Supervisory_Organization_Reference = wdRef( + 'Organization_Reference_ID', + input.newSupervisoryOrgId + ) + } + if (input.newPositionId) { + changeJobDetailData.Proposed_Position_Reference = wdRef('Position_ID', input.newPositionId) + } + const jobDetailsData: Record = {} + if (input.newJobProfileId) { + jobDetailsData.Job_Profile_Reference = wdRef('Job_Profile_ID', input.newJobProfileId) + } + if (input.newLocationId) { + jobDetailsData.Location_Reference = wdRef('Location_ID', input.newLocationId) + } + if (Object.keys(jobDetailsData).length > 0) { + changeJobDetailData.Job_Details_Data = jobDetailsData + } + + const client = await createClient(input, 'staffing', signal) + const [result] = await client.Change_JobAsync({ + Business_Process_Parameters: { Auto_Complete: true, Run_Now: true }, + Change_Job_Data: { + Worker_Reference: wdRef('Employee_ID', input.workerId), + Effective_Date: input.effectiveDate, + Change_Job_Detail_Data: changeJobDetailData, + }, + }) + signal?.throwIfAborted() + return { + success: true as const, + output: { + eventId: extractRefId(result?.Event_Reference), + workerId: input.workerId, + effectiveDate: input.effectiveDate, + }, + } +} + +export async function executeWorkdayGetCompensation( + input: WorkdayGetCompensationBody, + signal?: AbortSignal +) { + const client = await createClient(input, 'humanResources', signal) + const [result] = await client.Get_WorkersAsync({ + Request_References: { + Worker_Reference: { + ID: { attributes: { 'wd:type': 'Employee_ID' }, $value: input.workerId }, + }, + }, + Response_Group: { Include_Reference: true, Include_Compensation: true }, + }) + signal?.throwIfAborted() + const worker = + normalizeSoapArray( + result?.Response_Data?.Worker as WorkdayWorkerSoap | WorkdayWorkerSoap[] | undefined + )[0] ?? null + const compensationData = worker?.Worker_Data?.Compensation_Data + const mapPlan = (plan: WorkdayCompensationPlanSoap) => ({ + id: extractRefId(plan.Compensation_Plan_Reference) ?? null, + planName: plan.Compensation_Plan_Reference?.attributes?.Descriptor ?? null, + amount: + parseSoapNumber(plan.Amount) ?? + parseSoapNumber(plan.Per_Unit_Amount) ?? + parseSoapNumber(plan.Individual_Target_Amount) ?? + null, + currency: extractRefId(plan.Currency_Reference) ?? null, + frequency: extractRefId(plan.Frequency_Reference) ?? null, + }) + const planTypeKeys: (keyof WorkdayCompensationDataSoap)[] = [ + 'Employee_Base_Pay_Plan_Assignment_Data', + 'Employee_Salary_Unit_Plan_Assignment_Data', + 'Employee_Bonus_Plan_Assignment_Data', + 'Employee_Allowance_Plan_Assignment_Data', + 'Employee_Commission_Plan_Assignment_Data', + 'Employee_Stock_Plan_Assignment_Data', + 'Employee_Period_Salary_Plan_Assignment_Data', + ] + const compensationPlans = planTypeKeys.flatMap((key) => + normalizeSoapArray(compensationData?.[key]).map(mapPlan) + ) + return { success: true as const, output: { compensationPlans } } +} + +export async function executeWorkdayTerminate(input: WorkdayTerminateBody, signal?: AbortSignal) { + const client = await createClient(input, 'staffing', signal) + const [result] = await client.Terminate_EmployeeAsync({ + Business_Process_Parameters: { Auto_Complete: true, Run_Now: true }, + Terminate_Employee_Data: { + Employee_Reference: wdRef('Employee_ID', input.workerId), + Termination_Date: input.terminationDate, + Terminate_Event_Data: { + Primary_Reason_Reference: wdRef('Termination_Subcategory_ID', input.reason), + Last_Day_of_Work: input.lastDayOfWork ?? input.terminationDate, + Notification_Date: input.notificationDate ?? input.terminationDate, + }, + }, + }) + signal?.throwIfAborted() + return { + success: true as const, + output: { + eventId: extractRefId(result?.Event_Reference), + workerId: input.workerId, + terminationDate: input.terminationDate, + }, + } +} diff --git a/apps/sim/lib/internal/workflows/read-definition.test.ts b/apps/sim/lib/internal/workflows/read-definition.test.ts new file mode 100644 index 00000000000..fe725bac4e1 --- /dev/null +++ b/apps/sim/lib/internal/workflows/read-definition.test.ts @@ -0,0 +1,149 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockBindInternalExecutorDelegation, mockReadWorkflowDefinition } = vi.hoisted(() => ({ + mockBindInternalExecutorDelegation: vi.fn(), + mockReadWorkflowDefinition: vi.fn(), +})) + +vi.mock('@/lib/auth/internal-delegation', () => ({ + bindInternalExecutorDelegation: mockBindInternalExecutorDelegation, +})) + +vi.mock('@/lib/workflows/application/read-workflow-definition', () => ({ + readWorkflowDefinition: { execute: mockReadWorkflowDefinition }, +})) + +import { readWorkflowDefinitionAsExecutor } from '@/lib/internal/workflows/read-definition' +import { WORKFLOW_DELEGATION_AUDIENCE } from '@/lib/workflows/application/authorization' + +describe('readWorkflowDefinitionAsExecutor', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('binds the trusted workflow execution origin before reading the child', async () => { + const principal = { + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'workspace-1', + delegationId: 'delegation-1', + audience: WORKFLOW_DELEGATION_AUDIENCE, + issuedAt: new Date(), + expiresAt: new Date(Date.now() + 60_000), + delegationContext: { + kind: 'workflow_execution', + workflowId: 'parent-workflow', + executionId: 'execution-1', + }, + } + const definition = { workflow: { id: 'child-workflow' }, state: { blocks: {} } } + mockBindInternalExecutorDelegation.mockResolvedValue(principal) + mockReadWorkflowDefinition.mockResolvedValue(definition) + + const result = await readWorkflowDefinitionAsExecutor({ + origin: { + subjectUserId: 'user-1', + workflowId: 'parent-workflow', + executionId: 'execution-1', + }, + workflowId: 'child-workflow', + state: 'deployed', + }) + + expect(result).toBe(definition) + expect(mockBindInternalExecutorDelegation).toHaveBeenCalledWith( + expect.objectContaining({ + serviceId: 'executor', + subjectUserId: 'user-1', + workflowId: 'parent-workflow', + executionId: 'execution-1', + delegationId: expect.any(String), + issuedAt: expect.any(Date), + expiresAt: expect.any(Date), + }), + { audience: WORKFLOW_DELEGATION_AUDIENCE } + ) + expect(mockReadWorkflowDefinition).toHaveBeenCalledWith({ + principal, + input: { workflowId: 'child-workflow', state: 'deployed' }, + }) + }) + + it('preserves an actorless principal and current workflow authority', async () => { + const sourcePrincipal = { + kind: 'system' as const, + serviceId: 'internal' as const, + workspaceId: 'workspace-1', + workflowId: 'parent-workflow', + } + const delegatedPrincipal = { + kind: 'delegated' as const, + serviceId: 'executor' as const, + workspaceId: 'workspace-1', + delegationId: 'delegation-1', + audience: WORKFLOW_DELEGATION_AUDIENCE, + issuedAt: new Date(), + expiresAt: new Date(Date.now() + 60_000), + delegationContext: { + kind: 'workflow_execution' as const, + workflowId: 'parent-workflow', + executionId: 'execution-1', + principal: sourcePrincipal, + currentWorkflow: { + workflowId: 'parent-workflow', + mode: 'draft' as const, + }, + }, + } + mockBindInternalExecutorDelegation.mockResolvedValue(delegatedPrincipal) + mockReadWorkflowDefinition.mockResolvedValue({ workflow: {}, state: null }) + + await readWorkflowDefinitionAsExecutor({ + origin: { + workflowId: 'parent-workflow', + executionId: 'execution-1', + principal: sourcePrincipal, + currentWorkflow: { workflowId: 'parent-workflow', mode: 'draft' }, + }, + workflowId: 'child-workflow', + state: 'draft', + }) + + expect(mockBindInternalExecutorDelegation).toHaveBeenCalledWith( + expect.objectContaining({ + serviceId: 'executor', + workflowId: 'parent-workflow', + executionId: 'execution-1', + principal: sourcePrincipal, + currentWorkflow: { workflowId: 'parent-workflow', mode: 'draft' }, + }), + { audience: WORKFLOW_DELEGATION_AUDIENCE } + ) + expect(mockBindInternalExecutorDelegation.mock.calls[0][0]).not.toHaveProperty('subjectUserId') + expect(mockReadWorkflowDefinition).toHaveBeenCalledWith({ + principal: delegatedPrincipal, + input: { workflowId: 'child-workflow', state: 'draft' }, + }) + }) + + it('rejects a subject that conflicts with the preserved workflow principal', async () => { + await expect( + readWorkflowDefinitionAsExecutor({ + origin: { + subjectUserId: 'user-2', + workflowId: 'parent-workflow', + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + }, + workflowId: 'child-workflow', + state: 'draft', + }) + ).rejects.toThrow('Executor subject does not match its workflow principal') + + expect(mockBindInternalExecutorDelegation).not.toHaveBeenCalled() + expect(mockReadWorkflowDefinition).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/workflows/read-definition.ts b/apps/sim/lib/internal/workflows/read-definition.ts new file mode 100644 index 00000000000..96d0bcb4cac --- /dev/null +++ b/apps/sim/lib/internal/workflows/read-definition.ts @@ -0,0 +1,84 @@ +import { + resolvePrincipalSubject, + type WorkflowExecutionAuthority, + type WorkflowExecutionPrincipal, +} from '@sim/auth/principal' +import { generateId } from '@sim/utils/id' +import { bindInternalExecutorDelegation } from '@/lib/auth/internal-delegation' +import { WORKFLOW_DELEGATION_AUDIENCE } from '@/lib/workflows/application/authorization' +import { + type ReadWorkflowDefinitionInput, + readWorkflowDefinition, +} from '@/lib/workflows/application/read-workflow-definition' + +export interface ExecutorWorkflowDefinitionOrigin { + subjectUserId?: string + workflowId: string + executionId?: string + principal?: WorkflowExecutionPrincipal + currentWorkflow?: WorkflowExecutionAuthority +} + +export interface ReadWorkflowDefinitionAsExecutorInput { + origin: ExecutorWorkflowDefinitionOrigin + workflowId: string + state: ReadWorkflowDefinitionInput['state'] +} + +const EXECUTOR_DELEGATION_TTL_MS = 5 * 60 * 1000 + +function resolveExecutorSubject(origin: ExecutorWorkflowDefinitionOrigin): string | undefined { + const principalSubject = origin.principal ? resolvePrincipalSubject(origin.principal) : null + if (principalSubject?.kind === 'external_user' && origin.subjectUserId) { + throw new Error('External workflow subjects cannot be represented as Sim users') + } + if (!principalSubject && origin.principal && origin.subjectUserId) { + throw new Error('Actorless workflow principals cannot be represented as Sim users') + } + if ( + principalSubject?.kind === 'sim_user' && + origin.subjectUserId && + origin.subjectUserId !== principalSubject.userId + ) { + throw new Error('Executor subject does not match its workflow principal') + } + + const subjectUserId = + principalSubject?.kind === 'sim_user' ? principalSubject.userId : origin.subjectUserId + if (!subjectUserId && !origin.principal) { + throw new Error('Executor workflow definition read requires a workflow principal or subject') + } + return subjectUserId +} + +async function createWorkflowDefinitionExecutorPrincipal(origin: ExecutorWorkflowDefinitionOrigin) { + const issuedAt = new Date() + const subjectUserId = resolveExecutorSubject(origin) + return bindInternalExecutorDelegation( + { + serviceId: 'executor', + ...(subjectUserId ? { subjectUserId } : {}), + workflowId: origin.workflowId, + ...(origin.executionId ? { executionId: origin.executionId } : {}), + ...(origin.principal ? { principal: origin.principal } : {}), + ...(origin.currentWorkflow ? { currentWorkflow: origin.currentWorkflow } : {}), + delegationId: generateId(), + issuedAt, + expiresAt: new Date(issuedAt.getTime() + EXECUTOR_DELEGATION_TTL_MS), + }, + { audience: WORKFLOW_DELEGATION_AUDIENCE } + ) +} + +export async function readWorkflowDefinitionAsExecutor({ + origin, + workflowId, + state, +}: ReadWorkflowDefinitionAsExecutorInput) { + const principal = await createWorkflowDefinitionExecutorPrincipal(origin) + + return readWorkflowDefinition.execute({ + principal, + input: { workflowId, state }, + }) +} diff --git a/apps/sim/lib/internal/workflows/read-tool-enrichment.test.ts b/apps/sim/lib/internal/workflows/read-tool-enrichment.test.ts new file mode 100644 index 00000000000..868714d5cc6 --- /dev/null +++ b/apps/sim/lib/internal/workflows/read-tool-enrichment.test.ts @@ -0,0 +1,125 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockReadWorkflowDefinitionAsExecutor } = vi.hoisted(() => ({ + mockReadWorkflowDefinitionAsExecutor: vi.fn(), +})) + +vi.mock('@/lib/internal/workflows/read-definition', () => ({ + readWorkflowDefinitionAsExecutor: mockReadWorkflowDefinitionAsExecutor, +})) + +import { + readWorkflowInputFieldsForTool, + readWorkflowMetadataForTool, +} from '@/lib/internal/workflows/read-tool-enrichment' + +describe('workflow tool enrichment authority', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('derives target draft authority from the verified human execution principal', async () => { + mockReadWorkflowDefinitionAsExecutor.mockResolvedValue({ + workflow: { name: 'Child workflow', description: 'Runs the child' }, + state: { blocks: {} }, + }) + + await expect( + readWorkflowMetadataForTool('child-workflow', { + userId: 'billing-owner', + workflowId: 'parent-workflow', + executionId: 'execution-1', + executorDelegationOrigin: { + workflowId: 'parent-workflow', + executionId: 'execution-1', + principal: { kind: 'session', userId: 'actual-user', sessionId: 'session-1' }, + currentWorkflow: { workflowId: 'parent-workflow', mode: 'draft' }, + }, + }) + ).resolves.toEqual({ name: 'Child workflow', description: 'Runs the child' }) + + expect(mockReadWorkflowDefinitionAsExecutor).toHaveBeenCalledWith({ + origin: { subjectUserId: 'actual-user', workflowId: 'child-workflow' }, + workflowId: 'child-workflow', + state: 'draft', + }) + }) + + it('preserves deployed authority instead of reinterpreting a compatibility user as actor', async () => { + mockReadWorkflowDefinitionAsExecutor.mockResolvedValue({ + workflow: { name: 'Child workflow', description: null }, + state: { blocks: {} }, + }) + const principal = { + kind: 'system' as const, + serviceId: 'schedule' as const, + workspaceId: 'workspace-1', + workflowId: 'parent-workflow', + } + const currentWorkflow = { + workflowId: 'parent-workflow', + mode: 'deployment' as const, + deploymentVersionId: 'deployment-1', + } + + await expect( + readWorkflowInputFieldsForTool('child-workflow', { + userId: 'billing-owner', + workflowId: 'parent-workflow', + executionId: 'execution-1', + executorDelegationOrigin: { + workflowId: 'parent-workflow', + executionId: 'execution-1', + principal, + currentWorkflow, + }, + }) + ).resolves.toEqual([]) + + expect(mockReadWorkflowDefinitionAsExecutor).toHaveBeenCalledWith({ + origin: { + workflowId: 'parent-workflow', + executionId: 'execution-1', + principal, + currentWorkflow, + }, + workflowId: 'child-workflow', + state: 'deployed', + }) + }) + + it('rejects actorless draft enrichment', async () => { + await expect( + readWorkflowMetadataForTool('child-workflow', { + userId: 'billing-owner', + workflowId: 'parent-workflow', + executorDelegationOrigin: { + workflowId: 'parent-workflow', + principal: { + kind: 'system', + serviceId: 'internal', + workspaceId: 'workspace-1', + workflowId: 'parent-workflow', + }, + currentWorkflow: { workflowId: 'parent-workflow', mode: 'draft' }, + }, + }) + ).rejects.toThrow('Actorless workflow enrichment requires deployed execution authority') + + expect(mockReadWorkflowDefinitionAsExecutor).not.toHaveBeenCalled() + }) + + it('fails closed when execution authority is absent', async () => { + await expect( + readWorkflowMetadataForTool('child-workflow', { + userId: 'billing-owner', + workflowId: 'parent-workflow', + }) + ).rejects.toThrow('Workflow enrichment requires trusted execution authority') + + expect(mockReadWorkflowDefinitionAsExecutor).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/internal/workflows/read-tool-enrichment.ts b/apps/sim/lib/internal/workflows/read-tool-enrichment.ts new file mode 100644 index 00000000000..9ec04b735ff --- /dev/null +++ b/apps/sim/lib/internal/workflows/read-tool-enrichment.ts @@ -0,0 +1,49 @@ +import { resolveExecutorOriginSubject } from '@/lib/internal/principals/executor' +import { readWorkflowDefinitionAsExecutor } from '@/lib/internal/workflows/read-definition' +import { extractInputFieldsFromBlocks } from '@/lib/workflows/input-format' +import type { ExecutorDelegationOrigin } from '@/executor/types' + +export interface WorkflowToolEnrichmentContext { + userId?: string + workflowId?: string + executionId?: string + executorDelegationOrigin?: ExecutorDelegationOrigin +} + +async function readWorkflowForTool(workflowId: string, context: WorkflowToolEnrichmentContext) { + const origin = context.executorDelegationOrigin + if (!origin) { + throw new Error('Workflow enrichment requires trusted execution authority') + } + const subjectUserId = resolveExecutorOriginSubject(origin) + if (subjectUserId) { + return readWorkflowDefinitionAsExecutor({ + origin: { subjectUserId, workflowId }, + workflowId, + state: 'draft', + }) + } + if (origin.currentWorkflow?.mode !== 'deployment') { + throw new Error('Actorless workflow enrichment requires deployed execution authority') + } + return readWorkflowDefinitionAsExecutor({ origin, workflowId, state: 'deployed' }) +} + +export async function readWorkflowMetadataForTool( + workflowId: string, + context: WorkflowToolEnrichmentContext +): Promise<{ name: string; description: string | null }> { + const { workflow } = await readWorkflowForTool(workflowId, context) + return { + name: workflow.name || 'Workflow', + description: workflow.description || null, + } +} + +export async function readWorkflowInputFieldsForTool( + workflowId: string, + context: WorkflowToolEnrichmentContext +): Promise> { + const { state } = await readWorkflowForTool(workflowId, context) + return extractInputFieldsFromBlocks(state?.blocks ?? {}) +} diff --git a/apps/sim/lib/internal/zoho-desk/errors.ts b/apps/sim/lib/internal/zoho-desk/errors.ts new file mode 100644 index 00000000000..a8a099eac19 --- /dev/null +++ b/apps/sim/lib/internal/zoho-desk/errors.ts @@ -0,0 +1,9 @@ +export class ZohoDeskOperationError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'ZohoDeskOperationError' + } +} diff --git a/apps/sim/lib/internal/zoho-desk/execute-tool.test.ts b/apps/sim/lib/internal/zoho-desk/execute-tool.test.ts new file mode 100644 index 00000000000..907d649af75 --- /dev/null +++ b/apps/sim/lib/internal/zoho-desk/execute-tool.test.ts @@ -0,0 +1,60 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ getZohoDeskAttachment: vi.fn() })) + +vi.mock('@/lib/internal/zoho-desk/operations', () => ({ + getZohoDeskAttachment: mocks.getZohoDeskAttachment, + MAX_ZOHO_DESK_ATTACHMENT_BYTES: 7 * 1024 * 1024, +})) + +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' +import { ZohoDeskOperationError } from '@/lib/internal/zoho-desk/errors' +import { executeZohoDeskTool } from '@/lib/internal/zoho-desk/execute-tool' + +function request(overrides: Partial = {}): InternalToolOperationCall { + return { + toolId: 'zoho_desk_get_attachment', + input: { + accessToken: 'token', + orgId: 'org-1', + href: 'https://desk.zoho.com/api/v1/tickets/1/attachments/2/content', + }, + headers: new Headers(), + context: createExecutionContext({ workflowId: 'workflow-1' }), + requestId: 'request-1', + ...overrides, + } +} + +describe('executeZohoDeskTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.getZohoDeskAttachment.mockResolvedValue({ + success: true, + output: { file: { name: 'file.pdf', mimeType: 'application/pdf', data: 'YQ==' } }, + }) + }) + + it('dispatches the typed operation with cancellation', async () => { + const controller = new AbortController() + const response = await executeZohoDeskTool(request({ signal: controller.signal })) + + expect(response.status).toBe(200) + expect(mocks.getZohoDeskAttachment).toHaveBeenCalledWith( + expect.objectContaining({ orgId: 'org-1' }), + { signal: controller.signal } + ) + }) + + it('preserves operation status', async () => { + mocks.getZohoDeskAttachment.mockRejectedValue( + new ZohoDeskOperationError('Invalid attachment href', 400) + ) + const response = await executeZohoDeskTool(request()) + expect(response.status).toBe(400) + }) +}) diff --git a/apps/sim/lib/internal/zoho-desk/execute-tool.ts b/apps/sim/lib/internal/zoho-desk/execute-tool.ts new file mode 100644 index 00000000000..0af245bbe06 --- /dev/null +++ b/apps/sim/lib/internal/zoho-desk/execute-tool.ts @@ -0,0 +1,54 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { z } from 'zod' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { ZohoDeskOperationError } from '@/lib/internal/zoho-desk/errors' +import { + getZohoDeskAttachment, + MAX_ZOHO_DESK_ATTACHMENT_BYTES, +} from '@/lib/internal/zoho-desk/operations' + +const inputSchema = z.object({ + accessToken: z.string().min(1), + apiDomain: z.string().optional(), + orgId: z.string().min(1), + href: z.string().min(1), + fileName: z.string().optional(), +}) + +export const executeZohoDeskTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'zoho_desk_get_attachment') { + return Response.json( + { success: false, error: `Unsupported Zoho Desk tool: ${request.toolId}` }, + { status: 500 } + ) + } + const parsed = inputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + try { + return Response.json( + await getZohoDeskAttachment(parsed.data, { + signal: request.signal, + }) + ) + } catch (error) { + request.signal?.throwIfAborted() + if (isPayloadSizeLimitError(error)) { + return Response.json( + { + success: false, + error: `Attachment exceeds the ${Math.floor(MAX_ZOHO_DESK_ATTACHMENT_BYTES / (1024 * 1024))} MB download limit`, + }, + { status: 413 } + ) + } + const status = error instanceof ZohoDeskOperationError ? error.status : 500 + return Response.json( + { success: false, error: getErrorMessage(error, 'Failed to download attachment') }, + { status } + ) + } +} diff --git a/apps/sim/lib/internal/zoho-desk/operations.ts b/apps/sim/lib/internal/zoho-desk/operations.ts new file mode 100644 index 00000000000..21279980c70 --- /dev/null +++ b/apps/sim/lib/internal/zoho-desk/operations.ts @@ -0,0 +1,75 @@ +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' +import { ZohoDeskOperationError } from '@/lib/internal/zoho-desk/errors' +import { isZohoHost } from '@/tools/zoho_desk/host-allowlist' +import type { ZohoDeskGetAttachmentParams } from '@/tools/zoho_desk/types' +import { + buildZohoDeskHeaders, + deriveAttachmentName, + getZohoDeskApiBase, + resolveZohoAttachmentUrl, +} from '@/tools/zoho_desk/utils' + +export const MAX_ZOHO_DESK_ATTACHMENT_BYTES = 7 * 1024 * 1024 + +export interface ZohoDeskOperationContext { + signal?: AbortSignal +} + +export async function getZohoDeskAttachment( + input: ZohoDeskGetAttachmentParams, + context: ZohoDeskOperationContext +): Promise<{ + success: true + output: { file: { data: string; mimeType: string; name: string } } +}> { + context.signal?.throwIfAborted() + let downloadUrl: URL + try { + downloadUrl = resolveZohoAttachmentUrl( + input.href, + getZohoDeskApiBase({ apiDomain: input.apiDomain }) + ) + } catch { + throw new ZohoDeskOperationError('Invalid attachment href', 400) + } + if (downloadUrl.protocol !== 'https:' || !isZohoHost(downloadUrl.hostname)) { + throw new ZohoDeskOperationError('Attachment href must be an https Zoho URL', 400) + } + + const response = await secureFetchWithValidation(downloadUrl.toString(), { + method: 'GET', + headers: buildZohoDeskHeaders({ accessToken: input.accessToken, orgId: input.orgId }), + timeout: 30_000, + maxResponseBytes: MAX_ZOHO_DESK_ATTACHMENT_BYTES, + stripAuthOnRedirect: true, + signal: context.signal, + }) + if (!response.ok) { + throw new ZohoDeskOperationError( + `Failed to download attachment (HTTP ${response.status})`, + response.status >= 400 && response.status < 500 ? response.status : 502 + ) + } + if (response.status !== 200) { + throw new ZohoDeskOperationError( + `Attachment returned no content (HTTP ${response.status})`, + 502 + ) + } + const buffer = Buffer.from(await response.arrayBuffer()) + context.signal?.throwIfAborted() + return { + success: true, + output: { + file: { + data: buffer.toString('base64'), + mimeType: response.headers.get('content-type') || 'application/octet-stream', + name: deriveAttachmentName( + input.fileName, + response.headers.get('content-disposition'), + downloadUrl.pathname + ), + }, + }, + } +} diff --git a/apps/sim/lib/internal/zoom/errors.ts b/apps/sim/lib/internal/zoom/errors.ts new file mode 100644 index 00000000000..615560fb337 --- /dev/null +++ b/apps/sim/lib/internal/zoom/errors.ts @@ -0,0 +1,9 @@ +export class ZoomOperationError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'ZoomOperationError' + } +} diff --git a/apps/sim/lib/internal/zoom/execute-tool.test.ts b/apps/sim/lib/internal/zoom/execute-tool.test.ts new file mode 100644 index 00000000000..dbeecd0211f --- /dev/null +++ b/apps/sim/lib/internal/zoom/execute-tool.test.ts @@ -0,0 +1,55 @@ +/** + * @vitest-environment node + */ +import { createExecutionContext } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ getZoomMeetingRecordings: vi.fn() })) + +vi.mock('@/lib/internal/zoom/operations', () => ({ + getZoomMeetingRecordings: mocks.getZoomMeetingRecordings, +})) + +import type { InternalToolOperationCall } from '@/lib/internal/tool-operations/types' +import { ZoomOperationError } from '@/lib/internal/zoom/errors' +import { executeZoomTool } from '@/lib/internal/zoom/execute-tool' + +function request(overrides: Partial = {}): InternalToolOperationCall { + return { + toolId: 'zoom_get_meeting_recordings', + input: { accessToken: 'token', meetingId: 'meeting-1' }, + headers: new Headers(), + context: createExecutionContext({ workflowId: 'workflow-1' }), + requestId: 'request-1', + ...overrides, + } +} + +describe('executeZoomTool', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.getZoomMeetingRecordings.mockResolvedValue({ + success: true, + output: { recording: { recording_files: [] } }, + }) + }) + + it('dispatches with defaults and the operation signal', async () => { + const controller = new AbortController() + const response = await executeZoomTool(request({ signal: controller.signal })) + + expect(response.status).toBe(200) + expect(mocks.getZoomMeetingRecordings).toHaveBeenCalledWith( + { accessToken: 'token', meetingId: 'meeting-1', downloadFiles: false }, + { requestId: 'request-1', signal: controller.signal } + ) + }) + + it('preserves operation statuses', async () => { + mocks.getZoomMeetingRecordings.mockRejectedValue(new ZoomOperationError('too large', 413)) + + const response = await executeZoomTool(request()) + + expect(response.status).toBe(413) + }) +}) diff --git a/apps/sim/lib/internal/zoom/execute-tool.ts b/apps/sim/lib/internal/zoom/execute-tool.ts new file mode 100644 index 00000000000..803acb22ac3 --- /dev/null +++ b/apps/sim/lib/internal/zoom/execute-tool.ts @@ -0,0 +1,44 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { z } from 'zod' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { ZoomOperationError } from '@/lib/internal/zoom/errors' +import { getZoomMeetingRecordings } from '@/lib/internal/zoom/operations' + +const inputSchema = z.object({ + accessToken: z.string().min(1, 'Access token is required'), + meetingId: z.string().min(1, 'Meeting ID is required'), + includeFolderItems: z.boolean().optional(), + ttl: z.number().max(604800).optional(), + downloadFiles: z.boolean().default(false), +}) + +export const executeZoomTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (request.toolId !== 'zoom_get_meeting_recordings') { + return Response.json( + { success: false, error: `Unsupported Zoom tool: ${request.toolId}` }, + { status: 500 } + ) + } + const parsed = inputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json({ success: false, error: 'Invalid request data' }, { status: 400 }) + } + try { + return Response.json( + await getZoomMeetingRecordings(parsed.data, { + requestId: request.requestId, + signal: request.signal, + }) + ) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof ZoomOperationError) { + return Response.json({ success: false, error: error.message }, { status: error.status }) + } + return Response.json( + { success: false, error: getErrorMessage(error, 'Unknown error occurred') }, + { status: 500 } + ) + } +} diff --git a/apps/sim/lib/internal/zoom/operations.test.ts b/apps/sim/lib/internal/zoom/operations.test.ts new file mode 100644 index 00000000000..d21429d74d8 --- /dev/null +++ b/apps/sim/lib/internal/zoom/operations.test.ts @@ -0,0 +1,91 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' + +const mocks = vi.hoisted(() => ({ + secureFetchWithPinnedIP: vi.fn(), + validateUrlWithDNS: vi.fn(), +})) + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + secureFetchWithPinnedIP: mocks.secureFetchWithPinnedIP, + validateUrlWithDNS: mocks.validateUrlWithDNS, +})) + +vi.mock('@/lib/uploads/shared/types', () => ({ MAX_BUFFERED_TRANSFER_BYTES: 5 })) + +import { getZoomMeetingRecordings } from '@/lib/internal/zoom/operations' + +describe('getZoomMeetingRecordings', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.validateUrlWithDNS.mockResolvedValue({ isValid: true, resolvedIP: '203.0.113.1' }) + }) + + it('downloads sequentially and rejects cumulative recording bytes', async () => { + mocks.secureFetchWithPinnedIP + .mockResolvedValueOnce( + Response.json({ + recording_files: [ + { id: 'one', download_url: 'https://files.example/one' }, + { id: 'two', download_url: 'https://files.example/two' }, + ], + }) + ) + .mockResolvedValueOnce(new Response('one')) + .mockResolvedValueOnce(new Response('two')) + + await expect( + getZoomMeetingRecordings( + { + accessToken: 'token', + meetingId: 'meeting-1', + downloadFiles: true, + }, + { requestId: 'request-1' } + ) + ).rejects.toMatchObject({ status: 413 }) + expect(mocks.secureFetchWithPinnedIP).toHaveBeenCalledTimes(3) + expect(mocks.secureFetchWithPinnedIP).toHaveBeenNthCalledWith( + 3, + 'https://files.example/two', + '203.0.113.1', + expect.objectContaining({ maxResponseBytes: 2 }) + ) + }) + + it('streams each recording within the remaining aggregate byte budget', async () => { + mocks.secureFetchWithPinnedIP + .mockResolvedValueOnce( + Response.json({ + recording_files: [{ id: 'one', download_url: 'https://files.example/one' }], + }) + ) + .mockRejectedValueOnce( + new PayloadSizeLimitError({ + label: 'response body', + maxBytes: 5, + observedBytes: 6, + }) + ) + + await expect( + getZoomMeetingRecordings( + { + accessToken: 'token', + meetingId: 'meeting-1', + downloadFiles: true, + }, + { requestId: 'request-1' } + ) + ).rejects.toMatchObject({ status: 413 }) + expect(mocks.secureFetchWithPinnedIP).toHaveBeenNthCalledWith( + 2, + 'https://files.example/one', + '203.0.113.1', + expect.objectContaining({ maxResponseBytes: 5 }) + ) + }) +}) diff --git a/apps/sim/lib/internal/zoom/operations.ts b/apps/sim/lib/internal/zoom/operations.ts new file mode 100644 index 00000000000..9bf64c3e8ff --- /dev/null +++ b/apps/sim/lib/internal/zoom/operations.ts @@ -0,0 +1,182 @@ +import { createLogger } from '@sim/logger' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' +import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' +import { ZoomOperationError } from '@/lib/internal/zoom/errors' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { getExtensionFromMimeType } from '@/lib/uploads/utils/file-utils' +import type { ZoomGetMeetingRecordingsParams } from '@/tools/zoom/types' + +const logger = createLogger('ZoomOperations') + +interface ZoomRecordingFile { + id?: string + meeting_id?: string + recording_start?: string + recording_end?: string + file_type?: string + file_extension?: string + file_size?: number + play_url?: string + download_url?: string + status?: string + recording_type?: string +} + +interface ZoomRecordingsResponse { + uuid?: string + id?: string | number + account_id?: string + host_id?: string + topic?: string + type?: number + start_time?: string + duration?: number + total_size?: number + recording_count?: number + share_url?: string + recording_files?: ZoomRecordingFile[] +} + +interface ZoomErrorResponse { + message?: string +} + +export interface ZoomOperationContext { + requestId: string + signal?: AbortSignal +} + +export async function getZoomMeetingRecordings( + input: ZoomGetMeetingRecordingsParams, + context: ZoomOperationContext +): Promise<{ + success: true + output: { + recording: ZoomRecordingsResponse & { recording_files: ZoomRecordingFile[] } + files?: Array<{ name: string; mimeType: string; data: string; size: number }> + } +}> { + context.signal?.throwIfAborted() + const query = new URLSearchParams() + if (input.includeFolderItems != null) { + query.set('include_folder_items', String(input.includeFolderItems)) + } + if (input.ttl) query.set('ttl', String(input.ttl)) + const baseUrl = `https://api.zoom.us/v2/meetings/${encodeURIComponent(input.meetingId)}/recordings` + const apiUrl = query.size > 0 ? `${baseUrl}?${query}` : baseUrl + const validation = await validateUrlWithDNS(apiUrl, 'apiUrl') + context.signal?.throwIfAborted() + if (!validation.isValid || !validation.resolvedIP) { + throw new ZoomOperationError(validation.error || 'Invalid Zoom API URL', 400) + } + + const response = await secureFetchWithPinnedIP(apiUrl, validation.resolvedIP, { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${input.accessToken}`, + }, + signal: context.signal, + }) + if (!response.ok) { + const errorData = (await response.json().catch(() => ({}))) as ZoomErrorResponse + throw new ZoomOperationError(errorData.message || `Zoom API error: ${response.status}`, 400) + } + const data = (await response.json()) as ZoomRecordingsResponse + const files: Array<{ name: string; mimeType: string; data: string; size: number }> = [] + let bufferedBytes = 0 + + if (input.downloadFiles && Array.isArray(data.recording_files)) { + for (const file of data.recording_files) { + if (!file.download_url) continue + context.signal?.throwIfAborted() + try { + const remainingBytes = MAX_BUFFERED_TRANSFER_BYTES - bufferedBytes + if (remainingBytes <= 0) { + throw new ZoomOperationError( + `Downloaded recordings exceed the ${MAX_BUFFERED_TRANSFER_BYTES}-byte execution limit`, + 413 + ) + } + const fileValidation = await validateUrlWithDNS(file.download_url, 'downloadUrl') + if (!fileValidation.isValid || !fileValidation.resolvedIP) continue + const downloadResponse = await secureFetchWithPinnedIP( + file.download_url, + fileValidation.resolvedIP, + { + method: 'GET', + headers: { Authorization: `Bearer ${input.accessToken}` }, + maxResponseBytes: remainingBytes, + signal: context.signal, + } + ) + if (!downloadResponse.ok) continue + const buffer = Buffer.from(await downloadResponse.arrayBuffer()) + bufferedBytes += buffer.length + if (bufferedBytes > MAX_BUFFERED_TRANSFER_BYTES) { + throw new ZoomOperationError( + `Downloaded recordings exceed the ${MAX_BUFFERED_TRANSFER_BYTES}-byte execution limit`, + 413 + ) + } + const mimeType = downloadResponse.headers.get('content-type') || 'application/octet-stream' + const extension = + file.file_extension?.toLowerCase() || getExtensionFromMimeType(mimeType) || 'dat' + files.push({ + name: `zoom-recording-${file.id || file.recording_start || Date.now()}.${extension}`, + mimeType, + data: buffer.toString('base64'), + size: buffer.length, + }) + } catch (error) { + context.signal?.throwIfAborted() + if (error instanceof ZoomOperationError) throw error + if (isPayloadSizeLimitError(error)) { + throw new ZoomOperationError( + `Downloaded recordings exceed the ${MAX_BUFFERED_TRANSFER_BYTES}-byte execution limit`, + 413 + ) + } + logger.warn(`[${context.requestId}] Failed to download Zoom recording file`, { + fileId: file.id, + }) + } + } + } + + return { + success: true, + output: { + recording: { + uuid: data.uuid, + id: data.id, + account_id: data.account_id, + host_id: data.host_id, + topic: data.topic, + type: data.type, + start_time: data.start_time, + duration: data.duration, + total_size: data.total_size, + recording_count: data.recording_count, + share_url: data.share_url, + recording_files: (data.recording_files || []).map((file) => ({ + id: file.id, + meeting_id: file.meeting_id, + recording_start: file.recording_start, + recording_end: file.recording_end, + file_type: file.file_type, + file_extension: file.file_extension, + file_size: file.file_size, + play_url: file.play_url, + download_url: file.download_url, + status: file.status, + recording_type: file.recording_type, + })), + }, + files: files.length > 0 ? files : undefined, + }, + } +} diff --git a/apps/sim/lib/internal/zoominfo/client.ts b/apps/sim/lib/internal/zoominfo/client.ts new file mode 100644 index 00000000000..b908cfce267 --- /dev/null +++ b/apps/sim/lib/internal/zoominfo/client.ts @@ -0,0 +1,207 @@ +import { createHash } from 'node:crypto' +import { createLogger } from '@sim/logger' +import { + MAX_JSON_API_RESPONSE_BYTES, + secureFetchWithValidation, +} from '@/lib/core/security/input-validation.server' +import { + assertSafeZoomInfoUrl, + ZOOMINFO_API_BASE, + ZOOMINFO_TOKEN_URL, + type ZoomInfoAuth, + type ZoomInfoProviderRequest, +} from '@/lib/internal/zoominfo/schema' + +const logger = createLogger('ZoomInfoClient') + +const OUTBOUND_FETCH_TIMEOUT_MS = 30_000 +const MAX_TOKEN_RESPONSE_BYTES = 256 * 1024 +const TOKEN_CACHE_MAX_ENTRIES = 500 +const TOKEN_SAFETY_WINDOW_MS = 60_000 + +interface CachedToken { + accessToken: string + expiresAt: number +} + +interface ZoomInfoInvocation { + status: number + body: unknown +} + +const TOKEN_CACHE = new Map() + +export class ZoomInfoOperationError extends Error { + constructor( + message: string, + readonly status: number, + readonly providerStatus?: number + ) { + super(message) + this.name = 'ZoomInfoOperationError' + } +} + +function tokenCacheKey(auth: ZoomInfoAuth): string { + const secretHash = createHash('sha256').update(auth.clientSecret).digest('hex').slice(0, 16) + return `${auth.clientId}::${secretHash}` +} + +function rememberToken(key: string, token: CachedToken): void { + if (TOKEN_CACHE.has(key)) TOKEN_CACHE.delete(key) + TOKEN_CACHE.set(key, token) + while (TOKEN_CACHE.size > TOKEN_CACHE_MAX_ENTRIES) { + const oldestKey = TOKEN_CACHE.keys().next().value + if (oldestKey === undefined) break + TOKEN_CACHE.delete(oldestKey) + } +} + +async function fetchAccessToken( + auth: ZoomInfoAuth, + requestId: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const cacheKey = tokenCacheKey(auth) + const cached = TOKEN_CACHE.get(cacheKey) + if (cached && cached.expiresAt - TOKEN_SAFETY_WINDOW_MS > Date.now()) { + return cached.accessToken + } + + const tokenUrl = assertSafeZoomInfoUrl(ZOOMINFO_TOKEN_URL, 'tokenUrl').toString() + const basic = Buffer.from(`${auth.clientId}:${auth.clientSecret}`).toString('base64') + const response = await secureFetchWithValidation( + tokenUrl, + { + method: 'POST', + headers: { + Authorization: `Basic ${basic}`, + 'Content-Type': 'application/x-www-form-urlencoded', + Accept: 'application/json', + }, + body: new URLSearchParams({ grant_type: 'client_credentials' }).toString(), + timeout: OUTBOUND_FETCH_TIMEOUT_MS, + maxResponseBytes: MAX_TOKEN_RESPONSE_BYTES, + signal, + }, + 'tokenUrl' + ) + signal?.throwIfAborted() + + if (!response.ok) { + const text = await response.text().catch(() => '') + logger.warn('ZoomInfo token fetch failed', { requestId, status: response.status, error: text }) + throw new ZoomInfoOperationError(`ZoomInfo token request failed: HTTP ${response.status}`, 500) + } + const data = (await response.json()) as { access_token?: string; expires_in?: number } + if (!data.access_token) { + throw new ZoomInfoOperationError('ZoomInfo token response missing access_token', 500) + } + rememberToken(cacheKey, { + accessToken: data.access_token, + expiresAt: Date.now() + (data.expires_in ?? 3300) * 1000, + }) + return data.access_token +} + +function buildApiUrl(input: ZoomInfoProviderRequest): string { + const subPath = input.path.startsWith('/') ? input.path : `/${input.path}` + const url = `${ZOOMINFO_API_BASE}${subPath}` + if (!input.query || Object.keys(input.query).length === 0) return url + const search = new URLSearchParams() + for (const [key, value] of Object.entries(input.query)) search.append(key, String(value)) + const queryString = search.toString() + if (!queryString) return url + return url.includes('?') ? `${url}&${queryString}` : `${url}?${queryString}` +} + +function extractZoomInfoError(body: unknown, status: number): string { + if (body && typeof body === 'object') { + const obj = body as Record + if (obj.error && typeof obj.error === 'object') { + const error = obj.error as Record + const message = typeof error.message === 'string' ? error.message : '' + const code = typeof error.code === 'string' ? error.code : '' + if (message) return code ? `[${code}] ${message}` : message + } + if (typeof obj.error === 'string' && obj.error.length > 0) { + const description = + typeof obj.error_description === 'string' ? `: ${obj.error_description}` : '' + return `${obj.error}${description}` + } + if (typeof obj.message === 'string' && obj.message.length > 0) return obj.message + if (Array.isArray(obj.errors) && obj.errors.length > 0) { + return obj.errors + .map((entry) => { + if (!entry || typeof entry !== 'object') return String(entry) + const error = entry as Record + const title = typeof error.title === 'string' ? error.title : '' + const detail = typeof error.detail === 'string' ? `: ${error.detail}` : '' + return `${title}${detail}`.trim() + }) + .filter(Boolean) + .join('; ') + } + } + if (typeof body === 'string' && body.length > 0) return body + return `ZoomInfo request failed with HTTP ${status}` +} + +async function invokeZoomInfo( + input: ZoomInfoProviderRequest, + accessToken: string, + signal?: AbortSignal +): Promise { + const url = assertSafeZoomInfoUrl(buildApiUrl(input), 'apiUrl').toString() + const hasBody = input.body !== undefined && input.body !== null + const headers: Record = { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + } + if (hasBody) headers['Content-Type'] = 'application/json' + const response = await secureFetchWithValidation( + url, + { + method: input.method, + headers, + body: hasBody + ? typeof input.body === 'string' + ? input.body + : JSON.stringify(input.body) + : undefined, + timeout: OUTBOUND_FETCH_TIMEOUT_MS, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + signal, + }, + 'apiUrl' + ) + signal?.throwIfAborted() + const raw = await response.text() + let body: unknown = null + if (raw.length > 0) { + try { + body = JSON.parse(raw) + } catch { + body = raw + } + } + return { status: response.status, body } +} + +export async function requestZoomInfo( + input: ZoomInfoProviderRequest, + requestId: string, + signal?: AbortSignal +): Promise<{ status: number; data: unknown }> { + const accessToken = await fetchAccessToken(input, requestId, signal) + const invocation = await invokeZoomInfo(input, accessToken, signal) + if (invocation.status < 200 || invocation.status >= 300) { + throw new ZoomInfoOperationError( + extractZoomInfoError(invocation.body, invocation.status), + invocation.status, + invocation.status + ) + } + return { status: invocation.status, data: invocation.status === 204 ? null : invocation.body } +} diff --git a/apps/sim/lib/internal/zoominfo/execute-tool.ts b/apps/sim/lib/internal/zoominfo/execute-tool.ts new file mode 100644 index 00000000000..b12d0baef6e --- /dev/null +++ b/apps/sim/lib/internal/zoominfo/execute-tool.ts @@ -0,0 +1,81 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { getValidationErrorMessage } from '@/lib/api/server' +import { DEFAULT_MAX_JSON_BODY_BYTES } from '@/lib/api/server/validation' +import type { InternalToolOperationHandler } from '@/lib/internal/tool-operations/types' +import { ZoomInfoOperationError } from '@/lib/internal/zoominfo/client' +import { executeZoomInfoOperation } from '@/lib/internal/zoominfo/operations' +import { zoomInfoToolInputSchema } from '@/lib/internal/zoominfo/schema' + +const logger = createLogger('ZoomInfoToolExecution') + +const TOOL_IDS = new Set([ + 'zoominfo_enrich_companies', + 'zoominfo_enrich_contacts', + 'zoominfo_search_companies', + 'zoominfo_search_contacts', + 'zoominfo_search_intent', + 'zoominfo_search_news', +]) + +function exceedsInputCap(input: unknown): boolean { + try { + return Buffer.byteLength(JSON.stringify(input) ?? '') > DEFAULT_MAX_JSON_BODY_BYTES + } catch { + return true + } +} + +export const executeZoomInfoTool: InternalToolOperationHandler = async (request) => { + request.signal?.throwIfAborted() + if (!TOOL_IDS.has(request.toolId)) { + return Response.json( + { success: false, error: `Unsupported ZoomInfo tool: ${request.toolId}` }, + { status: 500 } + ) + } + if (!request.context.userId) { + return Response.json({ success: false, error: 'Authentication required' }, { status: 401 }) + } + if (exceedsInputCap(request.input)) { + return Response.json( + { + success: false, + error: `Request body exceeds the maximum allowed size of ${DEFAULT_MAX_JSON_BODY_BYTES} bytes`, + }, + { status: 413 } + ) + } + const parsed = zoomInfoToolInputSchema.safeParse(request.input) + if (!parsed.success) { + return Response.json( + { success: false, error: getValidationErrorMessage(parsed.error, 'Validation failed') }, + { status: 400 } + ) + } + try { + const output = await executeZoomInfoOperation( + request.toolId, + parsed.data, + request.requestId, + request.signal + ) + request.signal?.throwIfAborted() + return Response.json({ success: true, output }) + } catch (error) { + request.signal?.throwIfAborted() + if (error instanceof ZoomInfoOperationError) { + return Response.json( + { + success: false, + error: error.message, + ...(error.providerStatus === undefined ? {} : { status: error.providerStatus }), + }, + { status: error.status } + ) + } + const message = getErrorMessage(error, 'Unknown error occurred') + logger.error('ZoomInfo operation failed', { error: message, requestId: request.requestId }) + return Response.json({ success: false, error: message }, { status: 500 }) + } +} diff --git a/apps/sim/lib/internal/zoominfo/operations.test.ts b/apps/sim/lib/internal/zoominfo/operations.test.ts new file mode 100644 index 00000000000..2376d907e6c --- /dev/null +++ b/apps/sim/lib/internal/zoominfo/operations.test.ts @@ -0,0 +1,95 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { requestZoomInfo } = vi.hoisted(() => ({ requestZoomInfo: vi.fn() })) + +vi.mock('@/lib/internal/zoominfo/client', () => ({ requestZoomInfo })) + +import { executeZoomInfoOperation } from '@/lib/internal/zoominfo/operations' + +const AUTH = { clientId: 'client-1', clientSecret: 'secret-1' } + +describe('executeZoomInfoOperation', () => { + beforeEach(() => { + vi.clearAllMocks() + requestZoomInfo.mockResolvedValue({ status: 200, data: { data: [] } }) + }) + + it('builds the documented company-search provider request after operation admission', async () => { + await executeZoomInfoOperation( + 'zoominfo_search_companies', + { + ...AUTH, + companyName: 'Sim', + industryCodes: '["software","saas"]', + employeeRangeMin: 10, + page: 2, + rpp: 50, + sortBy: 'name', + sortOrder: 'desc', + }, + 'request-1' + ) + + expect(requestZoomInfo).toHaveBeenCalledWith( + { + ...AUTH, + path: '/data/v1/companies/search', + method: 'POST', + query: { 'page[number]': 2, 'page[size]': 50, sort: '-name' }, + body: { + data: { + type: 'CompanySearch', + attributes: { + companyName: 'Sim', + industryCodes: 'software,saas', + employeeRangeMin: '10', + }, + }, + }, + }, + 'request-1', + undefined + ) + }) + + it('enforces the documented 25-item enrichment cap before provider submission', async () => { + await expect( + executeZoomInfoOperation( + 'zoominfo_enrich_contacts', + { ...AUTH, matchPersonInput: JSON.stringify(Array.from({ length: 26 }, () => ({}))) }, + 'request-2' + ) + ).rejects.toThrow('matchPersonInput supports a maximum of 25 entries per request') + expect(requestZoomInfo).not.toHaveBeenCalled() + }) + + it('caps search cardinality before provider submission', async () => { + await expect( + executeZoomInfoOperation( + 'zoominfo_search_companies', + { ...AUTH, companyName: 'Sim', rpp: 101 }, + 'request-3' + ) + ).rejects.toThrow('rpp must be an integer between 1 and 100') + expect(requestZoomInfo).not.toHaveBeenCalled() + }) + + it('forwards cancellation through to the provider client', async () => { + const controller = new AbortController() + await executeZoomInfoOperation( + 'zoominfo_search_news', + { ...AUTH, categories: 'funding' }, + 'request-4', + controller.signal + ) + + expect(requestZoomInfo).toHaveBeenCalledWith( + expect.objectContaining({ path: '/data/v1/news/search' }), + 'request-4', + controller.signal + ) + }) +}) diff --git a/apps/sim/lib/internal/zoominfo/operations.ts b/apps/sim/lib/internal/zoominfo/operations.ts new file mode 100644 index 00000000000..5e0e6f54cec --- /dev/null +++ b/apps/sim/lib/internal/zoominfo/operations.ts @@ -0,0 +1,315 @@ +import { getErrorMessage } from '@sim/utils/errors' +import { requestZoomInfo } from '@/lib/internal/zoominfo/client' +import { + type ZoomInfoEnrichCompaniesInput, + type ZoomInfoEnrichContactsInput, + type ZoomInfoProviderRequest, + type ZoomInfoSearchCompaniesInput, + type ZoomInfoSearchContactsInput, + type ZoomInfoSearchIntentInput, + type ZoomInfoSearchNewsInput, + type ZoomInfoToolInput, + zoomInfoEnrichCompaniesInputSchema, + zoomInfoEnrichContactsInputSchema, + zoomInfoSearchCompaniesInputSchema, + zoomInfoSearchContactsInputSchema, + zoomInfoSearchIntentInputSchema, + zoomInfoSearchNewsInputSchema, +} from '@/lib/internal/zoominfo/schema' + +const DEFAULT_CONTACT_OUTPUT_FIELDS = [ + 'id', + 'firstName', + 'lastName', + 'email', + 'phone', + 'mobilePhone', + 'jobTitle', + 'jobFunction', + 'managementLevel', + 'city', + 'state', + 'country', + 'contactAccuracyScore', + 'validDate', + 'lastUpdatedDate', + 'companyId', + 'companyName', + 'companyWebsite', + 'companyPhone', +] + +const DEFAULT_COMPANY_OUTPUT_FIELDS = [ + 'id', + 'name', + 'website', + 'domainList', + 'ticker', + 'revenue', + 'revenueRange', + 'employeeCount', + 'employeeRange', + 'primaryIndustry', + 'industries', + 'street', + 'city', + 'state', + 'zipCode', + 'country', + 'phone', + 'foundedYear', + 'companyStatus', + 'socialMediaUrls', + 'logo', + 'description', +] + +function parseJsonField(value: unknown, fieldName: string): T { + if (typeof value !== 'string') return value as T + const trimmed = value.trim() + if (!trimmed) throw new Error(`${fieldName} is required`) + try { + return JSON.parse(trimmed) as T + } catch (error) { + throw new Error(`${fieldName} must be valid JSON: ${getErrorMessage(error)}`) + } +} + +function parseCsvOrJson(value: unknown, fieldName: string): string[] | undefined { + if (value === undefined || value === null) return undefined + if (Array.isArray(value)) return value.map(String) + if (typeof value !== 'string') return undefined + const trimmed = value.trim() + if (!trimmed) return undefined + if (trimmed.startsWith('[')) { + try { + const parsed = JSON.parse(trimmed) + if (!Array.isArray(parsed)) throw new Error(`${fieldName} JSON must be an array of strings`) + return parsed.map(String) + } catch (error) { + throw new Error(`${fieldName} must be valid JSON: ${getErrorMessage(error)}`) + } + } + return trimmed + .split(',') + .map((entry) => entry.trim()) + .filter(Boolean) +} + +function toCsvStringOrUndefined(value: unknown, fieldName: string): string | undefined { + const values = parseCsvOrJson(value, fieldName) + return values && values.length > 0 ? values.join(',') : undefined +} + +function toNumberOrUndefined(value: unknown): number | undefined { + if (value === undefined || value === null || value === '') return undefined + const number = Number(value) + return Number.isFinite(number) ? number : undefined +} + +function authInput(input: { clientId: string; clientSecret: string }) { + return { clientId: input.clientId, clientSecret: input.clientSecret } +} + +function paginationQuery(page: unknown, rpp: unknown): Record | undefined { + const query: Record = {} + const pageNumber = toNumberOrUndefined(page) + const pageSize = toNumberOrUndefined(rpp) + if (pageNumber !== undefined) query['page[number]'] = pageNumber + if (pageSize !== undefined) { + if (!Number.isInteger(pageSize) || pageSize < 1 || pageSize > 100) { + throw new Error('rpp must be an integer between 1 and 100') + } + query['page[size]'] = pageSize + } + return Object.keys(query).length > 0 ? query : undefined +} + +function buildSearchCompanies(input: ZoomInfoSearchCompaniesInput): ZoomInfoProviderRequest { + const attributes: Record = {} + if (input.companyName) attributes.companyName = input.companyName + if (input.companyWebsite) attributes.companyWebsite = input.companyWebsite + const companyTicker = parseCsvOrJson(input.companyTicker, 'companyTicker') + if (companyTicker) attributes.companyTicker = companyTicker + const industryCodes = toCsvStringOrUndefined(input.industryCodes, 'industryCodes') + if (industryCodes) attributes.industryCodes = industryCodes + if (input.country) attributes.country = input.country + if (input.state) attributes.state = input.state + if (input.metroRegion) attributes.metroRegion = input.metroRegion + const revenueMin = toNumberOrUndefined(input.revenueMin) + if (revenueMin !== undefined) attributes.revenueMin = revenueMin + const revenueMax = toNumberOrUndefined(input.revenueMax) + if (revenueMax !== undefined) attributes.revenueMax = revenueMax + const employeeRangeMin = toNumberOrUndefined(input.employeeRangeMin) + if (employeeRangeMin !== undefined) attributes.employeeRangeMin = String(employeeRangeMin) + const employeeRangeMax = toNumberOrUndefined(input.employeeRangeMax) + if (employeeRangeMax !== undefined) attributes.employeeRangeMax = String(employeeRangeMax) + if (input.excludeDefunctCompanies !== undefined) { + attributes.excludeDefunctCompanies = input.excludeDefunctCompanies + } + const query: Record = paginationQuery(input.page, input.rpp) ?? {} + if (input.sortBy) query.sort = `${input.sortOrder === 'desc' ? '-' : ''}${input.sortBy}` + return { + ...authInput(input), + path: '/data/v1/companies/search', + method: 'POST', + query: Object.keys(query).length > 0 ? query : undefined, + body: { data: { type: 'CompanySearch', attributes } }, + } +} + +function buildSearchContacts(input: ZoomInfoSearchContactsInput): ZoomInfoProviderRequest { + const attributes: Record = {} + if (input.firstName) attributes.firstName = input.firstName + if (input.lastName) attributes.lastName = input.lastName + if (input.fullName) attributes.fullName = input.fullName + if (input.emailAddress) attributes.emailAddress = input.emailAddress + if (input.jobTitle) attributes.jobTitle = input.jobTitle + const managementLevel = toCsvStringOrUndefined(input.managementLevel, 'managementLevel') + if (managementLevel) attributes.managementLevel = managementLevel + const department = toCsvStringOrUndefined(input.department, 'department') + if (department) attributes.department = department + if (input.companyId) attributes.companyId = input.companyId + if (input.companyName) attributes.companyName = input.companyName + const minimumScore = toNumberOrUndefined(input.contactAccuracyScoreMin) + if (minimumScore !== undefined) attributes.contactAccuracyScoreMin = String(minimumScore) + const requiredFields = toCsvStringOrUndefined(input.requiredFields, 'requiredFields') + if (requiredFields) attributes.requiredFields = requiredFields + if (input.excludePartialProfiles !== undefined) { + attributes.excludePartialProfiles = input.excludePartialProfiles + } + const query: Record = paginationQuery(input.page, input.rpp) ?? {} + if (input.sortBy) query.sort = `${input.sortOrder === 'desc' ? '-' : ''}${input.sortBy}` + return { + ...authInput(input), + path: '/data/v1/contacts/search', + method: 'POST', + query: Object.keys(query).length > 0 ? query : undefined, + body: { data: { type: 'ContactSearch', attributes } }, + } +} + +function buildEnrichCompanies(input: ZoomInfoEnrichCompaniesInput): ZoomInfoProviderRequest { + const matchCompanyInput = parseJsonField(input.matchCompanyInput, 'matchCompanyInput') + if (!Array.isArray(matchCompanyInput) || matchCompanyInput.length === 0) { + throw new Error('matchCompanyInput must be a non-empty JSON array') + } + if (matchCompanyInput.length > 25) { + throw new Error('matchCompanyInput supports a maximum of 25 entries per request') + } + return { + ...authInput(input), + path: '/data/v1/companies/enrich', + method: 'POST', + body: { + data: { + type: 'CompanyEnrich', + attributes: { + matchCompanyInput, + outputFields: + parseCsvOrJson(input.outputFields, 'outputFields') ?? DEFAULT_COMPANY_OUTPUT_FIELDS, + }, + }, + }, + } +} + +function buildEnrichContacts(input: ZoomInfoEnrichContactsInput): ZoomInfoProviderRequest { + const matchPersonInput = parseJsonField(input.matchPersonInput, 'matchPersonInput') + if (!Array.isArray(matchPersonInput) || matchPersonInput.length === 0) { + throw new Error('matchPersonInput must be a non-empty JSON array') + } + if (matchPersonInput.length > 25) { + throw new Error('matchPersonInput supports a maximum of 25 entries per request') + } + const attributes: Record = { + matchPersonInput, + outputFields: + parseCsvOrJson(input.outputFields, 'outputFields') ?? DEFAULT_CONTACT_OUTPUT_FIELDS, + } + const requiredFields = parseCsvOrJson(input.requiredFields, 'requiredFields') + if (requiredFields) attributes.requiredFields = requiredFields + return { + ...authInput(input), + path: '/data/v1/contacts/enrich', + method: 'POST', + body: { data: { type: 'ContactEnrich', attributes } }, + } +} + +function buildSearchIntent(input: ZoomInfoSearchIntentInput): ZoomInfoProviderRequest { + const topics = parseCsvOrJson(input.topics, 'topics') + if (!topics || topics.length === 0) throw new Error('topics is required') + if (topics.length > 50) throw new Error('topics supports a maximum of 50 entries per request') + const attributes: Record = { topics } + if (input.signalStartDate) attributes.signalStartDate = input.signalStartDate + if (input.signalEndDate) attributes.signalEndDate = input.signalEndDate + const scoreMin = toNumberOrUndefined(input.signalScoreMin) + if (scoreMin !== undefined) attributes.signalScoreMin = scoreMin + const scoreMax = toNumberOrUndefined(input.signalScoreMax) + if (scoreMax !== undefined) attributes.signalScoreMax = scoreMax + if (input.audienceStrengthMin) attributes.audienceStrengthMin = input.audienceStrengthMin + if (input.audienceStrengthMax) attributes.audienceStrengthMax = input.audienceStrengthMax + if (input.findRecommendedContacts !== undefined) { + attributes.findRecommendedContacts = input.findRecommendedContacts + } + if (input.country) attributes.country = input.country + if (input.state) attributes.state = input.state + const industryCodes = toCsvStringOrUndefined(input.industryCodes, 'industryCodes') + if (industryCodes) attributes.industryCodes = industryCodes + return { + ...authInput(input), + path: '/data/v1/intent/search', + method: 'POST', + query: paginationQuery(input.page, input.rpp), + body: { data: { type: 'IntentSearch', attributes } }, + } +} + +function buildSearchNews(input: ZoomInfoSearchNewsInput): ZoomInfoProviderRequest { + const attributes: Record = {} + const categories = parseCsvOrJson(input.categories, 'categories') + if (categories) attributes.categories = categories + const urls = parseCsvOrJson(input.url, 'url') + if (urls) attributes.url = urls + if (input.pageDateMin) attributes.pageDateMin = input.pageDateMin + if (input.pageDateMax) attributes.pageDateMax = input.pageDateMax + if (Object.keys(attributes).length === 0) { + throw new Error('Provide at least one of: categories, url, pageDateMin, pageDateMax') + } + return { + ...authInput(input), + path: '/data/v1/news/search', + method: 'POST', + query: paginationQuery(input.page, input.rpp), + body: { data: { type: 'NewsSearch', attributes } }, + } +} + +function buildProviderRequest(toolId: string, input: ZoomInfoToolInput): ZoomInfoProviderRequest { + switch (toolId) { + case 'zoominfo_search_companies': + return buildSearchCompanies(zoomInfoSearchCompaniesInputSchema.parse(input)) + case 'zoominfo_search_contacts': + return buildSearchContacts(zoomInfoSearchContactsInputSchema.parse(input)) + case 'zoominfo_enrich_companies': + return buildEnrichCompanies(zoomInfoEnrichCompaniesInputSchema.parse(input)) + case 'zoominfo_enrich_contacts': + return buildEnrichContacts(zoomInfoEnrichContactsInputSchema.parse(input)) + case 'zoominfo_search_intent': + return buildSearchIntent(zoomInfoSearchIntentInputSchema.parse(input)) + case 'zoominfo_search_news': + return buildSearchNews(zoomInfoSearchNewsInputSchema.parse(input)) + default: + throw new Error(`Unsupported ZoomInfo tool: ${toolId}`) + } +} + +export async function executeZoomInfoOperation( + toolId: string, + input: ZoomInfoToolInput, + requestId: string, + signal?: AbortSignal +) { + return requestZoomInfo(buildProviderRequest(toolId, input), requestId, signal) +} diff --git a/apps/sim/lib/internal/zoominfo/schema.ts b/apps/sim/lib/internal/zoominfo/schema.ts new file mode 100644 index 00000000000..60f57e34a1f --- /dev/null +++ b/apps/sim/lib/internal/zoominfo/schema.ts @@ -0,0 +1,146 @@ +import { isPrivateIpHost } from '@sim/security/ssrf' +import { z } from 'zod' + +export const ZOOMINFO_API_BASE = 'https://api.zoominfo.com/gtm' +export const ZOOMINFO_TOKEN_URL = `${ZOOMINFO_API_BASE}/oauth/v1/token` + +export const zoomInfoAuthSchema = z.object({ + clientId: z.string().min(1, 'clientId is required'), + clientSecret: z.string().min(1, 'clientSecret is required'), +}) + +export const zoomInfoProviderRequestSchema = zoomInfoAuthSchema.extend({ + path: z + .string() + .min(1, 'path is required') + .refine( + (path) => + !path.split(/[/\\]/).some((segment) => segment === '..' || segment === '.') && + !path.includes('#') && + !/%(?:2[eEfF]|5[cC]|23)/.test(path), + { + message: + 'path must not contain ".." or "." segments, "#", or percent-encoded path/fragment characters', + } + ), + method: z.enum(['GET', 'POST', 'PUT', 'PATCH', 'DELETE']).default('POST'), + query: z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])).optional(), + body: z.unknown().optional(), +}) + +export type ZoomInfoAuth = z.output +export const zoomInfoToolInputSchema = zoomInfoAuthSchema.passthrough() + +const optionalToolField = z.unknown().optional() + +export const zoomInfoSearchCompaniesInputSchema = zoomInfoAuthSchema.extend({ + companyName: optionalToolField, + companyWebsite: optionalToolField, + companyTicker: optionalToolField, + industryCodes: optionalToolField, + country: optionalToolField, + state: optionalToolField, + metroRegion: optionalToolField, + revenueMin: optionalToolField, + revenueMax: optionalToolField, + employeeRangeMin: optionalToolField, + employeeRangeMax: optionalToolField, + excludeDefunctCompanies: optionalToolField, + page: optionalToolField, + rpp: optionalToolField, + sortBy: optionalToolField, + sortOrder: optionalToolField, +}) + +export const zoomInfoSearchContactsInputSchema = zoomInfoAuthSchema.extend({ + firstName: optionalToolField, + lastName: optionalToolField, + fullName: optionalToolField, + emailAddress: optionalToolField, + jobTitle: optionalToolField, + managementLevel: optionalToolField, + department: optionalToolField, + companyId: optionalToolField, + companyName: optionalToolField, + contactAccuracyScoreMin: optionalToolField, + requiredFields: optionalToolField, + excludePartialProfiles: optionalToolField, + page: optionalToolField, + rpp: optionalToolField, + sortBy: optionalToolField, + sortOrder: optionalToolField, +}) + +export const zoomInfoEnrichCompaniesInputSchema = zoomInfoAuthSchema.extend({ + matchCompanyInput: z.unknown(), + outputFields: optionalToolField, +}) + +export const zoomInfoEnrichContactsInputSchema = zoomInfoAuthSchema.extend({ + matchPersonInput: z.unknown(), + outputFields: optionalToolField, + requiredFields: optionalToolField, +}) + +export const zoomInfoSearchIntentInputSchema = zoomInfoAuthSchema.extend({ + topics: z.unknown(), + signalStartDate: optionalToolField, + signalEndDate: optionalToolField, + signalScoreMin: optionalToolField, + signalScoreMax: optionalToolField, + audienceStrengthMin: optionalToolField, + audienceStrengthMax: optionalToolField, + findRecommendedContacts: optionalToolField, + country: optionalToolField, + state: optionalToolField, + industryCodes: optionalToolField, + page: optionalToolField, + rpp: optionalToolField, +}) + +export const zoomInfoSearchNewsInputSchema = zoomInfoAuthSchema.extend({ + categories: optionalToolField, + url: optionalToolField, + pageDateMin: optionalToolField, + pageDateMax: optionalToolField, + page: optionalToolField, + rpp: optionalToolField, +}) + +export type ZoomInfoProviderRequest = z.output +export type ZoomInfoToolInput = z.output +export type ZoomInfoSearchCompaniesInput = z.output +export type ZoomInfoSearchContactsInput = z.output +export type ZoomInfoEnrichCompaniesInput = z.output +export type ZoomInfoEnrichContactsInput = z.output +export type ZoomInfoSearchIntentInput = z.output +export type ZoomInfoSearchNewsInput = z.output + +const FORBIDDEN_HOSTS = new Set([ + 'localhost', + '0.0.0.0', + '127.0.0.1', + '169.254.169.254', + 'metadata.google.internal', + 'metadata', + '[::1]', + '[::]', +]) + +export function assertSafeZoomInfoUrl(rawUrl: string, label: string): URL { + let parsed: URL + try { + parsed = new URL(rawUrl) + } catch { + throw new Error(`${label} must be a valid URL`) + } + if (parsed.protocol !== 'https:') throw new Error(`${label} must use https://`) + const host = parsed.hostname.toLowerCase() + if (FORBIDDEN_HOSTS.has(host)) throw new Error(`${label} host is not allowed`) + if (isPrivateIpHost(host)) + throw new Error(`${label} host is not allowed (private/loopback range)`) + if (host !== 'api.zoominfo.com') { + throw new Error(`${label} host must be api.zoominfo.com`) + } + return parsed +} diff --git a/apps/sim/lib/knowledge/api/internal-route.test.ts b/apps/sim/lib/knowledge/api/internal-route.test.ts new file mode 100644 index 00000000000..3b04901a13c --- /dev/null +++ b/apps/sim/lib/knowledge/api/internal-route.test.ts @@ -0,0 +1,125 @@ +/** + * @vitest-environment node + */ +import type { WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' +import { NextRequest } from 'next/server' +import { describe, expect, it } from 'vitest' +import { + BILLING_ATTRIBUTION_HEADER, + serializeBillingAttributionHeader, +} from '@/lib/billing/core/billing-attribution' +import { + internalKnowledgeProvenanceUserId, + resolveInternalKnowledgeBillingAttribution, +} from '@/lib/knowledge/api/internal-route' +import { resolveKnowledgeAttributedUserId } from '@/lib/knowledge/application/billing' + +const BILLING_ATTRIBUTION = { + actorUserId: 'execution-billing-actor-1', + billedAccountUserId: 'billing-owner-1', + billingEntity: { type: 'user' as const, id: 'billing-owner-1' }, + billingPeriod: { + start: '2026-08-01T00:00:00.000Z', + end: '2026-09-01T00:00:00.000Z', + }, + organizationId: null, + payerSubscription: null, + workspaceId: 'workspace-1', +} + +function request(): NextRequest { + return new NextRequest('http://localhost/api/knowledge/search', { + headers: { + [BILLING_ATTRIBUTION_HEADER]: serializeBillingAttributionHeader(BILLING_ATTRIBUTION), + }, + }) +} + +function executorPrincipal( + originalPrincipal: NonNullable< + WorkflowExecutionDelegatedPrincipal['delegationContext'] + >['principal'] +): WorkflowExecutionDelegatedPrincipal { + return { + kind: 'delegated', + serviceId: 'executor', + workspaceId: 'workspace-1', + delegationId: 'executor-1', + audience: 'sim:knowledge', + issuedAt: new Date('2026-08-01T00:00:00.000Z'), + expiresAt: new Date('2099-01-01T00:00:00.000Z'), + delegationContext: { + kind: 'workflow_execution', + workflowId: 'workflow-1', + currentWorkflow: { + workflowId: 'workflow-1', + mode: 'deployment', + deploymentVersionId: 'deployment-1', + }, + ...(originalPrincipal ? { principal: originalPrincipal } : {}), + }, + } +} + +describe('internal Knowledge execution attribution', () => { + it.each([ + { + name: 'generic webhook', + principal: { + kind: 'system' as const, + serviceId: 'webhook' as const, + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + webhookId: 'webhook-1', + provider: 'generic', + }, + }, + { + name: 'Slack webhook', + principal: { + kind: 'system' as const, + serviceId: 'webhook' as const, + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + webhookId: 'webhook-1', + provider: 'slack', + subject: { + kind: 'external_user' as const, + provider: 'slack', + tenantId: 'tenant-1', + subjectId: 'subject-1', + }, + }, + }, + ])('restores billing and provenance attribution for a $name execution', async ({ principal }) => { + const executor = executorPrincipal(principal) + + await expect( + resolveInternalKnowledgeBillingAttribution(request(), executor, 'workspace-1') + ).resolves.toEqual(BILLING_ATTRIBUTION) + expect(internalKnowledgeProvenanceUserId(request().headers, executor, 'workspace-1')).toBe( + 'billing-owner-1' + ) + expect( + resolveKnowledgeAttributedUserId(executor, { + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + billedAccountUserId: 'billing-owner-1', + }) + ).toBe('billing-owner-1') + }) + + it('rejects a billing snapshot from another workspace', async () => { + const principal = executorPrincipal({ + kind: 'system', + serviceId: 'schedule', + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + }) + + await expect( + resolveInternalKnowledgeBillingAttribution(request(), principal, 'workspace-2') + ).rejects.toThrow('does not match the authenticated request scope') + }) +}) diff --git a/apps/sim/lib/knowledge/api/internal-route.ts b/apps/sim/lib/knowledge/api/internal-route.ts index fb5fdd6212f..68fa01cd54f 100644 --- a/apps/sim/lib/knowledge/api/internal-route.ts +++ b/apps/sim/lib/knowledge/api/internal-route.ts @@ -1,6 +1,7 @@ import { type Principal, requirePrincipalSubjectUserId, + resolvePrincipalSubject, type SessionPrincipal, } from '@sim/auth/principal' import type { NextRequest } from 'next/server' @@ -16,7 +17,7 @@ import { type DocumentData, documentDataSchema } from '@/lib/api/contracts/knowl import { type TagDefinitionData, tagDefinitionDataSchema } from '@/lib/api/contracts/knowledge/tags' import { AuthType, type AuthTypeValue } from '@/lib/auth/hybrid' import { - requireBillingAttributionHeader, + requireWorkspaceBillingAttributionHeader, resolveBillingAttribution, } from '@/lib/billing/core/billing-attribution' import { PlatformEvents } from '@/lib/core/telemetry' @@ -34,6 +35,20 @@ export function internalKnowledgeActorUserId(principal: Principal): string { return requirePrincipalSubjectUserId(principal) } +export function internalKnowledgeProvenanceUserId( + headers: Headers, + principal: Principal, + workspaceId: string | undefined +): string { + if (principal.kind !== 'delegated') return internalKnowledgeActorUserId(principal) + const subject = resolvePrincipalSubject(principal) + if (subject?.kind === 'sim_user') return subject.userId + if (!workspaceId) { + throw new Error('Delegated Knowledge provenance requires a workspace scope') + } + return requireWorkspaceBillingAttributionHeader(headers, { workspaceId }).billedAccountUserId +} + export function internalKnowledgeAuthType(principal: Principal): AuthTypeValue { return principal.kind === 'delegated' ? AuthType.INTERNAL_JWT : AuthType.SESSION } @@ -43,10 +58,18 @@ export async function resolveInternalKnowledgeBillingAttribution( principal: Principal, workspaceId: string ) { - const actorUserId = internalKnowledgeActorUserId(principal) - return await (principal.kind === 'delegated' - ? requireBillingAttributionHeader(request.headers, { actorUserId, workspaceId }) - : resolveBillingAttribution({ actorUserId, workspaceId })) + if (principal.kind === 'delegated') { + return requireWorkspaceBillingAttributionHeader(request.headers, { workspaceId }) + } + return await resolveBillingAttribution({ + actorUserId: internalKnowledgeActorUserId(principal), + workspaceId, + }) +} + +function internalKnowledgeAnalyticsUserId(principal: Principal): string | null { + const subject = resolvePrincipalSubject(principal) + return subject?.kind === 'sim_user' ? subject.userId : null } function serializeDate(date: Date | string): string { @@ -243,7 +266,6 @@ export const internalKnowledgeAnalytics = { } | { kind: 'bulk'; workspaceId?: string; data: { total: number }; knowledgeBaseId?: string } }): void { - const userId = internalKnowledgeActorUserId(principal) const documentCount = result.kind === 'bulk' ? result.data.total : 1 const knowledgeBaseId = result.kind === 'single' ? result.data.knowledgeBaseId : result.knowledgeBaseId @@ -258,6 +280,8 @@ export const internalKnowledgeAnalytics = { ? { mimeType: result.data.mimeType, fileSize: result.data.fileSize } : { recipe: input.processingOptions?.recipe }), }) + const userId = internalKnowledgeAnalyticsUserId(principal) + if (!userId) return captureServerEvent( userId, 'knowledge_base_document_uploaded', @@ -297,8 +321,10 @@ export const internalKnowledgeAnalytics = { }): void { const workspaceId = result.workspaceId if (!workspaceId) throw new Error('Deleted document result is missing its workspace scope') + const userId = internalKnowledgeAnalyticsUserId(principal) + if (!userId) return captureServerEvent( - internalKnowledgeActorUserId(principal), + userId, 'knowledge_base_document_deleted', { knowledge_base_id: result.knowledgeBaseId, workspace_id: workspaceId }, { groups: { workspace: workspaceId } } @@ -319,8 +345,10 @@ export const internalKnowledgeAnalytics = { } } }): void { + const userId = internalKnowledgeAnalyticsUserId(principal) + if (!userId) return captureServerEvent( - internalKnowledgeActorUserId(principal), + userId, 'knowledge_base_connector_added', { knowledge_base_id: connector.knowledgeBaseId, @@ -350,8 +378,10 @@ export const internalKnowledgeAnalytics = { if (!result.workspaceId) { throw new Error('Deleted connector result is missing its workspace analytics scope') } + const userId = internalKnowledgeAnalyticsUserId(principal) + if (!userId) return captureServerEvent( - internalKnowledgeActorUserId(principal), + userId, 'knowledge_base_connector_removed', { knowledge_base_id: result.knowledgeBaseId, @@ -377,8 +407,10 @@ export const internalKnowledgeAnalytics = { if (!result.workspaceId) { throw new Error('Synced connector result is missing its workspace analytics scope') } + const userId = internalKnowledgeAnalyticsUserId(principal) + if (!userId) return captureServerEvent( - internalKnowledgeActorUserId(principal), + userId, 'knowledge_base_connector_synced', { knowledge_base_id: result.knowledgeBaseId, diff --git a/apps/sim/lib/knowledge/api/secret-provenance.ts b/apps/sim/lib/knowledge/api/secret-provenance.ts new file mode 100644 index 00000000000..4ca15750404 --- /dev/null +++ b/apps/sim/lib/knowledge/api/secret-provenance.ts @@ -0,0 +1,255 @@ +import type { InternalJsonResponseFinalization } from '@/lib/api/server/routes/internal-json-route' +import { AuthType, type AuthTypeValue } from '@/lib/auth/hybrid' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { + createDurableSecretProvenanceRegistry, + type DurableSecretProvenance, + durableSecretProvenanceFromPrivateBundle, + EXACT_EMPTY_DURABLE_SECRET_PROVENANCE, +} from '@/lib/execution/durable-secret-provenance' +import { + inspectPrivateSecretProvenanceRequest, + isPrivateSecretProvenanceBundleV1, +} from '@/lib/execution/model-input-provenance' +import { + negotiatePrivateToolMetadataResponse, + RESOLVED_SECRET_PROVENANCE_FIELD, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, + serializePrivateToolMetadataResponseEnvelope, +} from '@/lib/execution/private-tool-metadata' +import { + importKnowledgePersistedResponseSecretProvenance, + type KnowledgeDocumentSourceValue, + type KnowledgeDocumentWriteSecretProvenance, +} from '@/lib/knowledge/secret-provenance' +import { + knowledgeDocumentContentSelectionKey, + knowledgeDocumentFilenameSelectionKey, + knowledgeDocumentTagValueSelectionKey, + parseKnowledgeDocumentTagProvenanceTargets, +} from '@/lib/knowledge/secret-provenance-selection' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' + +function invalidKnowledgeProvenanceResponse(): Response { + return Response.json({ error: 'Invalid knowledge secret provenance' }, { status: 400 }) +} + +function rejectInvalidKnowledgeProvenance(): never { + throw new OrchestrationError('validation', 'Invalid knowledge secret provenance') +} + +function finalizeKnowledgeMetadataEnvelope( + envelope: ReturnType +): InternalJsonResponseFinalization { + return { + bodyFields: { + [RESOLVED_SECRET_PROVENANCE_FIELD]: envelope.body[RESOLVED_SECRET_PROVENANCE_FIELD], + }, + headers: envelope.headers, + } +} + +type KnowledgeWriteProvenanceResolution = + | { success: true; provenances?: DurableSecretProvenance[] } + | { success: false; response: Response } + +/** Resolves private document/chunk write selections after auth and workspace authorization. */ +export function resolveKnowledgeWriteSecretProvenance(options: { + headers: Headers + payload: unknown + authType: AuthTypeValue | undefined + userId: string + workspaceId?: string + selectionKeys: readonly string[] +}): KnowledgeWriteProvenanceResolution { + const inspection = inspectPrivateSecretProvenanceRequest(options.headers, options.payload) + if (inspection.status === 'unsupported') { + return options.authType === AuthType.INTERNAL_JWT + ? { success: true } + : { + success: true, + provenances: options.selectionKeys.map(() => EXACT_EMPTY_DURABLE_SECRET_PROVENANCE), + } + } + if (inspection.status !== 'verified' || options.authType !== AuthType.INTERNAL_JWT) { + return { success: false, response: invalidKnowledgeProvenanceResponse() } + } + if (!isPrivateSecretProvenanceBundleV1(inspection.value)) { + return { success: false, response: invalidKnowledgeProvenanceResponse() } + } + if (!inspection.value.complete) { + return { + success: true, + provenances: options.selectionKeys.map(() => ({ status: 'unknown' })), + } + } + if (inspection.value.selections.length !== options.selectionKeys.length) { + return { success: false, response: invalidKnowledgeProvenanceResponse() } + } + const provenances = options.selectionKeys.map((selectionKey) => + durableSecretProvenanceFromPrivateBundle(inspection.value, selectionKey, { + userId: options.userId, + ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), + }) + ) + if (provenances.some((provenance) => provenance === undefined)) { + return { success: false, response: invalidKnowledgeProvenanceResponse() } + } + return { success: true, provenances: provenances as DurableSecretProvenance[] } +} + +type KnowledgeDocumentWriteProvenanceResolution = + | { success: true; provenances?: KnowledgeDocumentWriteSecretProvenance[] } + | { success: false; response: Response } + +/** Resolves provenance for durable document fields; persisted tag names remain raw and untracked. */ +export function resolveKnowledgeDocumentWriteSecretProvenance(options: { + headers: Headers + payload: unknown + authType: AuthTypeValue | undefined + userId: string + workspaceId?: string + documents: readonly { documentTagsData?: string }[] +}): KnowledgeDocumentWriteProvenanceResolution { + const tagTargets = options.documents.map((document) => + parseKnowledgeDocumentTagProvenanceTargets(document.documentTagsData) + ) + const selectionKeys = options.documents.flatMap((_document, documentIndex) => [ + knowledgeDocumentFilenameSelectionKey(documentIndex), + knowledgeDocumentContentSelectionKey(documentIndex), + ...tagTargets[documentIndex].map((_tag, tagIndex) => + knowledgeDocumentTagValueSelectionKey(documentIndex, tagIndex) + ), + ]) + const resolved = resolveKnowledgeWriteSecretProvenance({ + headers: options.headers, + payload: options.payload, + authType: options.authType, + userId: options.userId, + ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), + selectionKeys, + }) + if (!resolved.success) return resolved + if (!resolved.provenances) return { success: true } + + let provenanceIndex = 0 + const provenances: KnowledgeDocumentWriteSecretProvenance[] = [] + for (const tags of tagTargets) { + const filename = resolved.provenances[provenanceIndex++] + const content = resolved.provenances[provenanceIndex++] + const tagProvenances: KnowledgeDocumentWriteSecretProvenance['tags'][number][] = [] + for (const tag of tags) { + const tagValue = resolved.provenances[provenanceIndex++] + tagProvenances.push({ tagName: tag.tagName, provenance: tagValue }) + } + provenances.push({ filename, content, tags: tagProvenances }) + } + return { success: true, provenances } +} + +/** Finalizes private provenance after the functional Knowledge response passes its contract. */ +export async function finalizeKnowledgeProvenanceResponse(options: { + headers: Headers + authType: AuthTypeValue | undefined + userId: string + workspaceId?: string + body: Record + provenances: readonly DurableSecretProvenance[] +}): Promise { + const negotiation = negotiatePrivateToolMetadataResponse( + options.headers, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, + options.authType === AuthType.INTERNAL_JWT + ) + if (negotiation.status === 'not-requested') return {} + if (negotiation.status === 'rejected') rejectInvalidKnowledgeProvenance() + const registry = new ResolvedSecretTraceRegistry([], { + userId: options.userId, + ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), + }) + for (const provenance of options.provenances) { + if (provenance.status === 'unknown') { + registry.markIncomplete('durable-provenance-unknown') + break + } + const sourceRegistry = await createDurableSecretProvenanceRegistry(provenance, { + userId: options.userId, + ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), + }) + if (sourceRegistry) registry.mergeToolCallRegistry(sourceRegistry) + } + const envelope = serializePrivateToolMetadataResponseEnvelope( + options.body, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, + registry.exportCommittedProvenanceForValue(options.body) + ) + return finalizeKnowledgeMetadataEnvelope(envelope) +} + +/** Serializes an already-populated request registry as private response metadata. */ +export function finalizeKnowledgeRegistryResponse(options: { + headers: Headers + authType: AuthTypeValue | undefined + body: Record + registry: ResolvedSecretTraceRegistry +}): InternalJsonResponseFinalization { + const negotiation = negotiatePrivateToolMetadataResponse( + options.headers, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, + options.authType === AuthType.INTERNAL_JWT + ) + if (negotiation.status === 'not-requested') return {} + if (negotiation.status === 'rejected') rejectInvalidKnowledgeProvenance() + const envelope = serializePrivateToolMetadataResponseEnvelope( + options.body, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, + options.registry.exportCommittedProvenanceForValue(options.body) + ) + return finalizeKnowledgeMetadataEnvelope(envelope) +} + +/** Emits private response provenance for a bounded exact snapshot of persisted KB rows. */ +export async function finalizeKnowledgePersistedResponse(options: { + headers: Headers + authType: AuthTypeValue | undefined + userId: string + workspaceId?: string + body: Record + documents?: readonly { + id: string + source: KnowledgeDocumentSourceValue + value: unknown + }[] + chunks?: readonly { + id: string + documentId: string + content: string + value: unknown + }[] +}): Promise { + const negotiation = negotiatePrivateToolMetadataResponse( + options.headers, + RESOLVED_SECRET_PROVENANCE_METADATA_V1, + options.authType === AuthType.INTERNAL_JWT + ) + if (negotiation.status === 'not-requested') return {} + if (negotiation.status === 'rejected') rejectInvalidKnowledgeProvenance() + + const registry = new ResolvedSecretTraceRegistry([], { + userId: options.userId, + ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), + }) + await importKnowledgePersistedResponseSecretProvenance({ + registry, + documents: options.documents, + chunks: options.chunks, + ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), + actorUserId: options.userId, + }) + return finalizeKnowledgeRegistryResponse({ + headers: options.headers, + authType: options.authType, + body: options.body, + registry, + }) +} diff --git a/apps/sim/lib/knowledge/documents/document-processor-secret-provenance.test.ts b/apps/sim/lib/knowledge/documents/document-processor-secret-provenance.test.ts index d3386f69946..2d8bc1e55a1 100644 --- a/apps/sim/lib/knowledge/documents/document-processor-secret-provenance.test.ts +++ b/apps/sim/lib/knowledge/documents/document-processor-secret-provenance.test.ts @@ -7,11 +7,13 @@ const { mockDownloadFileFromUrl, mockGenerateInternalToken, mockGetInternalApiBaseUrl, + mockExecuteMistralParse, mockParseBuffer, } = vi.hoisted(() => ({ mockDownloadFileFromUrl: vi.fn(), mockGenerateInternalToken: vi.fn(), mockGetInternalApiBaseUrl: vi.fn(), + mockExecuteMistralParse: vi.fn(), mockParseBuffer: vi.fn(), })) @@ -29,6 +31,10 @@ vi.mock('@/lib/file-parsers', () => ({ isSupportedFileType: (extension: string) => ['pdf', 'docx', 'txt', 'csv'].includes(extension), })) +vi.mock('@/lib/internal/mistral/operations', () => ({ + executeMistralParse: mockExecuteMistralParse, +})) + vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ downloadFileFromUrl: mockDownloadFileFromUrl, })) @@ -51,6 +57,13 @@ describe('knowledge document model-input provenance', () => { }) mockGenerateInternalToken.mockResolvedValue('internal-token') mockGetInternalApiBaseUrl.mockReturnValue('http://sim.local') + mockExecuteMistralParse.mockResolvedValue({ + success: true, + output: { + pages: [{ markdown: 'Extracted document text' }], + usage_info: { pages_processed: 1 }, + }, + }) }) afterEach(() => { @@ -128,17 +141,6 @@ describe('knowledge document model-input provenance', () => { MISTRAL_API_KEY: 'mistral-key', }) mockDownloadFileFromUrl.mockResolvedValue(Buffer.from('not-a-real-pdf')) - const fetchMock = vi.fn().mockResolvedValue( - new Response( - JSON.stringify({ - pages: [{ markdown: 'Extracted document text' }], - usage_info: { pages_processed: 1 }, - }), - { status: 200, headers: { 'Content-Type': 'application/json' } } - ) - ) - vi.stubGlobal('fetch', fetchMock) - const processed = await runWithKnowledgeModelInputProvenance( undefined, () => @@ -155,15 +157,15 @@ describe('knowledge document model-input provenance', () => { ) expect(processed.metadata.processingMethod).toBe('mistral-ocr') - expect(fetchMock).toHaveBeenCalledOnce() - const [endpoint, init] = fetchMock.mock.calls[0] as [string, RequestInit] - expect(endpoint).toBe('http://sim.local/api/tools/mistral/parse') - const headers = new Headers(init.headers) - expect(headers.get('authorization')).toBe('Bearer internal-token') + expect(mockExecuteMistralParse).toHaveBeenCalledOnce() + const [requestBody, context] = mockExecuteMistralParse.mock.calls[0] as [ + Record, + { headers: Headers }, + ] + const headers = context.headers expect(headers.get('x-sim-private-model-input-provenance')).toBe( 'resolved-secret-provenance-v1' ) - const requestBody = JSON.parse(String(init.body)) as Record expect(requestBody[RESOLVED_SECRET_PROVENANCE_FIELD]).toEqual({ version: 1, complete: true, diff --git a/apps/sim/lib/knowledge/documents/document-processor.ts b/apps/sim/lib/knowledge/documents/document-processor.ts index 271eb445f67..68ffa2346c4 100644 --- a/apps/sim/lib/knowledge/documents/document-processor.ts +++ b/apps/sim/lib/knowledge/documents/document-processor.ts @@ -1,6 +1,7 @@ import { randomBytes } from 'crypto' import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' import { PDFDocument } from 'pdf-lib' import { getBYOKKey } from '@/lib/api-key/byok' import { @@ -22,9 +23,16 @@ import { isPayloadSizeLimitError, readResponseTextWithLimit, } from '@/lib/core/utils/stream-limits' +import { + addModelInputProvenanceToRequest, + createModelInputProvenanceRequestMetadata, +} from '@/lib/execution/model-input-provenance' import { parseBuffer } from '@/lib/file-parsers' import { decodeDataUriWithinLimit } from '@/lib/file-parsers/data-uri' import type { FileParseMetadata, FileParseResult } from '@/lib/file-parsers/types' +import { MistralOperationError } from '@/lib/internal/mistral/errors' +import { mistralParseInputSchema } from '@/lib/internal/mistral/input' +import { executeMistralParse } from '@/lib/internal/mistral/operations' import { MAX_DOCUMENT_CHUNKS, PermanentDocumentProcessingError, @@ -54,7 +62,6 @@ import { getFileExtension, isInternalFileUrl } from '@/lib/uploads/utils/file-ut import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server' import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' import { mistralParserTool } from '@/tools/mistral/parser' -import { prepareToolRequest } from '@/tools/request-transport' const logger = createLogger('DocumentProcessor') @@ -847,28 +854,47 @@ async function executeMistralOCRRequest( ): Promise { return retryWithExponentialBackoff( async () => { - const request = prepareToolRequest( - mistralParserTool, - params, - getKnowledgeOpaqueModelInputRegistry() + const input = mistralParseInputSchema.parse(mistralParserTool.operation.input(params)) + const headers = new Headers() + const modelInput = mistralParserTool.operation.modelInput + const inputPaths = + modelInput?.mode === 'private-provenance' ? modelInput.inputPaths(params) : [] + const metadata = createModelInputProvenanceRequestMetadata( + getKnowledgeOpaqueModelInputRegistry(), + inputPaths ) - let { url } = request - - if (request.isInternalRoute) { - const { getInternalApiBaseUrl } = await import('@/lib/core/utils/urls') - url = `${getInternalApiBaseUrl()}${url}` - } - - const { headers } = request - - if (request.isInternalRoute) { - const { generateInternalToken } = await import('@/lib/auth/internal') - const internalToken = await generateInternalToken(userId) - headers.set('Authorization', `Bearer ${internalToken}`) + const operationInput = mistralParseInputSchema.parse( + addModelInputProvenanceToRequest(input, headers, metadata) + ) + const controller = new AbortController() + const timeoutId = setTimeout(() => controller.abort(), TIMEOUTS.MISTRAL_OCR_API) + try { + try { + const result = await executeMistralParse(operationInput, { + headers, + maxResponseBytes: MAX_OCR_RESPONSE_BYTES, + requestId: generateId(), + signal: controller.signal, + trustedCaller: 'knowledge-ingestion', + userId, + }) + return Response.json(result) + } catch (error) { + if (controller.signal.aborted) throw new Error('OCR API request timed out') + if (error instanceof MistralOperationError) { + if (error.status === 413) { + throw new PermanentDocumentProcessingError( + 'document_complexity_limit', + 'The OCR provider rejected this document because the request was too large. Split or optimize the document and retry.' + ) + } + throw new APIError(`OCR failed: ${error.status}`, error.status) + } + throw error + } + } finally { + clearTimeout(timeoutId) } - - if (!request.body) throw new Error('Mistral parser request body is unavailable') - return makeOCRRequest(url, headers, request.body) }, { maxRetries: 3, initialDelayMs: 1000, maxDelayMs: 10000 } ) diff --git a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts index 4306fe7f2b2..b450f60897d 100644 --- a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts +++ b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts @@ -8,12 +8,14 @@ */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -const { mockParseBuffer, mockDownload, mockToken, mockBaseUrl } = vi.hoisted(() => ({ - mockParseBuffer: vi.fn(), - mockDownload: vi.fn(), - mockToken: vi.fn(), - mockBaseUrl: vi.fn(), -})) +const { mockParseBuffer, mockDownload, mockToken, mockBaseUrl, mockExecuteMistralParse } = + vi.hoisted(() => ({ + mockParseBuffer: vi.fn(), + mockDownload: vi.fn(), + mockToken: vi.fn(), + mockBaseUrl: vi.fn(), + mockExecuteMistralParse: vi.fn(), + })) vi.mock('@/lib/auth/internal', () => ({ generateInternalToken: mockToken })) vi.mock('@/lib/core/utils/urls', async (importOriginal) => ({ @@ -26,8 +28,12 @@ vi.mock('@/lib/file-parsers', () => ({ isSupportedFileType: (extension: string) => ['pdf'].includes(extension), })) vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ downloadFileFromUrl: mockDownload })) +vi.mock('@/lib/internal/mistral/operations', () => ({ + executeMistralParse: mockExecuteMistralParse, +})) import { env } from '@/lib/core/config/env' +import { MistralOperationError } from '@/lib/internal/mistral/errors' import { PermanentDocumentProcessingError } from '@/lib/knowledge/documents/document-processing-error' import { processDocument } from '@/lib/knowledge/documents/document-processor' import { runWithKnowledgeModelInputProvenance } from '@/lib/knowledge/model-input-provenance' @@ -63,6 +69,16 @@ describe('PDF OCR triage', () => { mockDownload.mockResolvedValue(Buffer.from('%PDF-1.7')) mockToken.mockResolvedValue('internal-token') mockBaseUrl.mockReturnValue('http://sim.local') + mockExecuteMistralParse.mockImplementation(async () => { + const response = await fetch('https://api.mistral.ai/v1/ocr', { method: 'POST' }) + if (!response.ok) { + throw new MistralOperationError(response.status, { + success: false, + error: `Mistral API error: ${response.statusText}`, + }) + } + return { success: true, output: await response.json() } + }) }) afterEach(() => { diff --git a/apps/sim/lib/knowledge/documents/utils.test.ts b/apps/sim/lib/knowledge/documents/utils.test.ts index 150f44af886..260743cf5d9 100644 --- a/apps/sim/lib/knowledge/documents/utils.test.ts +++ b/apps/sim/lib/knowledge/documents/utils.test.ts @@ -667,6 +667,24 @@ describe('retryWithExponentialBackoff retry budget', () => { expect(operation).toHaveBeenCalledTimes(3) }) + it('cancels a retry wait without starting another attempt', async () => { + const controller = new AbortController() + const operation = vi + .fn() + .mockRejectedValue(Object.assign(new Error('service unavailable'), { status: 503 })) + const result = retryWithExponentialBackoff(operation, { + maxRetries: 2, + initialDelayMs: 60_000, + signal: controller.signal, + }) + + await vi.waitFor(() => expect(operation).toHaveBeenCalledOnce()) + controller.abort() + + await expect(result).rejects.toMatchObject({ name: 'AbortError' }) + expect(operation).toHaveBeenCalledOnce() + }) + it.each([ { retryBudgetMs: Number.NaN }, { retryBudgetMs: Number.POSITIVE_INFINITY }, diff --git a/apps/sim/lib/knowledge/documents/utils.ts b/apps/sim/lib/knowledge/documents/utils.ts index 90abe05c0c6..5a54d0d7204 100644 --- a/apps/sim/lib/knowledge/documents/utils.ts +++ b/apps/sim/lib/knowledge/documents/utils.ts @@ -1,6 +1,6 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' -import { sleep } from '@sim/utils/helpers' +import { interruptibleSleep } from '@sim/utils/helpers' import { randomFloat } from '@sim/utils/random' import { parseRetryAfter } from '@sim/utils/retry' import { truncate } from '@sim/utils/string' @@ -39,6 +39,8 @@ type RetryableError = | { status?: number; message?: string; headers?: HeaderReader } export interface RetryOptions { + /** Cancels the current retry cycle, including waits between attempts. */ + signal?: AbortSignal maxRetries?: number initialDelayMs?: number maxDelayMs?: number @@ -340,6 +342,7 @@ export async function retryWithExponentialBackoff( retryBudgetMs, backoffMultiplier = 2, retryCondition = isRetryableError, + signal, } = options const maxRetryAfterMs = options.maxRetryAfterMs ?? retryBudgetMs ?? maxDelayMs @@ -366,6 +369,7 @@ export async function retryWithExponentialBackoff( let delay = initialDelayMs for (let attempt = 0; attempt <= maxRetries; attempt++) { + signal?.throwIfAborted() try { logger.debug(`Executing operation attempt ${attempt + 1}/${maxRetries + 1}`) const result = await operation() @@ -432,7 +436,8 @@ export async function retryWithExponentialBackoff( `Retrying in ${Math.round(actualDelay)}ms (attempt ${attempt + 1}/${maxRetries + 1})${retryAfterMs ? ' (server-stated)' : ''}` ) - await sleep(actualDelay) + await interruptibleSleep(actualDelay, signal) + signal?.throwIfAborted() // Exponential backoff (skip if we used Retry-After) if (!retryAfterMs) { diff --git a/apps/sim/lib/logs/api/route-policies.test.ts b/apps/sim/lib/logs/api/route-policies.test.ts new file mode 100644 index 00000000000..5c096657cd9 --- /dev/null +++ b/apps/sim/lib/logs/api/route-policies.test.ts @@ -0,0 +1,126 @@ +/** + * @vitest-environment node + */ + +import { resetEnvMock } from '@sim/testing' +import { NextRequest } from 'next/server' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockBindDelegation, mockGetSession } = vi.hoisted(() => ({ + mockBindDelegation: vi.fn(), + mockGetSession: vi.fn(), +})) + +vi.mock('@/lib/auth', () => ({ getSession: mockGetSession })) +vi.mock('@/lib/auth/internal-delegation', () => ({ + bindInternalExecutorDelegation: mockBindDelegation, + InvalidInternalDelegationBindingError: class InvalidInternalDelegationBindingError extends Error {}, +})) +vi.unmock('@/lib/auth/internal') + +import { generateInternalDelegationToken } from '@/lib/auth/internal' +import { internalLogsSessionOrExecutorAuth } from '@/lib/logs/api/route-policies' + +afterAll(resetEnvMock) + +describe('internal logs route authentication', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetSession.mockResolvedValue(null) + mockBindDelegation.mockImplementation(async (delegation, options) => ({ + kind: 'delegated', + serviceId: 'executor', + subjectUserId: delegation.subjectUserId, + workspaceId: 'canonical-workspace', + delegationId: delegation.delegationId, + audience: options.audience, + issuedAt: delegation.issuedAt, + expiresAt: delegation.expiresAt, + resourceScope: options.resourceScope, + delegationContext: { + kind: 'workflow_execution', + workflowId: delegation.workflowId, + ...(delegation.executionId ? { executionId: delegation.executionId } : {}), + }, + })) + }) + + it('preserves the signed execution origin when the route names a log ID', async () => { + const token = await generateInternalDelegationToken({ + subjectUserId: 'user-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }) + + const principal = await internalLogsSessionOrExecutorAuth.authenticate( + new NextRequest('http://localhost/api/logs/log-1', { + headers: { authorization: `Bearer ${token}` }, + }), + { id: 'log-1' } + ) + + expect(principal).toMatchObject({ + kind: 'delegated', + workspaceId: 'canonical-workspace', + resourceScope: { executionId: 'execution-1' }, + delegationContext: { executionId: 'execution-1' }, + }) + }) + + it('keeps workflow-scoped executor tokens unscoped to one execution', async () => { + const token = await generateInternalDelegationToken({ + subjectUserId: 'user-1', + workflowId: 'workflow-1', + }) + + const principal = await internalLogsSessionOrExecutorAuth.authenticate( + new NextRequest('http://localhost/api/logs/log-1', { + headers: { authorization: `Bearer ${token}` }, + }), + { id: 'log-1' } + ) + + expect(principal.resourceScope).toBeUndefined() + }) + + it('rejects an executor delegation without canonical workflow execution context', async () => { + mockBindDelegation.mockResolvedValueOnce({ + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'canonical-workspace', + delegationId: 'delegation-1', + audience: 'sim:logs', + issuedAt: new Date('2026-08-27T00:00:00.000Z'), + expiresAt: new Date('2026-08-27T00:05:00.000Z'), + }) + const token = await generateInternalDelegationToken({ + subjectUserId: 'user-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + }) + + await expect( + internalLogsSessionOrExecutorAuth.authenticate( + new NextRequest('http://localhost/api/logs/log-1', { + headers: { authorization: `Bearer ${token}` }, + }), + { id: 'log-1' } + ) + ).rejects.toThrow('Executor log delegation is missing its canonical workflow execution context') + }) + + it('preserves browser session principals', async () => { + mockGetSession.mockResolvedValue({ + user: { id: 'user-1' }, + session: { id: 'session-1' }, + }) + + await expect( + internalLogsSessionOrExecutorAuth.authenticate( + new NextRequest('http://localhost/api/logs/log-1'), + { id: 'log-1' } + ) + ).resolves.toEqual({ kind: 'session', userId: 'user-1', sessionId: 'session-1' }) + }) +}) diff --git a/apps/sim/lib/logs/api/route-policies.ts b/apps/sim/lib/logs/api/route-policies.ts index 1f97c566656..19c685eaa87 100644 --- a/apps/sim/lib/logs/api/route-policies.ts +++ b/apps/sim/lib/logs/api/route-policies.ts @@ -1,7 +1,31 @@ import { + createInternalSessionOrExecutorAuth, createV2ResourceConcealmentPolicy, v2OrchestrationErrorPolicy, } from '@/lib/api/server/routes' +import { LOGS_DELEGATION_AUDIENCE } from '@/lib/logs/application/authorization' + +const internalLogsSessionOrExecutorAuthBase = createInternalSessionOrExecutorAuth({ + audience: LOGS_DELEGATION_AUDIENCE, +}) + +export const internalLogsSessionOrExecutorAuth = { + async authenticate( + ...args: Parameters + ) { + const principal = await internalLogsSessionOrExecutorAuthBase.authenticate(...args) + if (principal.kind !== 'delegated') return principal + + const { delegationContext } = principal + if (!delegationContext) { + throw new Error('Executor log delegation is missing its canonical workflow execution context') + } + const { executionId } = delegationContext + return executionId + ? { ...principal, resourceScope: { ...principal.resourceScope, executionId } } + : principal + }, +} /** * `GET /logs` and `GET /billing/logs` both take a caller-named `workspaceId` and diff --git a/apps/sim/lib/logs/application/authorization.ts b/apps/sim/lib/logs/application/authorization.ts new file mode 100644 index 00000000000..81f587bb07e --- /dev/null +++ b/apps/sim/lib/logs/application/authorization.ts @@ -0,0 +1,38 @@ +import type { Principal } from '@sim/auth/principal' +import { + DelegatedWorkspaceAuthorizationError, + NoWorkspaceAccessError, + type WorkspaceAuthorizationContext, + type WorkspaceDelegationPolicy, +} from '@/lib/core/application' + +export const LOGS_DELEGATION_AUDIENCE = 'sim:logs' + +export interface LogAuthorizationContext extends WorkspaceAuthorizationContext { + executionId?: string +} + +export const logDelegationPolicy: WorkspaceDelegationPolicy = { + audience: LOGS_DELEGATION_AUDIENCE, + isWithinScope( + principal: Extract, + context: LogAuthorizationContext + ) { + if (principal.serviceId !== 'executor') return true + return principal.resourceScope?.executionId === undefined + ? true + : principal.resourceScope.executionId === context.executionId + }, +} + +export function logDelegationAuthorization() { + return { + delegation: logDelegationPolicy as WorkspaceDelegationPolicy, + } +} + +export function isConcealedLogAuthorizationError(error: unknown): boolean { + return ( + error instanceof NoWorkspaceAccessError || error instanceof DelegatedWorkspaceAuthorizationError + ) +} diff --git a/apps/sim/lib/logs/application/get-public-log.ts b/apps/sim/lib/logs/application/get-public-log.ts index 5f68e0753be..d99e2703d19 100644 --- a/apps/sim/lib/logs/application/get-public-log.ts +++ b/apps/sim/lib/logs/application/get-public-log.ts @@ -4,6 +4,7 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' import { MAX_FOLDERS_PER_WORKSPACE } from '@/lib/folders/constants' import { ROOT_FOLDER_PATH } from '@/lib/folders/paths' import { loadActiveFolderPathIndex } from '@/lib/folders/queries' +import { logDelegationAuthorization } from '@/lib/logs/application/authorization' import { logOperations } from '@/lib/logs/application/operations' import { buildCostLedger } from '@/lib/logs/cost-ledger' import { materializeExecutionDataForDisplay } from '@/lib/logs/execution/trace-store' @@ -86,7 +87,7 @@ export const getPublicLog = defineAuthorizedWorkspaceUseCase({ if (!workspace) throw new OrchestrationError('not_found', 'Log not found') return { ...workspace, executionId: scope.executionId, workflowId: scope.workflowId } }, - authorizationOptions: {}, + authorizationOptions: logDelegationAuthorization(), execute: async ({ principal, context }): Promise => { const log = await getPublicWorkflowLog( { column: 'executionId', value: context.executionId }, diff --git a/apps/sim/lib/logs/application/list-logs.ts b/apps/sim/lib/logs/application/list-logs.ts new file mode 100644 index 00000000000..9fed67ced18 --- /dev/null +++ b/apps/sim/lib/logs/application/list-logs.ts @@ -0,0 +1,41 @@ +import type { ListLogsResponse } from '@/lib/api/contracts/logs' +import { defineAuthorizedWorkspaceUseCase, type OperationUseCase } from '@/lib/core/application' +import { asOrchestrationError } from '@/lib/core/orchestration/types' +import { + isConcealedLogAuthorizationError, + logDelegationAuthorization, +} from '@/lib/logs/application/authorization' +import { logOperations } from '@/lib/logs/application/operations' +import { type ListLogsParams, readLogs } from '@/lib/logs/list-logs' +import { resolveActiveWorkspaceApplicationContext } from '@/lib/workspaces/application/workspace-context' + +const authorizedListLogsUseCase = defineAuthorizedWorkspaceUseCase({ + operation: logOperations.list, + resolveContext: ({ input }: { input: ListLogsParams }) => + resolveActiveWorkspaceApplicationContext(input.workspaceId), + authorizationOptions: logDelegationAuthorization(), + async execute({ input, context }) { + return readLogs({ ...input, workspaceId: context.workspaceId }) + }, +}) + +export const listLogsUseCase: OperationUseCase< + typeof logOperations.list, + ListLogsParams, + ListLogsResponse +> = { + operation: logOperations.list, + async execute(args) { + try { + return await authorizedListLogsUseCase.execute(args) + } catch (error) { + if ( + isConcealedLogAuthorizationError(error) || + asOrchestrationError(error)?.code === 'not_found' + ) { + return { data: [], nextCursor: null } + } + throw error + } + }, +} diff --git a/apps/sim/lib/logs/application/list-public-logs.ts b/apps/sim/lib/logs/application/list-public-logs.ts index fd92332c6af..729f015c6c9 100644 --- a/apps/sim/lib/logs/application/list-public-logs.ts +++ b/apps/sim/lib/logs/application/list-public-logs.ts @@ -2,6 +2,7 @@ import type { CursorKey, ListSortOrder } from '@/lib/api/list-query' import { defineAuthorizedWorkspaceUseCase } from '@/lib/core/application' import { OrchestrationError } from '@/lib/core/orchestration/types' import { MATERIALIZE_CONCURRENCY, mapWithConcurrency } from '@/lib/core/utils/concurrency' +import { logDelegationAuthorization } from '@/lib/logs/application/authorization' import { logOperations } from '@/lib/logs/application/operations' import { materializeExecutionDataForDisplay } from '@/lib/logs/execution/trace-store' import { resolveLogFolderScope } from '@/lib/logs/folder-scope' @@ -47,7 +48,7 @@ export const listPublicLogs = defineAuthorizedWorkspaceUseCase({ if (!context) throw new OrchestrationError('not_found', 'Workspace not found') return context }, - authorizationOptions: {}, + authorizationOptions: logDelegationAuthorization(), execute: async ({ principal, input, context }): Promise => { const folderScope = input.folderPaths ? await resolveLogFolderScope(context.workspaceId, input.folderPaths) diff --git a/apps/sim/lib/logs/application/operations.test.ts b/apps/sim/lib/logs/application/operations.test.ts new file mode 100644 index 00000000000..bf7b9bec922 --- /dev/null +++ b/apps/sim/lib/logs/application/operations.test.ts @@ -0,0 +1,55 @@ +/** + * @vitest-environment node + */ + +import type { WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' +import { describe, expect, it } from 'vitest' +import { logDelegationPolicy } from '@/lib/logs/application/authorization' +import { logOperations } from '@/lib/logs/application/operations' + +const EXECUTOR_PRINCIPAL: WorkflowExecutionDelegatedPrincipal = { + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'workspace-1', + delegationId: 'delegation-1', + audience: 'sim:logs', + issuedAt: new Date('2026-08-27T00:00:00.000Z'), + expiresAt: new Date('2026-08-27T00:05:00.000Z'), + delegationContext: { kind: 'workflow_execution', workflowId: 'workflow-1' }, + resourceScope: { executionId: 'execution-1' }, +} + +describe('logs operation registry', () => { + it('admits executor delegation only to the three semantic read operations it needs', () => { + expect(logOperations.list.delegatedServices).toEqual(['copilot', 'executor']) + expect(logOperations.readDetail.delegatedServices).toEqual(['copilot', 'executor']) + expect(logOperations.readExecutionSnapshot.delegatedServices).toEqual(['executor']) + expect(logOperations.readStats.delegatedServices).toBeUndefined() + + for (const operation of Object.values(logOperations)) { + expect(operation.minimumRole).toBe('read') + } + }) + + it('binds scoped executor reads to the canonical execution context', () => { + const workspaceContext = { + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + } + + expect( + logDelegationPolicy.isWithinScope(EXECUTOR_PRINCIPAL, { + ...workspaceContext, + executionId: 'execution-1', + }) + ).toBe(true) + expect( + logDelegationPolicy.isWithinScope(EXECUTOR_PRINCIPAL, { + ...workspaceContext, + executionId: 'execution-2', + }) + ).toBe(false) + }) +}) diff --git a/apps/sim/lib/logs/application/operations.ts b/apps/sim/lib/logs/application/operations.ts index 22b7f2debc5..2d84652e988 100644 --- a/apps/sim/lib/logs/application/operations.ts +++ b/apps/sim/lib/logs/application/operations.ts @@ -1,13 +1,17 @@ import { defineWorkspaceOperation } from '@/lib/core/application' const PUBLIC_API_PRINCIPAL_KINDS = ['personal_api_key', 'workspace_api_key'] as const +const LOG_READER_PRINCIPAL_POLICY = { + principalKinds: ['session', 'personal_api_key', 'workspace_api_key', 'delegated'], + delegatedServices: ['copilot', 'executor'], +} as const export const logOperations = { list: defineWorkspaceOperation({ id: 'logs.list', minimumRole: 'read', workspaceApiKey: 'allow', - principalKinds: PUBLIC_API_PRINCIPAL_KINDS, + ...LOG_READER_PRINCIPAL_POLICY, }), readStats: defineWorkspaceOperation({ id: 'logs.read_stats', @@ -19,6 +23,13 @@ export const logOperations = { id: 'logs.read_detail', minimumRole: 'read', workspaceApiKey: 'allow', - principalKinds: PUBLIC_API_PRINCIPAL_KINDS, + ...LOG_READER_PRINCIPAL_POLICY, + }), + readExecutionSnapshot: defineWorkspaceOperation({ + id: 'logs.read_execution_snapshot', + minimumRole: 'read', + workspaceApiKey: 'deny', + principalKinds: ['session', 'delegated'], + delegatedServices: ['executor'], }), } as const diff --git a/apps/sim/lib/logs/application/public-log-use-cases.test.ts b/apps/sim/lib/logs/application/public-log-use-cases.test.ts index 4b5ba8013c1..320c0e65bf9 100644 --- a/apps/sim/lib/logs/application/public-log-use-cases.test.ts +++ b/apps/sim/lib/logs/application/public-log-use-cases.test.ts @@ -134,7 +134,7 @@ describe('public log application use cases', () => { mocks.materialize.mockResolvedValue({ finalOutput: { ok: true } }) }) - it('rejects unsupported principals before resolving the run', async () => { + it('allows the internal session surface through the shared read operation', async () => { const principal: SessionPrincipal = { kind: 'session', userId: 'user-1', @@ -143,9 +143,9 @@ describe('public log application use cases', () => { await expect( getPublicLog.execute({ principal, input: { runId: 'run-1' } }) - ).rejects.toMatchObject({ code: 'forbidden' }) - expect(mocks.getLogScope).not.toHaveBeenCalled() - expect(mocks.getLog).not.toHaveBeenCalled() + ).resolves.toMatchObject({ log: { executionId: 'run-1' } }) + expect(mocks.getLogScope).toHaveBeenCalledWith('run-1') + expect(mocks.getLog).toHaveBeenCalledOnce() }) it('derives workspace and materialization scope from the canonical run', async () => { diff --git a/apps/sim/lib/logs/application/read-execution-snapshot.ts b/apps/sim/lib/logs/application/read-execution-snapshot.ts new file mode 100644 index 00000000000..e5b645710c0 --- /dev/null +++ b/apps/sim/lib/logs/application/read-execution-snapshot.ts @@ -0,0 +1,231 @@ +import { requirePrincipalSubjectUserId } from '@sim/auth/principal' +import { db } from '@sim/db' +import { jobExecutionLogs, workflowExecutionLogs, workflowExecutionSnapshots } from '@sim/db/schema' +import { eq, inArray } from 'drizzle-orm' +import type { ExecutionSnapshotData } from '@/lib/api/contracts/logs' +import { defineAuthorizedWorkspaceUseCase, type OperationUseCase } from '@/lib/core/application' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { + isConcealedLogAuthorizationError, + logDelegationAuthorization, +} from '@/lib/logs/application/authorization' +import { logOperations } from '@/lib/logs/application/operations' +import { hydrateChildTraces } from '@/lib/logs/execution/hydrate-child-traces' +import { materializeExecutionData } from '@/lib/logs/execution/trace-store' +import type { TraceSpan, WorkflowExecutionLog } from '@/lib/logs/types' +import { + type ActiveWorkspaceApplicationContext, + resolveActiveWorkspaceApplicationContext, +} from '@/lib/workspaces/application/workspace-context' + +const CHILD_SNAPSHOT_QUERY_CHUNK_SIZE = 1_000 + +interface WorkflowExecutionRecord { + kind: 'workflow' + id: string + workflowId: string | null + workspaceId: string + executionId: string + stateSnapshotId: string + trigger: string | null + startedAt: Date + endedAt: Date | null + totalDurationMs: number | null + costTotal: string | null + executionData: unknown +} + +interface JobExecutionRecord { + kind: 'job' + id: string + workspaceId: string + executionId: string + trigger: string + startedAt: Date + endedAt: Date | null + totalDurationMs: number | null + cost: unknown +} + +interface ExecutionSnapshotContext extends ActiveWorkspaceApplicationContext { + executionId: string + record: WorkflowExecutionRecord | JobExecutionRecord +} + +export interface ReadExecutionSnapshotInput { + executionId: string + signal?: AbortSignal +} + +async function resolveExecutionSnapshotContext( + input: ReadExecutionSnapshotInput +): Promise { + input.signal?.throwIfAborted() + const { executionId } = input + const [workflowRecord] = await db + .select({ + id: workflowExecutionLogs.id, + workflowId: workflowExecutionLogs.workflowId, + workspaceId: workflowExecutionLogs.workspaceId, + executionId: workflowExecutionLogs.executionId, + stateSnapshotId: workflowExecutionLogs.stateSnapshotId, + trigger: workflowExecutionLogs.trigger, + startedAt: workflowExecutionLogs.startedAt, + endedAt: workflowExecutionLogs.endedAt, + totalDurationMs: workflowExecutionLogs.totalDurationMs, + costTotal: workflowExecutionLogs.costTotal, + executionData: workflowExecutionLogs.executionData, + }) + .from(workflowExecutionLogs) + .where(eq(workflowExecutionLogs.executionId, executionId)) + .limit(1) + input.signal?.throwIfAborted() + + if (workflowRecord) { + const workspace = await resolveActiveWorkspaceApplicationContext(workflowRecord.workspaceId) + input.signal?.throwIfAborted() + return { + ...workspace, + executionId, + record: { kind: 'workflow', ...workflowRecord }, + } + } + + const [jobRecord] = await db + .select({ + id: jobExecutionLogs.id, + workspaceId: jobExecutionLogs.workspaceId, + executionId: jobExecutionLogs.executionId, + trigger: jobExecutionLogs.trigger, + startedAt: jobExecutionLogs.startedAt, + endedAt: jobExecutionLogs.endedAt, + totalDurationMs: jobExecutionLogs.totalDurationMs, + cost: jobExecutionLogs.cost, + }) + .from(jobExecutionLogs) + .where(eq(jobExecutionLogs.executionId, executionId)) + .limit(1) + input.signal?.throwIfAborted() + + if (!jobRecord) throw new OrchestrationError('not_found', 'Workflow execution not found') + const workspace = await resolveActiveWorkspaceApplicationContext(jobRecord.workspaceId) + input.signal?.throwIfAborted() + return { ...workspace, executionId, record: { kind: 'job', ...jobRecord } } +} + +function collectChildSnapshotIds(traceSpans: TraceSpan[]): string[] { + const ids = new Set() + const pending = [...traceSpans] + while (pending.length > 0) { + const span = pending.pop() + if (!span) continue + if (typeof span.childWorkflowSnapshotId === 'string') ids.add(span.childWorkflowSnapshotId) + if (span.children?.length) pending.push(...span.children) + } + return [...ids] +} + +const authorizedReadExecutionSnapshotUseCase = defineAuthorizedWorkspaceUseCase({ + operation: logOperations.readExecutionSnapshot, + resolveContext: ({ input }: { input: ReadExecutionSnapshotInput }) => + resolveExecutionSnapshotContext(input), + authorizationOptions: logDelegationAuthorization(), + async execute({ principal, input, context }): Promise { + input.signal?.throwIfAborted() + const record = context.record + if (record.kind === 'job') { + return { + executionId: record.executionId, + workflowId: null, + workflowState: null, + childWorkflowSnapshots: {}, + executionMetadata: { + trigger: record.trigger, + startedAt: record.startedAt.toISOString(), + endedAt: record.endedAt?.toISOString(), + totalDurationMs: record.totalDurationMs, + cost: record.cost || null, + }, + } + } + + const [snapshot] = await db + .select() + .from(workflowExecutionSnapshots) + .where(eq(workflowExecutionSnapshots.id, record.stateSnapshotId)) + .limit(1) + if (!snapshot) { + throw new OrchestrationError('not_found', 'Workflow state snapshot not found') + } + + const executionData = (await materializeExecutionData( + record.executionData as Record | null, + { + workspaceId: context.workspaceId, + workflowId: record.workflowId, + executionId: record.executionId, + } + )) as WorkflowExecutionLog['executionData'] + const traceSpans = (executionData?.traceSpans as TraceSpan[]) || [] + if (traceSpans.length > 0) { + await hydrateChildTraces(traceSpans, { + viewerUserId: requirePrincipalSubjectUserId(principal), + }) + } + + const childSnapshotIds = collectChildSnapshotIds(traceSpans) + const childWorkflowSnapshots: Array<{ id: string; stateData: unknown }> = [] + for (let index = 0; index < childSnapshotIds.length; index += CHILD_SNAPSHOT_QUERY_CHUNK_SIZE) { + input.signal?.throwIfAborted() + childWorkflowSnapshots.push( + ...(await db + .select({ + id: workflowExecutionSnapshots.id, + stateData: workflowExecutionSnapshots.stateData, + }) + .from(workflowExecutionSnapshots) + .where( + inArray( + workflowExecutionSnapshots.id, + childSnapshotIds.slice(index, index + CHILD_SNAPSHOT_QUERY_CHUNK_SIZE) + ) + )) + ) + } + + input.signal?.throwIfAborted() + return { + executionId: record.executionId, + workflowId: record.workflowId, + workflowState: snapshot.stateData as Record, + childWorkflowSnapshots: Object.fromEntries( + childWorkflowSnapshots.map((child) => [child.id, child.stateData]) + ), + executionMetadata: { + trigger: record.trigger, + startedAt: record.startedAt.toISOString(), + endedAt: record.endedAt?.toISOString(), + totalDurationMs: record.totalDurationMs, + cost: record.costTotal != null ? { total: Number(record.costTotal) } : null, + }, + } + }, +}) + +export const readExecutionSnapshotUseCase: OperationUseCase< + typeof logOperations.readExecutionSnapshot, + ReadExecutionSnapshotInput, + ExecutionSnapshotData +> = { + operation: logOperations.readExecutionSnapshot, + async execute(args) { + try { + return await authorizedReadExecutionSnapshotUseCase.execute(args) + } catch (error) { + if (isConcealedLogAuthorizationError(error)) { + throw new OrchestrationError('not_found', 'Workflow execution not found') + } + throw error + } + }, +} diff --git a/apps/sim/lib/logs/application/read-log-detail.ts b/apps/sim/lib/logs/application/read-log-detail.ts new file mode 100644 index 00000000000..3af63a0dc53 --- /dev/null +++ b/apps/sim/lib/logs/application/read-log-detail.ts @@ -0,0 +1,109 @@ +import { requirePrincipalSubjectUserId } from '@sim/auth/principal' +import { db } from '@sim/db' +import { jobExecutionLogs, workflowExecutionLogs } from '@sim/db/schema' +import { eq } from 'drizzle-orm' +import type { WorkflowLogDetail } from '@/lib/api/contracts/logs' +import { defineAuthorizedWorkspaceUseCase, type OperationUseCase } from '@/lib/core/application' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { + isConcealedLogAuthorizationError, + logDelegationAuthorization, +} from '@/lib/logs/application/authorization' +import { logOperations } from '@/lib/logs/application/operations' +import { readLogDetail } from '@/lib/logs/fetch-log-detail' +import { + type ActiveWorkspaceApplicationContext, + resolveActiveWorkspaceApplicationContext, +} from '@/lib/workspaces/application/workspace-context' + +export interface ReadLogDetailInput { + workspaceId: string + lookupColumn: 'id' | 'executionId' + lookupValue: string + signal?: AbortSignal +} + +interface ReadLogDetailContext extends ActiveWorkspaceApplicationContext { + executionId: string +} + +async function resolveReadLogDetailContext( + input: ReadLogDetailInput +): Promise { + input.signal?.throwIfAborted() + const workflowLookup = + input.lookupColumn === 'id' + ? eq(workflowExecutionLogs.id, input.lookupValue) + : eq(workflowExecutionLogs.executionId, input.lookupValue) + const [workflowRecord] = await db + .select({ + workspaceId: workflowExecutionLogs.workspaceId, + executionId: workflowExecutionLogs.executionId, + }) + .from(workflowExecutionLogs) + .where(workflowLookup) + .limit(1) + input.signal?.throwIfAborted() + + let record = workflowRecord + if (!record) { + const jobLookup = + input.lookupColumn === 'id' + ? eq(jobExecutionLogs.id, input.lookupValue) + : eq(jobExecutionLogs.executionId, input.lookupValue) + const [jobRecord] = await db + .select({ + workspaceId: jobExecutionLogs.workspaceId, + executionId: jobExecutionLogs.executionId, + }) + .from(jobExecutionLogs) + .where(jobLookup) + .limit(1) + record = jobRecord + input.signal?.throwIfAborted() + } + + if (!record || record.workspaceId !== input.workspaceId) { + throw new OrchestrationError('not_found', 'Not found') + } + const workspace = await resolveActiveWorkspaceApplicationContext(record.workspaceId) + input.signal?.throwIfAborted() + return { ...workspace, executionId: record.executionId } +} + +const authorizedReadLogDetailUseCase = defineAuthorizedWorkspaceUseCase({ + operation: logOperations.readDetail, + resolveContext: ({ input }: { input: ReadLogDetailInput }) => resolveReadLogDetailContext(input), + authorizationOptions: logDelegationAuthorization(), + async execute({ principal, input, context }) { + input.signal?.throwIfAborted() + const detail = await readLogDetail({ + viewerUserId: requirePrincipalSubjectUserId(principal), + workspaceId: context.workspaceId, + lookupColumn: input.lookupColumn, + lookupValue: input.lookupValue, + signal: input.signal, + }) + input.signal?.throwIfAborted() + if (!detail) throw new OrchestrationError('not_found', 'Not found') + return { detail } + }, +}) + +export const readLogDetailUseCase: OperationUseCase< + typeof logOperations.readDetail, + ReadLogDetailInput, + { detail: WorkflowLogDetail } +> = { + operation: logOperations.readDetail, + async execute(args) { + try { + return await authorizedReadLogDetailUseCase.execute(args) + } catch (error) { + if (isConcealedLogAuthorizationError(error)) { + throw new OrchestrationError('not_found', 'Not found') + } + throw error + } + }, +} diff --git a/apps/sim/lib/logs/cost-ledger.ts b/apps/sim/lib/logs/cost-ledger.ts index ad1012af706..b23ce9670e6 100644 --- a/apps/sim/lib/logs/cost-ledger.ts +++ b/apps/sim/lib/logs/cost-ledger.ts @@ -1,7 +1,8 @@ import { db } from '@sim/db' import { usageLog } from '@sim/db/schema' -import { and, eq } from 'drizzle-orm' +import { and, eq, notInArray } from 'drizzle-orm' import type { CostLedger } from '@/lib/api/contracts/logs' +import { UNBILLED_USAGE_CATEGORIES } from '@/lib/billing/core/usage-log' /** * The itemized billing lines for one run, or `null` when the run has no ledger. @@ -16,6 +17,10 @@ import type { CostLedger } from '@/lib/api/contracts/logs' * row per billed event and a run can bill the same model many times; token * counts take the maximum rather than the sum, matching how they are reported * per call rather than accumulated. + * + * Unbilled categories are excluded: this is what the run *cost*, so a BYOK model + * — which Sim does not charge for — is not a line here. That usage is reported by + * the organization usage panel instead. */ export async function buildCostLedger(executionId: string): Promise { const rows = await db @@ -26,7 +31,13 @@ export async function buildCostLedger(executionId: string): Promise { const statsLog = logger.withMetadata({ workflowId: workflowId ?? undefined, executionId }) @@ -1570,7 +1578,18 @@ export class ExecutionLogger implements IExecutionLoggerService { target: number metadata?: ModelUsageMetadata | null } + /** + * Model usage Sim does not charge for — a call funded by the customer's own + * provider key. `notBilledCost()` zeroes the cost but leaves the span's token + * counts intact, so these carry real volume with `cost: 0`. Recorded for the + * organization usage panel; they are never a charge and never a delta. + */ + type UnbilledLine = { + description: string + metadata: ModelUsageMetadata + } const targets: TargetLine[] = [] + const unbilledLines: UnbilledLine[] = [] const workflowLedgerModels = costSummary.workflowLedgerModels ?? costSummary.models ?? {} const totalModelCost = Object.values(costSummary.models ?? {}).reduce( (sum, model) => sum + model.total, @@ -1603,6 +1622,14 @@ export class ExecutionLogger implements IExecutionLoggerService { modelData.toolCost > 0 && { toolCost: modelData.toolCost }), }, }) + } else if (modelData.tokens.input > 0 || modelData.tokens.output > 0) { + unbilledLines.push({ + description: modelName, + metadata: { + inputTokens: modelData.tokens.input, + outputTokens: modelData.tokens.output, + }, + }) } } @@ -1620,11 +1647,22 @@ export class ExecutionLogger implements IExecutionLoggerService { } } + // Unbilled rows are reporting-only, so they must never be the reason a run + // demands billing attribution it does not have: a BYOK-only run without + // attribution has to bail exactly as it did before unbilled capture existed, + // or it starts throwing where it previously succeeded. Terminal-only because + // these lines carry no cost delta to reconcile — they are written once, with + // the run's cumulative tokens. + const canRecordUnbilled = + isTerminalBoundary && + unbilledLines.length > 0 && + (!workflowRecord.workspaceId || Boolean(billingContext)) + // Bail before requiring billing attribution: a run with no billable target // (e.g. a preprocessing-gated run that never executed) writes no ledger row // either way, so demanding attribution here would raise a lost-revenue // error for a charge that does not exist. - if (targets.length === 0) { + if (targets.length === 0 && !canRecordUnbilled) { statsLog.debug('No cost to record') return 0 } @@ -1682,6 +1720,29 @@ export class ExecutionLogger implements IExecutionLoggerService { return entries } + /** + * Zero-cost lines for BYOK models not already recorded for this execution. + * Unlike the cost path, *presence* — not amount — is the idempotency signal, + * because these lines never change value once written. The `eventKey` + + * `onConflictDoNothing` is the real guard; this filter only avoids a + * pointless insert on a retried terminal boundary. + */ + const buildUnbilledEntries = (recordedKeys: ReadonlySet) => + unbilledLines + .filter((line) => !recordedKeys.has(`model_unbilled::${line.description}`)) + .map((line) => ({ + category: 'model_unbilled' as const, + source: 'workflow' as const, + description: line.description, + cost: 0, + eventKey: stableEventKey({ + executionId: executionId ?? '', + category: 'model_unbilled', + description: line.description, + }), + metadata: line.metadata, + })) + if (executionId) { // Serialize concurrent completion boundaries for this execution so the // read-then-insert reconciliation cannot race. pg_advisory_xact_lock is @@ -1711,18 +1772,20 @@ export class ExecutionLogger implements IExecutionLoggerService { .groupBy(usageLog.category, usageLog.description) const alreadyBilled = new Map() + const recordedKeys = new Set() for (const row of billedRows) { - alreadyBilled.set( - `${row.category}::${row.description}`, - Number.parseFloat(row.cost ?? '0') - ) + const key = `${row.category}::${row.description}` + alreadyBilled.set(key, Number.parseFloat(row.cost ?? '0')) + recordedKeys.add(key) } const entries = buildDeltaEntries(alreadyBilled) - if (entries.length > 0) { + const unbilledEntries = canRecordUnbilled ? buildUnbilledEntries(recordedKeys) : [] + const allEntries = [...entries, ...unbilledEntries] + if (allEntries.length > 0) { await recordUsage({ userId, - entries, + entries: allEntries, workspaceId: workflowRecord.workspaceId ?? undefined, workflowId, executionId, @@ -1730,6 +1793,8 @@ export class ExecutionLogger implements IExecutionLoggerService { billingEntity: resolvedBillingContext.billingEntity, billingPeriod: resolvedBillingContext.billingPeriod, }) + // Billable deltas only: unbilled lines cost 0, and the caller drives + // usage-threshold math off this number. recordedIncrement = entries.reduce((acc, e) => acc + e.cost, 0) // Refine cost_total to the EXACT post-reconciliation ledger sum, @@ -1739,24 +1804,33 @@ export class ExecutionLogger implements IExecutionLoggerService { // the prior workflow-source sum plus the deltas just inserted. This // supersedes the main-transaction GREATEST baseline except when the // display total contains Mothership cost owned by Go update-cost. - const ledgerSum = - [...alreadyBilled.values()].reduce((acc, v) => acc + v, 0) + recordedIncrement - const displayedCostTotal = - externallyLedgeredModelCost > 0 ? costSummary.totalCost : ledgerSum - await tx - .update(workflowExecutionLogs) - .set({ costTotal: displayedCostTotal.toString() }) - .where(eq(workflowExecutionLogs.executionId, executionId)) + // + // Gated on billable deltas: a boundary that only wrote zero-cost + // unbilled rows has changed no cost, and must not restate cost_total. + if (entries.length > 0) { + const ledgerSum = + [...alreadyBilled.values()].reduce((acc, v) => acc + v, 0) + recordedIncrement + const displayedCostTotal = + externallyLedgeredModelCost > 0 ? costSummary.totalCost : ledgerSum + await tx + .update(workflowExecutionLogs) + .set({ costTotal: displayedCostTotal.toString() }) + .where(eq(workflowExecutionLogs.executionId, executionId)) + } } }) } else { // No execution scope to reconcile/lock against (not expected at a // workflow completion): record the full targets directly. const entries = buildDeltaEntries(new Map()) - if (entries.length > 0) { + const allEntries = [ + ...entries, + ...(canRecordUnbilled ? buildUnbilledEntries(new Set()) : []), + ] + if (allEntries.length > 0) { await recordUsage({ userId, - entries, + entries: allEntries, workspaceId: workflowRecord.workspaceId ?? undefined, workflowId, billingEntity: resolvedBillingContext.billingEntity, diff --git a/apps/sim/lib/logs/execution/logging-factory.test.ts b/apps/sim/lib/logs/execution/logging-factory.test.ts index 7dabdc6955b..45477d09a4c 100644 --- a/apps/sim/lib/logs/execution/logging-factory.test.ts +++ b/apps/sim/lib/logs/execution/logging-factory.test.ts @@ -197,6 +197,34 @@ describe('calculateCostSummary', () => { expect(result.models['gpt-4'].total).toBe(0.03) }) + test('keeps tokens for a zero-cost BYOK span so unbilled usage stays reportable', () => { + // A BYOK model resolves through notBilledCost(), which zeroes every cost field + // but leaves `cost` DEFINED and the token counts intact. hasBillableCost() tests + // `cost !== undefined`, not `cost.total > 0`, which is the only reason this span + // reaches the summary at all. Narrowing that predicate would silently stop the + // organization usage panel from ever seeing BYOK volume. + const traceSpans = [ + { + id: 'span-1', + name: 'Agent Block', + type: 'agent', + model: 'claude-sonnet-4', + cost: { input: 0, output: 0, total: 0 }, + tokens: { input: 1200, output: 340, total: 1540 }, + }, + ] + + const result = calculateCostSummary(traceSpans) + + expect(result.totalCost).toBe(BASE_EXECUTION_CHARGE) + expect(result.workflowLedgerModels['claude-sonnet-4']).toMatchObject({ + total: 0, + tokens: { input: 1200, output: 340, total: 1540 }, + }) + expect(result.totalPromptTokens).toBe(1200) + expect(result.totalCompletionTokens).toBe(340) + }) + test('should calculate cost from multiple spans', () => { const traceSpans = [ { diff --git a/apps/sim/lib/logs/fetch-log-detail.test.ts b/apps/sim/lib/logs/fetch-log-detail.test.ts index 7eb064ae99e..259eb9b8fa2 100644 --- a/apps/sim/lib/logs/fetch-log-detail.test.ts +++ b/apps/sim/lib/logs/fetch-log-detail.test.ts @@ -7,14 +7,9 @@ import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ - checkWorkspaceAccess: vi.fn(), materializeExecutionData: vi.fn(), })) -vi.mock('@/lib/workspaces/permissions/utils', () => ({ - checkWorkspaceAccess: mocks.checkWorkspaceAccess, -})) - vi.mock('@/lib/logs/execution/trace-store', () => ({ materializeExecutionDataForDisplay: mocks.materializeExecutionData, })) @@ -23,13 +18,12 @@ vi.mock('@/lib/logs/execution-origin', () => ({ workflowExecutionOriginSql: () => ({ as: () => ({}) }), })) -import { fetchLogDetail } from '@/lib/logs/fetch-log-detail' +import { readLogDetail } from '@/lib/logs/fetch-log-detail' -describe('fetchLogDetail', () => { +describe('readLogDetail', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() - mocks.checkWorkspaceAccess.mockResolvedValue({ hasAccess: true }) mocks.materializeExecutionData.mockResolvedValue({}) }) @@ -69,8 +63,8 @@ describe('fetchLogDetail', () => { ]) queueTableRows(usageLog, []) - const result = await fetchLogDetail({ - userId: 'user-1', + const result = await readLogDetail({ + viewerUserId: 'user-1', workspaceId: 'workspace-1', lookupColumn: 'id', lookupValue: 'log-1', diff --git a/apps/sim/lib/logs/fetch-log-detail.ts b/apps/sim/lib/logs/fetch-log-detail.ts index 40846ec8ecf..ef5423b94a9 100644 --- a/apps/sim/lib/logs/fetch-log-detail.ts +++ b/apps/sim/lib/logs/fetch-log-detail.ts @@ -7,6 +7,7 @@ import { workflowExecutionLogs, } from '@sim/db/schema' import { and, eq, type SQL } from 'drizzle-orm' +import { type WorkflowLogDetail, workflowLogDetailSchema } from '@/lib/api/contracts/logs' import { buildCostLedger } from '@/lib/logs/cost-ledger' import { hydrateChildTraces } from '@/lib/logs/execution/hydrate-child-traces' import { @@ -18,7 +19,6 @@ import { import { materializeExecutionDataForDisplay } from '@/lib/logs/execution/trace-store' import { workflowExecutionOriginSql } from '@/lib/logs/execution-origin' import type { TraceSpan } from '@/lib/logs/types' -import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils' type LookupColumn = 'id' | 'executionId' @@ -29,30 +29,29 @@ export function jobCostTotal(raw: unknown): { total: number } | null { } interface FetchLogDetailArgs { - userId: string + viewerUserId: string workspaceId: string lookupColumn: LookupColumn lookupValue: string + signal?: AbortSignal } /** - * Shared loader for the workflow-log detail shape returned by the by-id and - * by-execution routes. Returns `null` when no matching row exists in either - * the workflow-execution or job-execution tables for this user + workspace. + * Canonical workflow-log detail loader after workspace authorization. Returns + * `null` when no matching row exists in either execution-log table. * * For in-flight (running/pending) executions, live progress markers are merged * from Redis, since they are only folded into the row at a terminal/pause * boundary. */ -export async function fetchLogDetail({ - userId, +export async function readLogDetail({ + viewerUserId, workspaceId, lookupColumn, lookupValue, -}: FetchLogDetailArgs) { - const access = await checkWorkspaceAccess(workspaceId, userId) - if (!access.hasAccess) return null - + signal, +}: FetchLogDetailArgs): Promise { + signal?.throwIfAborted() const workflowMatch: SQL = lookupColumn === 'id' ? eq(workflowExecutionLogs.id, lookupValue) @@ -97,6 +96,7 @@ export async function fetchLogDetail({ .leftJoin(pausedExecutions, eq(pausedExecutions.executionId, workflowExecutionLogs.executionId)) .where(and(workflowMatch, eq(workflowExecutionLogs.workspaceId, workspaceId))) .limit(1) + signal?.throwIfAborted() const log = rows[0] @@ -123,6 +123,7 @@ export async function fetchLogDetail({ // Cost is sourced exclusively from the usage_log ledger (itemized breakdown) // and its cost_total projection (run total). The cost jsonb is never read. const costLedger = await buildCostLedger(log.executionId) + signal?.throwIfAborted() const totalDollars = costLedger?.total ?? (log.costTotal != null ? Number(log.costTotal) : null) // Trace spans / heavy execution data may live in object storage; resolve the @@ -133,20 +134,23 @@ export async function fetchLogDetail({ workspaceId, workflowId: log.workflowId, executionId: log.executionId, - userId, + userId: viewerUserId, } ) + signal?.throwIfAborted() // A custom block's child ran in another workspace and kept its spans on its // own log row. Join in the ones whose publisher opened them to consumers. if (Array.isArray(executionData?.traceSpans)) { - await hydrateChildTraces(executionData.traceSpans as TraceSpan[], { viewerUserId: userId }) + await hydrateChildTraces(executionData.traceSpans as TraceSpan[], { viewerUserId }) + signal?.throwIfAborted() } const liveMarkers = log.status === 'running' || log.status === 'pending' || log.status === 'redacting' ? ((await getProgressMarkers(log.executionId)) ?? {}) : {} + signal?.throwIfAborted() const rowMarkers = (executionData ?? {}) as ExecutionProgressMarkers const mergedStartedBlock = pickLatestStartedMarker( liveMarkers.lastStartedBlock, @@ -157,7 +161,7 @@ export async function fetchLogDetail({ rowMarkers.lastCompletedBlock ) - return { + return workflowLogDetailSchema.parse({ id: log.id, workflowId: log.workflowId, executionId: log.executionId, @@ -188,7 +192,7 @@ export async function fetchLogDetail({ enhanced: true as const, }, files: log.files ?? null, - } + }) } const jobMatch: SQL = @@ -213,6 +217,7 @@ export async function fetchLogDetail({ .from(jobExecutionLogs) .where(and(jobMatch, eq(jobExecutionLogs.workspaceId, workspaceId))) .limit(1) + signal?.throwIfAborted() const jobLog = jobRows[0] if (!jobLog) return null @@ -223,10 +228,11 @@ export async function fetchLogDetail({ workspaceId, workflowId: null, executionId: jobLog.executionId, - userId, + userId: viewerUserId, } ) - return { + signal?.throwIfAborted() + return workflowLogDetailSchema.parse({ id: jobLog.id, workflowId: null, executionId: jobLog.executionId, @@ -250,5 +256,5 @@ export async function fetchLogDetail({ enhanced: true as const, }, files: null, - } + }) } diff --git a/apps/sim/lib/logs/list-logs.test.ts b/apps/sim/lib/logs/list-logs.test.ts index 978794b0988..75da730b2f0 100644 --- a/apps/sim/lib/logs/list-logs.test.ts +++ b/apps/sim/lib/logs/list-logs.test.ts @@ -44,19 +44,8 @@ vi.mock('@/lib/logs/folder-expansion', () => ({ expandFolderIdsWithDescendants: vi.fn(async (_ws: string, ids: string | undefined) => ids), })) -// listLogs gates workspace access at entry; the resolver is tested separately. -vi.mock('@/lib/workspaces/permissions/utils', () => ({ - checkWorkspaceAccess: vi.fn(async () => ({ - exists: true, - hasAccess: true, - canWrite: true, - canAdmin: true, - workspace: { id: 'ws-1', name: 'Test', ownerId: 'user-1', organizationId: null }, - })), -})) - import type { ListLogsParams } from './list-logs' -import { listLogs } from './list-logs' +import { readLogs } from './list-logs' import { decodeLogSortCursor } from './sort-cursor' afterAll(resetDbChainMock) @@ -120,7 +109,7 @@ function baseParams(overrides: Partial = {}): ListLogsParams { } as ListLogsParams } -describe('listLogs', () => { +describe('readLogs', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() @@ -130,7 +119,7 @@ describe('listLogs', () => { queueTableRows(workflowExecutionLogs, [workflowRow()]) queueTableRows(jobExecutionLogs, [jobRow()]) - const result = await listLogs(baseParams(), 'user-1') + const result = await readLogs(baseParams()) expect(result.data).toHaveLength(2) const wf = result.data.find((r) => r.id === 'log-1')! @@ -158,7 +147,7 @@ describe('listLogs', () => { ]) queueTableRows(jobExecutionLogs, []) - const result = await listLogs(baseParams(), 'user-1') + const result = await readLogs(baseParams()) expect(result.data[0]).toMatchObject({ executionId: 'exec-1', @@ -175,7 +164,7 @@ describe('listLogs', () => { ]) queueTableRows(jobExecutionLogs, []) - const result = await listLogs(baseParams({ limit: 1 }), 'user-1') + const result = await readLogs(baseParams({ limit: 1 })) expect(result.data).toHaveLength(1) expect(result.nextCursor).not.toBeNull() @@ -186,7 +175,7 @@ describe('listLogs', () => { it('excludes job logs when a workflow-specific filter is present', async () => { queueTableRows(workflowExecutionLogs, [workflowRow()]) - const result = await listLogs(baseParams({ workflowIds: 'wf-1' }), 'user-1') + const result = await readLogs(baseParams({ workflowIds: 'wf-1' })) // Only the workflow query runs; the job query is Promise.resolve([]). expect(dbChainMockFns.select).toHaveBeenCalledTimes(1) diff --git a/apps/sim/lib/logs/list-logs.ts b/apps/sim/lib/logs/list-logs.ts index 60d4f6309d7..074cdd4753b 100644 --- a/apps/sim/lib/logs/list-logs.ts +++ b/apps/sim/lib/logs/list-logs.ts @@ -38,26 +38,19 @@ import { decodeLogSortCursor, encodeLogSortCursor, } from '@/lib/logs/sort-cursor' -import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils' -export type ListLogsParams = z.output +export type ListLogsParams = z.output & { + signal?: AbortSignal +} type SortBy = 'date' | 'duration' | 'cost' | 'status' type SortOrder = 'asc' | 'desc' /** - * Shared logs list query used by the `/api/logs` route and the copilot `query_logs` - * tool. Builds the workflow + job execution-log query (cursor pagination, sort, - * level running/pending logic, job-log merge) from the shared filter params. The - * caller is responsible for authenticating `userId`; this function enforces - * workspace permission via the `permissions` join. + * Canonical logs list query after workspace authorization. */ -export async function listLogs(params: ListLogsParams, userId: string): Promise { - const access = await checkWorkspaceAccess(params.workspaceId, userId) - if (!access.hasAccess) { - return { data: [], nextCursor: null } - } - +export async function readLogs(params: ListLogsParams): Promise { + params.signal?.throwIfAborted() const sortBy = params.sortBy as SortBy const sortOrder = params.sortOrder as SortOrder const cursor = params.cursor ? decodeLogSortCursor(params.cursor) : null @@ -67,6 +60,7 @@ export async function listLogs(params: ListLogsParams, userId: string): Promise< const folderIds = params.folderIds ? await expandFolderIdsWithDescendants(params.workspaceId, params.folderIds) : params.folderIds + params.signal?.throwIfAborted() const p: ListLogsParams = { ...params, folderIds } const workflowSortExpr: SQL = (() => { @@ -315,6 +309,7 @@ export async function listLogs(params: ListLogsParams, userId: string): Promise< : Promise.resolve([]) const [workflowRows, jobRows] = await Promise.all([workflowQuery, jobQuery]) + params.signal?.throwIfAborted() type RowWithSort = { id: string @@ -458,9 +453,11 @@ export async function listLogs(params: ListLogsParams, userId: string): Promise< .where(and(...jobFilterConditions)) : Promise.resolve([{ count: 0 }]) const [workflowCount, jobCount] = await Promise.all([workflowCountQuery, jobCountQuery]) + params.signal?.throwIfAborted() total = Number(workflowCount[0]?.count ?? 0) + Number(jobCount[0]?.count ?? 0) } + params.signal?.throwIfAborted() return { data: page.map((row) => row.summary), nextCursor, diff --git a/apps/sim/lib/logs/stats-logs.ts b/apps/sim/lib/logs/stats-logs.ts index 6a8abc1c6ab..6d98e2a5174 100644 --- a/apps/sim/lib/logs/stats-logs.ts +++ b/apps/sim/lib/logs/stats-logs.ts @@ -1,6 +1,7 @@ import { dbReplica } from '@sim/db' import { workflow, workflowExecutionLogs } from '@sim/db/schema' import { and, eq, sql } from 'drizzle-orm' +import { assertValidTimezone } from '@/lib/core/utils/timezone' import { buildFilterConditions } from '@/lib/logs/filters' import { expandFolderIdsWithDescendants } from '@/lib/logs/folder-expansion' import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils' @@ -58,14 +59,6 @@ export interface LogStatsResponse { const MAX_WORKFLOWS = 100 -function assertValidTimezone(timezone: string): void { - try { - new Intl.DateTimeFormat('en-US', { timeZone: timezone }) - } catch { - throw new Error(`Invalid timezone: ${timezone}. Use an IANA name like "America/Los_Angeles".`) - } -} - interface StatsAccumulator { executions: number byStatus: Record diff --git a/apps/sim/lib/mcp/application/context.ts b/apps/sim/lib/mcp/application/context.ts new file mode 100644 index 00000000000..104cd533350 --- /dev/null +++ b/apps/sim/lib/mcp/application/context.ts @@ -0,0 +1,32 @@ +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { getWorkspaceMcpServer, type McpServerRow } from '@/lib/mcp/queries' +import { loadActiveWorkspaceContext } from '@/lib/uploads/contexts/workspace' + +export interface McpWorkspaceContext { + workspaceId: string + workspaceOrganizationId: string | null + allowPersonalApiKeys: boolean + billedAccountUserId: string +} + +export interface McpServerContext extends McpWorkspaceContext { + server: McpServerRow +} + +export async function resolveMcpWorkspaceContext( + workspaceId: string +): Promise { + const context = await loadActiveWorkspaceContext(workspaceId) + if (!context) throw new OrchestrationError('not_found', 'Workspace not found') + return context +} + +export async function resolveMcpServerContext( + workspaceId: string, + serverId: string +): Promise { + const workspace = await resolveMcpWorkspaceContext(workspaceId) + const server = await getWorkspaceMcpServer({ workspaceId: workspace.workspaceId, serverId }) + if (!server) throw new OrchestrationError('not_found', 'MCP server not found') + return { ...workspace, server } +} diff --git a/apps/sim/lib/mcp/application/execute-tool.test.ts b/apps/sim/lib/mcp/application/execute-tool.test.ts new file mode 100644 index 00000000000..c775680d98d --- /dev/null +++ b/apps/sim/lib/mcp/application/execute-tool.test.ts @@ -0,0 +1,224 @@ +/** + * @vitest-environment node + */ +import { + PrincipalSubjectUserRequiredError, + type WorkflowExecutionDelegatedPrincipal, +} from '@sim/auth/principal' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + loadContext: vi.fn(), + getServer: vi.fn(), + resolvePermission: vi.fn(), + assertPermissionsAllowed: vi.fn(), + discoverServerTools: vi.fn(), + executeTool: vi.fn(), + telemetry: vi.fn(), +})) + +vi.mock('@/lib/uploads/contexts/workspace', () => ({ + loadActiveWorkspaceContext: mocks.loadContext, +})) +vi.mock('@/lib/mcp/queries', () => ({ getWorkspaceMcpServer: mocks.getServer })) +vi.mock('@sim/platform-authz/workspace', () => ({ + permissionSatisfies: (actual: string | null, required: string) => + actual === 'admin' || actual === required || (actual === 'write' && required === 'read'), + resolveEffectiveWorkspacePermission: mocks.resolvePermission, +})) +vi.mock('@/ee/access-control/utils/permission-check', () => ({ + assertPermissionsAllowed: mocks.assertPermissionsAllowed, + McpToolsNotAllowedError: class McpToolsNotAllowedError extends Error {}, +})) +vi.mock('@/lib/mcp/service', () => ({ + mcpService: { + discoverServerTools: mocks.discoverServerTools, + executeTool: mocks.executeTool, + }, +})) +vi.mock('@/lib/core/telemetry', () => ({ + PlatformEvents: { mcpToolExecuted: mocks.telemetry }, +})) + +import { executeMcpToolUseCase } from '@/lib/mcp/application/execute-tool' + +const WORKSPACE = { + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + billedAccountUserId: 'owner-1', +} +const SERVER = { + id: 'mcp-server-1', + workspaceId: WORKSPACE.workspaceId, + enabled: true, +} +const PRINCIPAL: WorkflowExecutionDelegatedPrincipal = { + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: WORKSPACE.workspaceId, + delegationId: 'delegation-1', + audience: 'sim:mcp-servers', + issuedAt: new Date('2026-08-27T00:00:00.000Z'), + expiresAt: new Date('2099-08-27T00:05:00.000Z'), + delegationContext: { kind: 'workflow_execution', workflowId: 'workflow-1' }, +} +const ACTORLESS_PRINCIPAL: WorkflowExecutionDelegatedPrincipal = { + kind: 'delegated', + serviceId: 'executor', + workspaceId: WORKSPACE.workspaceId, + delegationId: 'delegation-system', + audience: 'sim:mcp-servers', + issuedAt: new Date('2026-08-27T00:00:00.000Z'), + expiresAt: new Date('2099-08-27T00:05:00.000Z'), + delegationContext: { + kind: 'workflow_execution', + workflowId: 'workflow-1', + currentWorkflow: { + workflowId: 'workflow-1', + mode: 'deployment', + deploymentVersionId: 'deployment-1', + }, + principal: { + kind: 'system', + serviceId: 'schedule', + workspaceId: WORKSPACE.workspaceId, + workflowId: 'workflow-1', + }, + }, +} + +describe('executeMcpToolUseCase', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.loadContext.mockResolvedValue(WORKSPACE) + mocks.getServer.mockResolvedValue(SERVER) + mocks.resolvePermission.mockResolvedValue('read') + mocks.discoverServerTools.mockResolvedValue([ + { + name: 'lookup', + inputSchema: { + type: 'object', + required: ['count'], + properties: { + count: { type: 'integer' }, + enabled: { type: 'boolean' }, + tags: { type: 'array' }, + }, + }, + }, + ]) + mocks.executeTool.mockResolvedValue({ content: [{ type: 'text', text: 'done' }] }) + }) + + it('authorizes, coerces the discovered schema, and preserves execution context', async () => { + const provenance = vi.fn() + const signal = new AbortController().signal + const result = await executeMcpToolUseCase.execute({ + principal: PRINCIPAL, + input: { + workspaceId: WORKSPACE.workspaceId, + serverId: SERVER.id, + toolName: 'lookup', + arguments: { count: '2', enabled: 'true', tags: 'a,b' }, + callChain: ['workflow-parent', 'workflow-1'], + timeoutMs: 12_000, + signal, + onResolvedSecretTraceProvenance: provenance, + }, + }) + + expect(result).toEqual({ + success: true, + output: { content: [{ type: 'text', text: 'done' }] }, + }) + expect(mocks.assertPermissionsAllowed).toHaveBeenCalledWith({ + userId: 'user-1', + workspaceId: WORKSPACE.workspaceId, + toolKind: 'mcp', + }) + expect(mocks.executeTool).toHaveBeenCalledWith( + 'user-1', + SERVER.id, + { + name: 'lookup', + arguments: { count: 2, enabled: true, tags: ['a', 'b'] }, + }, + WORKSPACE.workspaceId, + { 'X-Sim-Via': 'workflow-parent,workflow-1' }, + provenance, + { signal, timeoutMs: 12_000 } + ) + expect(mocks.telemetry).toHaveBeenCalledOnce() + }) + + it('rejects foreign or missing servers before permission and provider work', async () => { + mocks.getServer.mockResolvedValueOnce(null) + + await expect( + executeMcpToolUseCase.execute({ + principal: PRINCIPAL, + input: { + workspaceId: WORKSPACE.workspaceId, + serverId: 'mcp-foreign', + toolName: 'lookup', + }, + }) + ).rejects.toMatchObject({ code: 'not_found', message: 'MCP server not found' }) + + expect(mocks.assertPermissionsAllowed).not.toHaveBeenCalled() + expect(mocks.executeTool).not.toHaveBeenCalled() + }) + + it('does not invent a user for actorless system execution', async () => { + await expect( + executeMcpToolUseCase.execute({ + principal: ACTORLESS_PRINCIPAL, + input: { + workspaceId: WORKSPACE.workspaceId, + serverId: SERVER.id, + toolName: 'lookup', + }, + }) + ).rejects.toEqual(new PrincipalSubjectUserRequiredError('delegated')) + + expect(mocks.assertPermissionsAllowed).not.toHaveBeenCalled() + expect(mocks.discoverServerTools).not.toHaveBeenCalled() + expect(mocks.executeTool).not.toHaveBeenCalled() + }) + + it('does not execute when schema validation fails', async () => { + await expect( + executeMcpToolUseCase.execute({ + principal: PRINCIPAL, + input: { + workspaceId: WORKSPACE.workspaceId, + serverId: SERVER.id, + toolName: 'lookup', + arguments: { enabled: true }, + }, + }) + ).rejects.toMatchObject({ code: 'validation', message: 'Invalid tool arguments' }) + + expect(mocks.executeTool).not.toHaveBeenCalled() + }) + + it('never retries a submitted tool call after an ambiguous provider failure', async () => { + mocks.executeTool.mockRejectedValueOnce(new Error('socket hang up')) + + await expect( + executeMcpToolUseCase.execute({ + principal: PRINCIPAL, + input: { + workspaceId: WORKSPACE.workspaceId, + serverId: SERVER.id, + toolName: 'lookup', + arguments: { count: 1 }, + }, + }) + ).rejects.toThrow('socket hang up') + + expect(mocks.executeTool).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/mcp/application/execute-tool.ts b/apps/sim/lib/mcp/application/execute-tool.ts new file mode 100644 index 00000000000..be542e71f0d --- /dev/null +++ b/apps/sim/lib/mcp/application/execute-tool.ts @@ -0,0 +1,208 @@ +import { requirePrincipalSubjectUserId } from '@sim/auth/principal' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { defineAuthorizedWorkspaceUseCase } from '@/lib/core/application' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { SIM_VIA_HEADER, serializeCallChain } from '@/lib/execution/call-chain' +import { mcpServerDelegationPolicy } from '@/lib/mcp/application/authorization' +import { resolveMcpServerContext } from '@/lib/mcp/application/context' +import { mcpServerOperations } from '@/lib/mcp/application/operations' +import { mcpService } from '@/lib/mcp/service' +import type { McpTool, McpToolCall, McpToolResult } from '@/lib/mcp/types' +import { + assertPermissionsAllowed, + McpToolsNotAllowedError, +} from '@/ee/access-control/utils/permission-check' +import type { ResolvedSecretTraceProvenanceV1 } from '@/executor/utils/resolved-secret-trace-registry' + +const logger = createLogger('McpToolExecution') + +interface SchemaProperty { + type: 'string' | 'number' | 'integer' | 'boolean' | 'object' | 'array' +} + +export interface ExecuteMcpToolInput { + workspaceId: string + serverId: string + toolName: string + arguments?: Record + callChain?: string[] + timeoutMs?: number + signal?: AbortSignal + onResolvedSecretTraceProvenance?: (provenance: ResolvedSecretTraceProvenanceV1) => void +} + +export type ExecuteMcpToolResult = + | { success: true; output: McpToolResult } + | { success: false; error: string } + +function hasType(value: unknown): value is SchemaProperty { + return typeof value === 'object' && value !== null && 'type' in value +} + +function coerceToolArguments( + tool: McpTool, + input: Record +): Record { + const result = { ...input } + if (!tool.inputSchema?.properties) return result + + for (const [name, property] of Object.entries(tool.inputSchema.properties)) { + if (!hasType(property)) continue + const value = result[name] + if (value === undefined || value === null) continue + + if ((property.type === 'number' || property.type === 'integer') && typeof value === 'string') { + const numberValue = + property.type === 'integer' ? Number.parseInt(value) : Number.parseFloat(value) + if (!Number.isNaN(numberValue)) result[name] = numberValue + continue + } + if (property.type === 'boolean' && typeof value === 'string') { + if (value.toLowerCase() === 'true') result[name] = true + if (value.toLowerCase() === 'false') result[name] = false + continue + } + if (property.type !== 'array' || typeof value !== 'string') continue + + const trimmed = value.trim() + if (!trimmed) { + result[name] = [] + continue + } + try { + const parsed: unknown = JSON.parse(trimmed) + result[name] = Array.isArray(parsed) ? parsed : [parsed] + } catch { + result[name] = trimmed.includes(',') + ? trimmed + .split(',') + .map((item) => item.trim()) + .filter(Boolean) + : [trimmed] + } + } + + return result +} + +function validateToolArguments(tool: McpTool, args: Record): void { + const schema = tool.inputSchema + if (!schema) return + + for (const requiredProperty of schema.required ?? []) { + if (!(requiredProperty in args)) { + throw new OrchestrationError('validation', 'Invalid tool arguments') + } + } + + for (const [name, property] of Object.entries(schema.properties ?? {})) { + const value = args[name] + if (value === undefined || !hasType(property)) continue + const isValid = + (property.type === 'string' && typeof value === 'string') || + (property.type === 'number' && typeof value === 'number') || + (property.type === 'integer' && typeof value === 'number' && Number.isInteger(value)) || + (property.type === 'boolean' && typeof value === 'boolean') || + (property.type === 'object' && + typeof value === 'object' && + value !== null && + !Array.isArray(value)) || + (property.type === 'array' && Array.isArray(value)) + if (!isValid) throw new OrchestrationError('validation', 'Invalid tool arguments') + } +} + +function transformToolResult(result: McpToolResult): ExecuteMcpToolResult { + if (!result.isError) return { success: true, output: result } + const firstContent = Array.isArray(result.content) ? result.content[0] : undefined + const errorText = + firstContent && typeof firstContent === 'object' && typeof firstContent.text === 'string' + ? firstContent.text.trim() + : '' + return { success: false, error: errorText || 'Tool execution failed' } +} + +export { McpToolsNotAllowedError } + +export const executeMcpToolUseCase = defineAuthorizedWorkspaceUseCase({ + operation: mcpServerOperations.executeTool, + resolveContext: ({ input }: { input: ExecuteMcpToolInput }) => + resolveMcpServerContext(input.workspaceId, input.serverId), + authorizationOptions: { delegation: mcpServerDelegationPolicy }, + async execute({ principal, input, context }): Promise { + input.signal?.throwIfAborted() + const userId = requirePrincipalSubjectUserId(principal) + await assertPermissionsAllowed({ + userId, + workspaceId: context.workspaceId, + toolKind: 'mcp', + }) + input.signal?.throwIfAborted() + + let tool: McpTool | undefined + let args = { ...input.arguments } + try { + const tools = await mcpService.discoverServerTools( + userId, + context.server.id, + context.workspaceId, + 'cache-aside', + input.onResolvedSecretTraceProvenance, + { signal: input.signal } + ) + tool = tools.find((candidate) => candidate.name === input.toolName) + if (!tool) { + throw new OrchestrationError('not_found', 'Tool not found on the specified server') + } + args = coerceToolArguments(tool, args) + } catch (error) { + input.signal?.throwIfAborted() + if (error instanceof OrchestrationError) throw error + logger.warn('Failed to discover MCP tools for validation; proceeding without schema', { + error: getErrorMessage(error), + serverId: context.server.id, + toolName: input.toolName, + }) + } + + if (tool) validateToolArguments(tool, args) + input.signal?.throwIfAborted() + const toolCall: McpToolCall = { name: input.toolName, arguments: args } + const extraHeaders = + input.callChain && input.callChain.length > 0 + ? { [SIM_VIA_HEADER]: serializeCallChain(input.callChain) } + : undefined + const providerResult = await mcpService.executeTool( + userId, + context.server.id, + toolCall, + context.workspaceId, + extraHeaders, + input.onResolvedSecretTraceProvenance, + { signal: input.signal, timeoutMs: input.timeoutMs } + ) + input.signal?.throwIfAborted() + const result = transformToolResult(providerResult) + if (!result.success) return result + + try { + const { PlatformEvents } = await import('@/lib/core/telemetry') + PlatformEvents.mcpToolExecuted({ + serverId: context.server.id, + toolName: input.toolName, + status: 'success', + workspaceId: context.workspaceId, + }) + } catch (error) { + logger.warn('Failed to record MCP tool execution telemetry', { + error: getErrorMessage(error), + serverId: context.server.id, + toolName: input.toolName, + workspaceId: context.workspaceId, + }) + } + + return result + }, +}) diff --git a/apps/sim/lib/mcp/application/operations.test.ts b/apps/sim/lib/mcp/application/operations.test.ts index cde261b8daf..75218e8f402 100644 --- a/apps/sim/lib/mcp/application/operations.test.ts +++ b/apps/sim/lib/mcp/application/operations.test.ts @@ -9,7 +9,17 @@ describe('MCP server operation registry', () => { expect(mcpServerOperations.discoverTools).toMatchObject({ workspaceApiKey: 'deny', principalKinds: ['session', 'personal_api_key', 'delegated'], - delegatedServices: ['copilot'], + delegatedServices: ['copilot', 'executor'], + }) + }) + + it('admits only the executor delegation for tool execution', () => { + expect(mcpServerOperations.executeTool).toMatchObject({ + id: 'mcp_servers.tools.execute', + minimumRole: 'read', + workspaceApiKey: 'deny', + principalKinds: ['delegated'], + delegatedServices: ['executor'], }) }) diff --git a/apps/sim/lib/mcp/application/operations.ts b/apps/sim/lib/mcp/application/operations.ts index c26cd09b2f9..dc4ed4d2335 100644 --- a/apps/sim/lib/mcp/application/operations.ts +++ b/apps/sim/lib/mcp/application/operations.ts @@ -8,6 +8,14 @@ const HUMAN_PRINCIPAL_POLICY = { principalKinds: ['session', 'personal_api_key', 'delegated'], delegatedServices: ['copilot'], } as const +const DISCOVERY_PRINCIPAL_POLICY = { + principalKinds: ['session', 'personal_api_key', 'delegated'], + delegatedServices: ['copilot', 'executor'], +} as const +const EXECUTION_PRINCIPAL_POLICY = { + principalKinds: ['delegated'], + delegatedServices: ['executor'], +} as const export const mcpServerOperations = { list: defineWorkspaceOperation({ @@ -20,7 +28,13 @@ export const mcpServerOperations = { id: 'mcp_servers.tools.discover', minimumRole: 'read', workspaceApiKey: 'deny', - ...HUMAN_PRINCIPAL_POLICY, + ...DISCOVERY_PRINCIPAL_POLICY, + }), + executeTool: defineWorkspaceOperation({ + id: 'mcp_servers.tools.execute', + minimumRole: 'read', + workspaceApiKey: 'deny', + ...EXECUTION_PRINCIPAL_POLICY, }), /** * Publishing a workflow as an MCP server was reachable only through Copilot, diff --git a/apps/sim/lib/mcp/application/use-cases.ts b/apps/sim/lib/mcp/application/use-cases.ts index fd5568ae2da..e35a74c6606 100644 --- a/apps/sim/lib/mcp/application/use-cases.ts +++ b/apps/sim/lib/mcp/application/use-cases.ts @@ -6,6 +6,12 @@ import { defineAuthorizedWorkspaceUseCase, ForbiddenOperationError } from '@/lib import { OrchestrationError } from '@/lib/core/orchestration/types' import { sanitizeUrlForLog } from '@/lib/core/utils/logging' import { mcpServerDelegationPolicy } from '@/lib/mcp/application/authorization' +import { + type McpServerContext, + type McpWorkspaceContext, + resolveMcpServerContext, + resolveMcpWorkspaceContext, +} from '@/lib/mcp/application/context' import { mcpServerOperations } from '@/lib/mcp/application/operations' import { applyMcpServerMutationEffects, @@ -16,7 +22,6 @@ import { } from '@/lib/mcp/orchestration' import { getMcpServerIdState, - getWorkspaceMcpServer, listWorkspaceMcpServers, type McpServerRow, type McpServerSortBy, @@ -24,38 +29,10 @@ import { import { mcpService } from '@/lib/mcp/service' import type { McpAuthType } from '@/lib/mcp/types' import { generateMcpServerId } from '@/lib/mcp/utils' -import { loadActiveWorkspaceContext } from '@/lib/uploads/contexts/workspace' type McpServerTransport = McpServerRow['transport'] type McpWriteSource = 'api' | 'settings' | 'tool_input' -interface McpWorkspaceContext { - workspaceId: string - workspaceOrganizationId: string | null - allowPersonalApiKeys: boolean - billedAccountUserId: string -} - -interface McpServerContext extends McpWorkspaceContext { - server: McpServerRow -} - -async function resolveWorkspaceContext(workspaceId: string): Promise { - const context = await loadActiveWorkspaceContext(workspaceId) - if (!context) throw new OrchestrationError('not_found', 'Workspace not found') - return context -} - -async function resolveServerContext( - workspaceId: string, - serverId: string -): Promise { - const workspace = await resolveWorkspaceContext(workspaceId) - const server = await getWorkspaceMcpServer({ workspaceId: workspace.workspaceId, serverId }) - if (!server) throw new OrchestrationError('not_found', 'MCP server not found') - return { ...workspace, server } -} - function requireSuccessfulResult( result: PerformMcpServerResult, fallback: string @@ -96,7 +73,7 @@ export interface ListMcpServersInput { export const listMcpServersUseCase = defineAuthorizedWorkspaceUseCase({ operation: mcpServerOperations.list, resolveContext: ({ input }: { input: ListMcpServersInput }) => - resolveWorkspaceContext(input.workspaceId), + resolveMcpWorkspaceContext(input.workspaceId), authorizationOptions, async execute({ input, context }) { const page = await listWorkspaceMcpServers({ ...input, workspaceId: context.workspaceId }) @@ -117,7 +94,7 @@ export interface DiscoverMcpToolsInput { export const discoverMcpToolsUseCase = defineAuthorizedWorkspaceUseCase({ operation: mcpServerOperations.discoverTools, resolveContext: ({ input }: { input: DiscoverMcpToolsInput }) => - resolveWorkspaceContext(input.workspaceId), + resolveMcpWorkspaceContext(input.workspaceId), authorizationOptions, async execute({ principal, input, context }) { const tools = await mcpService.discoverTools( @@ -146,7 +123,7 @@ export interface DiscoverMcpServerToolsInput { * Shares `mcp_servers.tools.discover` with the workspace-wide discovery: both * resolve the acting user's own OAuth credentials against a third-party server, * which is why that operation denies workspace API keys. Resolving the server - * through {@link resolveServerContext} first is what makes an id from another + * through {@link resolveMcpServerContext} first is what makes an id from another * workspace a not-found rather than an upstream connection attempt. * * The pass is also the only thing that writes the server row's @@ -156,7 +133,7 @@ export interface DiscoverMcpServerToolsInput { export const discoverMcpServerToolsUseCase = defineAuthorizedWorkspaceUseCase({ operation: mcpServerOperations.discoverTools, resolveContext: ({ input }: { input: DiscoverMcpServerToolsInput }) => - resolveServerContext(input.workspaceId, input.serverId), + resolveMcpServerContext(input.workspaceId, input.serverId), authorizationOptions, async execute({ principal, input, context }) { /** @@ -196,7 +173,7 @@ export interface GetMcpServerInput { export const getMcpServerUseCase = defineAuthorizedWorkspaceUseCase({ operation: mcpServerOperations.read, resolveContext: ({ input }: { input: GetMcpServerInput }) => - resolveServerContext(input.workspaceId, input.serverId), + resolveMcpServerContext(input.workspaceId, input.serverId), authorizationOptions, async execute({ context }) { return { server: context.server } @@ -283,7 +260,7 @@ function createAudit( export const createMcpServerUseCase = defineAuthorizedWorkspaceUseCase({ operation: mcpServerOperations.create, resolveContext: ({ input }: { input: SaveMcpServerInput }) => - resolveWorkspaceContext(input.workspaceId), + resolveMcpWorkspaceContext(input.workspaceId), authorizationOptions, async execute({ principal, input, context }) { const serverId = generateMcpServerId(context.workspaceId, input.url) @@ -327,7 +304,7 @@ export const createMcpServerUseCase = defineAuthorizedWorkspaceUseCase({ export const registerMcpServerUseCase = defineAuthorizedWorkspaceUseCase({ operation: mcpServerOperations.register, resolveContext: ({ input }: { input: SaveMcpServerInput }) => - resolveWorkspaceContext(input.workspaceId), + resolveMcpWorkspaceContext(input.workspaceId), authorizationOptions, execute: ({ principal, input, context }) => saveMcpServer({ principal, input, context }), projectAudit: ({ input, result }) => createAudit(input, result), @@ -404,7 +381,7 @@ function updateAudit( export const updateMcpServerUseCase = defineAuthorizedWorkspaceUseCase({ operation: mcpServerOperations.update, resolveContext: ({ input }: { input: UpdateMcpServerInput }) => - resolveServerContext(input.workspaceId, input.serverId), + resolveMcpServerContext(input.workspaceId, input.serverId), authorizationOptions, async execute({ principal, input, context }) { if (input.url !== undefined && input.url !== context.server.url) { @@ -423,7 +400,7 @@ export const updateMcpServerUseCase = defineAuthorizedWorkspaceUseCase({ export const reconfigureMcpServerUseCase = defineAuthorizedWorkspaceUseCase({ operation: mcpServerOperations.reconfigure, resolveContext: ({ input }: { input: UpdateMcpServerInput }) => - resolveServerContext(input.workspaceId, input.serverId), + resolveMcpServerContext(input.workspaceId, input.serverId), authorizationOptions, execute: ({ principal, input, context }) => updateMcpServer({ principal, input, context }), projectAudit: ({ input, result }) => updateAudit(input, result), @@ -440,7 +417,7 @@ export interface DeleteMcpServerInput { export const deleteMcpServerUseCase = defineAuthorizedWorkspaceUseCase({ operation: mcpServerOperations.delete, resolveContext: ({ input }: { input: DeleteMcpServerInput }) => - resolveServerContext(input.workspaceId, input.serverId), + resolveMcpServerContext(input.workspaceId, input.serverId), authorizationOptions, async execute({ principal, input, context }) { const attribution = resolvePrincipalAttribution(principal, { diff --git a/apps/sim/lib/mcp/client.ts b/apps/sim/lib/mcp/client.ts index e8030595c13..5aefff7fc08 100644 --- a/apps/sim/lib/mcp/client.ts +++ b/apps/sim/lib/mcp/client.ts @@ -62,6 +62,12 @@ function classifyConnectionOutcome( interface McpClientConnectOptions { isCancelled?: () => boolean + signal?: AbortSignal +} + +interface McpToolCallOptions { + signal?: AbortSignal + timeoutMs?: number } export class McpClient { @@ -176,8 +182,9 @@ export class McpClient { try { await this.client.connect(this.transport, { timeout: timeoutMs, + signal: options.signal, }) - if (options.isCancelled?.()) { + if (options.signal?.aborted || options.isCancelled?.()) { await this.client.close().catch((error) => { logger.warn(`Error closing cancelled connection to ${this.config.name}:`, error) }) @@ -268,7 +275,7 @@ export class McpClient { return { ...this.connectionStatus } } - async listTools(): Promise { + async listTools(signal?: AbortSignal): Promise { if (!this.isConnected) { throw new McpConnectionError('Not connected to server', this.config.name) } @@ -313,6 +320,7 @@ export class McpClient { timeout: Math.min(idleTimeoutMs, remainingMs), maxTotalTimeout: remainingMs, resetTimeoutOnProgress: true, + signal, onprogress: (progress) => { logger.debug(`Tool discovery progress from ${this.config.name}`, { serverId: this.config.id, @@ -377,6 +385,7 @@ export class McpClient { return tools } catch (error) { + signal?.throwIfAborted() logger.error(`Failed to list tools from server ${this.config.name}`, { serverId: this.config.id, phase: 'tools/list', @@ -395,7 +404,7 @@ export class McpClient { } } - async callTool(toolCall: McpToolCall): Promise { + async callTool(toolCall: McpToolCall, options: McpToolCallOptions = {}): Promise { if (!this.isConnected) { throw new McpConnectionError('Not connected to server', this.config.name) } @@ -429,7 +438,13 @@ export class McpClient { const sdkResult = await this.client.callTool( { name: toolCall.name, arguments: toolCall.arguments }, undefined, - { timeout: getMaxExecutionTimeout() } + { + timeout: + options.timeoutMs !== undefined && options.timeoutMs > 0 + ? options.timeoutMs + : getMaxExecutionTimeout(), + signal: options.signal, + } ) return sdkResult as McpToolResult diff --git a/apps/sim/lib/mcp/service.ts b/apps/sim/lib/mcp/service.ts index 2776a731ffa..bb5914156c8 100644 --- a/apps/sim/lib/mcp/service.ts +++ b/apps/sim/lib/mcp/service.ts @@ -5,7 +5,7 @@ import { db } from '@sim/db' import { mcpServers } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { sleep } from '@sim/utils/helpers' +import { interruptibleSleep } from '@sim/utils/helpers' import { backoffWithJitter } from '@sim/utils/retry' import { truncate } from '@sim/utils/string' import { and, eq, isNull, lte, or, sql } from 'drizzle-orm' @@ -62,6 +62,14 @@ const FAILURE_CACHE_SENTINEL: McpTool[] = [] type ResolvedSecretTraceProvenanceCallback = (provenance: ResolvedSecretTraceProvenanceV1) => void +interface McpRequestOptions { + signal?: AbortSignal +} + +interface McpToolExecutionOptions extends McpRequestOptions { + timeoutMs?: number +} + function reportRetainedClientProvenance( provenance: unknown, userId: string, @@ -392,7 +400,8 @@ class McpService { config: McpServerConfig, resolvedIP: string | null, userId?: string, - resolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceV1 + resolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceV1, + signal?: AbortSignal ): Promise { const securityPolicy = { requireConsent: true, @@ -408,7 +417,7 @@ class McpService { resolvedIP: resolvedIP ?? undefined, resolvedSecretTraceProvenance, }) - await client.connect() + await client.connect({ signal }) return client } @@ -440,7 +449,7 @@ class McpService { resolvedIP: resolvedIP ?? undefined, resolvedSecretTraceProvenance, }) - await client.connect() + await client.connect({ signal }) return client }) } @@ -464,7 +473,8 @@ class McpService { userId: string, workspaceId: string, extraHeaders?: Record, - onResolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceCallback + onResolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceCallback, + signal?: AbortSignal ): () => Promise { return async () => { const { @@ -480,7 +490,13 @@ class McpService { if (extraHeaders) { resolvedConfig.headers = { ...resolvedConfig.headers, ...extraHeaders } } - return this.createClient(resolvedConfig, resolvedIP, userId, resolvedSecretTraceProvenance) + return this.createClient( + resolvedConfig, + resolvedIP, + userId, + resolvedSecretTraceProvenance, + signal + ) } } @@ -495,9 +511,11 @@ class McpService { config: McpServerConfig, userId: string, workspaceId: string, - onResolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceCallback + onResolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceCallback, + signal?: AbortSignal ): Promise { for (let attempt = 0; ; attempt++) { + signal?.throwIfAborted() try { return await this.withServerClient( { @@ -505,7 +523,14 @@ class McpService { serverId: config.id, allowPool: true, }, - this.buildClient(config, userId, workspaceId, undefined, onResolvedSecretTraceProvenance), + this.buildClient( + config, + userId, + workspaceId, + undefined, + onResolvedSecretTraceProvenance, + signal + ), (client) => { reportRetainedClientProvenance( client.getResolvedSecretTraceProvenance?.(), @@ -513,10 +538,11 @@ class McpService { workspaceId, onResolvedSecretTraceProvenance ) - return client.listTools() + return client.listTools(signal) } ) } catch (error) { + signal?.throwIfAborted() if (attempt === 0 && isAuthError(error) && config.authType !== 'oauth') continue throw error } @@ -574,13 +600,15 @@ class McpService { toolCall: McpToolCall, workspaceId: string, extraHeaders?: Record, - onResolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceCallback + onResolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceCallback, + options: McpToolExecutionOptions = {} ): Promise { const requestId = generateRequestId() const maxRetries = 2 const reportProvenance = createInvocationProvenanceReporter(onResolvedSecretTraceProvenance) for (let attempt = 0; attempt < maxRetries; attempt++) { + options.signal?.throwIfAborted() try { logger.info( `[${requestId}] Executing MCP tool ${toolCall.name} on server ${serverId} for user ${userId}${attempt > 0 ? ` (attempt ${attempt + 1})` : ''}` @@ -603,7 +631,8 @@ class McpService { userId, workspaceId, hasExtraHeaders ? extraHeaders : undefined, - reportProvenance + reportProvenance, + options.signal ), (client) => { reportRetainedClientProvenance( @@ -612,12 +641,13 @@ class McpService { workspaceId, reportProvenance ) - return client.callTool(toolCall) + return client.callTool(toolCall, options) } ) logger.info(`[${requestId}] Successfully executed tool ${toolCall.name}`) return result } catch (error) { + options.signal?.throwIfAborted() // A stale session (400/404) or a rotated/revoked credential (401) is rejected // before the tool runs, so retrying on a fresh connection is safe and recovers // the request. Timeouts/resets are NOT retried — the tool may have executed. @@ -626,7 +656,8 @@ class McpService { `[${requestId}] Retryable connection error executing tool ${toolCall.name}, retrying (attempt ${attempt + 1}):`, error ) - await sleep(100) + await interruptibleSleep(100, options.signal) + options.signal?.throwIfAborted() continue } throw error @@ -1007,15 +1038,17 @@ class McpService { serverId: string, workspaceId: string, refresh: McpDiscoveryRefresh = 'cache-aside', - onResolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceCallback + onResolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceCallback, + options: McpRequestOptions = {} ): Promise { - if (onResolvedSecretTraceProvenance) { + if (onResolvedSecretTraceProvenance || options.signal) { return this.discoverServerToolsImpl( userId, serverId, workspaceId, refresh, - createInvocationProvenanceReporter(onResolvedSecretTraceProvenance) + createInvocationProvenanceReporter(onResolvedSecretTraceProvenance), + options.signal ) } @@ -1028,6 +1061,7 @@ class McpService { serverId, workspaceId, refresh, + undefined, undefined ).finally(() => { this.inflightServerDiscovery.delete(inflightKey) @@ -1041,8 +1075,10 @@ class McpService { serverId: string, workspaceId: string, refresh: McpDiscoveryRefresh, - onResolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceCallback + onResolvedSecretTraceProvenance?: ResolvedSecretTraceProvenanceCallback, + signal?: AbortSignal ): Promise { + signal?.throwIfAborted() const requestId = generateRequestId() const discoveryStartedAt = new Date() const maxRetries = 2 @@ -1065,6 +1101,7 @@ class McpService { } for (let attempt = 0; attempt < maxRetries; attempt++) { + signal?.throwIfAborted() let authType: McpServerConfig['authType'] try { logger.info( @@ -1081,7 +1118,8 @@ class McpService { config, userId, workspaceId, - onResolvedSecretTraceProvenance + onResolvedSecretTraceProvenance, + signal ) logger.info(`[${requestId}] Discovered ${tools.length} tools from server ${config.name}`) await Promise.allSettled([ @@ -1099,12 +1137,17 @@ class McpService { ]) return tools } catch (error) { + signal?.throwIfAborted() if (isRetryableDiscoveryError(error) && attempt < maxRetries - 1) { logger.warn( `[${requestId}] Transient error discovering tools from server ${serverId}, retrying (attempt ${attempt + 1}):`, error ) - await sleep(backoffWithJitter(attempt + 1, null, { baseMs: 250, maxMs: 2000 })) + await interruptibleSleep( + backoffWithJitter(attempt + 1, null, { baseMs: 250, maxMs: 2000 }), + signal + ) + signal?.throwIfAborted() continue } // Drop positive cache so a follow-up doesn't return stale tools. diff --git a/apps/sim/lib/media/falai-video.ts b/apps/sim/lib/media/falai-video.ts index 4b6a9413783..b0e1ff86781 100644 --- a/apps/sim/lib/media/falai-video.ts +++ b/apps/sim/lib/media/falai-video.ts @@ -22,8 +22,7 @@ interface FalVideoModelConfig { supportsPromptOptimizer?: boolean } -// Endpoints mirror app/api/tools/video/route.ts (FALAI_MODEL_CONFIGS), scoped to -// the latest-gen models the generate_video tool exposes. +/** Latest-generation models exposed by the Copilot video operation. */ const VIDEO_MODELS: Record = { 'veo-3.1': { endpoint: 'fal-ai/veo3.1', diff --git a/apps/sim/lib/memory/application/authorization.ts b/apps/sim/lib/memory/application/authorization.ts new file mode 100644 index 00000000000..d845c9f2a5b --- /dev/null +++ b/apps/sim/lib/memory/application/authorization.ts @@ -0,0 +1,10 @@ +import type { WorkspaceDelegationPolicy } from '@/lib/core/application' +import type { ActiveWorkspaceApplicationContext } from '@/lib/workspaces/application/workspace-context' + +export const MEMORY_DELEGATION_AUDIENCE = 'sim:memory' + +export const memoryDelegationPolicy: WorkspaceDelegationPolicy = + { + audience: MEMORY_DELEGATION_AUDIENCE, + isWithinScope: () => true, + } diff --git a/apps/sim/lib/memory/application/operations.test.ts b/apps/sim/lib/memory/application/operations.test.ts new file mode 100644 index 00000000000..7823931cfa2 --- /dev/null +++ b/apps/sim/lib/memory/application/operations.test.ts @@ -0,0 +1,21 @@ +/** + * @vitest-environment node + */ + +import { describe, expect, it } from 'vitest' +import { memoryOperations } from '@/lib/memory/application/operations' + +describe('memory operation registry', () => { + it('admits only executor delegation with semantic read and write roles', () => { + expect(memoryOperations.list.minimumRole).toBe('read') + expect(memoryOperations.read.minimumRole).toBe('read') + expect(memoryOperations.append.minimumRole).toBe('write') + expect(memoryOperations.delete.minimumRole).toBe('write') + + for (const operation of Object.values(memoryOperations)) { + expect(operation.principalKinds).toEqual(['delegated']) + expect(operation.delegatedServices).toEqual(['executor']) + expect(operation.workspaceApiKey).toBe('deny') + } + }) +}) diff --git a/apps/sim/lib/memory/application/operations.ts b/apps/sim/lib/memory/application/operations.ts new file mode 100644 index 00000000000..253df541c3d --- /dev/null +++ b/apps/sim/lib/memory/application/operations.ts @@ -0,0 +1,31 @@ +import { defineWorkspaceOperation } from '@/lib/core/application' + +const MEMORY_EXECUTOR_PRINCIPAL_POLICY = { + principalKinds: ['delegated'], + delegatedServices: ['executor'], +} as const + +function readOperation(id: Id) { + return defineWorkspaceOperation({ + id, + minimumRole: 'read', + workspaceApiKey: 'deny', + ...MEMORY_EXECUTOR_PRINCIPAL_POLICY, + }) +} + +function writeOperation(id: Id) { + return defineWorkspaceOperation({ + id, + minimumRole: 'write', + workspaceApiKey: 'deny', + ...MEMORY_EXECUTOR_PRINCIPAL_POLICY, + }) +} + +export const memoryOperations = { + list: readOperation('memory.list'), + read: readOperation('memory.read'), + append: writeOperation('memory.append'), + delete: writeOperation('memory.delete'), +} as const diff --git a/apps/sim/lib/memory/application/use-cases.test.ts b/apps/sim/lib/memory/application/use-cases.test.ts new file mode 100644 index 00000000000..f30c95699ba --- /dev/null +++ b/apps/sim/lib/memory/application/use-cases.test.ts @@ -0,0 +1,175 @@ +/** + * @vitest-environment node + */ + +import type { WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' +import { dbChainMock, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' + +const mocks = vi.hoisted(() => ({ + loadWorkspace: vi.fn(), + resolvePermission: vi.fn(), + reportUnrecorded: vi.fn(), + readBoundProvenance: vi.fn(), +})) + +vi.mock('@sim/db', () => ({ ...dbChainMock, ...schemaMock })) + +vi.mock('@sim/platform-authz/workspace', () => ({ + permissionSatisfies: (actual: string | null, required: string) => { + const rank = { read: 1, write: 2, admin: 3 } as const + return ( + actual !== null && rank[actual as keyof typeof rank] >= rank[required as keyof typeof rank] + ) + }, + resolveEffectiveWorkspacePermission: mocks.resolvePermission, +})) + +vi.mock('@/lib/billing/core/billing-attribution', () => ({ + assertBillingAttributionSnapshot: (value: unknown) => value, +})) + +vi.mock('@/lib/execution/durable-secret-provenance-enforcement', () => ({ + isDurableSecretProvenanceEnforced: () => false, + reportUnrecordedDurableProvenance: mocks.reportUnrecorded, +})) + +vi.mock('@/lib/memory/secret-provenance', () => ({ + readBoundMemorySecretProvenance: mocks.readBoundProvenance, + replaceMemorySecretProvenanceInTx: vi.fn(), +})) + +vi.mock('@/lib/workspaces/application/workspace-context', () => ({ + resolveActiveWorkspaceApplicationContext: mocks.loadWorkspace, +})) + +import { listMemoriesUseCase } from '@/lib/memory/application/use-cases' + +const WORKSPACE_ID = 'workspace-canonical' +const BILLING_OWNER_ID = 'billing-owner' +const BILLING_ATTRIBUTION: BillingAttributionSnapshot = { + actorUserId: BILLING_OWNER_ID, + workspaceId: WORKSPACE_ID, + organizationId: null, + billedAccountUserId: BILLING_OWNER_ID, + billingEntity: { type: 'user', id: BILLING_OWNER_ID }, + billingPeriod: { + start: '2026-08-01T00:00:00.000Z', + end: '2026-09-01T00:00:00.000Z', + }, + payerSubscription: null, +} + +const ACTORLESS_DEPLOYED_PRINCIPAL: WorkflowExecutionDelegatedPrincipal = { + kind: 'delegated', + serviceId: 'executor', + workspaceId: WORKSPACE_ID, + delegationId: 'delegation-1', + audience: 'sim:memory', + issuedAt: new Date(Date.now() - 1_000), + expiresAt: new Date(Date.now() + 60_000), + delegationContext: { + kind: 'workflow_execution', + workflowId: 'workflow-1', + executionId: 'execution-1', + principal: { + kind: 'system', + serviceId: 'schedule', + workspaceId: WORKSPACE_ID, + workflowId: 'workflow-1', + }, + currentWorkflow: { + workflowId: 'workflow-1', + mode: 'deployment', + deploymentVersionId: 'deployment-1', + }, + }, +} + +describe('Memory application use cases', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mocks.loadWorkspace.mockResolvedValue({ + workspaceId: WORKSPACE_ID, + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + billedAccountUserId: BILLING_OWNER_ID, + }) + mocks.readBoundProvenance.mockReturnValue({ status: 'unknown' }) + }) + + it('authorizes an actorless deployment before using signed billing for legacy provenance', async () => { + const record = { + id: 'memory-1', + key: 'conversation-1', + data: [{ role: 'user', content: 'hello' }], + secretProvenanceVersion: null, + } + queueTableRows(schemaMock.memory, [record]) + queueTableRows(schemaMock.memorySecretProvenance, []) + const resolveBillingAttribution = vi.fn(async () => BILLING_ATTRIBUTION) + + const result = await listMemoriesUseCase.execute({ + principal: ACTORLESS_DEPLOYED_PRINCIPAL, + input: { + workspaceId: WORKSPACE_ID, + limit: 50, + includePersistedSecretProvenance: true, + resolveBillingAttribution, + }, + }) + + expect(mocks.loadWorkspace.mock.invocationCallOrder[0]).toBeLessThan( + resolveBillingAttribution.mock.invocationCallOrder[0] + ) + expect(mocks.resolvePermission).not.toHaveBeenCalled() + expect(resolveBillingAttribution).toHaveBeenCalledWith(WORKSPACE_ID) + expect(result.provenanceScope).toEqual({ + userId: BILLING_OWNER_ID, + workspaceId: WORKSPACE_ID, + }) + expect(mocks.reportUnrecorded).toHaveBeenCalledWith({ + surface: 'memory', + cause: 'durable-provenance-unknown', + affectedCount: 1, + workspaceId: WORKSPACE_ID, + actorUserId: BILLING_OWNER_ID, + }) + }) + + it('rejects billing attribution outside the authorized canonical workspace', async () => { + const record = { + id: 'memory-1', + key: 'conversation-1', + data: [{ role: 'user', content: 'hello' }], + secretProvenanceVersion: null, + } + queueTableRows(schemaMock.memory, [record]) + const resolveBillingAttribution = vi.fn( + async (): Promise => ({ + ...BILLING_ATTRIBUTION, + workspaceId: 'workspace-other', + }) + ) + + await expect( + listMemoriesUseCase.execute({ + principal: ACTORLESS_DEPLOYED_PRINCIPAL, + input: { + workspaceId: WORKSPACE_ID, + limit: 50, + includePersistedSecretProvenance: true, + resolveBillingAttribution, + }, + }) + ).rejects.toThrow('Memory billing attribution does not match its canonical workspace') + + expect(mocks.loadWorkspace.mock.invocationCallOrder[0]).toBeLessThan( + resolveBillingAttribution.mock.invocationCallOrder[0] + ) + expect(mocks.readBoundProvenance).not.toHaveBeenCalled() + expect(mocks.reportUnrecorded).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/memory/application/use-cases.ts b/apps/sim/lib/memory/application/use-cases.ts new file mode 100644 index 00000000000..441405fcee7 --- /dev/null +++ b/apps/sim/lib/memory/application/use-cases.ts @@ -0,0 +1,405 @@ +import { + type Principal, + resolvePrincipalAttribution, + resolvePrincipalSubject, +} from '@sim/auth/principal' +import { db } from '@sim/db' +import { memory, memorySecretProvenance } from '@sim/db/schema' +import { getPostgresErrorCode } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { and, eq, inArray, isNull, like, sql } from 'drizzle-orm' +import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' +import { assertBillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' +import { defineAuthorizedWorkspaceUseCase } from '@/lib/core/application' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { + type DurableSecretProvenance, + mergeDurableSecretProvenance, +} from '@/lib/execution/durable-secret-provenance' +import { + isDurableSecretProvenanceEnforced, + reportUnrecordedDurableProvenance, +} from '@/lib/execution/durable-secret-provenance-enforcement' +import { memoryDelegationPolicy } from '@/lib/memory/application/authorization' +import { memoryOperations } from '@/lib/memory/application/operations' +import { + readBoundMemorySecretProvenance, + replaceMemorySecretProvenanceInTx, +} from '@/lib/memory/secret-provenance' +import { resolveActiveWorkspaceApplicationContext } from '@/lib/workspaces/application/workspace-context' + +const PRIVATE_MEMORY_QUERY_CHUNK_SIZE = 1_000 +const MAX_MEMORY_LIST_LIMIT = 1_000 + +export interface MemoryRecord { + id: string + key: string + data: unknown + secretProvenanceVersion: number | null +} + +export interface MemoryReadProvenance { + data: unknown + provenance: DurableSecretProvenance +} + +interface WorkspaceInput { + workspaceId: string +} + +export interface MemoryLegacyProvenanceScope { + userId: string + workspaceId: string +} + +interface ReadProvenanceInput { + includePersistedSecretProvenance?: boolean + resolveBillingAttribution?: (workspaceId: string) => Promise + signal?: AbortSignal +} + +async function resolveMemoryLegacyProvenanceScope( + principal: Principal, + workspaceId: string, + resolveBillingAttribution?: ReadProvenanceInput['resolveBillingAttribution'] +): Promise { + const subject = resolvePrincipalSubject(principal) + if (subject?.kind === 'sim_user') return { userId: subject.userId, workspaceId } + + const billingAttribution = resolveBillingAttribution + ? assertBillingAttributionSnapshot(await resolveBillingAttribution(workspaceId)) + : undefined + if (billingAttribution && billingAttribution.workspaceId !== workspaceId) { + throw new Error('Memory billing attribution does not match its canonical workspace') + } + const { attributedUserId } = resolvePrincipalAttribution(principal, { + workspaceBillingOwnerUserId: billingAttribution?.billedAccountUserId, + }) + return { userId: attributedUserId, workspaceId } +} + +function memoryMessageError(data: unknown): string | null { + const messages = Array.isArray(data) ? data : [data] + for (const message of messages) { + if (!message || typeof message !== 'object') { + return 'Memory requires messages with role and content' + } + const role = 'role' in message ? message.role : undefined + if (role && !['user', 'assistant', 'system'].includes(String(role))) { + return 'Message role must be user, assistant, or system' + } + if (!role || !('content' in message) || !message.content) { + return 'Memory requires messages with role and content' + } + } + return null +} + +async function loadReadProvenance( + records: MemoryRecord[], + scope: MemoryLegacyProvenanceScope, + signal?: AbortSignal +): Promise { + if (records.length === 0) return [] + + const recordsById = new Map() + for (const record of records) { + const matching = recordsById.get(record.id) ?? [] + matching.push(record) + recordsById.set(record.id, matching) + } + + const result: MemoryReadProvenance[] = [] + const ids = [...recordsById.keys()] + const enforced = isDurableSecretProvenanceEnforced('memory') + let unrecordedCount = 0 + + for (let index = 0; index < ids.length; index += PRIVATE_MEMORY_QUERY_CHUNK_SIZE) { + signal?.throwIfAborted() + const pageIds = ids.slice(index, index + PRIVATE_MEMORY_QUERY_CHUNK_SIZE) + const sidecars = await db + .select() + .from(memorySecretProvenance) + .where(inArray(memorySecretProvenance.memoryId, pageIds)) + const sidecarById = new Map(sidecars.map((sidecar) => [sidecar.memoryId, sidecar])) + + for (const memoryId of pageIds) { + for (const record of recordsById.get(memoryId) ?? []) { + const sidecar = sidecarById.get(memoryId) + const provenance = readBoundMemorySecretProvenance({ + secretProvenanceVersion: record.secretProvenanceVersion, + data: record.data, + provenanceContentHash: sidecar?.contentHash ?? null, + status: sidecar?.status ?? null, + entries: sidecar?.entries, + }) + if (provenance.status === 'unknown' && !enforced) unrecordedCount += 1 + result.push({ data: record.data, provenance }) + } + } + } + + if (unrecordedCount > 0) { + reportUnrecordedDurableProvenance({ + surface: 'memory', + cause: 'durable-provenance-unknown', + affectedCount: unrecordedCount, + workspaceId: scope.workspaceId, + actorUserId: scope.userId, + }) + } + + return result +} + +async function readResultProvenance( + records: MemoryRecord[], + principal: Principal, + workspaceId: string, + input: ReadProvenanceInput, + existingScope?: MemoryLegacyProvenanceScope +): Promise<{ + readProvenance?: MemoryReadProvenance[] + provenanceScope?: MemoryLegacyProvenanceScope +}> { + if (!input.includePersistedSecretProvenance) return {} + const provenanceScope = + existingScope ?? + (await resolveMemoryLegacyProvenanceScope( + principal, + workspaceId, + input.resolveBillingAttribution + )) + return { + readProvenance: await loadReadProvenance(records, provenanceScope, input.signal), + provenanceScope, + } +} + +export interface ListMemoriesInput extends WorkspaceInput, ReadProvenanceInput { + query?: string | null + limit: number +} + +export const listMemoriesUseCase = defineAuthorizedWorkspaceUseCase({ + operation: memoryOperations.list, + resolveContext: ({ input }: { input: ListMemoriesInput }) => + resolveActiveWorkspaceApplicationContext(input.workspaceId), + authorizationOptions: { delegation: memoryDelegationPolicy }, + async execute({ principal, input, context }) { + if (!Number.isInteger(input.limit) || input.limit < 1 || input.limit > MAX_MEMORY_LIST_LIMIT) { + throw new OrchestrationError('validation', 'Invalid memory list limit') + } + input.signal?.throwIfAborted() + const conditions = [isNull(memory.deletedAt), eq(memory.workspaceId, context.workspaceId)] + if (input.query) conditions.push(like(memory.key, `%${input.query}%`)) + const records = await db + .select() + .from(memory) + .where(and(...conditions)) + .orderBy(memory.createdAt) + .limit(input.limit) + input.signal?.throwIfAborted() + const provenance = await readResultProvenance(records, principal, context.workspaceId, input) + return { + records, + ...provenance, + } + }, +}) + +export interface ReadMemoryInput extends WorkspaceInput, ReadProvenanceInput { + key: string +} + +export const readMemoryUseCase = defineAuthorizedWorkspaceUseCase({ + operation: memoryOperations.read, + resolveContext: ({ input }: { input: ReadMemoryInput }) => + resolveActiveWorkspaceApplicationContext(input.workspaceId), + authorizationOptions: { delegation: memoryDelegationPolicy }, + async execute({ principal, input, context }) { + input.signal?.throwIfAborted() + const records = await db + .select() + .from(memory) + .where( + and( + eq(memory.key, input.key), + eq(memory.workspaceId, context.workspaceId), + isNull(memory.deletedAt) + ) + ) + .orderBy(memory.createdAt) + .limit(1) + input.signal?.throwIfAborted() + const provenance = await readResultProvenance(records, principal, context.workspaceId, input) + return { + record: records[0] ?? null, + ...provenance, + } + }, +}) + +export interface AppendMemoryInput extends WorkspaceInput, ReadProvenanceInput { + key: string + data: unknown + writeProvenance?: DurableSecretProvenance + resolveWriteProvenance?: ( + scope: MemoryLegacyProvenanceScope + ) => DurableSecretProvenance | undefined +} + +export const appendMemoryUseCase = defineAuthorizedWorkspaceUseCase({ + operation: memoryOperations.append, + resolveContext: ({ input }: { input: AppendMemoryInput }) => + resolveActiveWorkspaceApplicationContext(input.workspaceId), + authorizationOptions: { delegation: memoryDelegationPolicy }, + async execute({ principal, input, context }) { + if (!input.key) throw new OrchestrationError('validation', 'Memory key is required') + if (!input.data) throw new OrchestrationError('validation', 'Memory data is required') + const messageError = memoryMessageError(input.data) + if (messageError) throw new OrchestrationError('validation', messageError) + + input.signal?.throwIfAborted() + const provenanceScope = input.resolveWriteProvenance + ? await resolveMemoryLegacyProvenanceScope( + principal, + context.workspaceId, + input.resolveBillingAttribution + ) + : undefined + const writeProvenance = + input.resolveWriteProvenance && provenanceScope + ? input.resolveWriteProvenance(provenanceScope) + : input.writeProvenance + const initialData = Array.isArray(input.data) ? input.data : [input.data] + const now = new Date() + const id = `mem_${generateId().replace(/-/g, '')}` + + try { + await db.transaction(async (tx) => { + const [existing] = await tx + .select({ + id: memory.id, + data: memory.data, + secretProvenanceVersion: memory.secretProvenanceVersion, + }) + .from(memory) + .where(and(eq(memory.workspaceId, context.workspaceId), eq(memory.key, input.key))) + .limit(1) + .for('update') + + let previousProvenance: DurableSecretProvenance | undefined + if (existing && writeProvenance) { + const [sidecar] = await tx + .select() + .from(memorySecretProvenance) + .where(eq(memorySecretProvenance.memoryId, existing.id)) + .limit(1) + previousProvenance = readBoundMemorySecretProvenance({ + secretProvenanceVersion: existing.secretProvenanceVersion, + data: existing.data, + provenanceContentHash: sidecar?.contentHash ?? null, + status: sidecar?.status ?? null, + entries: sidecar?.entries, + }) + } + + const [written] = await tx + .insert(memory) + .values({ + id, + workspaceId: context.workspaceId, + key: input.key, + data: initialData, + secretProvenanceVersion: writeProvenance ? 1 : null, + createdAt: now, + updatedAt: now, + }) + .onConflictDoUpdate({ + target: [memory.workspaceId, memory.key], + set: { + data: sql`${memory.data} || ${JSON.stringify(initialData)}::jsonb`, + secretProvenanceVersion: writeProvenance + ? 1 + : (existing?.secretProvenanceVersion ?? null), + updatedAt: now, + }, + }) + .returning({ id: memory.id, data: memory.data }) + + if (writeProvenance) { + await replaceMemorySecretProvenanceInTx( + tx, + written.id, + written.data, + previousProvenance + ? mergeDurableSecretProvenance(previousProvenance, writeProvenance) + : writeProvenance + ) + } + }) + } catch (error) { + if (getPostgresErrorCode(error) === '23505') { + throw new OrchestrationError('conflict', 'Memory with this key already exists') + } + throw error + } + + input.signal?.throwIfAborted() + const records = await db + .select() + .from(memory) + .where( + and( + eq(memory.key, input.key), + eq(memory.workspaceId, context.workspaceId), + isNull(memory.deletedAt) + ) + ) + .orderBy(memory.createdAt) + .limit(1) + const record = records[0] + if (!record) throw new Error('Failed to retrieve memory after creation/update') + input.signal?.throwIfAborted() + const provenance = await readResultProvenance( + records, + principal, + context.workspaceId, + input, + provenanceScope + ) + return { + record, + ...provenance, + } + }, +}) + +export interface DeleteMemoryInput extends WorkspaceInput { + key: string + signal?: AbortSignal +} + +export const deleteMemoryUseCase = defineAuthorizedWorkspaceUseCase({ + operation: memoryOperations.delete, + resolveContext: ({ input }: { input: DeleteMemoryInput }) => + resolveActiveWorkspaceApplicationContext(input.workspaceId), + authorizationOptions: { delegation: memoryDelegationPolicy }, + async execute({ input, context }) { + if (!input.key) throw new OrchestrationError('validation', 'conversationId must be provided') + input.signal?.throwIfAborted() + const deleted = await db + .delete(memory) + .where( + and( + eq(memory.key, input.key), + eq(memory.workspaceId, context.workspaceId), + isNull(memory.deletedAt) + ) + ) + .returning({ id: memory.id }) + input.signal?.throwIfAborted() + return { deletedCount: deleted.length } + }, +}) diff --git a/apps/sim/lib/microsoft-word/graph.server.test.ts b/apps/sim/lib/microsoft-word/graph.server.test.ts deleted file mode 100644 index eb6c6d28a35..00000000000 --- a/apps/sim/lib/microsoft-word/graph.server.test.ts +++ /dev/null @@ -1,194 +0,0 @@ -/** - * @vitest-environment node - */ -import { inputValidationMock, inputValidationMockFns } from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) - -import { replaceContentIfUnchanged } from '@/lib/microsoft-word/graph.server' - -const { mockValidateUrlWithDNS, mockSecureFetchWithPinnedIP } = inputValidationMockFns - -const BASE_PATH = 'https://graph.microsoft.com/v1.0/me/drive/items/doc-abc' -const UPLOAD_URL = 'https://sn3302.up.1drv.com/up/session-abc' - -/** Graph's `createUploadSession` response. */ -function sessionResponse() { - const body = { uploadUrl: UPLOAD_URL, expirationDateTime: '2026-01-01T00:00:00Z' } - return { - ok: true, - status: 200, - statusText: '', - headers: new Headers(), - body: null, - text: async () => JSON.stringify(body), - json: async () => body, - arrayBuffer: async () => new ArrayBuffer(0), - } -} - -/** A 202 acknowledging a non-final fragment; carries no driveItem. */ -function fragmentAccepted() { - const body = { nextExpectedRanges: ['1-'] } - return { - ok: true, - status: 202, - statusText: 'Accepted', - headers: new Headers(), - body: null, - text: async () => JSON.stringify(body), - json: async () => body, - arrayBuffer: async () => new ArrayBuffer(0), - } -} - -/** The final fragment's response, carrying the completed driveItem. */ -function completedItem() { - const body = { id: 'doc-abc', name: 'notes.docx', size: 123 } - return { - ok: true, - status: 201, - statusText: 'Created', - headers: new Headers(), - body: null, - text: async () => JSON.stringify(body), - json: async () => body, - arrayBuffer: async () => new ArrayBuffer(0), - } -} - -function preconditionFailed() { - return { - ok: false, - status: 412, - statusText: 'Precondition Failed', - headers: new Headers(), - body: null, - text: async () => '', - json: async () => ({}), - arrayBuffer: async () => new ArrayBuffer(0), - } -} - -/** Parses a `Content-Range: bytes {start}-{end}/{total}` header. */ -function parseRange(header: string): { start: number; end: number; total: number } { - const [range, total] = header.replace('bytes ', '').split('/') - const [start, end] = range.split('-').map(Number) - return { start, end, total: Number(total) } -} - -beforeEach(() => { - // reset, not clear: an unconsumed mockResolvedValueOnce queue would otherwise - // leak into the next test and make its result meaningless. - mockSecureFetchWithPinnedIP.mockReset() - mockValidateUrlWithDNS.mockReset() - mockValidateUrlWithDNS.mockResolvedValue({ - isValid: true, - resolvedIP: '93.184.216.34', - originalHostname: 'graph.microsoft.com', - }) -}) - -describe('replaceContentIfUnchanged', () => { - it('sends a small package as a single fragment', async () => { - mockSecureFetchWithPinnedIP - .mockResolvedValueOnce(sessionResponse()) - .mockResolvedValueOnce(completedItem()) - - await replaceContentIfUnchanged(BASE_PATH, 'token', Buffer.alloc(1024), 'tag-1') - - const puts = mockSecureFetchWithPinnedIP.mock.calls.filter((c) => c[2]?.method === 'PUT') - expect(puts).toHaveLength(1) - expect(parseRange(puts[0][2].headers['Content-Range'])).toEqual({ - start: 0, - end: 1023, - total: 1024, - }) - }) - - it('splits a package larger than one fragment into contiguous ordered ranges', async () => { - // Graph rejects a single upload request at or above 60 MiB, and documents - // will not always fit: content is read under a 100 MB ceiling. - const size = 25 * 1024 * 1024 - mockSecureFetchWithPinnedIP - .mockResolvedValueOnce(sessionResponse()) - .mockResolvedValueOnce(fragmentAccepted()) - .mockResolvedValueOnce(fragmentAccepted()) - .mockResolvedValueOnce(completedItem()) - - const item = await replaceContentIfUnchanged(BASE_PATH, 'token', Buffer.alloc(size), 'tag-1') - expect(item.id).toBe('doc-abc') - - const puts = mockSecureFetchWithPinnedIP.mock.calls.filter((c) => c[2]?.method === 'PUT') - expect(puts).toHaveLength(3) - - let expectedStart = 0 - for (const put of puts) { - const { start, end, total } = parseRange(put[2].headers['Content-Range']) - expect(total).toBe(size) - expect(start).toBe(expectedStart) - // Every fragment but the last must be a multiple of 320 KiB. - const length = end - start + 1 - expect(Number(put[2].headers['Content-Length'])).toBe(length) - if (end !== size - 1) expect(length % (320 * 1024)).toBe(0) - expectedStart = end + 1 - } - expect(expectedStart).toBe(size) - }) - - it('never sends the bearer token to the preauthenticated upload URL', async () => { - mockSecureFetchWithPinnedIP - .mockResolvedValueOnce(sessionResponse()) - .mockResolvedValueOnce(completedItem()) - - await replaceContentIfUnchanged(BASE_PATH, 'token', Buffer.alloc(64), 'tag-1') - - const put = mockSecureFetchWithPinnedIP.mock.calls.find((c) => c[2]?.method === 'PUT') - expect(put?.[0]).toBe(UPLOAD_URL) - expect(put?.[2].headers.Authorization).toBeUndefined() - }) - - it('carries the precondition on the session and maps its rejection to a conflict', async () => { - mockSecureFetchWithPinnedIP.mockResolvedValueOnce(preconditionFailed()) - - await expect( - replaceContentIfUnchanged(BASE_PATH, 'token', Buffer.alloc(64), 'tag-1') - ).rejects.toMatchObject({ status: 409 }) - - const session = mockSecureFetchWithPinnedIP.mock.calls[0] - expect(session[0]).toBe(`${BASE_PATH}/createUploadSession`) - expect(session[2].headers['if-match']).toBe('tag-1') - // Rejected before any bytes left the process. - expect(mockSecureFetchWithPinnedIP.mock.calls.some((c) => c[2]?.method === 'PUT')).toBe(false) - }) - - it('maps a conflict raised part-way through the fragments to the same error', async () => { - mockSecureFetchWithPinnedIP - .mockResolvedValueOnce(sessionResponse()) - .mockResolvedValueOnce(fragmentAccepted()) - .mockResolvedValueOnce(preconditionFailed()) - - await expect( - replaceContentIfUnchanged(BASE_PATH, 'token', Buffer.alloc(25 * 1024 * 1024), 'tag-1') - ).rejects.toMatchObject({ status: 409 }) - }) - - it('fails loudly when the session response carries no upload URL', async () => { - const body = { expirationDateTime: '2026-01-01T00:00:00Z' } - mockSecureFetchWithPinnedIP.mockResolvedValueOnce({ - ok: true, - status: 200, - statusText: '', - headers: new Headers(), - body: null, - text: async () => JSON.stringify(body), - json: async () => body, - arrayBuffer: async () => new ArrayBuffer(0), - }) - - await expect( - replaceContentIfUnchanged(BASE_PATH, 'token', Buffer.alloc(64), 'tag-1') - ).rejects.toThrow(/did not return an upload URL/) - }) -}) diff --git a/apps/sim/lib/microsoft-word/graph.server.ts b/apps/sim/lib/microsoft-word/graph.server.ts deleted file mode 100644 index 3c0f440e1b9..00000000000 --- a/apps/sim/lib/microsoft-word/graph.server.ts +++ /dev/null @@ -1,358 +0,0 @@ -import { - secureFetchWithPinnedIP, - validateUrlWithDNS, -} from '@/lib/core/security/input-validation.server' -import { DOCX_MIME_TYPE } from '@/lib/microsoft-word/document.server' -import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' -import { parseGraphErrorMessage } from '@/tools/microsoft_excel/utils' -import type { MicrosoftWordDocumentMetadata } from '@/tools/microsoft_word/types' - -/** Microsoft Graph `driveItem` fields the Word routes project. */ -interface GraphDriveItem { - id?: string - name?: string - size?: number - webUrl?: string - createdDateTime?: string - lastModifiedDateTime?: string - /** An eTag for the item's content, unchanged when only metadata changes. */ - cTag?: string - /** An eTag for the whole item, metadata included. */ - eTag?: string - file?: { mimeType?: string } - folder?: Record -} - -/** - * The token that identifies the exact content an edit was based on. - * - * `cTag` is the right one: Graph documents it as "an eTag for the content of the - * item" that does not move when only metadata changes, so a rename will not - * spuriously abort an edit. `eTag` is the fallback for the shapes where Graph - * omits `cTag`. - * - * @see https://learn.microsoft.com/en-us/graph/api/resources/driveitem - */ -export function getContentTag(item: GraphDriveItem): string | undefined { - return item.cTag ?? item.eTag -} - -/** Thrown when Microsoft Graph rejects a request, carrying its HTTP status. */ -export class GraphRequestError extends Error { - constructor( - message: string, - readonly status: number - ) { - super(message) - this.name = 'GraphRequestError' - } -} - -/** - * Issues an IP-pinned request to a Microsoft Graph URL, rejecting the URL when - * DNS resolution points anywhere Sim must not reach. - */ -async function graphFetch( - url: string, - paramName: string, - options: Parameters[2] -) { - const validation = await validateUrlWithDNS(url, paramName) - if (!validation.isValid) { - throw new GraphRequestError(validation.error || `Invalid ${paramName}`, 400) - } - return secureFetchWithPinnedIP(url, validation.resolvedIP as string, options) -} - -/** Reads a Graph error body and raises it as a {@link GraphRequestError}. */ -async function raiseGraphError(response: { - status: number - statusText: string - text: () => Promise -}): Promise { - const errorText = await response.text().catch(() => '') - throw new GraphRequestError( - parseGraphErrorMessage(response.status, response.statusText, errorText), - response.status - ) -} - -/** Projects a Graph `driveItem` onto the metadata shape the Word tools return. */ -export function toDocumentMetadata( - item: GraphDriveItem, - fallbackId: string -): MicrosoftWordDocumentMetadata { - return { - documentId: item.id ?? fallbackId, - name: item.name ?? null, - mimeType: item.file?.mimeType ?? null, - webViewLink: item.webUrl ?? null, - size: item.size ?? null, - createdTime: item.createdDateTime ?? null, - modifiedTime: item.lastModifiedDateTime ?? null, - } -} - -/** - * Fetches a drive item's metadata and rejects folders, which have no document - * content and would otherwise fail later with an opaque Graph error. - * - * @see https://learn.microsoft.com/en-us/graph/api/driveitem-get - */ -export async function fetchDocumentItem( - basePath: string, - accessToken: string -): Promise { - const response = await graphFetch(basePath, 'documentUrl', { - headers: { Authorization: `Bearer ${accessToken}` }, - }) - - if (!response.ok) await raiseGraphError(response) - - const item = (await response.json()) as GraphDriveItem - if (item.folder && !item.file) { - throw new GraphRequestError( - `"${item.name ?? 'The selected item'}" is a folder, not a Word document`, - 400 - ) - } - if (!isWordDocument(item)) { - throw new GraphRequestError( - `"${item.name ?? 'The selected item'}" is not a Word document. Every Microsoft Word operation reads or writes a .docx file.`, - 400 - ) - } - return item -} - -/** - * Whether a drive item is a `.docx` package. - * - * The name suffix is accepted alongside the MIME type because Graph does not - * always populate `file.mimeType`. Getting this wrong is destructive rather than - * merely wrong: without the check, pointing Replace Content at a PDF would - * overwrite it with generated WordprocessingML bytes. - */ -function isWordDocument(item: GraphDriveItem): boolean { - return ( - item.file?.mimeType === DOCX_MIME_TYPE || Boolean(item.name?.toLowerCase().endsWith('.docx')) - ) -} - -/** - * Downloads a drive item's raw content. - * - * @see https://learn.microsoft.com/en-us/graph/api/driveitem-get-content - */ -export async function downloadDocumentContent( - basePath: string, - accessToken: string -): Promise { - const response = await graphFetch(`${basePath}/content`, 'documentContentUrl', { - headers: { Authorization: `Bearer ${accessToken}` }, - maxResponseBytes: MAX_FILE_SIZE, - // Graph redirects to a short-lived preauthenticated URL on another host that - // must not receive the bearer token. - stripAuthOnRedirect: true, - }) - - if (!response.ok) await raiseGraphError(response) - - return Buffer.from(await response.arrayBuffer()) -} - -/** - * Downloads a drive item converted to another format. - * - * @see https://learn.microsoft.com/en-us/graph/api/driveitem-get-content-format - */ -export async function downloadConvertedContent( - basePath: string, - accessToken: string, - format: 'pdf' -): Promise { - const response = await graphFetch(`${basePath}/content?format=${format}`, 'documentConvertUrl', { - headers: { Authorization: `Bearer ${accessToken}` }, - maxResponseBytes: MAX_FILE_SIZE, - stripAuthOnRedirect: true, - }) - - if (!response.ok) await raiseGraphError(response) - - return Buffer.from(await response.arrayBuffer()) -} - -/** Message shown when someone else changed the document mid-edit. */ -const CONFLICT_MESSAGE = - 'The document changed in OneDrive or SharePoint after Sim read it, so the edit was not applied and no other change was overwritten. Run the operation again to edit the current version.' - -/** Raised instead of overwriting a document that changed since it was read. */ -export function documentChangedError(): GraphRequestError { - return new GraphRequestError(CONFLICT_MESSAGE, 409) -} - -/** Message shown when the document carries no version to compare against. */ -const UNVERIFIABLE_MESSAGE = - 'Microsoft Graph did not report a version for this document, so Sim cannot confirm the edit would not overwrite someone else’s change and did not apply it. Use Replace Content if you intend to overwrite the document outright.' - -/** - * Raised when there is no version to compare, rather than writing unguarded. - * - * Graph returns `cTag` for every file and `eTag` for every drive item, so this - * should not be reachable in practice — but a read-modify-write that silently - * degrades to no protection is the failure this whole guard exists to prevent, - * so the missing-version path fails closed instead of proceeding. - */ -function unverifiableDocumentError(): GraphRequestError { - return new GraphRequestError(UNVERIFIABLE_MESSAGE, 409) -} - -/** - * Returns the content tag an edit must be based on, refusing the edit outright - * when the item carries none. - */ -export function requireContentTag(item: GraphDriveItem): string { - const tag = getContentTag(item) - if (!tag) { - throw unverifiableDocumentError() - } - return tag -} - -/** A Graph upload session, used for a precondition-checked content write. */ -interface GraphUploadSession { - uploadUrl?: string -} - -/** - * Replaces a document's content only if it still matches `expectedTag`. - * - * `PUT /items/{id}/content` documents no precondition — its request-headers - * table lists only `Authorization` and `Content-Type` — so a conditional write - * has to go through an upload session, whose `if-match` header Graph documents - * as returning `412 Precondition Failed` on a mismatch. That makes the check - * service-enforced rather than a client-side compare that could itself race. - * - * The residual window is small and stated honestly: the tag is evaluated when - * the session is created and the bytes commit on the following request. Closing - * it completely would need the deferred-commit form, whose conditional commit - * relies on `@microsoft.graph.sourceUrl` — which Microsoft documents as - * unsupported on OneDrive for Business and SharePoint Online, where these - * documents live. - * - * @see https://learn.microsoft.com/en-us/graph/api/driveitem-createuploadsession - * @see https://learn.microsoft.com/en-us/graph/api/resources/driveitem - */ -export async function replaceContentIfUnchanged( - basePath: string, - accessToken: string, - content: Buffer, - expectedTag: string -): Promise { - const sessionResponse = await graphFetch( - `${basePath}/createUploadSession`, - 'documentUploadSessionUrl', - { - method: 'POST', - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json', - 'if-match': expectedTag, - }, - body: '{}', - } - ) - - if (sessionResponse.status === 412) { - throw documentChangedError() - } - if (!sessionResponse.ok) await raiseGraphError(sessionResponse) - - const { uploadUrl } = (await sessionResponse.json()) as GraphUploadSession - if (!uploadUrl) { - throw new GraphRequestError('Microsoft Graph did not return an upload URL', 502) - } - - return uploadSessionBytes(uploadUrl, content) -} - -/** - * Byte size of each upload fragment. - * - * Graph caps a single upload request below 60 MiB, and requires every fragment - * of a split upload to be a multiple of 320 KiB — 10 MiB is both (327,680 × 32) - * and is inside the 5–10 MiB range Microsoft recommends. Documents are read - * under a 100 MB ceiling, so a single request would not always be enough. - */ -const UPLOAD_FRAGMENT_BYTES = 10 * 1024 * 1024 - -/** - * Sends the package to a session's upload URL, splitting it into sequential - * fragments. Graph answers `202 Accepted` for every fragment but the last, and - * returns the finished driveItem with the one that completes the file. - * - * The URL is preauthenticated and on another host; Graph documents that sending - * `Authorization` here can itself fail the request with a 401, so no bearer - * token is attached. - * - * @see https://learn.microsoft.com/en-us/graph/api/driveitem-createuploadsession - */ -async function uploadSessionBytes(uploadUrl: string, content: Buffer): Promise { - const total = content.length - - for (let start = 0; start < total; start += UPLOAD_FRAGMENT_BYTES) { - const end = Math.min(start + UPLOAD_FRAGMENT_BYTES, total) - 1 - const fragment = content.subarray(start, end + 1) - - const response = await graphFetch(uploadUrl, 'documentUploadUrl', { - method: 'PUT', - headers: { - 'Content-Length': String(fragment.length), - 'Content-Range': `bytes ${start}-${end}/${total}`, - }, - body: fragment, - }) - - if (response.status === 412 || response.status === 409) { - throw documentChangedError() - } - if (!response.ok) await raiseGraphError(response) - - // Every fragment but the last is acknowledged with 202 and no item body. - if (end === total - 1) { - return (await response.json()) as GraphDriveItem - } - } - - throw new GraphRequestError('Microsoft Graph did not complete the document upload', 502) -} - -/** - * Uploads bytes as a drive item's content and returns the resulting item. - * - * Unconditional by design: this backs creating a new document and the - * deliberate whole-document overwrite. An edit that must not clobber a - * concurrent change uses {@link replaceContentIfUnchanged} instead. - * - * @see https://learn.microsoft.com/en-us/graph/api/driveitem-put-content - */ -export async function uploadDocumentContent( - url: string, - accessToken: string, - content: Buffer, - mimeType: string -): Promise { - const response = await graphFetch(url, 'documentUploadUrl', { - method: 'PUT', - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': mimeType, - 'Content-Length': String(content.length), - }, - body: content, - }) - - if (!response.ok) await raiseGraphError(response) - - return (await response.json()) as GraphDriveItem -} diff --git a/apps/sim/lib/oauth/utils.test.ts b/apps/sim/lib/oauth/utils.test.ts index 2b6a3116c8b..ab0e34a9273 100644 --- a/apps/sim/lib/oauth/utils.test.ts +++ b/apps/sim/lib/oauth/utils.test.ts @@ -398,6 +398,28 @@ describe('getScopeDescription', () => { expect(getScopeDescription('account', 'reddit')).toBe('Update account preferences and settings') expect(getScopeDescription('account')).toBe('Update account preferences and settings') }) + + /** + * The consent screen is where a user decides what to grant, so a write scope + * has to read as one. `w_member_social` previously said 'Access LinkedIn + * profile', describing a posting grant as a profile read. + * + * The wording tracks LinkedIn's own: "Post, comment, and like posts on behalf + * of an authenticated member." It names all three verbs even though Sim only + * posts -- the label describes the grant the token carries, not Sim's current + * use of it, and LinkedIn's scopes cannot be sub-selected. + */ + it.concurrent('describes w_member_social as the write grant it is', () => { + const description = getScopeDescription('w_member_social', 'linkedin') + + expect(description).toBe('Post, comment, and like posts on your behalf') + expect(description).not.toMatch(/access .*profile/i) + }) + + it.concurrent('leaves the read-only LinkedIn scopes read-only', () => { + expect(getScopeDescription('profile', 'linkedin')).toBe('Access profile information') + expect(getScopeDescription('email', 'linkedin')).toBe('Access email address') + }) }) describe('parseProvider', () => { diff --git a/apps/sim/lib/oauth/utils.ts b/apps/sim/lib/oauth/utils.ts index 786ce8e46cb..9e6f6a3a494 100644 --- a/apps/sim/lib/oauth/utils.ts +++ b/apps/sim/lib/oauth/utils.ts @@ -385,7 +385,7 @@ export const SCOPE_DESCRIPTIONS: Record = { 'webhooks:full': 'Full access to manage Pipedrive webhooks', // LinkedIn scopes - w_member_social: 'Access LinkedIn profile', + w_member_social: 'Post, comment, and like posts on your behalf', // Instagram scopes (Business Login for Instagram) instagram_business_basic: 'Access Instagram professional profile and media', diff --git a/apps/sim/lib/table/api/row-route-policies.ts b/apps/sim/lib/table/api/row-route-policies.ts index 4a93c4e74c6..8eb5107aaf0 100644 --- a/apps/sim/lib/table/api/row-route-policies.ts +++ b/apps/sim/lib/table/api/row-route-policies.ts @@ -5,7 +5,7 @@ import { } from '@/lib/api/server/routes' import { internalTableErrorPolicies, v2TableErrorPolicies } from '@/lib/table/api/route-policies' import { TableRowProvenanceError } from '@/lib/table/application/row-secret-provenance' -import { TableRowsValidationError } from '@/lib/table/application/rows' +import { TableRowsValidationError, TableV2FeatureDisabledError } from '@/lib/table/application/rows' import { v2Error } from '@/app/api/v2/lib/response' export const v2TableRowsErrorPolicy = { @@ -35,3 +35,25 @@ export const internalTableRowsErrorPolicy = extendInternalErrorPolicy( ? internalErrorResponse(400, { error: error.message }) : null ) + +export const internalTableV2QueryErrorPolicy = extendInternalErrorPolicy( + internalTableRowsErrorPolicy, + (error) => { + if (error instanceof TableV2FeatureDisabledError) { + return internalErrorResponse(403, { + error: error.message, + code: 'tables_v2_disabled', + }) + } + if ( + error instanceof TableRowsValidationError && + typeof error.details === 'object' && + error.details !== null && + 'code' in error.details && + typeof error.details.code === 'string' + ) { + return internalErrorResponse(400, { error: error.message, code: error.details.code }) + } + return null + } +) diff --git a/apps/sim/lib/table/application/operations.test.ts b/apps/sim/lib/table/application/operations.test.ts index 29d9cde0dba..3d92657571d 100644 --- a/apps/sim/lib/table/application/operations.test.ts +++ b/apps/sim/lib/table/application/operations.test.ts @@ -75,6 +75,13 @@ describe('table operation registry', () => { // tools run under them, and a policy without `executor` fails every one of // those calls with a 403 while every route test still passes. const sharedToolOperations = new Set([ + tableOperations.list.id, + tableOperations.read.id, + tableOperations.create.id, + tableOperations.queryRows.id, + tableOperations.createRows.id, + tableOperations.updateRows.id, + tableOperations.deleteRows.id, tableOperations.createGroup.id, tableOperations.updateGroup.id, tableOperations.deleteGroup.id, diff --git a/apps/sim/lib/table/application/operations.ts b/apps/sim/lib/table/application/operations.ts index cb0e9b55025..209d11979a7 100644 --- a/apps/sim/lib/table/application/operations.ts +++ b/apps/sim/lib/table/application/operations.ts @@ -84,9 +84,9 @@ function delegatedWriteOperation(id: Id) { } export const tableOperations = { - list: readOperation('tables.list'), - read: readOperation('tables.read'), - create: writeOperation('tables.create'), + list: toolReadOperation('tables.list'), + read: toolReadOperation('tables.read'), + create: toolWriteOperation('tables.create'), update: writeOperation('tables.update'), delete: writeOperation('tables.delete'), restore: writeOperation('tables.restore'), @@ -119,15 +119,15 @@ export const tableOperations = { updateColumn: writeOperation('tables.columns.update'), deleteColumn: writeOperation('tables.columns.delete'), listRows: readOperation('tables.rows.list'), - queryRows: readOperation('tables.rows.query'), + queryRows: toolReadOperation('tables.rows.query'), searchRows: readOperation('tables.rows.search'), readRow: toolReadOperation('tables.rows.read'), - createRows: writeOperation('tables.rows.create'), + createRows: toolWriteOperation('tables.rows.create'), replaceRows: writeOperation('tables.rows.replace'), updateRow: toolWriteOperation('tables.rows.update'), - updateRows: writeOperation('tables.rows.update_many'), + updateRows: toolWriteOperation('tables.rows.update_many'), deleteRow: toolWriteOperation('tables.rows.delete'), - deleteRows: writeOperation('tables.rows.delete_many'), + deleteRows: toolWriteOperation('tables.rows.delete_many'), upsertRow: toolWriteOperation('tables.rows.upsert'), listViews: readOperation('tables.views.list'), readView: readOperation('tables.views.read'), diff --git a/apps/sim/lib/table/application/row-secret-provenance.test.ts b/apps/sim/lib/table/application/row-secret-provenance.test.ts index 100b56f2eeb..ae863ddc0c5 100644 --- a/apps/sim/lib/table/application/row-secret-provenance.test.ts +++ b/apps/sim/lib/table/application/row-secret-provenance.test.ts @@ -36,12 +36,28 @@ const SESSION = { kind: 'session' as const, userId: 'user-1', sessionId: 'sessio const EXECUTOR = { kind: 'delegated' as const, serviceId: 'executor' as const, - subjectUserId: 'user-1', workspaceId: 'workspace-1', delegationId: 'delegation-1', audience: 'table', issuedAt: new Date('2026-01-01'), - expiresAt: new Date('2026-01-02'), + expiresAt: new Date('2099-01-02'), + delegationContext: { + kind: 'workflow_execution' as const, + workflowId: 'workflow-1', + currentWorkflow: { + workflowId: 'workflow-1', + mode: 'deployment' as const, + deploymentVersionId: 'deployment-1', + }, + principal: { + kind: 'system' as const, + serviceId: 'webhook' as const, + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + webhookId: 'webhook-1', + provider: 'generic', + }, + }, } /** @@ -175,7 +191,7 @@ describe('row write provenance', () => { expect(stamps[0]).toEqual({ complete: true, columns: { col_aaa: traceProvenance() } }) }) - it('checks the scope against the acting principal, not a billing owner', () => { + it('checks a deployed actorless execution against its authorized workspace', () => { resolve({ principal: EXECUTOR, envelope: { @@ -191,7 +207,7 @@ describe('row write provenance', () => { expect(mocks.scopeCompatible).toHaveBeenCalledWith( { userId: 'billing-owner', workspaceId: 'workspace-1' }, - { userId: 'user-1', workspaceId: 'workspace-1' } + { workspaceId: 'workspace-1' } ) }) diff --git a/apps/sim/lib/table/application/row-secret-provenance.ts b/apps/sim/lib/table/application/row-secret-provenance.ts index 3911703ac42..e5a110a807a 100644 --- a/apps/sim/lib/table/application/row-secret-provenance.ts +++ b/apps/sim/lib/table/application/row-secret-provenance.ts @@ -1,4 +1,4 @@ -import { type Principal, requirePrincipalSubjectUserId } from '@sim/auth/principal' +import type { Principal } from '@sim/auth/principal' import { isPrivateSecretProvenanceScopeCompatible } from '@/lib/execution/durable-secret-provenance' import { isPrivateSecretProvenanceBundleV1 } from '@/lib/execution/model-input-provenance' import { buildIdByName } from '@/lib/table/column-keys' @@ -124,13 +124,11 @@ export function resolveRowWriteProvenance(options: { complete: true, columns: {}, })) - const subjectUserId = requirePrincipalSubjectUserId(principal) for (const selection of bundle.selections) { const touched = touchedBySelectionKey.get(selection.key) if ( !touched || !isPrivateSecretProvenanceScopeCompatible(selection.provenance.scope, { - userId: subjectUserId, workspaceId: options.workspaceId, }) ) { diff --git a/apps/sim/lib/table/application/rows.test.ts b/apps/sim/lib/table/application/rows.test.ts index 6ba968fa788..9e9c9e0dc71 100644 --- a/apps/sim/lib/table/application/rows.test.ts +++ b/apps/sim/lib/table/application/rows.test.ts @@ -32,6 +32,8 @@ const { mockGetRowSummaryById, mockLoadExecutionsForRow, mockLoadEnrichmentDetail, + mockIsFeatureEnabled, + mockGetWorkspaceOrganizationId, } = vi.hoisted(() => ({ mockReplaceRowsPrimitive: vi.fn(), mockDeleteRowsByIds: vi.fn(), @@ -59,6 +61,16 @@ const { mockGetRowSummaryById: vi.fn(), mockLoadExecutionsForRow: vi.fn(), mockLoadEnrichmentDetail: vi.fn(), + mockIsFeatureEnabled: vi.fn(), + mockGetWorkspaceOrganizationId: vi.fn(), +})) + +vi.mock('@/lib/core/config/feature-flags', () => ({ + isFeatureEnabled: mockIsFeatureEnabled, +})) + +vi.mock('@/lib/workspaces/utils', () => ({ + getWorkspaceOrganizationId: mockGetWorkspaceOrganizationId, })) vi.mock('@sim/audit', () => ({ @@ -199,6 +211,33 @@ const TABLE: TableDefinition = { } const PRINCIPAL = { kind: 'session' as const, userId: 'user-1', sessionId: 'session-1' } +const GENERIC_WEBHOOK_EXECUTOR = { + kind: 'delegated' as const, + serviceId: 'executor' as const, + workspaceId: TABLE.workspaceId, + delegationId: 'executor-1', + audience: 'sim:tables', + issuedAt: new Date('2026-01-01'), + expiresAt: new Date('2099-01-01'), + resourceScope: { tableId: TABLE.id }, + delegationContext: { + kind: 'workflow_execution' as const, + workflowId: 'workflow-1', + currentWorkflow: { + workflowId: 'workflow-1', + mode: 'deployment' as const, + deploymentVersionId: 'deployment-1', + }, + principal: { + kind: 'system' as const, + serviceId: 'webhook' as const, + workspaceId: TABLE.workspaceId, + workflowId: 'workflow-1', + webhookId: 'webhook-1', + provider: 'generic', + }, + }, +} /** * The active-table context every row command resolves before it does any work. @@ -366,7 +405,7 @@ describe('replaceProjectedWireRows application command', () => { ) expect(mockIsScopeCompatible).toHaveBeenCalledWith( { userId: 'user-1', workspaceId: TABLE.workspaceId }, - { userId: 'user-1', workspaceId: TABLE.workspaceId } + { workspaceId: TABLE.workspaceId } ) expect(mockReplaceRowsWithTx).toHaveBeenCalledWith( expect.anything(), @@ -404,7 +443,7 @@ describe('replaceProjectedWireRows application command', () => { expect(mockIsScopeCompatible).toHaveBeenCalledWith( { userId: 'user-1', workspaceId: 'workspace-other' }, - { userId: 'user-1', workspaceId: TABLE.workspaceId } + { workspaceId: TABLE.workspaceId } ) expect(mockReplaceRowsWithTx).toHaveBeenCalledWith( expect.anything(), @@ -601,6 +640,65 @@ describe('row query and upsert application semantics', () => { vi.clearAllMocks() mockResolvePermission.mockResolvedValue('write') mockResolveContext.mockResolvedValue(contextFor()) + mockIsFeatureEnabled.mockResolvedValue(true) + mockGetWorkspaceOrganizationId.mockResolvedValue('organization-1') + }) + + it('preserves storage-keyed predicates and sort specs for the session row route', async () => { + const storageTable = { + ...TABLE, + schema: { columns: [{ id: 'column_name', name: 'name', type: 'string' as const }] }, + } + mockResolveContext.mockResolvedValueOnce(contextFor(storageTable)) + mockQueryRows.mockResolvedValueOnce({ + rows: [], + rowCount: 0, + totalCount: null, + limit: 25, + offset: 0, + nextCursor: null, + }) + + await queryTableRows.execute({ + principal: PRINCIPAL, + input: { + tableId: TABLE.id, + predicate: { field: 'column_name', op: 'eq', value: 'Ada' }, + sort: [{ field: 'column_name', direction: 'asc' }], + legacyKeying: 'ids', + limit: 25, + }, + }) + + expect(mockQueryRows).toHaveBeenCalledWith( + storageTable, + expect.objectContaining({ + predicate: { field: 'column_name', op: 'eq', value: 'Ada' }, + sort: { column_name: 'asc' }, + }), + expect.any(String) + ) + }) + + it('checks the v2 feature gate against the canonical authorized workspace', async () => { + mockIsFeatureEnabled.mockResolvedValueOnce(false) + + await expect( + queryTableRows.execute({ + principal: PRINCIPAL, + input: { tableId: TABLE.id, limit: 25, requireV2Feature: true }, + }) + ).rejects.toMatchObject({ + code: 'forbidden', + message: 'The v2 table query API is not enabled for this workspace', + }) + + expect(mockGetWorkspaceOrganizationId).toHaveBeenCalledWith(TABLE.workspaceId) + expect(mockIsFeatureEnabled).toHaveBeenCalledWith('tables-v2-api', { + userId: PRINCIPAL.userId, + orgId: 'organization-1', + }) + expect(mockQueryRows).not.toHaveBeenCalled() }) it('rejects a malformed POST query cursor before querying storage', async () => { @@ -990,6 +1088,20 @@ describe('table row write secret provenance defaulting', () => { {} ) }) + + it('authorizes a generic webhook by deployment and uses the billing owner for storage attribution', async () => { + await upsertTableRow.execute({ + principal: GENERIC_WEBHOOK_EXECUTOR, + input: { tableId: TABLE.id, data: { name: 'Ada' } }, + }) + + expect(mockUpsertRow).toHaveBeenCalledWith( + expect.objectContaining({ userId: 'billing-owner-1' }), + TABLE, + expect.any(String), + {} + ) + }) }) /** diff --git a/apps/sim/lib/table/application/rows.ts b/apps/sim/lib/table/application/rows.ts index ebf52c121d6..7746b60db91 100644 --- a/apps/sim/lib/table/application/rows.ts +++ b/apps/sim/lib/table/application/rows.ts @@ -9,6 +9,7 @@ import { db } from '@sim/db' import { getRequestContext } from '@sim/logger' import { generateId } from '@sim/utils/id' import { isPlainRecord } from '@sim/utils/object' +import { isFeatureEnabled } from '@/lib/core/config/feature-flags' import { OrchestrationError } from '@/lib/core/orchestration/types' import { isPrivateSecretProvenanceScopeCompatible } from '@/lib/execution/durable-secret-provenance' import type { @@ -24,6 +25,7 @@ import type { TablePredicate, TableRow, TableRowSecretProvenanceWrite, + TableRowsCursor, } from '@/lib/table' import { batchInsertRows, @@ -55,12 +57,20 @@ import { type TableRowProvenanceEnvelope, } from '@/lib/table/application/row-secret-provenance' import { assertRowCapacity, notifyTableRowUsage } from '@/lib/table/billing' -import { buildColumnNameById, buildIdByName, unknownColumnNames } from '@/lib/table/column-keys' +import { + buildColumnNameById, + buildIdByName, + columnMatchesRef, + filterNamesToIds, + getColumnId, + sortNamesToIds, + unknownColumnNames, +} from '@/lib/table/column-keys' import { columnTypeOf } from '@/lib/table/column-types' import { TableQueryValidationError } from '@/lib/table/errors' import { signalTableRowsChanged, signalTableRowsChangedByActor } from '@/lib/table/events' import { CSV_MAX_BATCH_SIZE } from '@/lib/table/import' -import { predicateToFilter } from '@/lib/table/query-builder/converters' +import { isTablePredicate, predicateToFilter } from '@/lib/table/query-builder/converters' import { validatePredicate, validatePredicateShape, @@ -77,8 +87,9 @@ import { } from '@/lib/table/rows/secret-provenance' import type { FindRowMatch, RowWriteOptions } from '@/lib/table/rows/service' import { replaceTableRowsWithTx } from '@/lib/table/rows/service' -import { predicateToStorage } from '@/lib/table/select-values' +import { predicateToStorage, resolveFilterSelectValues } from '@/lib/table/select-values' import { coerceRowValues } from '@/lib/table/validation' +import { getWorkspaceOrganizationId } from '@/lib/workspaces/utils' import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' export class TableRowsValidationError extends OrchestrationError { @@ -91,6 +102,13 @@ export class TableRowsValidationError extends OrchestrationError { } } +export class TableV2FeatureDisabledError extends OrchestrationError { + constructor() { + super('forbidden', 'The v2 table query API is not enabled for this workspace') + this.name = 'TableV2FeatureDisabledError' + } +} + interface TableScopedInput { tableId: string assertedWorkspaceId?: string @@ -135,8 +153,8 @@ interface TableResult { type TableRowsProvenance = Awaited> async function loadAuthorizedRowsProvenance( - principal: Parameters[0], workspaceId: string, + attributedUserId: string, // The loader reads only id, updatedAt and the selected values, so a row // without its executions sidecar is enough — see `TABLE_ROW_SIDECAR_SELECTION`. rows: TableRowSummary[], @@ -149,7 +167,7 @@ async function loadAuthorizedRowsProvenance( // which is how the unmigrated `rows`/`query` routes have always behaved. rows.map((row) => ({ id: row.id, updatedAt: row.updatedAt, selectedValues: row.data })), { - userId: requirePrincipalSubjectUserId(principal), + userId: attributedUserId, workspaceId, } ) @@ -359,6 +377,28 @@ export function tablePredicateNamesToFilter( } } +function tableFilterToStorage( + filter: Filter | TablePredicate, + table: TableDefinition, + keying: TableRowDataKeying = 'names' +): Filter { + if (isTablePredicate(filter)) { + if (keying === 'names') return tablePredicateNamesToFilter(filter, table) + try { + validatePredicateShape(filter) + validateStoragePredicate(filter, table.schema.columns) + return predicateToFilter(filter) + } catch (error) { + rethrowQueryValidation(error) + } + } + if (keying === 'ids') return filter + return resolveFilterSelectValues( + filterNamesToIds(filter, buildIdByName(table.schema)), + table.schema.columns + ) +} + async function throwValidationResponse( validation: | { valid: true } @@ -423,9 +463,17 @@ export const listTableRows = defineAuthorizedTableUseCase({ export interface QueryTableRowsInput extends TableScopedInput, RunStateReadInput { predicate?: TablePredicate sort?: SortSpec + legacyFilter?: Filter + legacySort?: Sort + legacyKeying?: TableRowDataKeying limit?: number + offset?: number + after?: TableRowsCursor cursor?: string + columns?: string[] includeTotal?: boolean + allowExpandedLimit?: boolean + requireV2Feature?: boolean includePersistedSecretProvenance?: boolean } @@ -433,6 +481,8 @@ export interface QueryTableRowsResult extends TableResult { rows: TableRow[] rowCount: number totalCount: number | null + limit: number + offset: number nextCursor: string | null secretProvenance?: TableRowsProvenance } @@ -442,35 +492,89 @@ export const queryTableRows = defineAuthorizedTableUseCase({ resolveContext: ({ input }: { input: QueryTableRowsInput }) => resolveActiveTableContext(input), async execute({ principal, input, context }): Promise { try { - if (input.limit !== undefined) { + if (input.requireV2Feature) { + const orgId = await getWorkspaceOrganizationId(context.workspaceId) + if ( + !(await isFeatureEnabled('tables-v2-api', { + userId: requirePrincipalSubjectUserId(principal), + orgId, + })) + ) { + throw new TableV2FeatureDisabledError() + } + } + if (input.limit !== undefined && !input.allowExpandedLimit) { requireIntegerInRange(input.limit, 1, TABLE_LIMITS.MAX_QUERY_LIMIT, 'Limit') + } else if ( + input.limit !== undefined && + (!Number.isSafeInteger(input.limit) || input.limit < 1) + ) { + throw new TableRowsValidationError('Limit must be at least 1') + } + if (input.offset !== undefined && (!Number.isSafeInteger(input.offset) || input.offset < 0)) { + throw new TableRowsValidationError('Offset must be 0 or greater') } let predicate = input.predicate if (predicate) { - validatePredicate(predicate, context.table.schema.columns) - predicate = predicateToStorage(predicate, context.table.schema) + if (input.legacyKeying !== undefined) { + validatePredicateShape(predicate) + if (input.legacyKeying === 'names') { + predicate = predicateToStorage(predicate, context.table.schema) + } + validateStoragePredicate(predicate, context.table.schema.columns) + } else { + validatePredicate(predicate, context.table.schema.columns) + predicate = predicateToStorage(predicate, context.table.schema) + } } let sortSpec = input.sort if (sortSpec?.length) { - validateSortSpec(sortSpec, context.table.schema.columns) - sortSpec = sortSpecNamesToIds(sortSpec, buildIdByName(context.table.schema)) + if (input.legacyKeying !== undefined) { + if (input.legacyKeying === 'names') { + sortSpec = sortSpecNamesToIds(sortSpec, buildIdByName(context.table.schema)) + } + } else { + validateSortSpec(sortSpec, context.table.schema.columns) + sortSpec = sortSpecNamesToIds(sortSpec, buildIdByName(context.table.schema)) + } } - const sort: Sort | undefined = sortSpec?.length + let sort: Sort | undefined = sortSpec?.length ? Object.fromEntries(sortSpec.map((item) => [item.field, item.direction])) : undefined + if (input.legacySort) { + sort = + input.legacyKeying === 'ids' + ? input.legacySort + : sortNamesToIds(input.legacySort, buildIdByName(context.table.schema)) + } + const legacyFilter = input.legacyFilter + ? tableFilterToStorage(input.legacyFilter, context.table, input.legacyKeying ?? 'names') + : undefined const cursor = input.cursor ? decodeCursor(input.cursor) : undefined if (cursor) assertCursorQueryBinding(cursor, { sort, predicate }) + let columnIds: Set | undefined + if (input.columns?.length) { + columnIds = new Set() + for (const reference of input.columns) { + const column = context.table.schema.columns.find((candidate) => + columnMatchesRef(candidate, reference) + ) + if (column) columnIds.add(getColumnId(column)) + } + } const result = await queryRows( context.table, { predicate, + filter: legacyFilter, sort, limit: input.limit, - after: cursor?.after, - offset: cursor?.offset, + after: cursor?.after ?? input.after, + offset: cursor?.offset ?? input.offset, includeTotal: input.includeTotal ?? false, withExecutions: input.includeRunState ?? false, runStateBudgetBytes: TABLE_LIMITS.MAX_ROW_RUN_STATE_BYTES, + columnIds, }, requestId(input) ) @@ -478,8 +582,8 @@ export const queryTableRows = defineAuthorizedTableUseCase({ table: context.table, ...result, secretProvenance: await loadAuthorizedRowsProvenance( - principal, context.workspaceId, + actorUserId(principal, context.billedAccountUserId), result.rows, input.includePersistedSecretProvenance ), @@ -559,8 +663,8 @@ export const readTableRow = defineAuthorizedTableUseCase({ row, ...(runState ? { runState } : {}), secretProvenance: await loadAuthorizedRowsProvenance( - principal, context.workspaceId, + actorUserId(principal, context.billedAccountUserId), [row], input.includePersistedSecretProvenance ), @@ -620,6 +724,8 @@ interface CreateSingleTableRowInput extends TableScopedInput { afterRowId?: string beforeRowId?: string secretProvenance?: TableRowSecretProvenanceWrite + secretProvenanceEnvelope?: TableRowProvenanceEnvelope + includePersistedSecretProvenance?: boolean } interface CreateBatchTableRowsInput extends TableScopedInput { @@ -631,13 +737,15 @@ interface CreateBatchTableRowsInput extends TableScopedInput { rows: RowData[] orderKeys?: string[] secretProvenance?: Array + secretProvenanceEnvelope?: TableRowProvenanceEnvelope + includePersistedSecretProvenance?: boolean } export type CreateTableRowsInput = CreateSingleTableRowInput | CreateBatchTableRowsInput export type CreateTableRowsResult = - | (TableResult & { kind: 'single'; row: TableRow }) - | (TableResult & { kind: 'batch'; rows: TableRow[] }) + | (TableResult & { kind: 'single'; row: TableRow; secretProvenance?: TableRowsProvenance }) + | (TableResult & { kind: 'batch'; rows: TableRow[]; secretProvenance?: TableRowsProvenance }) export const createTableRows = defineAuthorizedTableUseCase({ operation: tableOperations.createRows, @@ -655,6 +763,13 @@ export const createTableRows = defineAuthorizedTableUseCase({ throw new TableRowsValidationError('Position must be 0 or greater') } const data = rowDataToStorage(input.data, context.table, input.dataKeying, input.strictWrite) + const secretProvenance = singleRowWriteProvenance({ + principal, + workspaceId: context.workspaceId, + table: context.table, + input, + storageData: data, + }) const writeOptions = rowWriteOptions(input) await throwValidationResponse( await validateRowData({ @@ -673,13 +788,23 @@ export const createTableRows = defineAuthorizedTableUseCase({ position: input.position, afterRowId: input.afterRowId, beforeRowId: input.beforeRowId, - secretProvenance: defaultedRowSecretProvenance(data, input.secretProvenance), + secretProvenance, }, context.table, requestId(input), writeOptions ) - return { kind: 'single', table: context.table, row } + return { + kind: 'single', + table: context.table, + row, + secretProvenance: await loadAuthorizedRowsProvenance( + context.workspaceId, + actorUserId(principal, context.billedAccountUserId), + [row], + input.includePersistedSecretProvenance + ), + } } if (input.rows.length < 1 || input.rows.length > TABLE_LIMITS.MAX_BATCH_INSERT_SIZE) { throw new TableRowsValidationError( @@ -693,6 +818,17 @@ export const createTableRows = defineAuthorizedTableUseCase({ throw new TableRowsValidationError('orderKeys must align one-to-one with rows') } const rows = rowsToStorage(input.rows, context.table, input.dataKeying, input.strictWrite) + const secretProvenance = input.secretProvenanceEnvelope + ? resolveRowWriteProvenance({ + envelope: input.secretProvenanceEnvelope, + principal, + workspaceId: context.workspaceId, + table: context.table, + keying: input.dataKeying, + wireRows: input.rows, + storageRows: rows, + }).stamps + : defaultedRowsSecretProvenance(rows, input.secretProvenance) const batchWriteOptions = rowWriteOptions(input) await throwValidationResponse( await validateBatchRows({ @@ -709,13 +845,23 @@ export const createTableRows = defineAuthorizedTableUseCase({ rows, userId, orderKeys: input.orderKeys, - secretProvenance: defaultedRowsSecretProvenance(rows, input.secretProvenance), + secretProvenance, }, context.table, requestId(input), batchWriteOptions ) - return { kind: 'batch', table: context.table, rows: created } + return { + kind: 'batch', + table: context.table, + rows: created, + secretProvenance: await loadAuthorizedRowsProvenance( + context.workspaceId, + actorUserId(principal, context.billedAccountUserId), + created, + input.includePersistedSecretProvenance + ), + } }, afterSuccess: ({ context, input, result }) => { // Narrowed on the input, not the result: only the single-row variant carries @@ -844,7 +990,6 @@ function projectedRowsForTable( function projectedRowsSecretProvenance( rows: RowData[], - principal: Parameters[0], workspaceId: string, policy: ReplaceProjectedWireRowsInput['secretProvenance'] ): TableRowSecretProvenanceWrite[] { @@ -853,7 +998,6 @@ function projectedRowsSecretProvenance( if (!registry) return rows.map(createUnknownTableRowSecretProvenance) const destinationScope = { - userId: requirePrincipalSubjectUserId(principal), workspaceId, } return rows.map((row) => { @@ -911,7 +1055,6 @@ export const replaceProjectedWireRows = defineAuthorizedTableUseCase({ userId: actorUserId(principal, context.billedAccountUserId), secretProvenance: projectedRowsSecretProvenance( provenanceRows, - principal, context.workspaceId, input.secretProvenance ), @@ -1017,8 +1160,8 @@ export const updateTableRow = defineAuthorizedTableUseCase({ row, changed: Object.keys(data).length > 0, secretProvenance: await loadAuthorizedRowsProvenance( - principal, context.workspaceId, + actorUserId(principal, context.billedAccountUserId), [row], input.includePersistedSecretProvenance ), @@ -1034,10 +1177,12 @@ export interface UpdateTableRowsInput extends TableScopedInput { strictWrite: boolean /** See {@link TableRowDataKeying}. Required so a new write surface must choose. */ dataKeying: TableRowDataKeying - filter: TablePredicate + filter: TablePredicate | Filter + filterKeying?: TableRowDataKeying data: RowData limit?: number secretProvenance?: TableRowSecretProvenanceWrite + secretProvenanceEnvelope?: TableRowProvenanceEnvelope } export interface UpdateTableRowsResult extends TableResult, BulkOperationResult {} @@ -1051,14 +1196,21 @@ export const updateTableRows = defineAuthorizedTableUseCase({ requireIntegerInRange(input.limit, 1, TABLE_LIMITS.MAX_BULK_OPERATION_SIZE, 'Limit') } const data = rowDataToStorage(input.data, context.table, input.dataKeying, input.strictWrite) + const secretProvenance = singleRowWriteProvenance({ + principal, + workspaceId: context.workspaceId, + table: context.table, + input, + storageData: data, + }) const result = await updateRowsByFilter( context.table, { - filter: tablePredicateNamesToFilter(input.filter, context.table), + filter: tableFilterToStorage(input.filter, context.table, input.filterKeying ?? 'names'), data, limit: input.limit, actorUserId: actorUserId(principal, context.billedAccountUserId), - secretProvenance: defaultedRowSecretProvenance(data, input.secretProvenance), + secretProvenance, }, requestId(input), rowWriteOptions(input) @@ -1080,6 +1232,7 @@ export interface BatchUpdateTableRowsInput extends TableScopedInput { dataKeying: TableRowDataKeying /** One merge patch per row. A row identifier may appear at most once. */ updates: readonly { rowId: string; data: RowData }[] + secretProvenanceEnvelope?: TableRowProvenanceEnvelope } export interface BatchUpdateTableRowsResult extends TableResult, BulkOperationResult {} @@ -1132,6 +1285,17 @@ export const batchUpdateTableRows = defineAuthorizedTableUseCase({ rowId: update.rowId, data: storageData[index], })) + const secretProvenance = input.secretProvenanceEnvelope + ? resolveRowWriteProvenance({ + envelope: input.secretProvenanceEnvelope, + principal, + workspaceId: context.workspaceId, + table: context.table, + keying: input.dataKeying, + wireRows: input.updates.map((update) => update.data), + storageRows: storageData, + }).stamps + : storageData.map(createExactEmptyTableRowSecretProvenance) const result = await batchUpdateRows( { tableId: context.tableId, @@ -1139,10 +1303,10 @@ export const batchUpdateTableRows = defineAuthorizedTableUseCase({ workspaceId: context.workspaceId, actorUserId: actorUserId(principal, context.billedAccountUserId), secretProvenanceByRowId: Object.fromEntries( - updates.map((update) => [ - update.rowId, - createExactEmptyTableRowSecretProvenance(update.data), - ]) + updates.flatMap((update, index) => { + const stamp = secretProvenance[index] + return stamp ? [[update.rowId, stamp]] : [] + }) ), }, context.table, @@ -1188,7 +1352,15 @@ export const deleteTableRow = defineAuthorizedTableUseCase({ }) export type DeleteTableRowsInput = TableScopedInput & - ({ kind: 'ids'; rowIds: string[] } | { kind: 'filter'; filter: TablePredicate; limit?: number }) + ( + | { kind: 'ids'; rowIds: string[] } + | { + kind: 'filter' + filter: TablePredicate | Filter + filterKeying?: TableRowDataKeying + limit?: number + } + ) export type DeleteTableRowsResult = TableResult & (({ kind: 'ids' } & BulkDeleteByIdsResult) | ({ kind: 'filter' } & BulkOperationResult)) @@ -1221,7 +1393,7 @@ export const deleteTableRows = defineAuthorizedTableUseCase({ const result = await deleteRowsByFilter( context.table, { - filter: tablePredicateNamesToFilter(input.filter, context.table), + filter: tableFilterToStorage(input.filter, context.table, input.filterKeying ?? 'names'), limit: input.limit, }, requestId(input) @@ -1308,8 +1480,8 @@ export const upsertTableRow = defineAuthorizedTableUseCase({ row: result.row, operation: result.operation, secretProvenance: await loadAuthorizedRowsProvenance( - principal, context.workspaceId, + actorUserId(principal, context.billedAccountUserId), [result.row], input.includePersistedSecretProvenance ), diff --git a/apps/sim/lib/table/application/tables.test.ts b/apps/sim/lib/table/application/tables.test.ts index ca3067a2bc2..405b1c73056 100644 --- a/apps/sim/lib/table/application/tables.test.ts +++ b/apps/sim/lib/table/application/tables.test.ts @@ -8,9 +8,12 @@ import type { TableDefinition } from '@/lib/table/types' const mocks = vi.hoisted(() => ({ audit: vi.fn(), getTableById: vi.fn(), + getLimits: vi.fn(), + listDefinitions: vi.fn(), loadFolderIndex: vi.fn(), queryTables: vi.fn(), resolveArchivedContext: vi.fn(), + resolveActiveContext: vi.fn(), resolveFolderPathFilter: vi.fn(), resolvePermission: vi.fn(), resolveWorkspaceContext: vi.fn(), @@ -45,7 +48,8 @@ vi.mock('@/lib/table', () => ({ createTable: vi.fn(), deleteTable: vi.fn(), getTableById: mocks.getTableById, - getWorkspaceTableLimits: vi.fn(), + getWorkspaceTableLimits: mocks.getLimits, + listTables: mocks.listDefinitions, moveTableToFolder: vi.fn(), queryTables: mocks.queryTables, renameTable: vi.fn(), @@ -54,7 +58,7 @@ vi.mock('@/lib/table', () => ({ })) vi.mock('@/lib/table/application/context', () => ({ - resolveActiveTableContext: vi.fn(), + resolveActiveTableContext: mocks.resolveActiveContext, resolveArchivedTableContext: mocks.resolveArchivedContext, resolveTableWorkspaceContext: mocks.resolveWorkspaceContext, })) @@ -76,7 +80,13 @@ vi.mock('@/lib/table/application/folder-paths', () => ({ vi.mock('@/lib/table/events', () => ({ signalTableSchemaChanged: mocks.signal })) -import { listTablesUseCase, restoreTableUseCase } from '@/lib/table/application/tables' +import { + listTableDefinitionsUseCase, + listTablesUseCase, + readTableDefinitionUseCase, + readTableDetailsUseCase, + restoreTableUseCase, +} from '@/lib/table/application/tables' const WORKSPACE = { workspaceId: 'workspace-1', @@ -197,6 +207,55 @@ describe('table list scope', () => { }) }) +describe('internal table compatibility reads', () => { + const active = { ...ARCHIVED, archivedAt: null } + + beforeEach(() => { + vi.clearAllMocks() + mocks.resolvePermission.mockResolvedValue('read') + mocks.resolveWorkspaceContext.mockResolvedValue(WORKSPACE) + mocks.resolveActiveContext.mockResolvedValue({ + ...WORKSPACE, + tableId: active.id, + table: active, + }) + mocks.listDefinitions.mockResolvedValue([active]) + mocks.getLimits.mockResolvedValue({ maxRowsPerTable: 2500 }) + }) + + it('lists definitions without materializing the workspace folder index', async () => { + const result = await listTableDefinitionsUseCase.execute({ + principal: PRINCIPAL, + input: { workspaceId: WORKSPACE.workspaceId, scope: 'all' }, + }) + + expect(mocks.listDefinitions).toHaveBeenCalledWith(WORKSPACE.workspaceId, { scope: 'all' }) + expect(result.tables).toEqual([active]) + expect(mocks.loadFolderIndex).not.toHaveBeenCalled() + }) + + it('reads schema-only metadata without loading folders or plan limits', async () => { + const result = await readTableDefinitionUseCase.execute({ + principal: PRINCIPAL, + input: { tableId: active.id, workspaceId: WORKSPACE.workspaceId }, + }) + + expect(result.table).toBe(active) + expect(mocks.loadFolderIndex).not.toHaveBeenCalled() + expect(mocks.getLimits).not.toHaveBeenCalled() + }) + + it('reads the live row limit without loading unrelated folder state', async () => { + const result = await readTableDetailsUseCase.execute({ + principal: PRINCIPAL, + input: { tableId: active.id, workspaceId: WORKSPACE.workspaceId }, + }) + + expect(result).toEqual({ table: active, maxRows: 2500 }) + expect(mocks.loadFolderIndex).not.toHaveBeenCalled() + }) +}) + /** * Without a restore, a headless `DELETE` was unrecoverable: the table is * archived, not erased, but nothing on the public surface could bring it back. diff --git a/apps/sim/lib/table/application/tables.ts b/apps/sim/lib/table/application/tables.ts index 001c3ba9f13..52974c43c4a 100644 --- a/apps/sim/lib/table/application/tables.ts +++ b/apps/sim/lib/table/application/tables.ts @@ -5,18 +5,24 @@ import type { CursorKey, ListSortOrder } from '@/lib/api/list-query' import { OrchestrationError } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { MAX_FOLDERS_PER_WORKSPACE } from '@/lib/folders/constants' -import { loadActiveFolderPathIndex, resolveFolderPathFilter } from '@/lib/folders/queries' +import { + findActiveFolder, + loadActiveFolderPathIndex, + resolveFolderPathFilter, +} from '@/lib/folders/queries' import { createTable, deleteTable, getTableById, getWorkspaceTableLimits, + listTables as listTableDefinitions, moveTableToFolder, queryTables, renameTable, restoreTable, type TableDefinition, type TableSchema, + type TableScope, updateTableDescription, } from '@/lib/table' import { defineAuthorizedTableUseCase } from '@/lib/table/application/authorized-table-use-case' @@ -41,10 +47,6 @@ export interface ListTablesInput { * its third value, `'all'`, would mix archived rows into a page projected by * the strict folder-path resolver, which throws on the dangling `folderId` a * folder archive leaves behind. - * - * The value set is the one `ListWorkflowsInput['scope']` accepts; the - * optionality is not, since that sibling requires a scope where this one - * defaults an absent scope through to the query. */ scope?: 'active' | 'archived' folderPath?: string @@ -93,12 +95,29 @@ export const listTablesUseCase = defineAuthorizedTableUseCase({ }, }) +export interface ListTableDefinitionsInput { + workspaceId: string + scope?: TableScope +} + +export const listTableDefinitionsUseCase = defineAuthorizedTableUseCase({ + operation: tableOperations.list, + resolveContext: ({ input }: { input: ListTableDefinitionsInput }) => + resolveTableWorkspaceContext(input.workspaceId), + async execute({ input, context }) { + return { + tables: await listTableDefinitions(context.workspaceId, { scope: input.scope }), + } + }, +}) + export interface CreateTableInput { workspaceId: string name: string description?: string schema: TableSchema folderPath?: string + folderId?: string | null initialRowCount?: number } @@ -111,8 +130,21 @@ export const createTableUseCase = defineAuthorizedTableUseCase({ workspaceBillingOwnerUserId: context.billedAccountUserId, }) const planLimits = await getWorkspaceTableLimits(context.workspaceId) - const resolution = await resolveTableFolderPath(context.workspaceId, input.folderPath ?? '/') - if (!resolution) throw new OrchestrationError('not_found', 'Folder not found') + const resolution = + input.folderId !== undefined + ? { + folderId: input.folderId, + index: await loadActiveFolderPathIndex(context.workspaceId, 'table', undefined, { + maxRows: MAX_FOLDERS_PER_WORKSPACE, + }), + } + : await resolveTableFolderPath(context.workspaceId, input.folderPath ?? '/') + if ( + !resolution || + (input.folderId && !(await findActiveFolder(input.folderId, context.workspaceId, 'table'))) + ) { + throw new OrchestrationError('not_found', 'Folder not found in this workspace') + } const table = await createTable( { @@ -168,6 +200,31 @@ export const readTableUseCase = defineAuthorizedTableUseCase({ }, }) +export const readTableDefinitionUseCase = defineAuthorizedTableUseCase({ + operation: tableOperations.read, + resolveContext: ({ input }: { input: ReadTableInput }) => + resolveActiveTableContext({ + tableId: input.tableId, + assertedWorkspaceId: input.workspaceId, + }), + async execute({ context }) { + return { table: context.table } + }, +}) + +export const readTableDetailsUseCase = defineAuthorizedTableUseCase({ + operation: tableOperations.read, + resolveContext: ({ input }: { input: ReadTableInput }) => + resolveActiveTableContext({ + tableId: input.tableId, + assertedWorkspaceId: input.workspaceId, + }), + async execute({ context }) { + const { maxRowsPerTable } = await getWorkspaceTableLimits(context.workspaceId) + return { table: context.table, maxRows: maxRowsPerTable } + }, +}) + export type AppliedTableUpdate = 'name' | 'description' | 'folderPath' export interface UpdateTableInput extends ReadTableInput { diff --git a/apps/sim/lib/table/events.attribution.test.ts b/apps/sim/lib/table/events.attribution.test.ts index d2616fa917d..3ed09ce7135 100644 --- a/apps/sim/lib/table/events.attribution.test.ts +++ b/apps/sim/lib/table/events.attribution.test.ts @@ -16,19 +16,16 @@ vi.setConfig({ testTimeout: 30_000 }) * That invariant lives in `hooks/queries/tables.ts` — nothing in the type system ties it to the * call site, so a well-meaning extra call would silently strand that client on stale rows. * - * Two lists are pinned, because a migrated single-row route now signals from inside its - * application use case rather than from the route. The call itself is no longer the decision: - * that use case is shared with `/api/v2` and Copilot, and it degrades to a broadcast whenever no - * actor is named. What actually selects the behavior is which surface supplies `actorClientId`, - * so that is pinned too and is the list to scrutinise. + * Two lists are pinned. All row mutations now signal from the shared application use case rather + * than duplicating that side effect in their route adapters. The call itself is no longer the + * decision: the use case is shared with `/api/v2` and Copilot, and it degrades to a broadcast + * whenever no actor is named. What actually selects the behavior is which surface supplies + * `actorClientId`, so that is pinned too and is the list to scrutinise. * * If you are here because it failed: adding a supplier means proving that surface's client hook * reconciles the write locally across every cached rows query. Removing one is always safe. */ -const ATTRIBUTED_CALL_SITES = [ - 'app/api/table/[tableId]/rows/route.ts', - 'lib/table/application/rows.ts', -] as const +const ATTRIBUTED_CALL_SITES = ['lib/table/application/rows.ts'] as const /** Surfaces that name the acting tab. See the note above — this is the real allowlist. */ const ACTOR_SUPPLYING_SURFACES = [ diff --git a/apps/sim/lib/table/secret-provenance-selection.test.ts b/apps/sim/lib/table/secret-provenance-selection.test.ts index d68ceb67435..130c29e08ae 100644 --- a/apps/sim/lib/table/secret-provenance-selection.test.ts +++ b/apps/sim/lib/table/secret-provenance-selection.test.ts @@ -1,11 +1,15 @@ /** * @vitest-environment node */ +import { filterUndefined } from '@sim/utils/object' import { describe, expect, it } from 'vitest' +import { + addModelInputProvenanceToRequest, + createPrivateSecretProvenanceRequestMetadata, +} from '@/lib/execution/model-input-provenance' import { PRIVATE_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' import { selectTableRowSecretProvenance } from '@/lib/table/secret-provenance-selection' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' -import { prepareToolRequest } from '@/tools/request-transport' import { tableBatchInsertRowsTool } from '@/tools/table/batch_insert_rows' interface TableWriteRequestBody { @@ -34,19 +38,26 @@ describe('selectTableRowSecretProvenance', () => { }) it('keeps selection keys aligned with the serialized request body', () => { - const request = prepareToolRequest( - tableBatchInsertRowsTool, - { - tableId: 'table-1', - rows: [{ email: 'user@example.com', status: 'queued', processed_at: undefined }], - _context: { workspaceId: 'workspace-1' }, - }, - new ResolvedSecretTraceRegistry([], { - userId: 'user-1', - workspaceId: 'workspace-1', - }) + const params = { + tableId: 'table-1', + rows: [{ email: 'user@example.com', status: 'queued', processed_at: undefined }], + _context: { workspaceId: 'workspace-1' }, + } + const registry = new ResolvedSecretTraceRegistry([], { + userId: 'user-1', + workspaceId: 'workspace-1', + }) + const input = tableBatchInsertRowsTool.operation.input(params) + const selections = tableBatchInsertRowsTool.operation.secretProvenance?.request?.(params) ?? [] + const payload = addModelInputProvenanceToRequest( + input, + new Headers(), + createPrivateSecretProvenanceRequestMetadata(registry, selections) ) - const body = JSON.parse(request.body ?? '') as TableWriteRequestBody + const body: TableWriteRequestBody = { + ...payload, + rows: payload.rows.map((row) => filterUndefined(row)), + } const wireSelectionKeys = body.rows.flatMap((row, rowIndex) => Object.keys(row).map((columnKey) => JSON.stringify([rowIndex, columnKey])) ) diff --git a/apps/sim/lib/table/service.ts b/apps/sim/lib/table/service.ts index 6063ec9be9c..cc35089085c 100644 --- a/apps/sim/lib/table/service.ts +++ b/apps/sim/lib/table/service.ts @@ -506,7 +506,7 @@ function workspaceTableLimitReached(maxTables: number): ForbiddenOperationError * Advisory table-quota check for a caller that is about to make the user pay * for work before {@link createTable} would run. * - * The authoritative check is the `FOR UPDATE` count inside `createTable`'s + * The authoritative check is the `FOR NO KEY UPDATE` count inside `createTable`'s * transaction and stays there — this one races, by construction, because the * ceiling can be reached (or cleared) during whatever the caller does next. It * exists so that "next" is not a multi-gigabyte upload: the CSV import used to @@ -613,12 +613,16 @@ export async function createTable( }) } - // Wrap count check, duplicate check, and insert in a transaction with FOR UPDATE - // to prevent TOCTOU race on the table count limit + // Wrap count check, duplicate check, and insert in a transaction with FOR NO KEY UPDATE + // to prevent TOCTOU race on the table count limit. The weaker lock still conflicts with + // itself, so table creations stay serialized, but it does not block unrelated inserts + // into the workspace's other child tables. See lib/billing/storage/tracking.ts. try { await db.transaction(async (trx) => { await setTableTxTimeouts(trx) - await trx.execute(sql`SELECT 1 FROM workspace WHERE id = ${data.workspaceId} FOR UPDATE`) + await trx.execute( + sql`SELECT 1 FROM workspace WHERE id = ${data.workspaceId} FOR NO KEY UPDATE` + ) const [{ count: existingCount }] = await trx .select({ count: count() }) diff --git a/apps/sim/lib/tools/falai-pricing.test.ts b/apps/sim/lib/tools/falai-pricing.test.ts index da230e998f3..09733dae07a 100644 --- a/apps/sim/lib/tools/falai-pricing.test.ts +++ b/apps/sim/lib/tools/falai-pricing.test.ts @@ -10,7 +10,9 @@ import { } from './falai-pricing' // Avoid the real inter-attempt backoff so the fallback path resolves instantly. -vi.mock('@sim/utils/helpers', () => ({ sleep: vi.fn().mockResolvedValue(undefined) })) +vi.mock('@sim/utils/helpers', () => ({ + interruptibleSleep: vi.fn().mockResolvedValue(undefined), +})) describe('getFalAICostMetadata fallback floor', () => { const originalFetch = global.fetch diff --git a/apps/sim/lib/tools/falai-pricing.ts b/apps/sim/lib/tools/falai-pricing.ts index 764a9e741a2..0695a7bf0d1 100644 --- a/apps/sim/lib/tools/falai-pricing.ts +++ b/apps/sim/lib/tools/falai-pricing.ts @@ -1,7 +1,12 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { sleep } from '@sim/utils/helpers' +import { interruptibleSleep } from '@sim/utils/helpers' import { isRecordLike } from '@sim/utils/object' +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + readResponseJsonWithLimit, + readResponseTextWithLimit, +} from '@/lib/core/utils/stream-limits' export const FALAI_HOSTED_KEY_MARKUP_MULTIPLIER = 1.5 export const FALAI_IMAGE_FALLBACK_PROVIDER_COST_DOLLARS = 0.05 @@ -9,6 +14,7 @@ export const FALAI_AUDIO_FALLBACK_PROVIDER_COST_DOLLARS = 0.02 export const FALAI_VIDEO_FALLBACK_PROVIDER_COST_DOLLARS = 0.25 const FALAI_BILLING_EVENT_ATTEMPTS = 2 const FALAI_BILLING_EVENT_RETRY_MS = 500 +const FALAI_PRICING_RESPONSE_MAX_BYTES = 2 * 1024 * 1024 const logger = createLogger('FalAIPricing') export interface FalAICostMetadata { @@ -83,7 +89,8 @@ function parseBillingEvent(value: unknown): FalAIBillingEvent | undefined { async function fetchFalAIBillingEvent( apiKey: string, - requestId: string + requestId: string, + signal?: AbortSignal ): Promise { const url = new URL('https://api.fal.ai/v1/models/billing-events') url.searchParams.set('request_id', requestId) @@ -95,8 +102,10 @@ async function fetchFalAIBillingEvent( headers: { Authorization: `Key ${apiKey}`, }, + signal, }) } catch (error) { + signal?.throwIfAborted() logger.warn('Failed to fetch Fal.ai billing event', { requestId, error: getErrorMessage(error, 'Unknown error'), @@ -106,7 +115,11 @@ async function fetchFalAIBillingEvent( if (!response.ok) return undefined - const data = await response.json().catch((error) => { + const data = await readResponseJsonWithLimit(response, { + maxBytes: FALAI_PRICING_RESPONSE_MAX_BYTES, + label: 'Fal.ai billing event response', + }).catch((error) => { + signal?.throwIfAborted() logger.warn('Failed to parse Fal.ai billing event response', { requestId, error: getErrorMessage(error, 'Unknown error'), @@ -120,7 +133,8 @@ async function fetchFalAIBillingEvent( async function estimateFalAICallCost( apiKey: string, - endpointId: string + endpointId: string, + signal?: AbortSignal ): Promise<{ costDollars?: number; error?: string }> { let response: Response try { @@ -138,17 +152,25 @@ async function estimateFalAICallCost( }, }, }), + signal, }) } catch (error) { + signal?.throwIfAborted() return { error: getErrorMessage(error, 'Unknown error') } } if (!response.ok) { - const error = await response.text().catch(() => '') + const error = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Fal.ai pricing estimate error response', + }).catch(() => '') return { error: `Fal.ai pricing estimate failed: ${response.status} ${error}` } } - const data = (await response.json()) as unknown + const data = await readResponseJsonWithLimit(response, { + maxBytes: FALAI_PRICING_RESPONSE_MAX_BYTES, + label: 'Fal.ai pricing estimate response', + }) const totalCost = isRecordLike(data) ? getNumber(data.total_cost) : undefined if (totalCost === undefined) { return { error: 'Fal.ai pricing estimate missing total_cost' } @@ -161,13 +183,16 @@ export async function getFalAICostMetadata({ apiKey, endpointId, requestId, + signal, }: { apiKey: string endpointId: string requestId: string + signal?: AbortSignal }): Promise { + signal?.throwIfAborted() for (let attempt = 0; attempt < FALAI_BILLING_EVENT_ATTEMPTS; attempt++) { - const event = await fetchFalAIBillingEvent(apiKey, requestId) + const event = await fetchFalAIBillingEvent(apiKey, requestId, signal) if (event) { return { endpointId: event.endpoint_id, @@ -182,11 +207,12 @@ export async function getFalAICostMetadata({ } if (attempt < FALAI_BILLING_EVENT_ATTEMPTS - 1) { - await sleep(FALAI_BILLING_EVENT_RETRY_MS) + await interruptibleSleep(FALAI_BILLING_EVENT_RETRY_MS, signal) + signal?.throwIfAborted() } } - const estimate = await estimateFalAICallCost(apiKey, endpointId) + const estimate = await estimateFalAICallCost(apiKey, endpointId, signal) if (estimate.costDollars !== undefined) { return { endpointId, diff --git a/apps/sim/lib/uploads/contexts/execution/execution-file-manager.ts b/apps/sim/lib/uploads/contexts/execution/execution-file-manager.ts index 5e3bdf77141..d4f4bea9cb9 100644 --- a/apps/sim/lib/uploads/contexts/execution/execution-file-manager.ts +++ b/apps/sim/lib/uploads/contexts/execution/execution-file-manager.ts @@ -140,9 +140,10 @@ export async function uploadExecutionFile( */ export async function downloadExecutionFile( userFile: UserFile, - options: { maxBytes?: number } = {} + options: { maxBytes?: number; signal?: AbortSignal } = {} ): Promise { logger.info(`Downloading execution file: ${userFile.name}`) + options.signal?.throwIfAborted() try { const StorageService = await getStorageService() @@ -150,13 +151,16 @@ export async function downloadExecutionFile( key: userFile.key, context: 'execution', ...(options.maxBytes === undefined ? {} : { maxBytes: options.maxBytes }), + ...(options.signal ? { signal: options.signal } : {}), }) + options.signal?.throwIfAborted() logger.info( `Successfully downloaded execution file: ${userFile.name} (${fileBuffer.length} bytes)` ) return fileBuffer } catch (error) { + options.signal?.throwIfAborted() if (isPayloadSizeLimitError(error)) { throw error } diff --git a/apps/sim/lib/uploads/core/storage-service.ts b/apps/sim/lib/uploads/core/storage-service.ts index 26e2a83d6f2..6a035a0ec4a 100644 --- a/apps/sim/lib/uploads/core/storage-service.ts +++ b/apps/sim/lib/uploads/core/storage-service.ts @@ -482,7 +482,8 @@ export async function createMultipartUpload(options: { * Download a file from the configured storage provider */ export async function downloadFile(options: DownloadFileOptions): Promise { - const { key, context, maxBytes } = options + const { key, context, maxBytes, signal } = options + signal?.throwIfAborted() if (context) { const config = getStorageConfig(context) @@ -490,25 +491,19 @@ export async function downloadFile(options: DownloadFileOptions): Promise { ) expect(mockDestroy).toHaveBeenCalledWith(expect.any(Error)) }) + + it('forwards cancellation to Azure and destroys the response stream', async () => { + const controller = new AbortController() + const mockDestroy = vi.fn() + const mockReadableStream = { + destroy: mockDestroy, + on: vi.fn(() => mockReadableStream), + off: vi.fn(() => mockReadableStream), + } + mockDownload.mockResolvedValueOnce({ readableStreamBody: mockReadableStream }) + + const download = downloadFromBlob('test-file-key', undefined, undefined, controller.signal) + await vi.waitFor(() => expect(mockReadableStream.on).toHaveBeenCalled()) + controller.abort(new Error('cancelled')) + + await expect(download).rejects.toThrow('cancelled') + expect(mockDownload).toHaveBeenCalledWith( + 0, + undefined, + expect.objectContaining({ abortSignal: controller.signal }) + ) + expect(mockDestroy).toHaveBeenCalledWith() + }) }) describe('headBlobObject', () => { diff --git a/apps/sim/lib/uploads/providers/blob/client.ts b/apps/sim/lib/uploads/providers/blob/client.ts index c35752963fe..dfba6eff794 100644 --- a/apps/sim/lib/uploads/providers/blob/client.ts +++ b/apps/sim/lib/uploads/providers/blob/client.ts @@ -352,10 +352,18 @@ export async function downloadFromBlob( maxBytes: number ): Promise +export async function downloadFromBlob( + key: string, + customConfig: BlobConfig | undefined, + maxBytes: number | undefined, + signal: AbortSignal | undefined +): Promise + export async function downloadFromBlob( key: string, customConfig?: BlobConfig, - maxBytes?: number + maxBytes?: number, + signal?: AbortSignal ): Promise { const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob') let blobServiceClient: BlobServiceClientType @@ -385,7 +393,9 @@ export async function downloadFromBlob( const containerClient = blobServiceClient.getContainerClient(containerName) const blockBlobClient = containerClient.getBlockBlobClient(key) - const downloadBlockBlobResponse = await blockBlobClient.download() + const downloadBlockBlobResponse = await blockBlobClient.download(0, undefined, { + abortSignal: signal, + }) if (maxBytes !== undefined && downloadBlockBlobResponse.contentLength !== undefined) { try { assertKnownSizeWithinLimit( @@ -410,6 +420,7 @@ export async function downloadFromBlob( { maxBytes: maxBytes ?? Number.MAX_SAFE_INTEGER, label: 'storage download', + signal, } ) diff --git a/apps/sim/lib/uploads/providers/gcs/client.test.ts b/apps/sim/lib/uploads/providers/gcs/client.test.ts index 99fa4f966ac..d8b8024a462 100644 --- a/apps/sim/lib/uploads/providers/gcs/client.test.ts +++ b/apps/sim/lib/uploads/providers/gcs/client.test.ts @@ -303,6 +303,27 @@ describe('GCS Client', () => { await expect(downloadFromGcs('test-file.txt')).rejects.toThrow('Stream error') }) + + it('destroys the response stream when cancelled', async () => { + const controller = new AbortController() + const stream = new Readable({ + read() {}, + }) + const destroy = vi.spyOn(stream, 'destroy') + mockFile.createReadStream.mockReturnValueOnce(stream) + + const download = downloadFromGcs( + 'test-file.txt', + { bucket: 'test-bucket' }, + undefined, + controller.signal + ) + await vi.waitFor(() => expect(mockFile.createReadStream).toHaveBeenCalled()) + controller.abort(new Error('cancelled')) + + await expect(download).rejects.toThrow('cancelled') + expect(destroy).toHaveBeenCalledWith() + }) }) describe('headGcsObject', () => { diff --git a/apps/sim/lib/uploads/providers/gcs/client.ts b/apps/sim/lib/uploads/providers/gcs/client.ts index abf0c3b5b4f..1966361f901 100644 --- a/apps/sim/lib/uploads/providers/gcs/client.ts +++ b/apps/sim/lib/uploads/providers/gcs/client.ts @@ -318,17 +318,27 @@ export async function downloadFromGcs( maxBytes: number ): Promise +export async function downloadFromGcs( + key: string, + customConfig: GcsConfig, + maxBytes: number | undefined, + signal: AbortSignal | undefined +): Promise + export async function downloadFromGcs( key: string, customConfig?: GcsConfig, - maxBytes?: number + maxBytes?: number, + signal?: AbortSignal ): Promise { + signal?.throwIfAborted() const config = customConfig || { bucket: GCS_CONFIG.bucket } const storage = await getGcsClient() const file = storage.bucket(config.bucket).file(key) if (maxBytes !== undefined) { const [fileMetadata] = await file.getMetadata() + signal?.throwIfAborted() const knownSize = Number(fileMetadata.size) if (Number.isFinite(knownSize)) { assertKnownSizeWithinLimit(knownSize, maxBytes, 'storage download') @@ -338,6 +348,7 @@ export async function downloadFromGcs( return readNodeStreamToBufferWithLimit(file.createReadStream(), { maxBytes: maxBytes ?? Number.MAX_SAFE_INTEGER, label: 'storage download', + signal, }) } diff --git a/apps/sim/lib/uploads/providers/s3/client.test.ts b/apps/sim/lib/uploads/providers/s3/client.test.ts index 5e1ddde0baf..b007ef03593 100644 --- a/apps/sim/lib/uploads/providers/s3/client.test.ts +++ b/apps/sim/lib/uploads/providers/s3/client.test.ts @@ -459,6 +459,36 @@ describe('S3 Client', () => { expect(mockDestroy).toHaveBeenCalledWith(expect.any(Error)) }) + it('forwards cancellation to the S3 request and stream reader', async () => { + const controller = new AbortController() + const mockDestroy = vi.fn() + const mockStream = { + destroy: mockDestroy, + on: vi.fn(() => mockStream), + off: vi.fn(() => mockStream), + } + mockSend.mockResolvedValueOnce({ + Body: mockStream, + $metadata: { httpStatusCode: 200 }, + }) + + const download = downloadFromS3( + 'test-file.txt', + { bucket: 'test-bucket', region: 'test-region' }, + undefined, + controller.signal + ) + await vi.waitFor(() => expect(mockStream.on).toHaveBeenCalled()) + controller.abort(new Error('cancelled')) + + await expect(download).rejects.toThrow('cancelled') + expect(mockSend).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ abortSignal: controller.signal }) + ) + expect(mockDestroy).toHaveBeenCalledWith() + }) + it('should handle S3 client errors', async () => { const error = new Error('Download failed') mockSend.mockRejectedValueOnce(error) diff --git a/apps/sim/lib/uploads/providers/s3/client.ts b/apps/sim/lib/uploads/providers/s3/client.ts index 942b5fdd709..d9e9dc2256b 100644 --- a/apps/sim/lib/uploads/providers/s3/client.ts +++ b/apps/sim/lib/uploads/providers/s3/client.ts @@ -227,10 +227,18 @@ export async function downloadFromS3( maxBytes: number ): Promise +export async function downloadFromS3( + key: string, + customConfig: S3Config, + maxBytes: number | undefined, + signal: AbortSignal | undefined +): Promise + export async function downloadFromS3( key: string, customConfig?: S3Config, - maxBytes?: number + maxBytes?: number, + signal?: AbortSignal ): Promise { const config = customConfig || { bucket: S3_CONFIG.bucket, region: S3_CONFIG.region } @@ -239,7 +247,7 @@ export async function downloadFromS3( Key: key, }) - const response = await getS3Client().send(command) + const response = await getS3Client().send(command, { abortSignal: signal }) if (maxBytes !== undefined && response.ContentLength !== undefined) { try { assertKnownSizeWithinLimit(response.ContentLength, maxBytes, 'storage download') @@ -254,6 +262,7 @@ export async function downloadFromS3( return readNodeStreamToBufferWithLimit(stream, { maxBytes: maxBytes ?? Number.MAX_SAFE_INTEGER, label: 'storage download', + signal, }) } diff --git a/apps/sim/lib/uploads/shared/types.ts b/apps/sim/lib/uploads/shared/types.ts index d1e3f8350f6..79ee500cc62 100644 --- a/apps/sim/lib/uploads/shared/types.ts +++ b/apps/sim/lib/uploads/shared/types.ts @@ -133,6 +133,7 @@ export interface DownloadFileOptions { key: string context?: StorageContext maxBytes?: number + signal?: AbortSignal } export interface DeleteFileOptions { diff --git a/apps/sim/lib/uploads/utils/file-schemas.ts b/apps/sim/lib/uploads/utils/file-schemas.ts index 36d85ba8a2f..53fb4be7b61 100644 --- a/apps/sim/lib/uploads/utils/file-schemas.ts +++ b/apps/sim/lib/uploads/utils/file-schemas.ts @@ -49,6 +49,21 @@ export const RawFileInputSchema = z export type RawFileInput = z.infer +/** Parses a resolved file object, including the JSON string produced by advanced-mode inputs. */ +export function parseRawFileInput(value: unknown): RawFileInput | null { + let candidate = value + if (typeof candidate === 'string') { + try { + candidate = JSON.parse(candidate) + } catch { + return null + } + } + + const parsed = RawFileInputSchema.safeParse(candidate) + return parsed.success ? parsed.data : null +} + export const RawFileInputArraySchema = z.array(RawFileInputSchema) export const FileInputSchema = z.union([RawFileInputSchema, z.string()]) diff --git a/apps/sim/lib/uploads/utils/file-utils.server.test.ts b/apps/sim/lib/uploads/utils/file-utils.server.test.ts index f93ce28d490..91670aeeb25 100644 --- a/apps/sim/lib/uploads/utils/file-utils.server.test.ts +++ b/apps/sim/lib/uploads/utils/file-utils.server.test.ts @@ -137,6 +137,20 @@ describe('downloadFileFromStorage size ceiling', () => { expect(mockDownloadFile).toHaveBeenCalledWith(expect.objectContaining({ maxBytes: 1024 })) }) + + it('forwards cancellation to the storage layer', async () => { + const controller = new AbortController() + mockDownloadFile.mockResolvedValue(Buffer.alloc(512)) + + await downloadFileFromStorage(fileOfSize(512), 'req-1', logger, { + maxBytes: 1024, + signal: controller.signal, + }) + + expect(mockDownloadFile).toHaveBeenCalledWith( + expect.objectContaining({ maxBytes: 1024, signal: controller.signal }) + ) + }) }) describe('downloadServableFilesWithinBudget', () => { diff --git a/apps/sim/lib/uploads/utils/file-utils.server.ts b/apps/sim/lib/uploads/utils/file-utils.server.ts index 579ad67f65d..faa41bf5bfb 100644 --- a/apps/sim/lib/uploads/utils/file-utils.server.ts +++ b/apps/sim/lib/uploads/utils/file-utils.server.ts @@ -273,7 +273,7 @@ export async function downloadFileFromUrl( } const { downloadFile } = await import('@/lib/uploads/core/storage-service') - return downloadFile({ key, context, maxBytes }) + return downloadFile({ key, context, maxBytes, signal }) } const urlValidation = await validateUrlWithDNS(fileUrl, 'fileUrl') @@ -357,9 +357,10 @@ export async function downloadFileFromStorage( userFile: UserFile, requestId: string, logger: Logger, - options: { maxBytes: number } + options: { maxBytes: number; signal?: AbortSignal } ): Promise { - const { maxBytes } = options + const { maxBytes, signal } = options + signal?.throwIfAborted() let buffer: Buffer assertKnownSizeWithinLimit(userFile.size, maxBytes, 'storage file download') @@ -368,7 +369,7 @@ export async function downloadFileFromStorage( const { downloadExecutionFile } = await import( '@/lib/uploads/contexts/execution/execution-file-manager' ) - buffer = await downloadExecutionFile(userFile, { maxBytes }) + buffer = await downloadExecutionFile(userFile, { maxBytes, signal }) } else if (userFile.key) { const context = resolveTrustedFileContext(userFile.key, userFile.context) logger.info(`[${requestId}] Downloading from ${context} storage: ${userFile.key}`) @@ -378,12 +379,14 @@ export async function downloadFileFromStorage( key: userFile.key, context, maxBytes, + signal, }) } else { throw new Error('File has no key - cannot download') } assertKnownSizeWithinLimit(buffer.length, maxBytes, 'storage file download') + signal?.throwIfAborted() return buffer } @@ -431,6 +434,7 @@ export async function downloadServableFileFromStorage( ): Promise { const buffer = await downloadFileFromStorage(userFile, requestId, logger, { maxBytes: options.maxBytes, + signal: options.signal, }) // The pdf model for pages: a page file stores its source and downloads diff --git a/apps/sim/lib/uploads/utils/file-utils.ts b/apps/sim/lib/uploads/utils/file-utils.ts index 425a110b104..39cfdc2a6aa 100644 --- a/apps/sim/lib/uploads/utils/file-utils.ts +++ b/apps/sim/lib/uploads/utils/file-utils.ts @@ -788,6 +788,11 @@ const PUBLIC_STORAGE_CONTEXTS = new Set([ 'workspace-logos', ]) +/** Whether a trusted storage context is world-readable. */ +export function isPublicStorageContext(context: StorageContext): boolean { + return PUBLIC_STORAGE_CONTEXTS.has(context) +} + /** * Resolve the storage context for a stored file from its trusted key prefix. * @@ -811,7 +816,7 @@ export function resolveTrustedFileContext(key: string, context?: string): Storag try { return inferContextFromKey(key) } catch (error) { - if (context && !PUBLIC_STORAGE_CONTEXTS.has(context as StorageContext)) { + if (context && !isPublicStorageContext(context as StorageContext)) { return context as StorageContext } throw error diff --git a/apps/sim/lib/uploads/utils/model-input.test.ts b/apps/sim/lib/uploads/utils/model-input.test.ts index 46d3b880db6..c761db80dd9 100644 --- a/apps/sim/lib/uploads/utils/model-input.test.ts +++ b/apps/sim/lib/uploads/utils/model-input.test.ts @@ -2,7 +2,12 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' -import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' +import { + addModelInputProvenanceToRequest, + createModelInputProvenanceRequestMetadata, + markModelInputProjected, + validateOpaqueModelInputProvenance, +} from '@/lib/execution/model-input-provenance' import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' import { applyProjectedModelVisibleFileNames, @@ -12,9 +17,25 @@ import { } from '@/lib/uploads/utils/model-input' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' import { a2aSendMessageTool } from '@/tools/a2a/send_message' -import { prepareToolRequest } from '@/tools/request-transport' +import { projectToolModelInputParams } from '@/tools/request-transport' import { visionTool } from '@/tools/vision/tool' +function prepareVisionOperation( + params: Parameters[0], + registry?: ResolvedSecretTraceRegistry +) { + const projected = projectToolModelInputParams(visionTool, params, registry) + const input = visionTool.operation.input(projected) + const inputPaths = visionTool.operation.modelInput?.privateInputPaths?.(projected) + const metadata = inputPaths + ? createModelInputProvenanceRequestMetadata(registry, inputPaths) + : undefined + const headers = new Headers() + const payload = addModelInputProvenanceToRequest(input, headers, metadata) + if (metadata) markModelInputProjected(headers) + return { headers, payload } +} + describe('model-bound file input selection', () => { it('omits internal storage keys and unrelated file metadata', () => { expect( @@ -159,8 +180,8 @@ describe('model-bound file input selection', () => { expect(applyProjectedModelVisibleFileNames(original, [{}])).toEqual(original) }) - it('preserves an optional undefined file name through tool request projection', () => { - const prepared = prepareToolRequest( + it('preserves an optional undefined file name through operation input projection', () => { + const projected = projectToolModelInputParams( a2aSendMessageTool, { agentUrl: 'https://agent.example', @@ -170,7 +191,7 @@ describe('model-bound file input selection', () => { new ResolvedSecretTraceRegistry() ) - expect(JSON.parse(prepared.body ?? '{}')).toEqual({ + expect(a2aSendMessageTool.operation.input(projected)).toEqual({ agentUrl: 'https://agent.example', message: 'Summarize the attachment', files: [{ key: 'workspace/ws-1/report.pdf' }], @@ -195,12 +216,11 @@ describe('server-resolved model file provenance', () => { 'https://files.example/document.png?token={{FILE_TOKEN}}' ) - const prepared = prepareToolRequest( - visionTool, + const prepared = prepareVisionOperation( { apiKey: 'key', imageUrl: locator, prompt: 'Describe this image' }, registry ) - const payload = JSON.parse(prepared.body ?? '{}') as Record + const { payload } = prepared expect(payload.imageUrl).toBe(locator) expect(payload[RESOLVED_SECRET_PROVENANCE_FIELD]).toEqual({ @@ -235,8 +255,7 @@ describe('server-resolved model file provenance', () => { '{{INLINE_BYTES}}' ) - const prepared = prepareToolRequest( - visionTool, + const prepared = prepareVisionOperation( { apiKey: 'key', imageFile: { @@ -250,7 +269,7 @@ describe('server-resolved model file provenance', () => { }, registry ) - const payload = JSON.parse(prepared.body ?? '{}') as Record + const { payload } = prepared expect( validateOpaqueModelInputProvenance({ @@ -267,12 +286,12 @@ describe('server-resolved model file provenance', () => { it('keeps headerless legacy file requests unchanged', () => { const locator = 'https://files.example/legacy.png' - const prepared = prepareToolRequest(visionTool, { + const prepared = prepareVisionOperation({ apiKey: 'key', imageUrl: locator, prompt: 'Describe this image', }) - const payload = JSON.parse(prepared.body ?? '{}') as Record + const { payload } = prepared expect(payload.imageUrl).toBe(locator) expect(payload).not.toHaveProperty(RESOLVED_SECRET_PROVENANCE_FIELD) diff --git a/apps/sim/lib/webhooks/execution-principal.test.ts b/apps/sim/lib/webhooks/execution-principal.test.ts new file mode 100644 index 00000000000..347da65270c --- /dev/null +++ b/apps/sim/lib/webhooks/execution-principal.test.ts @@ -0,0 +1,64 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + assertWebhookExecutionPrincipal, + createWebhookExecutionPrincipal, +} from '@/lib/webhooks/execution-principal' + +describe('webhook execution principals', () => { + it('represents a generic webhook without inventing a person', () => { + const principal = createWebhookExecutionPrincipal({ + webhookId: 'webhook-1', + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + provider: 'generic', + }) + + expect(principal.subject).toBeUndefined() + expect(() => + assertWebhookExecutionPrincipal(principal, { + webhookId: 'webhook-1', + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + provider: 'generic', + }) + ).not.toThrow() + }) + + it('preserves a verified external webhook actor', () => { + const subject = { + kind: 'external_user' as const, + provider: 'slack', + tenantId: 'tenant-1', + subjectId: 'subject-1', + } + const principal = createWebhookExecutionPrincipal({ + webhookId: 'webhook-1', + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + provider: 'slack', + subject, + }) + + expect(principal.subject).toEqual(subject) + }) + + it('rejects an external actor from another provider', () => { + expect(() => + createWebhookExecutionPrincipal({ + webhookId: 'webhook-1', + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + provider: 'generic', + subject: { + kind: 'external_user', + provider: 'slack', + tenantId: 'tenant-1', + subjectId: 'subject-1', + }, + }) + ).toThrow('Webhook execution subject provider must match the webhook provider') + }) +}) diff --git a/apps/sim/lib/workflows/application/operations.test.ts b/apps/sim/lib/workflows/application/operations.test.ts index cde476e6379..c79db22ca80 100644 --- a/apps/sim/lib/workflows/application/operations.test.ts +++ b/apps/sim/lib/workflows/application/operations.test.ts @@ -56,6 +56,34 @@ describe('workflow operation registry', () => { expect(Object.isFrozen(workflowOperations.moveBulk)).toBe(true) }) + it('admits executor delegation only to workflow deployment operations', () => { + for (const operation of [ + workflowOperations.deploy, + workflowOperations.undeploy, + workflowOperations.activateVersion, + ]) { + expect(operation).toMatchObject({ + minimumRole: 'admin', + workspaceApiKey: 'deny', + principalKinds: ['session', 'personal_api_key', 'delegated'], + delegatedServices: ['copilot', 'executor'], + }) + } + + for (const operation of [workflowOperations.listVersions, workflowOperations.readVersion]) { + expect(operation).toMatchObject({ + minimumRole: 'read', + workspaceApiKey: 'allow', + principalKinds: ['session', 'personal_api_key', 'workspace_api_key', 'delegated'], + delegatedServices: ['copilot', 'executor'], + }) + } + + expect(workflowOperations.deployChat.delegatedServices).toEqual(['copilot']) + expect(workflowOperations.undeployChat.delegatedServices).toEqual(['copilot']) + expect(workflowOperations.revertVersion.delegatedServices).toEqual(['copilot']) + }) + /** * Toggling unauthenticated public execution removes the authentication * requirement from a deployed workflow, so it takes an accountable human: diff --git a/apps/sim/lib/workflows/application/operations.ts b/apps/sim/lib/workflows/application/operations.ts index 75270135b24..87591f517bc 100644 --- a/apps/sim/lib/workflows/application/operations.ts +++ b/apps/sim/lib/workflows/application/operations.ts @@ -15,6 +15,11 @@ const HUMAN_WORKFLOW_PRINCIPAL_POLICY = { delegatedServices: ['copilot'], } as const +const WORKFLOW_DEPLOYMENT_PRINCIPAL_POLICY = { + principalKinds: ['session', 'personal_api_key', 'delegated'], + delegatedServices: ['copilot', 'executor'], +} as const + const COPILOT_WORKFLOW_PRINCIPAL_POLICY = { principalKinds: ['delegated'], delegatedServices: ['copilot'], @@ -231,13 +236,13 @@ export const workflowOperations = { id: 'workflows.deploy', minimumRole: 'admin', workspaceApiKey: 'deny', - ...HUMAN_WORKFLOW_PRINCIPAL_POLICY, + ...WORKFLOW_DEPLOYMENT_PRINCIPAL_POLICY, }), undeploy: defineWorkspaceOperation({ id: 'workflows.undeploy', minimumRole: 'admin', workspaceApiKey: 'deny', - ...HUMAN_WORKFLOW_PRINCIPAL_POLICY, + ...WORKFLOW_DEPLOYMENT_PRINCIPAL_POLICY, }), deployChat: defineWorkspaceOperation({ id: 'workflows.chat.deploy', @@ -269,7 +274,7 @@ export const workflowOperations = { id: 'workflows.versions.activate', minimumRole: 'admin', workspaceApiKey: 'deny', - ...HUMAN_WORKFLOW_PRINCIPAL_POLICY, + ...WORKFLOW_DEPLOYMENT_PRINCIPAL_POLICY, }), revertVersion: defineWorkspaceOperation({ id: 'workflows.versions.revert', @@ -287,13 +292,13 @@ export const workflowOperations = { id: 'workflows.versions.list', minimumRole: 'read', workspaceApiKey: 'allow', - ...ALL_WORKFLOW_PRINCIPAL_POLICY, + ...WORKFLOW_READ_PRINCIPAL_POLICY, }), readVersion: defineWorkspaceOperation({ id: 'workflows.versions.read', minimumRole: 'read', workspaceApiKey: 'allow', - ...ALL_WORKFLOW_PRINCIPAL_POLICY, + ...WORKFLOW_READ_PRINCIPAL_POLICY, }), compareReferences: defineWorkspaceOperation({ id: 'workflows.versions.compare_references', diff --git a/apps/sim/lib/workflows/application/run-workflow-from-copilot.test.ts b/apps/sim/lib/workflows/application/run-workflow-from-copilot.test.ts index 90d3bf6a3bf..593516912b4 100644 --- a/apps/sim/lib/workflows/application/run-workflow-from-copilot.test.ts +++ b/apps/sim/lib/workflows/application/run-workflow-from-copilot.test.ts @@ -68,6 +68,7 @@ import { runFromBlockFromCopilot, runWorkflowFromCopilot, } from '@/lib/workflows/application/run-workflow-from-copilot' +import { readAttemptedExecutionId } from '@/executor/utils/errors' const principal = { kind: 'delegated' as const, @@ -305,4 +306,89 @@ describe('Copilot workflow run application commands', () => { }) ).rejects.toThrow('database unavailable') }) + + /** + * A caller whose result was withheld decides about retry from one fact: whether a run + * exists. This layer owns that answer, because it is the last place that can distinguish + * "we never handed the work to the executor" from "we did". + * + * Deliberately coarse. A preflight refusal inside `executeWorkflow` also names the run, + * costing the caller one lookup; establishing anything finer would take a callback on + * every block of every execution in the product. + */ + describe('naming the run a failure belongs to', () => { + const runInput = { + workflowId: 'workflow-1', + useDraftState: true, + lifecycle, + hasWorkflowInput: false, + useMockPayload: true, + } + + const failWith = (input = runInput) => + runWorkflowFromCopilot.execute({ principal, input }).catch((thrown) => thrown) + + it('names the run once it has been handed to the executor', async () => { + mocks.executeWorkflow.mockRejectedValueOnce(new Error('database unavailable')) + + expect(readAttemptedExecutionId(await failWith())).toBe('child-execution-1') + }) + + /** + * Deliberate, and the one place this contract is deliberately coarse: `executeWorkflow` + * validates its own arguments before creating anything, and those failures still name + * the run. `attempted` means "zero or one executions exist under this id, resolve it", + * so the caller resolves, finds nothing, and retries — correct, at the cost of a lookup. + * + * Paying to avoid that lookup means an executor-side dispatch marker, which is a + * callback on every block of every execution in the product. It would also buy nothing: + * all four preflight throws are invariant violations — no workspace id, no billing + * attribution, no principal, attribution mismatch — so a retry fails identically. + */ + it('names the run for a failure inside the executor call, whatever its cause', async () => { + mocks.executeWorkflow.mockRejectedValueOnce( + new Error('Billing attribution is required for workspace execution') + ) + + expect(readAttemptedExecutionId(await failWith())).toBe('child-execution-1') + }) + + it('names the run when the crossing threw after it already returned', async () => { + // Only the post-run crossing throws; the catch re-enters this same method to record + // the failed crossing, and throwing again there would replace the error the id is on. + let crossings = 0 + const registry = { + exportProvenanceForValue: () => undefined, + beginPendingActivation: () => () => {}, + importCrossingProvenance: () => { + if (crossings++ === 0) throw new Error('crossing import failed') + }, + } + + const error = await failWith({ + ...runInput, + lifecycle: { resolvedSecretTraceRegistry: registry }, + } as typeof runInput) + + expect(readAttemptedExecutionId(error)).toBe('child-execution-1') + }) + + it('names nothing when admission refused the run before it could start', async () => { + mocks.admission.mockRejectedValueOnce(new Error('Usage limit exceeded')) + + const error = await failWith() + + expect(mocks.executeWorkflow).not.toHaveBeenCalled() + expect(readAttemptedExecutionId(error)).toBeUndefined() + }) + + it('names nothing when authorization refused the run', async () => { + mocks.permission.mockResolvedValue('read') + + const error = await failWith() + + expect(mocks.admission).not.toHaveBeenCalled() + expect(readAttemptedExecutionId(error)).toBeUndefined() + }) + }) }) diff --git a/apps/sim/lib/workflows/application/run-workflow-from-copilot.ts b/apps/sim/lib/workflows/application/run-workflow-from-copilot.ts index 32a4066041e..6467156f445 100644 --- a/apps/sim/lib/workflows/application/run-workflow-from-copilot.ts +++ b/apps/sim/lib/workflows/application/run-workflow-from-copilot.ts @@ -1,4 +1,5 @@ import { type Principal, requirePrincipalSubjectUserId } from '@sim/auth/principal' +import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { mergeSubblockStateWithValues } from '@sim/workflow-persistence/subblocks' @@ -28,6 +29,10 @@ import { } from '@/lib/workflows/triggers/run-options' import type { SerializableExecutionState } from '@/executor/execution/types' import type { ExecutionResult } from '@/executor/types' +import { attachAttemptedExecutionId } from '@/executor/utils/errors' + +const logger = createLogger('CopilotWorkflowRun') + import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' export interface CopilotWorkflowRunLifecycle { @@ -245,6 +250,17 @@ async function executeCopilotRun(params: { params.executionInput ) const completePendingActivation = registry?.beginPendingActivation() + /** + * The executor call is the first statement of this `try`, so everything caught below is + * post-dispatch by construction, while authorization, admission and provenance export all + * throw past this function having created nothing. That asymmetry is the whole of what a + * caller needs: no id means nothing exists, an id means resolve it before retrying. + * + * Deliberately no finer. Establishing whether a particular block ran would take a callback + * on every block of every execution in the product, to spare this one caller a lookup it + * can already make with the id it was handed. Keep the executor call first: anything + * inserted above it would be reported as a run that may exist. + */ try { const result = await executeWorkflow( { @@ -295,6 +311,19 @@ async function executeCopilotRun(params: { } return result } catch (error) { + /** + * `executeWorkflow` names the run itself once it crosses its own dispatch boundary, so + * preflight failures inside it correctly carry nothing. This covers only the window it + * cannot see: a failure after the run already returned, where the crossing import is + * what threw and an execution certainly exists. + */ + attachAttemptedExecutionId(error, childExecutionId) + /** + * Recovery must never replace the failure it is describing. Both steps below run only to + * record and release, and either throwing would propagate a different error — one the + * dispatched-run id was never recorded against — so an existing run would report itself + * as never started and invite the duplicate this id exists to prevent. + */ if (registry) { const executionResult = typeof error === 'object' && @@ -303,18 +332,34 @@ async function executeCopilotRun(params: { typeof error.executionResult === 'object' ? (error.executionResult as ExecutionResult) : undefined - await registry.importCrossingProvenance( - executionResult?.executionState?.resolvedSecretTraceProvenance, - { - output: executionResult?.output, - logs: executionResult?.logs, - error: executionResult?.error, - thrownMessage: toError(error).message, - }, - { trusted: true, origin: 'copilotWorkflowMutation.failedRunCrossing' } - ) + try { + await registry.importCrossingProvenance( + executionResult?.executionState?.resolvedSecretTraceProvenance, + { + output: executionResult?.output, + logs: executionResult?.logs, + error: executionResult?.error, + thrownMessage: toError(error).message, + }, + { trusted: true, origin: 'copilotWorkflowMutation.failedRunCrossing' } + ) + } catch (importError) { + logger.error('Failed to record provenance for a failed Copilot run', { + executionId: childExecutionId, + error: toError(importError).message, + }) + } + } + if (admission.targetReservation) { + try { + await releaseExecutionSlot(childExecutionId) + } catch (releaseError) { + logger.error('Failed to release the execution slot for a failed Copilot run', { + executionId: childExecutionId, + error: toError(releaseError).message, + }) + } } - if (admission.targetReservation) await releaseExecutionSlot(childExecutionId) throw error } finally { completePendingActivation?.() diff --git a/apps/sim/lib/workflows/application/workflow-deployments.test.ts b/apps/sim/lib/workflows/application/workflow-deployments.test.ts index add3175f73f..440d8bc4f37 100644 --- a/apps/sim/lib/workflows/application/workflow-deployments.test.ts +++ b/apps/sim/lib/workflows/application/workflow-deployments.test.ts @@ -189,30 +189,49 @@ describe('workflow deployment application use cases', () => { expect(mocks.deploy).not.toHaveBeenCalled() }) - it('rejects executor deployment transitions before canonical lookup', async () => { - await expect( - deployWorkflow.execute({ - principal: { + it('admits executor deployment transitions through canonical workflow authorization', async () => { + await deployWorkflow.execute({ + principal: { + kind: 'delegated', + serviceId: 'executor', + subjectUserId: 'user-1', + workspaceId: 'workspace-1', + delegationId: 'executor-1', + audience: 'sim:workflows', + issuedAt: new Date('2026-08-08T00:00:00Z'), + expiresAt: new Date('2999-08-08T00:00:00Z'), + delegationContext: { + kind: 'workflow_execution', + workflowId: 'origin-workflow', + executionId: 'execution-1', + }, + }, + input: { workflowId: 'workflow-1', requestId: 'request-1' }, + }) + + expect(mocks.resolveContext).toHaveBeenCalledWith({ + workflowId: 'workflow-1', + assertedWorkspaceId: undefined, + }) + expect(mocks.resolvePermission).toHaveBeenCalledWith('user-1', 'workspace-1', null, undefined, { + forUpdate: undefined, + }) + expect(mocks.assertMutable).toHaveBeenCalledWith('workflow-1') + expect(mocks.deploy).toHaveBeenCalledWith( + expect.objectContaining({ + workflowId: 'workflow-1', + userId: 'user-1', + actorId: 'user-1', + actor: { kind: 'delegated', serviceId: 'executor', subjectUserId: 'user-1', - workspaceId: 'workspace-1', delegationId: 'executor-1', - audience: 'sim:workflows', - issuedAt: new Date('2026-08-08T00:00:00Z'), - expiresAt: new Date('2999-08-08T00:00:00Z'), - delegationContext: { - kind: 'workflow_execution', - workflowId: 'workflow-1', - executionId: 'execution-1', - }, }, - input: { workflowId: 'workflow-1', requestId: 'request-1' }, + captureAnalytics: false, + requestId: 'request-1', }) - ).rejects.toMatchObject({ code: 'forbidden' }) - - expect(mocks.resolveContext).not.toHaveBeenCalled() - expect(mocks.deploy).not.toHaveBeenCalled() + ) }) it('requires current admin permission before deployment', async () => { diff --git a/apps/sim/lib/workflows/custom-tools/available-lookup.test.ts b/apps/sim/lib/workflows/custom-tools/available-lookup.test.ts new file mode 100644 index 00000000000..5fb6a6d59b5 --- /dev/null +++ b/apps/sim/lib/workflows/custom-tools/available-lookup.test.ts @@ -0,0 +1,86 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mocks } = vi.hoisted(() => ({ + mocks: { + select: vi.fn(), + }, +})) + +vi.mock('@sim/db', () => ({ + db: { select: mocks.select }, +})) + +import { getAvailableCustomTool } from '@/lib/workflows/custom-tools/operations' + +const workspaceTool = { + id: 'workspace-tool', + workspaceId: 'workspace-1', + userId: 'user-2', + title: 'lookup_order', +} +const personalTool = { + id: 'personal-tool', + workspaceId: null, + userId: 'user-1', + title: 'lookup_order', +} + +function selection(rows: unknown[]) { + const limit = vi.fn().mockResolvedValue(rows) + const where = vi.fn().mockReturnValue({ limit }) + const from = vi.fn().mockReturnValue({ where }) + return { from } +} + +describe('getAvailableCustomTool', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('returns the workspace tool without querying the personal fallback', async () => { + mocks.select.mockReturnValueOnce(selection([workspaceTool])) + + await expect( + getAvailableCustomTool({ + identifier: workspaceTool.title, + userId: personalTool.userId, + workspaceId: workspaceTool.workspaceId, + lookup: 'id_or_title', + }) + ).resolves.toEqual(workspaceTool) + + expect(mocks.select).toHaveBeenCalledTimes(1) + }) + + it('queries the authenticated subject personal fallback only after a workspace miss', async () => { + mocks.select.mockReturnValueOnce(selection([])).mockReturnValueOnce(selection([personalTool])) + + await expect( + getAvailableCustomTool({ + identifier: personalTool.id, + userId: personalTool.userId, + workspaceId: workspaceTool.workspaceId, + lookup: 'id', + }) + ).resolves.toEqual(personalTool) + + expect(mocks.select).toHaveBeenCalledTimes(2) + }) + + it('does not expose a personal fallback to an actorless workflow execution', async () => { + mocks.select.mockReturnValueOnce(selection([])) + + await expect( + getAvailableCustomTool({ + identifier: personalTool.id, + workspaceId: workspaceTool.workspaceId, + lookup: 'id', + }) + ).resolves.toBeNull() + + expect(mocks.select).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/sim/lib/workflows/custom-tools/operations.ts b/apps/sim/lib/workflows/custom-tools/operations.ts index 2cd4b4cda57..565e49735c1 100644 --- a/apps/sim/lib/workflows/custom-tools/operations.ts +++ b/apps/sim/lib/workflows/custom-tools/operations.ts @@ -275,60 +275,67 @@ export async function deleteWorkspaceCustomTool(params: { return deleted.length > 0 } -export async function getCustomToolById(params: { - toolId: string - userId: string - workspaceId?: string +export type AvailableCustomToolLookup = 'id' | 'id_or_title' + +export async function getAvailableCustomTool(params: { + identifier: string + userId?: string + workspaceId: string + lookup: AvailableCustomToolLookup }) { - const { toolId, userId, workspaceId } = params + const identifierCondition = + params.lookup === 'id' + ? eq(customTools.id, params.identifier) + : or(eq(customTools.id, params.identifier), eq(customTools.title, params.identifier)) - if (workspaceId) { - const workspaceTool = await db - .select() - .from(customTools) - .where(and(eq(customTools.id, toolId), eq(customTools.workspaceId, workspaceId))) - .limit(1) - if (workspaceTool[0]) return workspaceTool[0] - } + const workspaceTool = await db + .select() + .from(customTools) + .where(and(eq(customTools.workspaceId, params.workspaceId), identifierCondition)) + .limit(1) + if (workspaceTool[0]) return workspaceTool[0] + if (!params.userId) return null const legacyTool = await db .select() .from(customTools) .where( and( - eq(customTools.id, toolId), isNull(customTools.workspaceId), - eq(customTools.userId, userId) + eq(customTools.userId, params.userId), + identifierCondition ) ) .limit(1) return legacyTool[0] || null } -export async function getCustomToolByIdOrTitle(params: { - identifier: string +export async function getCustomToolById(params: { + toolId: string userId: string workspaceId?: string }) { - const { identifier, userId, workspaceId } = params - - const conditions = [or(eq(customTools.id, identifier), eq(customTools.title, identifier))] - - if (workspaceId) { - const workspaceTool = await db + if (!params.workspaceId) { + const [legacyTool] = await db .select() .from(customTools) - .where(and(eq(customTools.workspaceId, workspaceId), ...conditions)) + .where( + and( + eq(customTools.id, params.toolId), + isNull(customTools.workspaceId), + eq(customTools.userId, params.userId) + ) + ) .limit(1) - if (workspaceTool[0]) return workspaceTool[0] + return legacyTool ?? null } - const legacyTool = await db - .select() - .from(customTools) - .where(and(isNull(customTools.workspaceId), eq(customTools.userId, userId), ...conditions)) - .limit(1) - return legacyTool[0] || null + return getAvailableCustomTool({ + identifier: params.toolId, + userId: params.userId, + workspaceId: params.workspaceId, + lookup: 'id', + }) } export async function updateCustomTool(params: { diff --git a/apps/sim/lib/workflows/executor/execution-core.test.ts b/apps/sim/lib/workflows/executor/execution-core.test.ts index 20b2d2eee03..666493bf95a 100644 --- a/apps/sim/lib/workflows/executor/execution-core.test.ts +++ b/apps/sim/lib/workflows/executor/execution-core.test.ts @@ -1,3 +1,4 @@ +import type { WorkflowExecutionPrincipal } from '@sim/auth/principal' import { environmentUtilsMockFns, loggerMock, @@ -435,6 +436,107 @@ describe('executeWorkflowCore terminal finalization sequencing', () => { } ) + it.each([ + { + name: 'schedule', + principal: { + kind: 'system' as const, + serviceId: 'schedule' as const, + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + }, + triggerType: 'schedule', + isPublicApiAccess: false, + }, + { + name: 'webhook with a verified external subject', + principal: { + kind: 'system' as const, + serviceId: 'webhook' as const, + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + webhookId: 'webhook-1', + provider: 'slack', + subject: { + kind: 'external_user' as const, + provider: 'slack', + tenantId: 'team-1', + subjectId: 'slack-user-1', + }, + }, + triggerType: 'webhook', + isPublicApiAccess: false, + }, + { + name: 'workspace API key', + principal: { + kind: 'workspace_api_key' as const, + workspaceId: 'workspace-1', + keyId: 'workspace-key-1', + }, + triggerType: 'api', + isPublicApiAccess: false, + }, + { + name: 'anonymous public API', + principal: { + kind: 'system' as const, + serviceId: 'public_api' as const, + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + }, + triggerType: 'api', + isPublicApiAccess: true, + }, + ] satisfies Array<{ + name: string + principal: WorkflowExecutionPrincipal + triggerType: 'api' | 'schedule' | 'webhook' + isPublicApiAccess: boolean + }>)( + 'preserves the exact $name principal and deployed workflow authority in executor delegation', + async ({ principal, triggerType, isPublicApiAccess }) => { + executorExecuteMock.mockResolvedValue({ + success: true, + status: 'completed', + output: { done: true }, + logs: [], + metadata: { duration: 123, startTime: 'start', endTime: 'end' }, + }) + + const snapshot = createSnapshot() + await executeWorkflowCore({ + snapshot: { + ...snapshot, + metadata: { + ...snapshot.metadata, + userId: 'billing-actor', + principal, + triggerType, + useDraftState: false, + isPublicApiAccess, + }, + } as any, + callbacks: {}, + loggingSession: loggingSession as any, + }) + + const contextExtensions = executorConstructorMock.mock.calls[0]?.[0]?.contextExtensions + expect(contextExtensions.principal).toBe(principal) + expect(contextExtensions.executorDelegationOrigin.principal).toBe(principal) + expect(contextExtensions.executorDelegationOrigin).toEqual({ + workflowId: 'workflow-1', + executionId: 'execution-1', + principal, + currentWorkflow: { + workflowId: 'workflow-1', + mode: 'deployment', + deploymentVersionId: 'dep-1', + }, + }) + } + ) + it('starts logging with the workflow state that will be executed', async () => { const executedWorkflowState = { blocks: { diff --git a/apps/sim/lib/workflows/migrations/subblock-migrations.ts b/apps/sim/lib/workflows/migrations/subblock-migrations.ts index f6edd10576e..52f91909997 100644 --- a/apps/sim/lib/workflows/migrations/subblock-migrations.ts +++ b/apps/sim/lib/workflows/migrations/subblock-migrations.ts @@ -292,6 +292,13 @@ export const SUBBLOCK_ID_MIGRATIONS: Record { + it('preserves actorless workflow identity and deployment authority', () => { + const expiresAt = new Date(Date.now() + 60_000) + const delegationContext = { + kind: 'workflow_execution' as const, + workflowId: 'workflow-1', + executionId: 'execution-1', + principal: { + kind: 'system' as const, + serviceId: 'schedule' as const, + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + }, + currentWorkflow: { + workflowId: 'workflow-1', + mode: 'deployment' as const, + deploymentVersionId: 'deployment-1', + }, + } + + const rebound = rebindWorkspaceFileDelegatedPrincipal({ + principal: { + kind: 'delegated', + serviceId: 'executor', + workspaceId: 'workspace-1', + delegationId: 'function-1', + audience: 'sim:function-executions', + issuedAt: new Date(Date.now() - 1_000), + expiresAt, + delegationContext, + }, + workspaceId: 'workspace-1', + delegationId: 'file-1', + executionId: 'execution-1', + }) + + expect(rebound).toMatchObject({ + serviceId: 'executor', + workspaceId: 'workspace-1', + delegationId: 'file-1', + audience: WORKSPACE_FILES_DELEGATION_AUDIENCE, + resourceScope: { executionId: 'execution-1' }, + delegationContext, + }) + expect(rebound.expiresAt).toEqual(expiresAt) + expect(rebound).not.toHaveProperty('subjectUserId') + }) + + it('rejects a cross-workspace rebind', () => { + expect(() => + rebindWorkspaceFileDelegatedPrincipal({ + principal: { + kind: 'delegated', + serviceId: 'copilot', + subjectUserId: 'user-1', + workspaceId: 'workspace-1', + delegationId: 'copilot-1', + audience: 'sim:function-executions', + issuedAt: new Date(), + expiresAt: new Date(Date.now() + 60_000), + }, + workspaceId: 'workspace-2', + delegationId: 'file-1', + }) + ).toThrow('Workspace file delegation does not match its authorized workspace') + }) +}) diff --git a/apps/sim/lib/workspace-files/application/delegated-principal.ts b/apps/sim/lib/workspace-files/application/delegated-principal.ts index 7cb5040c86c..664fde027d0 100644 --- a/apps/sim/lib/workspace-files/application/delegated-principal.ts +++ b/apps/sim/lib/workspace-files/application/delegated-principal.ts @@ -37,3 +37,40 @@ export function createWorkspaceFileDelegatedPrincipal( }, } } + +export interface RebindWorkspaceFileDelegationInput { + principal: DelegatedPrincipal + workspaceId: string + delegationId: string + fileId?: string + chatId?: string + executionId?: string +} + +/** Rebinds an already-authorized service principal without changing its workflow actor. */ +export function rebindWorkspaceFileDelegatedPrincipal( + input: RebindWorkspaceFileDelegationInput +): DelegatedPrincipal { + if (input.principal.workspaceId !== input.workspaceId || !input.delegationId) { + throw new Error('Workspace file delegation does not match its authorized workspace') + } + const issuedAt = new Date() + return { + ...input.principal, + workspaceId: input.workspaceId, + delegationId: input.delegationId, + audience: WORKSPACE_FILES_DELEGATION_AUDIENCE, + issuedAt, + expiresAt: new Date( + Math.min( + input.principal.expiresAt.getTime(), + issuedAt.getTime() + WORKSPACE_FILE_DELEGATION_TTL_MS + ) + ), + resourceScope: { + ...(input.fileId ? { fileId: input.fileId } : {}), + ...(input.chatId ? { chatId: input.chatId } : {}), + ...(input.executionId ? { executionId: input.executionId } : {}), + }, + } +} diff --git a/apps/sim/lib/workspace-files/application/operations.test.ts b/apps/sim/lib/workspace-files/application/operations.test.ts index 26b65ebab15..d07f3edebfe 100644 --- a/apps/sim/lib/workspace-files/application/operations.test.ts +++ b/apps/sim/lib/workspace-files/application/operations.test.ts @@ -48,6 +48,7 @@ describe('file operation registry', () => { 'files.create', 'files.update_content', 'files.move', + 'files.share.read', 'files.share.update', 'files.folders.create', ]) diff --git a/apps/sim/lib/workspace-files/application/operations.ts b/apps/sim/lib/workspace-files/application/operations.ts index 1fad2018f4d..2752a576521 100644 --- a/apps/sim/lib/workspace-files/application/operations.ts +++ b/apps/sim/lib/workspace-files/application/operations.ts @@ -132,7 +132,7 @@ export const fileOperations = { id: 'files.share.read', minimumRole: 'read', workspaceApiKey: 'allow', - ...ALL_COPILOT_PRINCIPAL_POLICY, + ...ALL_FILE_TOOL_PRINCIPAL_POLICY, }), updateShare: defineWorkspaceOperation({ id: 'files.share.update', diff --git a/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.test.ts b/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.test.ts index ca1ca6b784d..d635047deb8 100644 --- a/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.test.ts +++ b/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.test.ts @@ -27,7 +27,10 @@ vi.mock('@sim/platform-authz/workspace', () => ({ })) import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' -import { readWorkspaceFileContentByKey } from '@/lib/workspace-files/application/read-workspace-file-content-by-key' +import { + readWorkspaceFileContentByKey, + readWorkspaceFileRecordByKey, +} from '@/lib/workspace-files/application/read-workspace-file-content-by-key' const principal = { kind: 'session' as const, userId: 'user-1', sessionId: 'session-1' } const context = { @@ -92,4 +95,74 @@ describe('readWorkspaceFileContentByKey', () => { ).rejects.toMatchObject({ code: 'not_found' }) expect(mocks.fetchContent).not.toHaveBeenCalled() }) + + it('authorizes an exact-key record read for a workspace API key without a human fallback', async () => { + await expect( + readWorkspaceFileRecordByKey.execute({ + principal: { + kind: 'workspace_api_key', + workspaceId: file.workspaceId, + keyId: 'key-1', + }, + input: { key: file.key, assertedWorkspaceId: file.workspaceId }, + }) + ).resolves.toEqual({ file }) + + expect(mocks.getMetadata).toHaveBeenCalledWith(file.key, 'workspace') + expect(mocks.loadContext).toHaveBeenCalledWith(file.id) + expect(mocks.resolvePermission).not.toHaveBeenCalled() + expect(mocks.getFile).toHaveBeenCalledWith(file.workspaceId, file.id, { + throwOnError: true, + }) + expect(mocks.fetchContent).not.toHaveBeenCalled() + }) + + it('authorizes an actorless deployment executor by its preserved workflow authority', async () => { + await expect( + readWorkspaceFileRecordByKey.execute({ + principal: { + kind: 'delegated', + serviceId: 'executor', + workspaceId: file.workspaceId, + delegationId: 'execution-file-read:request-1', + audience: 'sim:workspace-files', + issuedAt: new Date(Date.now() - 1_000), + expiresAt: new Date(Date.now() + 60_000), + delegationContext: { + kind: 'workflow_execution', + workflowId: 'workflow-1', + executionId: 'execution-1', + principal: { + kind: 'system', + serviceId: 'schedule', + workspaceId: file.workspaceId, + workflowId: 'workflow-1', + }, + currentWorkflow: { + workflowId: 'workflow-1', + mode: 'deployment', + deploymentVersionId: 'deployment-1', + }, + }, + }, + input: { key: file.key, assertedWorkspaceId: file.workspaceId }, + }) + ).resolves.toEqual({ file }) + + expect(mocks.resolvePermission).not.toHaveBeenCalled() + expect(mocks.fetchContent).not.toHaveBeenCalled() + }) + + it('conceals an exact key asserted under a different workspace before authorization', async () => { + await expect( + readWorkspaceFileRecordByKey.execute({ + principal, + input: { key: file.key, assertedWorkspaceId: 'workspace-other' }, + }) + ).rejects.toMatchObject({ code: 'not_found' }) + + expect(mocks.loadContext).not.toHaveBeenCalled() + expect(mocks.resolvePermission).not.toHaveBeenCalled() + expect(mocks.getFile).not.toHaveBeenCalled() + }) }) diff --git a/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.ts b/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.ts index 4a669fd940f..066c49022a2 100644 --- a/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.ts +++ b/apps/sim/lib/workspace-files/application/read-workspace-file-content-by-key.ts @@ -12,7 +12,7 @@ import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case' import { fileOperations } from '@/lib/workspace-files/application/operations' -export interface ReadWorkspaceFileContentByKeyInput { +export interface ReadWorkspaceFileByKeyInput { key: string assertedWorkspaceId?: string } @@ -22,40 +22,65 @@ export interface ReadWorkspaceFileContentByKeyResult { content: Buffer } +export interface ReadWorkspaceFileRecordByKeyResult { + file: WorkspaceFileRecord +} + +async function loadCurrentWorkspaceFileByKey( + input: ReadWorkspaceFileByKeyInput, + context: ActiveWorkspaceFileContext +): Promise { + const file = await getWorkspaceFile(context.workspaceId, context.fileId, { + throwOnError: true, + }) + if (!file || file.key !== input.key) throw new OrchestrationError('not_found', 'File not found') + return file +} + async function executeReadWorkspaceFileContentByKey({ input, context, }: AuthorizedWorkspaceUseCaseContext< typeof fileOperations.readContent, - ReadWorkspaceFileContentByKeyInput, + ReadWorkspaceFileByKeyInput, ActiveWorkspaceFileContext >): Promise { - const file = await getWorkspaceFile(context.workspaceId, context.fileId, { - throwOnError: true, - }) - if (!file || file.key !== input.key) throw new OrchestrationError('not_found', 'File not found') + const file = await loadCurrentWorkspaceFileByKey(input, context) return { file, content: await fetchWorkspaceFileBuffer(file, { maxBytes: MAX_BUFFERED_TRANSFER_BYTES }), } } -export const readWorkspaceFileContentByKey = defineAuthorizedWorkspaceFileUseCase({ +async function resolveWorkspaceFileByKeyContext({ + input, +}: { + input: ReadWorkspaceFileByKeyInput +}): Promise { + const metadata = await getFileMetadataByKey(input.key, 'workspace') + if ( + !metadata?.workspaceId || + (input.assertedWorkspaceId !== undefined && input.assertedWorkspaceId !== metadata.workspaceId) + ) { + throw new OrchestrationError('not_found', 'File not found') + } + const canonical = await loadActiveWorkspaceFileContext(metadata.id) + if (!canonical || canonical.workspaceId !== metadata.workspaceId) { + throw new OrchestrationError('not_found', 'File not found') + } + return canonical +} + +export const readWorkspaceFileRecordByKey = defineAuthorizedWorkspaceFileUseCase({ operation: fileOperations.readContent, - async resolveContext({ input }) { - const metadata = await getFileMetadataByKey(input.key, 'workspace') - if ( - !metadata?.workspaceId || - (input.assertedWorkspaceId !== undefined && - input.assertedWorkspaceId !== metadata.workspaceId) - ) { - throw new OrchestrationError('not_found', 'File not found') - } - const canonical = await loadActiveWorkspaceFileContext(metadata.id) - if (!canonical || canonical.workspaceId !== metadata.workspaceId) { - throw new OrchestrationError('not_found', 'File not found') - } - return canonical + resolveContext: resolveWorkspaceFileByKeyContext, + async execute({ input, context }): Promise { + return { file: await loadCurrentWorkspaceFileByKey(input, context) } }, +}) + +export const readWorkspaceFileContentByKey = defineAuthorizedWorkspaceFileUseCase({ + operation: fileOperations.readContent, + resolveContext: resolveWorkspaceFileByKeyContext, execute: executeReadWorkspaceFileContentByKey, }) diff --git a/apps/sim/lib/workspace-files/application/read-workspace-file-name-by-key.ts b/apps/sim/lib/workspace-files/application/read-workspace-file-name-by-key.ts new file mode 100644 index 00000000000..64f28a5e5a0 --- /dev/null +++ b/apps/sim/lib/workspace-files/application/read-workspace-file-name-by-key.ts @@ -0,0 +1,31 @@ +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { loadActiveWorkspaceContext } from '@/lib/uploads/contexts/workspace' +import { getFileMetadataByKey } from '@/lib/uploads/server/metadata' +import { isWorkspaceScopedContext } from '@/lib/uploads/shared/types' +import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case' +import { fileOperations } from '@/lib/workspace-files/application/operations' + +export interface ReadWorkspaceFileNameByKeyInput { + workspaceId: string + key: string +} + +export const readWorkspaceFileNameByKey = defineAuthorizedWorkspaceFileUseCase({ + operation: fileOperations.readMetadata, + async resolveContext({ input }: { input: ReadWorkspaceFileNameByKeyInput }) { + const workspace = await loadActiveWorkspaceContext(input.workspaceId) + if (!workspace) throw new OrchestrationError('not_found', 'Workspace not found') + return workspace + }, + async execute({ input, context }): Promise<{ name: string | null }> { + const metadata = await getFileMetadataByKey(input.key) + if ( + !metadata || + metadata.workspaceId !== context.workspaceId || + !isWorkspaceScopedContext(metadata.context) + ) { + return { name: null } + } + return { name: metadata.originalName } + }, +}) diff --git a/apps/sim/lib/workspace-files/application/read-workspace-file-secret-provenance.ts b/apps/sim/lib/workspace-files/application/read-workspace-file-secret-provenance.ts new file mode 100644 index 00000000000..45075640fa7 --- /dev/null +++ b/apps/sim/lib/workspace-files/application/read-workspace-file-secret-provenance.ts @@ -0,0 +1,35 @@ +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { getWorkspaceFile } from '@/lib/uploads/contexts/workspace/workspace-file-manager' +import { + getBoundWorkspaceFileSecretProvenance, + type WorkspaceFileSecretProvenance, +} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case' +import { fileOperations } from '@/lib/workspace-files/application/operations' +import { resolveActiveWorkspaceFileContext } from '@/lib/workspace-files/application/workspace-file-context' + +export interface ReadWorkspaceFileSecretProvenanceInput { + fileId: string + assertedWorkspaceId?: string +} + +export const readWorkspaceFileSecretProvenance = defineAuthorizedWorkspaceFileUseCase({ + operation: fileOperations.readContent, + resolveContext: ({ input }: { input: ReadWorkspaceFileSecretProvenanceInput }) => + resolveActiveWorkspaceFileContext(input), + async execute({ input, context }): Promise<{ + provenance: WorkspaceFileSecretProvenance + ownerUserId: string + }> { + const file = await getWorkspaceFile(context.workspaceId, context.fileId, { throwOnError: true }) + if (!file) throw new OrchestrationError('not_found', 'File not found') + return { + provenance: await getBoundWorkspaceFileSecretProvenance(context.workspaceId, { + fileId: file.id, + key: file.key, + context: 'workspace', + }), + ownerUserId: file.uploadedBy, + } + }, +}) diff --git a/apps/sim/lib/workspace-files/shell-layout.test.ts b/apps/sim/lib/workspace-files/shell-layout.test.ts index 493cabb462d..edb5e249d18 100644 --- a/apps/sim/lib/workspace-files/shell-layout.test.ts +++ b/apps/sim/lib/workspace-files/shell-layout.test.ts @@ -17,7 +17,6 @@ function runShell(source: string) { .replace(/^\n/, '') .replace(/<\/html>$/, '') const script = SIM_ARTIFACT_SHELL.replace(/^